From 94f798e217c29f613ba4e19754be3fbb51d1366f Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 07:04:43 +0000 Subject: [PATCH 01/20] Add counterfactual PointMass MVP --- .github/ISSUE_TEMPLATE/adapter_proposal.yml | 42 ++ .github/ISSUE_TEMPLATE/bug_report.yml | 35 ++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/metric_proposal.yml | 32 ++ .github/PULL_REQUEST_TEMPLATE.md | 20 + .github/workflows/ci.yml | 34 ++ CHANGELOG.md | 14 + CITATION.cff | 10 + CODE_OF_CONDUCT.md | 12 + CONTRIBUTING.md | 34 ++ LICENSE | 201 +++++++++ README.md | 116 ++++- SECURITY.md | 15 + docs/QUICKSTART.md | 48 ++ docs/rfcs/0001-scope-and-capabilities.md | 33 ++ docs/rfcs/0002-counterfactual-metrics.md | 34 ++ examples/pointmass-demo/report.md | 14 + examples/pointmass-demo/summary.json | 354 +++++++++++++++ pyproject.toml | 31 +- schemas/.gitkeep | 1 - schemas/capability-v0.1.schema.json | 20 + schemas/intervention-v0.1.schema.json | 44 ++ schemas/result-v0.1.schema.json | 37 ++ src/wamprobe/__init__.py | 4 +- src/wamprobe/__main__.py | 5 + src/wamprobe/adapters/__init__.py | 15 + src/wamprobe/adapters/baselines.py | 106 +++++ src/wamprobe/api/__init__.py | 24 + src/wamprobe/api/capabilities.py | 41 ++ src/wamprobe/api/errors.py | 13 + src/wamprobe/api/model.py | 36 ++ src/wamprobe/api/types.py | 111 +++++ src/wamprobe/benchmarks/__init__.py | 5 + src/wamprobe/benchmarks/pointmass.py | 74 ++++ src/wamprobe/cli.py | 69 +++ src/wamprobe/evaluation.py | 114 +++++ src/wamprobe/metrics.py | 84 ++++ src/wamprobe/reporting.py | 65 +++ tests/test_api.py | 29 ++ tests/test_cli.py | 40 ++ tests/test_evaluation.py | 37 ++ tests/test_import.py | 2 +- tests/test_pointmass.py | 28 ++ uv.lock | 467 ++++++++++++++++++++ 44 files changed, 2532 insertions(+), 23 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/adapter_proposal.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/metric_proposal.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 CITATION.cff create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 SECURITY.md create mode 100644 docs/QUICKSTART.md create mode 100644 docs/rfcs/0001-scope-and-capabilities.md create mode 100644 docs/rfcs/0002-counterfactual-metrics.md create mode 100644 examples/pointmass-demo/report.md create mode 100644 examples/pointmass-demo/summary.json delete mode 100644 schemas/.gitkeep create mode 100644 schemas/capability-v0.1.schema.json create mode 100644 schemas/intervention-v0.1.schema.json create mode 100644 schemas/result-v0.1.schema.json create mode 100644 src/wamprobe/__main__.py create mode 100644 src/wamprobe/adapters/__init__.py create mode 100644 src/wamprobe/adapters/baselines.py create mode 100644 src/wamprobe/api/__init__.py create mode 100644 src/wamprobe/api/capabilities.py create mode 100644 src/wamprobe/api/errors.py create mode 100644 src/wamprobe/api/model.py create mode 100644 src/wamprobe/api/types.py create mode 100644 src/wamprobe/benchmarks/__init__.py create mode 100644 src/wamprobe/benchmarks/pointmass.py create mode 100644 src/wamprobe/cli.py create mode 100644 src/wamprobe/evaluation.py create mode 100644 src/wamprobe/metrics.py create mode 100644 src/wamprobe/reporting.py create mode 100644 tests/test_api.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_evaluation.py create mode 100644 tests/test_pointmass.py create mode 100644 uv.lock diff --git a/.github/ISSUE_TEMPLATE/adapter_proposal.yml b/.github/ISSUE_TEMPLATE/adapter_proposal.yml new file mode 100644 index 0000000..a20ea4e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/adapter_proposal.yml @@ -0,0 +1,42 @@ +name: Adapter proposal +description: Propose integration of an open World Action Model +title: "[Adapter]: " +labels: ["area:adapter"] +body: + - type: input + id: repository + attributes: + label: Upstream repository + placeholder: https://github.com/owner/repository + validations: + required: true + - type: input + id: checkpoint + attributes: + label: Public checkpoint + validations: + required: true + - type: input + id: license + attributes: + label: Code and checkpoint licenses + validations: + required: true + - type: checkboxes + id: capabilities + attributes: + label: Exposed capabilities + options: + - label: Pixel future prediction + - label: State future prediction + - label: Latent future representation + - label: Action prediction + - label: Candidate action scoring + - label: World feature access + - type: textarea + id: smoke + attributes: + label: Smallest reproducible smoke path + description: Describe hardware, benchmark, and expected runtime. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..de4b0b6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,35 @@ +name: Bug report +description: Report reproducible incorrect behavior in WAMProbe +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: "Do not include credentials, private datasets, or model access tokens." + - type: textarea + id: description + attributes: + label: What happened? + description: Describe the observed and expected behavior. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Minimal reproduction + description: Include the command, resolved config, and smallest non-sensitive fixture. + validations: + required: true + - type: input + id: version + attributes: + label: WAMProbe version or commit + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Python, OS, hardware, and relevant adapter/upstream revisions. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d295553 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Security report + url: https://github.com/myheart521/WAMProbe/security/advisories/new + about: Report vulnerabilities or sensitive security concerns privately. diff --git a/.github/ISSUE_TEMPLATE/metric_proposal.yml b/.github/ISSUE_TEMPLATE/metric_proposal.yml new file mode 100644 index 0000000..d7b57c0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/metric_proposal.yml @@ -0,0 +1,32 @@ +name: Metric proposal +description: Propose a causal, dynamics, utility, uncertainty, or efficiency metric +title: "[Metric]: " +labels: ["area:metric", "needs-rfc"] +body: + - type: textarea + id: failure + attributes: + label: Failure mode + description: What concrete WAM failure does this metric detect? + validations: + required: true + - type: textarea + id: definition + attributes: + label: Definition and capability requirements + description: Include directionality, aggregation unit, and stochastic-model handling. + validations: + required: true + - type: textarea + id: gaming + attributes: + label: How can the metric be gamed? + validations: + required: true + - type: textarea + id: baselines + attributes: + label: Expected reference-baseline behavior + description: State the expected ordering for oracle, copy-last, wrong-direction, and action-agnostic baselines. + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..3f6588f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +## What changed + + + +## Why + + + +## Validation + +- [ ] `ruff format --check .` +- [ ] `ruff check .` +- [ ] `mypy` +- [ ] `pytest --cov=wamprobe --cov-report=term-missing` + +## Metric/adapter/benchmark checklist + +- [ ] Not applicable, or capability requirements are documented. +- [ ] Not applicable, or expected reference-baseline behavior is tested. +- [ ] Not applicable, or data/checkpoint license and upstream revision are documented. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f6921f8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + quality: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install project + run: python -m pip install -e '.[dev]' + - name: Check formatting + run: ruff format --check . + - name: Lint + run: ruff check . + - name: Type check + run: mypy + - name: Test with coverage + run: pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..772c93e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +All notable changes to WAMProbe will be documented here. + +## [Unreleased] + +### Added + +- Typed alpha `WAMAdapter` protocol and capability declaration. +- Analytic PointMass-2D paired counterfactual benchmark. +- Oracle, copy-last-frame, wrong-direction, and action-agnostic reference baselines. +- Action Dependence, Counterfactual Direction Accuracy, No-op Stability, state ADE, + and Top-1 Regret metrics. +- JSON/Markdown reports, schemas, CLI, CI, and contributor documentation. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..eb1c8c3 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,10 @@ +cff-version: 1.2.0 +message: "If you use WAMProbe, please cite the software release." +title: "WAMProbe: Counterfactual Evaluation for World Action Models" +type: software +version: 0.1.0-alpha +date-released: 2026-07-15 +license: Apache-2.0 +repository-code: "https://github.com/myheart521/WAMProbe" +authors: + - name: "WAMProbe contributors" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..2ba5382 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,12 @@ +# Code of conduct + +WAMProbe contributors are expected to keep discussions welcoming, technical, and respectful. +Harassment, discrimination, personal attacks, publication of private information, and +deliberate disruption are not acceptable. + +Maintainers may edit, hide, or remove contributions that violate these expectations and may +temporarily or permanently restrict participation. Report conduct concerns through a private +GitHub security/advisory channel when possible; do not disclose personal details publicly. + +This policy applies to repository issues, pull requests, discussions, and project-affiliated +community spaces. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b462031 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing to WAMProbe + +Thank you for helping make WAM evaluation more causal, reproducible, and control-grounded. + +## Development setup + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install -e '.[dev]' +``` + +Before opening a pull request, run: + +```bash +ruff format --check . +ruff check . +mypy +pytest --cov=wamprobe --cov-report=term-missing +``` + +## Contribution requirements + +- Follow test-driven development for behavior changes. +- Keep the core package CPU-only and avoid heavy mandatory dependencies. +- New adapters must publish a capability declaration and pin an upstream revision used by + their smoke test. +- New benchmarks must document generation, splits, licenses, and shared-state guarantees. +- New metrics must include a metric card, expected baseline ordering, and at least one + failure/anti-gaming test. +- Do not commit datasets, checkpoints, generated runs, tokens, or browser sessions. + +Please use focused commits and explain both the scientific reason and user-visible impact +of a change. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e60c58a --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 WAMProbe contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 2c10ae2..9ae69e0 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,111 @@ # WAMProbe -WAMProbe 是一个面向 World Action Model（WAM）的开源评测工具箱，重点评估： +[![CI](https://github.com/myheart521/WAMProbe/actions/workflows/ci.yml/badge.svg)](https://github.com/myheart521/WAMProbe/actions/workflows/ci.yml) +[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) +[![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE) -- 动作与预测未来之间的因果一致性； -- 动力学和时间一致性； -- 候选动作排序与闭环控制价值； -- 控制收益与推理成本之间的权衡。 +**Counterfactual evaluation for World Action Models.** -项目当前处于规划与初始化阶段，尚未发布可用 API。 +WAMProbe tests whether a World Action Model (WAM) predicts futures that are causally +controlled by the input action and useful for choosing robot actions. It complements +task-success and video-quality benchmarks with paired interventions: -## 项目文档 +```text +the same initial state +├── no-op → predicted future / true future +├── move left → predicted future / true future +├── move right → predicted future / true future +└── expert act → predicted future / true future +``` + +The project is in an early alpha. Its first runnable slice is deliberately small and +CPU-only: an analytic PointMass-2D benchmark verifies the public API and metrics against +an oracle and several intentionally broken baselines before expensive robot models are +integrated. + +## Why another evaluation project? + +A model can generate a realistic-looking success video while ignoring the candidate +action. WAMProbe separates three questions that should not be collapsed into one score: + +1. **Action dependence:** do different actions produce different predicted futures? +2. **Direction correctness:** do those differences agree with true dynamics? +3. **Control utility:** does the predicted future select a better candidate action? -- [详细开发规划](docs/WAMProbe_PLAN.md) +For example, the built-in `wrong-direction` baseline receives a high Action Dependence +score because it reacts to actions, but a negative Counterfactual Direction score because +it predicts the opposite motion. This sanity check prevents a superficially responsive +model from looking correct. -## 初始目录 +## Quick start + +```bash +git clone https://github.com/myheart521/WAMProbe.git +cd WAMProbe +python -m venv .venv +source .venv/bin/activate +python -m pip install -e . + +wamprobe demo --contexts 12 --seed 7 --output runs/pointmass-demo +``` + +The command creates: ```text -WAMProbe/ -├── configs/ # 模型、benchmark 和实验配置 -├── docs/ # 设计与规划文档 -├── schemas/ # 能力、数据和结果 JSON Schema -├── src/wamprobe/ # Python 包 -└── tests/ # 自动化测试 +runs/pointmass-demo/ +├── summary.json # versioned machine-readable results +└── report.md # human-readable metric comparison +``` + +See the committed [example report](examples/pointmass-demo/report.md) for the expected +baseline ordering. + +You can also run the module directly: + +```bash +PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo +``` + +## Current runnable scope + +- typed, model-agnostic `WAMAdapter` protocol; +- capability manifest data model and JSON Schema; +- paired PointMass-2D counterfactual interventions; +- oracle, copy-last-frame, wrong-direction, and action-agnostic baselines; +- Action Dependence, Counterfactual Direction Accuracy, No-op Stability, state ADE, + and Top-1 Regret; +- JSON and Markdown reports; +- Python 3.11–3.13 CI with linting, strict typing, and coverage. + +## Roadmap + +The next milestones are: + +1. context-block bootstrap confidence intervals; +2. a small image-rendered BlockPush benchmark; +3. the versioned intervention dataset loader; +4. LIBERO-CF-Mini paired simulator branches; +5. the first real WAM adapter, selected after checkpoint and license verification. + +See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), +[quick-start notes](docs/QUICKSTART.md), and [design RFCs](docs/rfcs/). + +## Development + +```bash +python -m pip install -e '.[dev]' +ruff format --check . +ruff check . +mypy +pytest --cov=wamprobe --cov-report=term-missing ``` -## 开发状态 +Contributions are welcome. New metrics must include a documented failure mode and a +sanity test against the reference baselines; see [CONTRIBUTING.md](CONTRIBUTING.md). + +## 中文说明 -第一阶段将实现 capability manifest、`WAMAdapter` 协议、Toy benchmark 和 reference baselines。具体范围与验收标准见详细规划文档。 +WAMProbe 不是新的 WAM 训练框架，而是一套“给 WAM 做反事实考试”的工具。它在完全 +相同的初始状态下执行多个不同动作，比较模型预测未来与模拟器真实未来，并检查这些 +预测是否能帮助机器人选出更好的动作。详细设计见 +[WAMProbe 开源项目规划](docs/WAMProbe_PLAN.md)。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..eba6849 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security policy + +## Supported versions + +WAMProbe is pre-release software. Security fixes are applied to the latest alpha branch. + +## Reporting a vulnerability + +Do not publish credentials, private datasets, model tokens, or exploit details in a public +issue. Use GitHub's private vulnerability reporting feature for this repository. If it is +not available, open a minimal issue requesting a private maintainer contact without including +the sensitive details. + +WAMProbe is a research evaluation tool, not a certified robot safety system. Its metrics and +action-selection demos must not be treated as a safety controller for real hardware. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md new file mode 100644 index 0000000..83119bd --- /dev/null +++ b/docs/QUICKSTART.md @@ -0,0 +1,48 @@ +# Quick start and metric interpretation + +## Run the diagnostic suite + +From a source checkout: + +```bash +PYTHONPATH=src python -m wamprobe demo \ + --contexts 12 \ + --seed 7 \ + --output runs/pointmass-demo +``` + +The suite creates multiple actions from each shared initial state. The goal is placed at +the final state of one directional action, so the simulator's optimal candidate is known +exactly. + +## Built-in baselines + +| Baseline | Behavior | Diagnostic purpose | +|---|---|---| +| `oracle-pointmass` | Uses exact dynamics | Expected upper bound | +| `copy-last-frame` | Predicts no movement | Detects action ignorance | +| `wrong-direction` | Applies the negative action | Separates dependence from correctness | +| `action-agnostic` | Moves to the goal for every action | Mimics a plausible success prior | + +## Metrics + +- **Action Dependence** measures endpoint separation across action branches. It does not + establish that the direction is correct. +- **Counterfactual Direction Accuracy** is mean cosine alignment between predicted and + true non-noop displacement. `1` is aligned, `0` is uninformative, and `-1` is reversed. +- **No-op Stability** is the fraction of no-op predictions whose final state remains at + the shared initial position. +- **State ADE** is average Euclidean state error over branches and time; lower is better. +- **Top-1 Regret** measures how much true return is lost by selecting the candidate that + looks best under the predicted future; lower is better. + +WAMProbe intentionally reports a metric profile, not a composite score. A model may be +action-dependent but physically wrong, or visually accurate but useless for candidate +selection. + +## What this demo does not prove + +PointMass-2D validates the evaluator and guards against metric bugs. It is not evidence +that the metrics transfer to manipulation videos. That requires paired simulator branches, +perception-based state extraction, stochastic generation analysis, and correlation with +closed-loop return; these are explicit later milestones. diff --git a/docs/rfcs/0001-scope-and-capabilities.md b/docs/rfcs/0001-scope-and-capabilities.md new file mode 100644 index 0000000..cf7f322 --- /dev/null +++ b/docs/rfcs/0001-scope-and-capabilities.md @@ -0,0 +1,33 @@ +# RFC 0001: Scope and capability-aware adapters + +- Status: Accepted for alpha +- Version: 0.1 + +## Decision + +WAMProbe treats World Action Models as a family rather than one architecture. An adapter +declares the future representation it exposes (`none`, `pixels`, `states`, or `latents`) +and whether it predicts actions, scores candidate actions, exposes world features, supports +batching, or provides deterministic seeding. + +Metrics must state the capabilities they require. Missing capabilities are skipped with an +explicit reason or rejected before evaluation; they are never silently approximated. + +## Alpha protocol + +The CPU-first alpha requires one method: + +```python +predict_future(context, action, *, horizon, seed) -> Trajectory2D +``` + +This narrow state-space contract establishes ID alignment, horizon validation, intervention +semantics, and deterministic tests. Pixel/latent batches and remote dependency isolation are +planned extensions that will preserve the same capability-routing principle. + +## Non-goals + +- defining a universal robot action space in this repository; +- forcing auxiliary-video WAMs to expose pixel rollouts; +- treating unsupported metrics as zero; +- ranking heterogeneous capability groups with one aggregate score. diff --git a/docs/rfcs/0002-counterfactual-metrics.md b/docs/rfcs/0002-counterfactual-metrics.md new file mode 100644 index 0000000..b68c39c --- /dev/null +++ b/docs/rfcs/0002-counterfactual-metrics.md @@ -0,0 +1,34 @@ +# RFC 0002: Counterfactual-first metric design + +- Status: Accepted for alpha +- Version: 0.1 + +## Problem + +Single recorded trajectories confound action effects with the initial scene and policy. A +realistic future prediction may come from a scene prior while ignoring the candidate action. + +## Decision + +The primary evaluation unit is an intervention group: one shared context paired with at +least two uniquely named actions. Ground-truth futures are generated by restoring the same +simulator state before each action branch. + +The alpha validates five complementary measurements: + +1. action dependence; +2. counterfactual direction correctness; +3. no-op stability; +4. state trajectory error; +5. candidate-selection regret. + +## Required sanity checks + +Every new metric must document what it measures, what can game it, and how the following +baselines should behave: exact oracle, copy-last, wrong-direction, and action-agnostic success +prior. Learned metrics must additionally pin model weights and training data. + +## No composite score in v0.1 + +Action responsiveness, physical accuracy, and control utility are distinct. WAMProbe will +publish a profile and paired confidence intervals before considering any aggregate score. diff --git a/examples/pointmass-demo/report.md b/examples/pointmass-demo/report.md new file mode 100644 index 0000000..ec3a2ad --- /dev/null +++ b/examples/pointmass-demo/report.md @@ -0,0 +1,14 @@ +# WAMProbe PointMass-2D Demo + +This report compares action-aware and deliberately broken reference baselines. + +| Model | Action Dependence | Counterfactual Direction | No-op Stability | State ADE ↓ | Top-1 Regret ↓ | +|---|---|---|---|---|---| +| oracle-pointmass | 1.0000 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | +| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.5000 | 1.0000 | +| wrong-direction | 1.0000 | -1.0000 | 1.0000 | 1.0000 | 2.0000 | +| action-agnostic | 0.0000 | 0.0000 | 0.0000 | 0.7286 | 1.0000 | + +A high Action Dependence score is not sufficient: the wrong-direction baseline +depends on the action but fails Counterfactual Direction Accuracy. This is why +WAMProbe reports a metric profile instead of a single aggregate score. diff --git a/examples/pointmass-demo/summary.json b/examples/pointmass-demo/summary.json new file mode 100644 index 0000000..5b79386 --- /dev/null +++ b/examples/pointmass-demo/summary.json @@ -0,0 +1,354 @@ +{ + "schema_version": "0.1", + "benchmark": "pointmass-2d-v0.1", + "results": [ + { + "schema_version": "0.1", + "model_id": "oracle-pointmass", + "benchmark": "pointmass-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 1.0, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "context-0000", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0 + }, + { + "context_id": "context-0001", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0 + }, + { + "context_id": "context-0002", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0 + }, + { + "context_id": "context-0003", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0 + }, + { + "context_id": "context-0004", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0 + }, + { + "context_id": "context-0005", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0 + }, + { + "context_id": "context-0006", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0 + }, + { + "context_id": "context-0007", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0 + }, + { + "context_id": "context-0008", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0 + }, + { + "context_id": "context-0009", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0 + }, + { + "context_id": "context-0010", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0 + }, + { + "context_id": "context-0011", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0 + } + ] + }, + { + "schema_version": "0.1", + "model_id": "copy-last-frame", + "benchmark": "pointmass-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "top1_regret": 1.0 + }, + "context_results": [ + { + "context_id": "context-0000", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0001", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0002", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0003", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + }, + { + "context_id": "context-0004", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0005", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0006", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0007", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + }, + { + "context_id": "context-0008", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0009", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0010", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0011", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + } + ] + }, + { + "schema_version": "0.1", + "model_id": "wrong-direction", + "benchmark": "pointmass-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 1.0, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "top1_regret": 2.0 + }, + "context_results": [ + { + "context_id": "context-0000", + "selected_action": "left", + "optimal_action": "right", + "top1_regret": 2.0 + }, + { + "context_id": "context-0001", + "selected_action": "right", + "optimal_action": "left", + "top1_regret": 2.0 + }, + { + "context_id": "context-0002", + "selected_action": "down", + "optimal_action": "up", + "top1_regret": 2.0 + }, + { + "context_id": "context-0003", + "selected_action": "up", + "optimal_action": "down", + "top1_regret": 2.0 + }, + { + "context_id": "context-0004", + "selected_action": "left", + "optimal_action": "right", + "top1_regret": 2.0 + }, + { + "context_id": "context-0005", + "selected_action": "right", + "optimal_action": "left", + "top1_regret": 2.0 + }, + { + "context_id": "context-0006", + "selected_action": "down", + "optimal_action": "up", + "top1_regret": 2.0 + }, + { + "context_id": "context-0007", + "selected_action": "up", + "optimal_action": "down", + "top1_regret": 2.0 + }, + { + "context_id": "context-0008", + "selected_action": "left", + "optimal_action": "right", + "top1_regret": 2.0 + }, + { + "context_id": "context-0009", + "selected_action": "right", + "optimal_action": "left", + "top1_regret": 2.0 + }, + { + "context_id": "context-0010", + "selected_action": "down", + "optimal_action": "up", + "top1_regret": 2.0 + }, + { + "context_id": "context-0011", + "selected_action": "up", + "optimal_action": "down", + "top1_regret": 2.0 + } + ] + }, + { + "schema_version": "0.1", + "model_id": "action-agnostic", + "benchmark": "pointmass-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "top1_regret": 1.0 + }, + "context_results": [ + { + "context_id": "context-0000", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0001", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0002", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0003", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + }, + { + "context_id": "context-0004", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0005", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0006", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0007", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + }, + { + "context_id": "context-0008", + "selected_action": "noop", + "optimal_action": "right", + "top1_regret": 1.0 + }, + { + "context_id": "context-0009", + "selected_action": "noop", + "optimal_action": "left", + "top1_regret": 1.0 + }, + { + "context_id": "context-0010", + "selected_action": "noop", + "optimal_action": "up", + "top1_regret": 1.0 + }, + { + "context_id": "context-0011", + "selected_action": "noop", + "optimal_action": "down", + "top1_regret": 1.0 + } + ] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index a6490b5..c6b2c45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,12 +4,23 @@ build-backend = "hatchling.build" [project] name = "wamprobe" -version = "0.0.0" +version = "0.1.0a0" description = "Causal consistency and control-utility evaluation for World Action Models" readme = "README.md" requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{name = "WAMProbe contributors"}] +keywords = ["robotics", "world-model", "world-action-model", "evaluation", "benchmark"] dependencies = [] +[project.urls] +Homepage = "https://github.com/myheart521/WAMProbe" +Repository = "https://github.com/myheart521/WAMProbe" +Issues = "https://github.com/myheart521/WAMProbe/issues" + +[project.scripts] +wamprobe = "wamprobe.cli:main" + [project.optional-dependencies] dev = [ "mypy", @@ -21,14 +32,30 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["src/wamprobe"] +[tool.hatch.build] +exclude = [ + "/.coverage", + "/.mypy_cache", + "/.pytest_cache", + "/.ruff_cache", + "/.venv", + "/dist", +] + [tool.pytest.ini_options] testpaths = ["tests"] +addopts = [ + "--strict-markers", +] [tool.ruff] line-length = 100 target-version = "py311" +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] + [tool.mypy] python_version = "3.11" strict = true -packages = ["wamprobe"] +files = ["src/wamprobe", "tests"] diff --git a/schemas/.gitkeep b/schemas/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/schemas/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/schemas/capability-v0.1.schema.json b/schemas/capability-v0.1.schema.json new file mode 100644 index 0000000..05502d2 --- /dev/null +++ b/schemas/capability-v0.1.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/myheart521/WAMProbe/schemas/capability-v0.1.schema.json", + "title": "WAMProbe model capability manifest v0.1", + "type": "object", + "additionalProperties": false, + "required": ["model_id", "future_representation"], + "properties": { + "model_id": {"type": "string", "minLength": 1}, + "future_representation": { + "enum": ["none", "pixels", "states", "latents"] + }, + "predicts_actions": {"type": "boolean", "default": false}, + "scores_actions": {"type": "boolean", "default": false}, + "exposes_world_features": {"type": "boolean", "default": false}, + "stochastic": {"type": "boolean", "default": false}, + "deterministic_seed": {"type": "boolean", "default": true}, + "supports_batching": {"type": "boolean", "default": false} + } +} diff --git a/schemas/intervention-v0.1.schema.json b/schemas/intervention-v0.1.schema.json new file mode 100644 index 0000000..4f47032 --- /dev/null +++ b/schemas/intervention-v0.1.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/myheart521/WAMProbe/schemas/intervention-v0.1.schema.json", + "title": "WAMProbe 2D intervention suite v0.1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "benchmark_id", "groups"], + "properties": { + "schema_version": {"const": "0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "groups": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["context_id", "position", "goal", "actions"], + "properties": { + "context_id": {"type": "string", "minLength": 1}, + "position": {"$ref": "#/$defs/vec2"}, + "goal": {"$ref": "#/$defs/vec2"}, + "actions": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "required": ["name", "delta"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "delta": {"$ref": "#/$defs/vec2"} + } + } + } + } + } + } + }, + "$defs": { + "vec2": { + "type": "array", + "prefixItems": [{"type": "number"}, {"type": "number"}], + "items": false + } + } +} diff --git a/schemas/result-v0.1.schema.json b/schemas/result-v0.1.schema.json new file mode 100644 index 0000000..15bcae3 --- /dev/null +++ b/schemas/result-v0.1.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/myheart521/WAMProbe/schemas/result-v0.1.schema.json", + "title": "WAMProbe evaluation summary v0.1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "benchmark", "results"], + "properties": { + "schema_version": {"const": "0.1"}, + "benchmark": {"type": "string", "minLength": 1}, + "results": { + "type": "array", + "items": { + "type": "object", + "required": [ + "schema_version", + "model_id", + "benchmark", + "contexts", + "metrics", + "context_results" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "model_id": {"type": "string", "minLength": 1}, + "benchmark": {"type": "string", "minLength": 1}, + "contexts": {"type": "integer", "minimum": 1}, + "metrics": { + "type": "object", + "additionalProperties": {"type": "number"} + }, + "context_results": {"type": "array"} + } + } + } + } +} diff --git a/src/wamprobe/__init__.py b/src/wamprobe/__init__.py index a2d5df8..52b2f4e 100644 --- a/src/wamprobe/__init__.py +++ b/src/wamprobe/__init__.py @@ -1,3 +1,3 @@ -"""WAMProbe: evaluation tools for World Action Models.""" +"""WAMProbe: counterfactual evaluation for World Action Models.""" -__version__ = "0.0.0" +__version__ = "0.1.0a0" diff --git a/src/wamprobe/__main__.py b/src/wamprobe/__main__.py new file mode 100644 index 0000000..c4c25a1 --- /dev/null +++ b/src/wamprobe/__main__.py @@ -0,0 +1,5 @@ +"""Allow ``python -m wamprobe`` execution.""" + +from wamprobe.cli import main + +raise SystemExit(main()) diff --git a/src/wamprobe/adapters/__init__.py b/src/wamprobe/adapters/__init__.py new file mode 100644 index 0000000..829dde7 --- /dev/null +++ b/src/wamprobe/adapters/__init__.py @@ -0,0 +1,15 @@ +"""Built-in model adapters and diagnostic baselines.""" + +from wamprobe.adapters.baselines import ( + ActionAgnosticAdapter, + CopyLastFrameAdapter, + OraclePointMassAdapter, + WrongDirectionAdapter, +) + +__all__ = [ + "ActionAgnosticAdapter", + "CopyLastFrameAdapter", + "OraclePointMassAdapter", + "WrongDirectionAdapter", +] diff --git a/src/wamprobe/adapters/baselines.py b/src/wamprobe/adapters/baselines.py new file mode 100644 index 0000000..2420774 --- /dev/null +++ b/src/wamprobe/adapters/baselines.py @@ -0,0 +1,106 @@ +"""Reference baselines used to sanity-check WAMProbe metrics.""" + +from __future__ import annotations + +from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.types import Action2D, Context2D, Trajectory2D, Vec2 +from wamprobe.benchmarks.pointmass import PointMass2D + + +class _StateBaseline: + model_id = "state-baseline" + + @property + def capabilities(self) -> ModelCapabilities: + return ModelCapabilities( + model_id=self.model_id, + future_representation=FutureRepresentation.STATES, + ) + + def close(self) -> None: + """Baselines own no external resources.""" + + +class OraclePointMassAdapter(_StateBaseline): + """Upper bound that uses the exact benchmark transition.""" + + model_id = "oracle-pointmass" + + def __init__(self, benchmark: PointMass2D) -> None: + self._benchmark = benchmark + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + del seed + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + return self._benchmark.rollout(context, action) + + +class CopyLastFrameAdapter(_StateBaseline): + """Action-ignoring baseline that repeats the current state.""" + + model_id = "copy-last-frame" + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + del seed + return Trajectory2D(context.context_id, action.name, (context.position,) * horizon) + + +class WrongDirectionAdapter(_StateBaseline): + """Action-dependent baseline that deliberately predicts the opposite direction.""" + + model_id = "wrong-direction" + + def __init__(self, benchmark: PointMass2D) -> None: + self._benchmark = benchmark + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + del seed + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + reversed_action = Action2D(action.name, action.delta * -1.0) + return self._benchmark.rollout(context, reversed_action) + + +class ActionAgnosticAdapter(_StateBaseline): + """Plausible-looking baseline that moves to the goal for every input action.""" + + model_id = "action-agnostic" + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + del seed + step = (context.goal - context.position) * (1.0 / horizon) + position = context.position + states: list[Vec2] = [] + for _ in range(horizon): + position = position + step + states.append(position) + return Trajectory2D(context.context_id, action.name, tuple(states)) diff --git a/src/wamprobe/api/__init__.py b/src/wamprobe/api/__init__.py new file mode 100644 index 0000000..d9cbbc1 --- /dev/null +++ b/src/wamprobe/api/__init__.py @@ -0,0 +1,24 @@ +"""Public, model-agnostic WAMProbe API.""" + +from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.model import WAMAdapter +from wamprobe.api.types import ( + Action2D, + Context2D, + InterventionGroup, + InterventionSuite, + Trajectory2D, + Vec2, +) + +__all__ = [ + "Action2D", + "Context2D", + "FutureRepresentation", + "InterventionGroup", + "InterventionSuite", + "ModelCapabilities", + "Trajectory2D", + "Vec2", + "WAMAdapter", +] diff --git a/src/wamprobe/api/capabilities.py b/src/wamprobe/api/capabilities.py new file mode 100644 index 0000000..87357c8 --- /dev/null +++ b/src/wamprobe/api/capabilities.py @@ -0,0 +1,41 @@ +"""Machine-readable model capability declarations.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from enum import StrEnum +from typing import Any + + +class FutureRepresentation(StrEnum): + """Representations a WAM may expose for predicted futures.""" + + NONE = "none" + PIXELS = "pixels" + STATES = "states" + LATENTS = "latents" + + +@dataclass(frozen=True, slots=True) +class ModelCapabilities: + """Capabilities used to route only compatible WAMProbe metrics.""" + + model_id: str + future_representation: FutureRepresentation + predicts_actions: bool = False + scores_actions: bool = False + exposes_world_features: bool = False + stochastic: bool = False + deterministic_seed: bool = True + supports_batching: bool = False + + def __post_init__(self) -> None: + if not self.model_id.strip(): + raise ValueError("model_id must not be empty") + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable representation.""" + + result = asdict(self) + result["future_representation"] = self.future_representation.value + return result diff --git a/src/wamprobe/api/errors.py b/src/wamprobe/api/errors.py new file mode 100644 index 0000000..6fd005e --- /dev/null +++ b/src/wamprobe/api/errors.py @@ -0,0 +1,13 @@ +"""WAMProbe-specific exception hierarchy.""" + + +class WAMProbeError(Exception): + """Base exception for expected WAMProbe failures.""" + + +class UnsupportedCapabilityError(WAMProbeError): + """Raised when a metric requests a capability the adapter does not expose.""" + + +class ValidationError(WAMProbeError): + """Raised when benchmark or model data violates the public contract.""" diff --git a/src/wamprobe/api/model.py b/src/wamprobe/api/model.py new file mode 100644 index 0000000..aefc26e --- /dev/null +++ b/src/wamprobe/api/model.py @@ -0,0 +1,36 @@ +"""Protocol implemented by World Action Model adapters.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from wamprobe.api.capabilities import ModelCapabilities +from wamprobe.api.types import Action2D, Context2D, Trajectory2D + + +@runtime_checkable +class WAMAdapter(Protocol): + """Minimal state-future adapter used by the CPU-first MVP.""" + + @property + def capabilities(self) -> ModelCapabilities: + """Describe outputs and runtime behavior exposed by the adapter.""" + + ... + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + """Predict a future trajectory for one context/action intervention.""" + + ... + + def close(self) -> None: + """Release resources owned by the adapter.""" + + ... diff --git a/src/wamprobe/api/types.py b/src/wamprobe/api/types.py new file mode 100644 index 0000000..ed21e81 --- /dev/null +++ b/src/wamprobe/api/types.py @@ -0,0 +1,111 @@ +"""Typed data carried between benchmarks, adapters, and metrics.""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import hypot + + +@dataclass(frozen=True, slots=True) +class Vec2: + """Small dependency-free two-dimensional vector.""" + + x: float + y: float + + def __add__(self, other: Vec2) -> Vec2: + return Vec2(self.x + other.x, self.y + other.y) + + def __sub__(self, other: Vec2) -> Vec2: + return Vec2(self.x - other.x, self.y - other.y) + + def __mul__(self, scalar: float) -> Vec2: + return Vec2(self.x * scalar, self.y * scalar) + + def norm(self) -> float: + """Return the Euclidean vector norm.""" + + return hypot(self.x, self.y) + + def dot(self, other: Vec2) -> float: + """Return the dot product with another vector.""" + + return self.x * other.x + self.y * other.y + + def to_list(self) -> list[float]: + """Return a JSON-compatible coordinate pair.""" + + return [self.x, self.y] + + +@dataclass(frozen=True, slots=True) +class Context2D: + """A shared initial state for a counterfactual intervention group.""" + + context_id: str + position: Vec2 + goal: Vec2 + + def __post_init__(self) -> None: + if not self.context_id.strip(): + raise ValueError("context_id must not be empty") + + +@dataclass(frozen=True, slots=True) +class Action2D: + """A named per-step displacement intervention.""" + + name: str + delta: Vec2 + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("action name must not be empty") + + +@dataclass(frozen=True, slots=True) +class Trajectory2D: + """A predicted or ground-truth state trajectory.""" + + context_id: str + action_name: str + states: tuple[Vec2, ...] + + def __post_init__(self) -> None: + if not self.states: + raise ValueError("trajectory must contain at least one state") + + @property + def final_state(self) -> Vec2: + """Return the final predicted state.""" + + return self.states[-1] + + +@dataclass(frozen=True, slots=True) +class InterventionGroup: + """Actions branched from exactly one shared context.""" + + context: Context2D + actions: tuple[Action2D, ...] + + def __post_init__(self) -> None: + if len(self.actions) < 2: + raise ValueError("an intervention group requires at least two actions") + names = [action.name for action in self.actions] + if len(names) != len(set(names)): + raise ValueError("action names in an intervention group must be unique") + + +@dataclass(frozen=True, slots=True) +class InterventionSuite: + """A versioned collection of counterfactual intervention groups.""" + + benchmark_id: str + groups: tuple[InterventionGroup, ...] + + def __post_init__(self) -> None: + if not self.benchmark_id.strip(): + raise ValueError("benchmark_id must not be empty") + if not self.groups: + raise ValueError("an intervention suite requires at least one group") diff --git a/src/wamprobe/benchmarks/__init__.py b/src/wamprobe/benchmarks/__init__.py new file mode 100644 index 0000000..48f1402 --- /dev/null +++ b/src/wamprobe/benchmarks/__init__.py @@ -0,0 +1,5 @@ +"""Built-in WAMProbe benchmarks.""" + +from wamprobe.benchmarks.pointmass import PointMass2D + +__all__ = ["PointMass2D"] diff --git a/src/wamprobe/benchmarks/pointmass.py b/src/wamprobe/benchmarks/pointmass.py new file mode 100644 index 0000000..ba84b39 --- /dev/null +++ b/src/wamprobe/benchmarks/pointmass.py @@ -0,0 +1,74 @@ +"""Analytic counterfactual benchmark for a two-dimensional point mass.""" + +from __future__ import annotations + +from random import Random + +from wamprobe.api.types import ( + Action2D, + Context2D, + InterventionGroup, + InterventionSuite, + Trajectory2D, + Vec2, +) + + +class PointMass2D: + """Known linear dynamics used to validate metrics before real WAMs.""" + + benchmark_id = "pointmass-2d-v0.1" + + def __init__(self, *, horizon: int = 4, action_scale: float = 0.25) -> None: + if horizon <= 0: + raise ValueError("horizon must be positive") + if action_scale <= 0: + raise ValueError("action_scale must be positive") + self.horizon = horizon + self.action_scale = action_scale + + def actions(self) -> tuple[Action2D, ...]: + """Return a deterministic, constraint-valid candidate action set.""" + + scale = self.action_scale + return ( + Action2D("noop", Vec2(0.0, 0.0)), + Action2D("right", Vec2(scale, 0.0)), + Action2D("left", Vec2(-scale, 0.0)), + Action2D("up", Vec2(0.0, scale)), + Action2D("down", Vec2(0.0, -scale)), + ) + + def rollout(self, context: Context2D, action: Action2D) -> Trajectory2D: + """Apply the action once per step using exact linear dynamics.""" + + position = context.position + states: list[Vec2] = [] + for _ in range(self.horizon): + position = position + action.delta + states.append(position) + return Trajectory2D(context.context_id, action.name, tuple(states)) + + def score(self, context: Context2D, trajectory: Trajectory2D) -> float: + """Return negative final distance to the context goal.""" + + return -(trajectory.final_state - context.goal).norm() + + def make_suite(self, *, contexts: int = 8, seed: int = 0) -> InterventionSuite: + """Build paired interventions whose optimal action is known exactly.""" + + if contexts <= 0: + raise ValueError("contexts must be positive") + + rng = Random(seed) + candidates = self.actions() + directional_actions = candidates[1:] + groups: list[InterventionGroup] = [] + for index in range(contexts): + start = Vec2(rng.uniform(-0.5, 0.5), rng.uniform(-0.5, 0.5)) + optimal = directional_actions[index % len(directional_actions)] + goal = start + optimal.delta * self.horizon + context = Context2D(f"context-{index:04d}", start, goal) + groups.append(InterventionGroup(context, candidates)) + + return InterventionSuite(self.benchmark_id, tuple(groups)) diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py new file mode 100644 index 0000000..ca84bc1 --- /dev/null +++ b/src/wamprobe/cli.py @@ -0,0 +1,69 @@ +"""Command-line entry point for the CPU-first WAMProbe MVP.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from pathlib import Path + +from wamprobe.adapters.baselines import ( + ActionAgnosticAdapter, + CopyLastFrameAdapter, + OraclePointMassAdapter, + WrongDirectionAdapter, +) +from wamprobe.api.model import WAMAdapter +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.evaluation import evaluate +from wamprobe.reporting import write_reports + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="wamprobe", + description="Counterfactual evaluation for World Action Models", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + demo = subparsers.add_parser("demo", help="run the PointMass-2D baseline comparison") + demo.add_argument("--contexts", type=int, default=12, help="number of shared contexts") + demo.add_argument("--seed", type=int, default=7, help="deterministic suite seed") + demo.add_argument("--horizon", type=int, default=4, help="prediction horizon") + demo.add_argument( + "--output", + type=Path, + default=Path("runs/pointmass-demo"), + help="directory for summary.json and report.md", + ) + return parser + + +def _run_demo(args: argparse.Namespace) -> int: + benchmark = PointMass2D(horizon=args.horizon) + suite = benchmark.make_suite(contexts=args.contexts, seed=args.seed) + adapters: list[WAMAdapter] = [ + OraclePointMassAdapter(benchmark), + CopyLastFrameAdapter(), + WrongDirectionAdapter(benchmark), + ActionAgnosticAdapter(), + ] + results = [evaluate(adapter, benchmark, suite, seed=args.seed) for adapter in adapters] + summary_path, report_path = write_reports( + args.output, + benchmark=benchmark.benchmark_id, + results=results, + ) + print("WAMProbe demo complete") + print(f"JSON: {summary_path}") + print(f"Markdown: {report_path}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + """Run WAMProbe and return a process exit code.""" + + parser = _build_parser() + args = parser.parse_args(argv) + if args.command == "demo": + return _run_demo(args) + parser.error(f"unsupported command: {args.command}") + return 2 diff --git a/src/wamprobe/evaluation.py b/src/wamprobe/evaluation.py new file mode 100644 index 0000000..97e7c87 --- /dev/null +++ b/src/wamprobe/evaluation.py @@ -0,0 +1,114 @@ +"""End-to-end counterfactual evaluation orchestration.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + +from wamprobe.api.model import WAMAdapter +from wamprobe.api.types import Context2D, InterventionSuite, Trajectory2D +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.metrics import ( + action_dependence, + counterfactual_direction_accuracy, + noop_stability, + state_ade, +) + + +@dataclass(frozen=True, slots=True) +class ContextResult: + """Candidate-selection result for one shared initial state.""" + + context_id: str + selected_action: str + optimal_action: str + top1_regret: float + + +@dataclass(frozen=True, slots=True) +class EvaluationResult: + """Aggregated metrics and context-level control decisions.""" + + model_id: str + benchmark: str + contexts: int + metrics: dict[str, float] + context_results: tuple[ContextResult, ...] + + def to_dict(self) -> dict[str, Any]: + """Return a stable JSON-compatible result.""" + + return { + "schema_version": "0.1", + "model_id": self.model_id, + "benchmark": self.benchmark, + "contexts": self.contexts, + "metrics": self.metrics, + "context_results": [asdict(result) for result in self.context_results], + } + + +def evaluate( + adapter: WAMAdapter, + benchmark: PointMass2D, + suite: InterventionSuite, + *, + seed: int = 0, +) -> EvaluationResult: + """Evaluate one adapter on paired point-mass interventions.""" + + if suite.benchmark_id != benchmark.benchmark_id: + raise ValueError("suite and benchmark IDs must match") + + grouped_samples: list[list[tuple[Context2D, Trajectory2D, Trajectory2D]]] = [] + context_results: list[ContextResult] = [] + for group in suite.groups: + samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]] = [] + predicted_scores: dict[str, float] = {} + true_scores: dict[str, float] = {} + for action in group.actions: + truth = benchmark.rollout(group.context, action) + predicted = adapter.predict_future( + group.context, + action, + horizon=benchmark.horizon, + seed=seed, + ) + if predicted.context_id != group.context.context_id: + raise ValueError("adapter returned a mismatched context_id") + if predicted.action_name != action.name: + raise ValueError("adapter returned a mismatched action_name") + samples.append((group.context, predicted, truth)) + predicted_scores[action.name] = benchmark.score(group.context, predicted) + true_scores[action.name] = benchmark.score(group.context, truth) + + selected = max(predicted_scores, key=predicted_scores.__getitem__) + optimal = max(true_scores, key=true_scores.__getitem__) + regret = true_scores[optimal] - true_scores[selected] + context_results.append( + ContextResult( + context_id=group.context.context_id, + selected_action=selected, + optimal_action=optimal, + top1_regret=regret, + ) + ) + grouped_samples.append(samples) + + flat_samples = [sample for samples in grouped_samples for sample in samples] + metrics = { + "action_dependence": action_dependence(grouped_samples), + "counterfactual_direction_accuracy": counterfactual_direction_accuracy(flat_samples), + "noop_stability": noop_stability(flat_samples), + "state_ade": state_ade(flat_samples), + "top1_regret": sum(result.top1_regret for result in context_results) / len(context_results), + } + adapter.close() + return EvaluationResult( + model_id=adapter.capabilities.model_id, + benchmark=benchmark.benchmark_id, + contexts=len(suite.groups), + metrics=metrics, + context_results=tuple(context_results), + ) diff --git a/src/wamprobe/metrics.py b/src/wamprobe/metrics.py new file mode 100644 index 0000000..f98649e --- /dev/null +++ b/src/wamprobe/metrics.py @@ -0,0 +1,84 @@ +"""Counterfactual, dynamics, and control-utility metrics.""" + +from __future__ import annotations + +from itertools import combinations + +from wamprobe.api.types import Context2D, Trajectory2D, Vec2 + + +def _displacement(context: Context2D, trajectory: Trajectory2D) -> Vec2: + return trajectory.final_state - context.position + + +def counterfactual_direction_accuracy( + samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]], +) -> float: + """Return mean cosine alignment between predicted and true non-noop displacements.""" + + scores: list[float] = [] + for context, predicted, truth in samples: + true_delta = _displacement(context, truth) + if true_delta.norm() <= 1e-12: + continue + predicted_delta = _displacement(context, predicted) + if predicted_delta.norm() <= 1e-12: + scores.append(0.0) + continue + scores.append( + predicted_delta.dot(true_delta) / (predicted_delta.norm() * true_delta.norm()) + ) + return sum(scores) / len(scores) if scores else 0.0 + + +def state_ade(samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]]) -> float: + """Return average state displacement error over all branches and time steps.""" + + errors: list[float] = [] + for _, predicted, truth in samples: + if len(predicted.states) != len(truth.states): + raise ValueError("predicted and true trajectories must have equal horizons") + errors.extend( + (predicted_state - true_state).norm() + for predicted_state, true_state in zip(predicted.states, truth.states, strict=True) + ) + return sum(errors) / len(errors) if errors else 0.0 + + +def action_dependence( + groups: list[list[tuple[Context2D, Trajectory2D, Trajectory2D]]], +) -> float: + """Measure whether future endpoints separate when actions change. + + The score is a capped ratio of mean predicted pairwise endpoint distance to + ground-truth pairwise endpoint distance. Direction correctness is intentionally + evaluated separately by :func:`counterfactual_direction_accuracy`. + """ + + group_scores: list[float] = [] + for samples in groups: + predicted_distances = [ + (left[1].final_state - right[1].final_state).norm() + for left, right in combinations(samples, 2) + ] + true_distances = [ + (left[2].final_state - right[2].final_state).norm() + for left, right in combinations(samples, 2) + ] + predicted_mean = sum(predicted_distances) / len(predicted_distances) + true_mean = sum(true_distances) / len(true_distances) + group_scores.append(min(1.0, predicted_mean / true_mean) if true_mean else 0.0) + return sum(group_scores) / len(group_scores) if group_scores else 0.0 + + +def noop_stability( + samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]], +) -> float: + """Return the fraction of no-op predictions that remain at the current state.""" + + noop_scores = [ + 1.0 if _displacement(context, predicted).norm() <= 1e-12 else 0.0 + for context, predicted, truth in samples + if truth.action_name == "noop" + ] + return sum(noop_scores) / len(noop_scores) if noop_scores else 0.0 diff --git a/src/wamprobe/reporting.py b/src/wamprobe/reporting.py new file mode 100644 index 0000000..afc96cc --- /dev/null +++ b/src/wamprobe/reporting.py @@ -0,0 +1,65 @@ +"""Machine- and human-readable report generation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from wamprobe.evaluation import EvaluationResult + +_METRIC_COLUMNS = ( + ("action_dependence", "Action Dependence"), + ("counterfactual_direction_accuracy", "Counterfactual Direction"), + ("noop_stability", "No-op Stability"), + ("state_ade", "State ADE ↓"), + ("top1_regret", "Top-1 Regret ↓"), +) + + +def render_markdown(results: list[EvaluationResult]) -> str: + """Render a compact comparison table.""" + + headers = ["Model", *(label for _, label in _METRIC_COLUMNS)] + lines = [ + "# WAMProbe PointMass-2D Demo", + "", + "This report compares action-aware and deliberately broken reference baselines.", + "", + "| " + " | ".join(headers) + " |", + "|" + "|".join("---" for _ in headers) + "|", + ] + for result in results: + values = [result.model_id] + values.extend(f"{result.metrics[key]:.4f}" for key, _ in _METRIC_COLUMNS) + lines.append("| " + " | ".join(values) + " |") + lines.extend( + [ + "", + "A high Action Dependence score is not sufficient: the wrong-direction baseline", + "depends on the action but fails Counterfactual Direction Accuracy. This is why", + "WAMProbe reports a metric profile instead of a single aggregate score.", + "", + ] + ) + return "\n".join(lines) + + +def write_reports( + output_dir: Path, + *, + benchmark: str, + results: list[EvaluationResult], +) -> tuple[Path, Path]: + """Write versioned JSON and Markdown reports.""" + + output_dir.mkdir(parents=True, exist_ok=True) + summary_path = output_dir / "summary.json" + report_path = output_dir / "report.md" + payload = { + "schema_version": "0.1", + "benchmark": benchmark, + "results": [result.to_dict() for result in results], + } + summary_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + report_path.write_text(render_markdown(results), encoding="utf-8") + return summary_path, report_path diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..ae34a4e --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,29 @@ +import pytest + +from wamprobe.api.types import Action2D, Context2D, InterventionGroup, Trajectory2D, Vec2 + + +def test_vec2_arithmetic_and_norm() -> None: + start = Vec2(1.0, -2.0) + delta = Vec2(0.5, 2.0) + + assert start + delta == Vec2(1.5, 0.0) + assert start - delta == Vec2(0.5, -4.0) + assert delta * 2 == Vec2(1.0, 4.0) + assert Vec2(3.0, 4.0).norm() == 5.0 + + +def test_trajectory_requires_at_least_one_state() -> None: + with pytest.raises(ValueError, match="at least one state"): + Trajectory2D(context_id="context-0", action_name="noop", states=()) + + +def test_intervention_group_rejects_duplicate_action_names() -> None: + context = Context2D("context-0", position=Vec2(0.0, 0.0), goal=Vec2(1.0, 0.0)) + duplicate_actions = ( + Action2D("right", Vec2(0.25, 0.0)), + Action2D("right", Vec2(0.5, 0.0)), + ) + + with pytest.raises(ValueError, match="unique"): + InterventionGroup(context=context, actions=duplicate_actions) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..cc9bfe8 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,40 @@ +import json +from pathlib import Path + +import pytest + +from wamprobe.cli import main + + +def test_demo_cli_writes_machine_and_human_readable_reports( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code = main( + [ + "demo", + "--contexts", + "5", + "--seed", + "9", + "--output", + str(tmp_path), + ] + ) + + assert exit_code == 0 + summary_path = tmp_path / "summary.json" + report_path = tmp_path / "report.md" + assert summary_path.is_file() + assert report_path.is_file() + + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert summary["benchmark"] == "pointmass-2d-v0.1" + assert {result["model_id"] for result in summary["results"]} == { + "action-agnostic", + "copy-last-frame", + "oracle-pointmass", + "wrong-direction", + } + assert "Counterfactual Direction" in report_path.read_text(encoding="utf-8") + assert "WAMProbe demo complete" in capsys.readouterr().out diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..322a024 --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,37 @@ +from wamprobe.adapters.baselines import ( + ActionAgnosticAdapter, + CopyLastFrameAdapter, + OraclePointMassAdapter, + WrongDirectionAdapter, +) +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.evaluation import evaluate + + +def test_reference_baselines_have_expected_metric_ordering() -> None: + benchmark = PointMass2D(horizon=4) + suite = benchmark.make_suite(contexts=12, seed=11) + + oracle = evaluate(OraclePointMassAdapter(benchmark), benchmark, suite) + copy_last = evaluate(CopyLastFrameAdapter(), benchmark, suite) + wrong = evaluate(WrongDirectionAdapter(benchmark), benchmark, suite) + action_agnostic = evaluate(ActionAgnosticAdapter(), benchmark, suite) + + assert oracle.metrics["counterfactual_direction_accuracy"] == 1.0 + assert wrong.metrics["counterfactual_direction_accuracy"] == -1.0 + assert oracle.metrics["state_ade"] == 0.0 + assert copy_last.metrics["action_dependence"] == 0.0 + assert action_agnostic.metrics["action_dependence"] == 0.0 + assert oracle.metrics["top1_regret"] == 0.0 + assert action_agnostic.metrics["top1_regret"] > oracle.metrics["top1_regret"] + + +def test_noop_stability_catches_action_agnostic_motion() -> None: + benchmark = PointMass2D(horizon=4) + suite = benchmark.make_suite(contexts=4, seed=3) + + oracle = evaluate(OraclePointMassAdapter(benchmark), benchmark, suite) + action_agnostic = evaluate(ActionAgnosticAdapter(), benchmark, suite) + + assert oracle.metrics["noop_stability"] == 1.0 + assert action_agnostic.metrics["noop_stability"] == 0.0 diff --git a/tests/test_import.py b/tests/test_import.py index 8038ed8..257f50b 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -2,4 +2,4 @@ def test_version_is_exposed() -> None: - assert wamprobe.__version__ == "0.0.0" + assert wamprobe.__version__ == "0.1.0a0" diff --git a/tests/test_pointmass.py b/tests/test_pointmass.py new file mode 100644 index 0000000..d6523ed --- /dev/null +++ b/tests/test_pointmass.py @@ -0,0 +1,28 @@ +from wamprobe.api.types import Action2D, Context2D, Vec2 +from wamprobe.benchmarks.pointmass import PointMass2D + + +def test_pointmass_rollout_applies_action_each_step() -> None: + benchmark = PointMass2D(horizon=4) + context = Context2D("context-0", position=Vec2(0.0, 0.0), goal=Vec2(1.0, 0.0)) + action = Action2D("right", Vec2(0.25, 0.0)) + + trajectory = benchmark.rollout(context, action) + + assert trajectory.states == ( + Vec2(0.25, 0.0), + Vec2(0.5, 0.0), + Vec2(0.75, 0.0), + Vec2(1.0, 0.0), + ) + assert benchmark.score(context, trajectory) == 0.0 + + +def test_counterfactual_suite_shares_context_and_has_noop() -> None: + benchmark = PointMass2D(horizon=4) + + suite = benchmark.make_suite(contexts=6, seed=7) + + assert len(suite.groups) == 6 + assert all(group.actions[0].name == "noop" for group in suite.groups) + assert all(len({action.name for action in group.actions}) == 5 for group in suite.groups) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..d67bc57 --- /dev/null +++ b/uv.lock @@ -0,0 +1,467 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version < '3.15'", +] + +[[package]] +name = "ast-serialize" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/ad/0d70a3a2d6e01968d985415259e8ec7ad3f777903f9b1c1f3c8c44642c60/ast_serialize-0.6.0.tar.gz", hash = "sha256:aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe", size = 61489, upload-time = "2026-06-30T20:02:55.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/12/3e5f575f156555547c250a8b0d1347517a3a20fc7f4492e9703a69d4f45e/ast_serialize-0.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:a7520b672827885bafeae7501f684d14d47d17e5f45256f9df547686cca52264", size = 1177640, upload-time = "2026-06-30T20:02:06.708Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a4/921a9e27951627983b0f368859ea00f8330a551dc0bf4c2fdcb11855a98b/ast_serialize-0.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a14191beec7e0c078d2fc1f6edc0aee88bcd4db9f18e1bc9f8052b559c22dddc", size = 1168111, upload-time = "2026-06-30T20:02:08.366Z" }, + { url = "https://files.pythonhosted.org/packages/00/69/950cf404de7b8782cf95e5c1237e25e2aa46177b287f39f9eeddf481fd6f/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32ef62ec34cf6be20ad77d4799556638fbdf187f3ae10698dfb20ef9f2c89516", size = 1227656, upload-time = "2026-06-30T20:02:09.843Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a8/46f8f6a6479d9d2273980957bb091a506c55f5b95d3c029ee58518a78407/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13b7769970a39983b0adf2f38917b1cd3b8946f76df045756c3d741bc689f089", size = 1227706, upload-time = "2026-06-30T20:02:11.367Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b9/9ac415bda0a40e49eab8fea3b2741c19c98bb84d57d62c4cfc6230eb67be/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f7a408601bb3edaefb3bc67a4c01f5235e3253653b6a5729a2ee2382b35341c", size = 1431705, upload-time = "2026-06-30T20:02:12.737Z" }, + { url = "https://files.pythonhosted.org/packages/e5/06/8807115d441444879f7561b5eede5ac18fc80392f11826d61ccf31f503b1/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8670bfa51208a2c0c8d138928e40e998fab158f9200d53bb80c088b5b8eda7b8", size = 1249533, upload-time = "2026-06-30T20:02:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c0/c2ba82ef9618650357d9421a1fdb27ffec862a7f57e8e2de82a3ccd11e12/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4826809eb8597a8cd59fd924b6d7c285b8969a1e0007e2cb652cab62376270f", size = 1252619, upload-time = "2026-06-30T20:02:16.219Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a7/fa31d52dd4102cede29fb9634e98d214129b2783b4f95528c6dc6a8f6587/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:577a6c189068686869f5f1ddc38363f3ae1808a4753b577266f9202071a7bb66", size = 1242983, upload-time = "2026-06-30T20:02:17.813Z" }, + { url = "https://files.pythonhosted.org/packages/b1/20/ddf742b5ad3c4bafd3466f2265037cfd99bc1b9a5ee46a5d58c90d523242/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:085de7f62dc9cc247eb01e965a362707d1d90b1d89a82c5bf78301a60a3c417b", size = 1296148, upload-time = "2026-06-30T20:02:19.146Z" }, + { url = "https://files.pythonhosted.org/packages/24/cb/9f6f217cce8b3b632c5568b478d195a35e79dce4dbe309438cb89ba6ea4f/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9f8a8b78b13173de6a9ec22111d9be674874cd5bdccda04f14ae5ebc2bef403a", size = 1403826, upload-time = "2026-06-30T20:02:20.696Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f8/9d16d4f0107a183924425cc0e7618d8bf76f96b45afa9ff19f924ed1ad57/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554", size = 1502943, upload-time = "2026-06-30T20:02:22.034Z" }, + { url = "https://files.pythonhosted.org/packages/80/dd/bbc1c38756350dddf7e24acae1c9482ef42051c267417e019aecc1ed4075/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0067b25fce104eaae5b88383de9ab803faeb671831e14ca698b771b356e2600f", size = 1497632, upload-time = "2026-06-30T20:02:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/42/7e/9daffefcf5b97e6bb4c3e0b3c024c1aee9722f23d3cf7cd2ff80d6fb4a40/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c617417f9cbb0cb144f6283c3cbe0d2e0f01beaf9f608f662b21191058a626ec", size = 1448858, upload-time = "2026-06-30T20:02:24.889Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1f/f9baaab81a677ea0af7d2458cac2f94ebcc85958f8a3c15ba9d9e5dab653/ast_serialize-0.6.0-cp314-cp314t-win32.whl", hash = "sha256:5337cb256dcea3df9288205213d1601581536526b8f4da44b6974f1180f3252a", size = 1052600, upload-time = "2026-06-30T20:02:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1f/41b535866519512d8cf6669cb2cff7823b7672bb6279c0333b4ff89d7d9f/ast_serialize-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d947e45cafc4b09bd7528917fa84c517654a43de173c79785574b7b3068ac24", size = 1095570, upload-time = "2026-06-30T20:02:27.639Z" }, + { url = "https://files.pythonhosted.org/packages/50/64/e472fe3e3a2d33d874b987e8518aedf24562919e3b6161a4fa1797e89c0f/ast_serialize-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6e15ec740436e1a0d62de848641abe5f3a2f89a7f94907d534795ac91bbacf14", size = 1067267, upload-time = "2026-06-30T20:02:28.949Z" }, + { url = "https://files.pythonhosted.org/packages/52/19/ac8348ae8711c9b5ae834634f635780cab62a0f5e6f988882e048b89c2ae/ast_serialize-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596", size = 1185367, upload-time = "2026-06-30T20:02:30.427Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f6/ec7ec652c51db77c2f61d8573338e13e4704303265ccc658cb4031d9f354/ast_serialize-0.6.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad", size = 1178657, upload-time = "2026-06-30T20:02:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/6f/02/613a7534a41d0122f37d1e0c64aa8ac78bfb831f8c92f6db057a311abb3c/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a", size = 1238620, upload-time = "2026-06-30T20:02:33.664Z" }, + { url = "https://files.pythonhosted.org/packages/4d/21/087957bba486242afc52f49b2d9e21c9dad00289356cf9efe67084015a9d/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081", size = 1236075, upload-time = "2026-06-30T20:02:34.936Z" }, + { url = "https://files.pythonhosted.org/packages/82/04/78128bbb170071c2c72a210a181f1c00e11cc1cec60a8beef747b07f9201/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77", size = 1441348, upload-time = "2026-06-30T20:02:36.245Z" }, + { url = "https://files.pythonhosted.org/packages/64/64/62fb99d6faf199b4c3e5b08a07136e9a0d7664bb249c6de3670e5b63e9b6/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790", size = 1258580, upload-time = "2026-06-30T20:02:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/b4d6c38e0ccd5e85dc54cecdf933a152c60b28fe5d993a6d8a72fa6d5896/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7", size = 1261693, upload-time = "2026-06-30T20:02:39.123Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/3676ca2191f39bafb75f93f99b2f429ec464586158fece2165f3572805dc/ast_serialize-0.6.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b", size = 1252517, upload-time = "2026-06-30T20:02:40.511Z" }, + { url = "https://files.pythonhosted.org/packages/f3/58/494ef8c4b4acb2f4a265ac934caf45f792a08fe27d6b853de35ad991941a/ast_serialize-0.6.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b", size = 1304843, upload-time = "2026-06-30T20:02:41.961Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f2/13736d920ab3d49bbee80ef1a277dd7b7aaf3b3545efd9d2a8114fe05525/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32", size = 1413698, upload-time = "2026-06-30T20:02:44.179Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5a/e046f3899e2acba4677d7427b76431443a1aa1a0e583dfb05b55b69d55cf/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b", size = 1512209, upload-time = "2026-06-30T20:02:45.584Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c7/e42aaca7bb2d22a7c06d5a8c7930086c5a334e93d716e6fa5e6647a4515f/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a", size = 1508464, upload-time = "2026-06-30T20:02:46.942Z" }, + { url = "https://files.pythonhosted.org/packages/95/93/5524a3dc6c3f593de3228ed9cbef73afa047625b7000ec21b7f58e6eb4d4/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e", size = 1457164, upload-time = "2026-06-30T20:02:48.294Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c0/36a6ffb4d653cf621427b4c4928671f53ad800c453474de2b82564a44ad9/ast_serialize-0.6.0-cp39-abi3-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b", size = 863014, upload-time = "2026-06-30T20:02:49.742Z" }, + { url = "https://files.pythonhosted.org/packages/09/c7/7d5ad8b49e1278e1c2a1e0274bd7850560b3f09313aa00c13bc8d5544792/ast_serialize-0.6.0-cp39-abi3-win32.whl", hash = "sha256:82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1", size = 1063165, upload-time = "2026-06-30T20:02:50.98Z" }, + { url = "https://files.pythonhosted.org/packages/47/ae/6710c14ecb276031cf10249f6adf5a59e2d3fdb3b5183bd59f70524067ee/ast_serialize-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e", size = 1101444, upload-time = "2026-06-30T20:02:52.554Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/83/6051c2a2feab48ae5bd27c84ef047191d2d4a3172f689e38eaa48ed17db1/coverage-7.15.1.tar.gz", hash = "sha256:165e9949eaf222ef1f018635d0d7f368a23bfe0212af558534c40d8c04686d67", size = 927640, upload-time = "2026-07-12T20:58:19.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/ee/5095e07a0986930174fc153fd0bb115f7f2724f5344cc672e016d4f23a4e/coverage-7.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6506330b4a8dcf53b95bd84d8d0e817107cdb3fc1438e835029cdf0bc6612eb0", size = 221320, upload-time = "2026-07-12T20:56:16.036Z" }, + { url = "https://files.pythonhosted.org/packages/43/7a/7e2598ce98433a214020a61c0755d5961f9f26df0c630dc73e06b86d3416/coverage-7.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8a51a8ec382c39d939cba0ab07ae949077ae4e842343bd4eed22d432358cea9", size = 221823, upload-time = "2026-07-12T20:56:17.54Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4f/67dac6a69139b490a239d91b6d5ef127982ffb89159769241656ab879ee5/coverage-7.15.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:18ea20e3922d7f8ca9e0ef1084408d08c4ad62d5e531cb9c1f6896a99297ebea", size = 252242, upload-time = "2026-07-12T20:56:18.868Z" }, + { url = "https://files.pythonhosted.org/packages/05/37/5e439725a96de592309725550c8df639c359c209dcb4b152ed46032d4c0d/coverage-7.15.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:aab9902a64b8390e3b56e539fddae1d79a267807fe5cb0c18d7d2f544ce867e2", size = 254153, upload-time = "2026-07-12T20:56:20.118Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1c/671ba9e15f9651a99962fb588a1fe4fb267cae4bb85f9bb4ac4413c6f7ef/coverage-7.15.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cc316264317b07a9e90d7f2b4188a15e36e9b54e651081b791b0515fa612a29", size = 256261, upload-time = "2026-07-12T20:56:21.682Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3c/88305322b4d015b4536b7174b8f9b87f850f01af9d89008cdbf984b65ff2/coverage-7.15.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d4e47e7eea81a8ccf060a07627654151d929da62c7b715738387c200905cec89", size = 258222, upload-time = "2026-07-12T20:56:22.962Z" }, + { url = "https://files.pythonhosted.org/packages/8d/20/d02e42333a57f266ded61ed4fb3821da1f2dc143734aaa3dcc9c117204c5/coverage-7.15.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6829d9a3b55ad2b73ef5fda8302e5be03683789e88b1a079dcf4a773229c21d", size = 252368, upload-time = "2026-07-12T20:56:24.475Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d0/94ebc010926a191d83e83b1f1236f8bd0d14d0eb98b5c324aa4be2589a8e/coverage-7.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:764e045811f9c8cda436641f3f088283351d331a519b5807f19041cd0a68da1c", size = 253953, upload-time = "2026-07-12T20:56:26.036Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a6/2c8553191da0c2da2c43ff294fb9bab57a5b0fae03560304a82a8b5addc3/coverage-7.15.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:09b3b088aa24489c4082bcc35fcc8224281ab94a653dfb6d3f0c8165b0d628ab", size = 252016, upload-time = "2026-07-12T20:56:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/17/99/20f47986d8360fa1a31d9858dd2ba0be009d44ecfa8d02120b1eb93c028b/coverage-7.15.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7911b02f57053adf8164ae63edb1c26574d24dfccabadc5268cf69310a69a358", size = 255783, upload-time = "2026-07-12T20:56:28.921Z" }, + { url = "https://files.pythonhosted.org/packages/17/36/fb61983b5259f78fa5e62ee74e91fe4de4c556fcbc9a3b9c9021c2f8b57b/coverage-7.15.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:664e279ed40599b8ed16f4db18d92a7e212c73129672bec8f5d96d4da48d2404", size = 251735, upload-time = "2026-07-12T20:56:30.465Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2e/d109d8d2d6c726ba2e78125297073918a2a91464f0ea777e5398fa3cf75c/coverage-7.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34fe7cf79d5f1f87f2e8ce7dc1c32950841f50e10d0120f263856acfad66de34", size = 252645, upload-time = "2026-07-12T20:56:32.076Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/5b3219cbb46135e14673427f2bf5890c4da4e6f655243692f3448aa3f42c/coverage-7.15.1-cp311-cp311-win32.whl", hash = "sha256:5e2d2536d2f57a354aa382ed303ac0e2e5c9522a508c05b998d26181b94163a7", size = 223414, upload-time = "2026-07-12T20:56:33.432Z" }, + { url = "https://files.pythonhosted.org/packages/5b/80/24e13ade0b6df8090e2492f9c55044bf1423f7ef28c76fc1af8c827a61cb/coverage-7.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:c337da8fca7ea93ab43f3868cfcde6cf6dad32c3906b273cfbad5d7390bc423b", size = 223888, upload-time = "2026-07-12T20:56:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9e/34659930ae380491af48e2f5f5f9a2e99cd9fb7de3d30f27be5ac5425137/coverage-7.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:db3403fdb7a94d5eb73e099befad8104d2a7d110a0f0d99df0de61c5d1fa756c", size = 223435, upload-time = "2026-07-12T20:56:36.302Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/32c1826309beaf4604c54accef108fdd611e5e5e93f2f5192f050cd5f6bd/coverage-7.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d9476292594309db922cc841dd13b303b3c388f4c25d279884f7e2341c681f80", size = 221497, upload-time = "2026-07-12T20:56:37.628Z" }, + { url = "https://files.pythonhosted.org/packages/db/5c/b88ce0d68fa550c7f3b58617fbf363bce64df5bf8295a01b627e4696e022/coverage-7.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c579056b0de461b3a62318b63d0b6ce90aed7f8158d3f00da094df82f29d189", size = 221854, upload-time = "2026-07-12T20:56:39.033Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/8509fd2a66fc4e0a829f76a0f0b1dc3cc163368352435b5f243168658077/coverage-7.15.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:23214bdbe226f2b0e9c66a7d6a1d59d4a88045dcf86e702cf0fe0d0935e3d615", size = 253359, upload-time = "2026-07-12T20:56:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/e5/81/c7009ed7ea9765adb2b9d095054d748266fae5f07ac6c5f925f33715fcde/coverage-7.15.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df164be93b46b4825cc39339440a05edc54c4d1d865ba4a60fd43d151a2a1cd3", size = 256096, upload-time = "2026-07-12T20:56:42.115Z" }, + { url = "https://files.pythonhosted.org/packages/21/52/dc8ee03968a5ba86e2da5aa48ddc9e3747bd65d63825fdce2d96acb9c5ff/coverage-7.15.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a524fca1a6f08927d9dc2d4c873cfb7bd7202c247f08b14bdc02424071b8b304", size = 257211, upload-time = "2026-07-12T20:56:43.513Z" }, + { url = "https://files.pythonhosted.org/packages/b8/27/95d7623908da8937deb53d48efcdbf423907a47540e63c62fa21372c652b/coverage-7.15.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d70f3542cd38de85a9e257dcb1ac4c1ab4b6d7d2c2a645809207556628755d1c", size = 259473, upload-time = "2026-07-12T20:56:44.974Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3b/730d761928de97d585465680b568ae69622fb40716babadeabffe75cb51b/coverage-7.15.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d78aa537237212c4313aabe5e964b66acc86350ed19ebc56a3e202df33b6077b", size = 253759, upload-time = "2026-07-12T20:56:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fc/6b9277acff1f9484b6c12857af5774689d1a6a95e13265f7405329d2f5da/coverage-7.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a318112bb4f79d9d04766196d5a3388caa825908a6a9b052aa87de3d9aea7c61", size = 255131, upload-time = "2026-07-12T20:56:48.073Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f2/c704f86129594ba34e25a64695d2068c71d51c2b98907184d716c94f4aec/coverage-7.15.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e55d24cada901963eed5bc89fa562aa033f0d84b9d3de4ecf363737c13aed11e", size = 253275, upload-time = "2026-07-12T20:56:49.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/29/80fee8af47de4a6dce71ccf2938491f444687a756af258a56d8469b8f1b0/coverage-7.15.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c78f0cea7275342cf2adc2ad5fdd0aafa106ad91e66d573568f2fcf62c41df5", size = 257345, upload-time = "2026-07-12T20:56:51.038Z" }, + { url = "https://files.pythonhosted.org/packages/20/21/a1e7d7ed1b48a8adf8fd5154d9e83fcc5ad8e6ff20ae00e44865057dce8d/coverage-7.15.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:86bd37eabe39977216f630a7fc1b698e7f5e81a191c7186013245c6c3d313f9d", size = 252844, upload-time = "2026-07-12T20:56:52.535Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8c/a4bc26e6ee207d412f3678f04d74be1550e83140563ca0e4997510579712/coverage-7.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6db15c217693bdc3ca0b84de1ba9afafe1c14c26a8a29d77f4ed0de2b6132e2", size = 254716, upload-time = "2026-07-12T20:56:53.968Z" }, + { url = "https://files.pythonhosted.org/packages/11/9d/8ad0266ecfada6353cf6627a1a02294cf55a907521b6ee0bd7b770cfd659/coverage-7.15.1-cp312-cp312-win32.whl", hash = "sha256:359f3fbe09a51500c51966596ee4ee4070b356552c70b3b2420eb200d68e0f76", size = 223554, upload-time = "2026-07-12T20:56:55.583Z" }, + { url = "https://files.pythonhosted.org/packages/81/6d/24224929e06c6e05a93f738bc5f9e8e6ab658f8f1d9b823e7b85430e28b8/coverage-7.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:fa75dc099c126e941a9c0baa8ebd2cbc78bd778687534fe410baf754f6d9e374", size = 224087, upload-time = "2026-07-12T20:56:57.041Z" }, + { url = "https://files.pythonhosted.org/packages/35/23/f81441dd01de88e53c97842e706907b307d9078918c3f4998b11e9ac7250/coverage-7.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:26f89cf6d0634375f454fa71057945ad18edb0f1607a90fecf22c57dc3dc289a", size = 223472, upload-time = "2026-07-12T20:56:58.594Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1e/6fa289d7993a2a39f1b283ddb58c4bfec80f7800be654b8ba8a9f6a07c63/coverage-7.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:71ac4ca1658ca99160fd58cc6967110e989c34b04627f24ed6ec9f70fb24571a", size = 221519, upload-time = "2026-07-12T20:57:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e1/0db4902a0588234a70ab0218073c0b20fbc5c740aa35f91d360160a2ebc9/coverage-7.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:26a40cbf2b13bd94af53ee02a424cb3bb96a9edfac0d00834bd068512a62714b", size = 221895, upload-time = "2026-07-12T20:57:01.867Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/3719783865092dac5e08df842730305ee9ab1973ae7ddb6fbdf27d401f30/coverage-7.15.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4c5a5eff4ad4f9f7088fd3fc7a66d98d06566ee294b3b053309fb0a3b45be1e", size = 252882, upload-time = "2026-07-12T20:57:03.459Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5e/caf3abbdbb22629626160ffc9c017eb995b7cb11c0be46b974834cef1792/coverage-7.15.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:962aa56c1c9b016d681265880eb6acc9966029d2c4c559319cc43a1abbb9b59a", size = 255479, upload-time = "2026-07-12T20:57:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f1/d60f375bfe095fef944f0f19427aefdbf9bdd5a9571c41a4bf6e2f5fdb81/coverage-7.15.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1678eb2dc57a8ce67601b029582ef6d41e9e6ca22692aaeccd4107e40f27386c", size = 256715, upload-time = "2026-07-12T20:57:06.446Z" }, + { url = "https://files.pythonhosted.org/packages/d7/17/8b0cbc90d02dc5adad4d9034c1824ec3fa567771b4c39d9c1e3f9b1431b8/coverage-7.15.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1174900a43f6f8c425fee10d7dbddc308adefcdc78aaced32357f5ab750a0e90", size = 258845, upload-time = "2026-07-12T20:57:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/92/29/c5e69f5fb75c322e9a3e4ef64d02eebfc3d66efceccc8514ff80a3c13a56/coverage-7.15.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98847557a6859cadf693792ce89f440cb89692993f60dc6d3a7e35f3d340216f", size = 253098, upload-time = "2026-07-12T20:57:09.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/57/21144252fdd0c01d707d48fbcea13a80b0b7c42ced3f299f885ab8978c3a/coverage-7.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8697b2edb57143546a24389efc11e1b000cd5800fc20d84f04edb601e4a7cfb8", size = 254844, upload-time = "2026-07-12T20:57:11.141Z" }, + { url = "https://files.pythonhosted.org/packages/59/2a/499a28a322b0ce6768328e6c5bb2e2ad00ac068a7c7adb2ecd8533c8c5d9/coverage-7.15.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6827ac0519be3fe91bf96b4060eb00d1d24f82649b29862cd75a3cfca248b02a", size = 252807, upload-time = "2026-07-12T20:57:12.678Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/928a95da5da8b60f2b00e1482c7787b3316188e6d2d227fb8e124ada43a1/coverage-7.15.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2de8ecbbc77c7e4d22572779920ed8979c69168675e96be3a548c996568c6c31", size = 256965, upload-time = "2026-07-12T20:57:14.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/10/889adbc1b8c9f866ed51e18a98bcafc0259fb9d29b81f50a719407c64ea8/coverage-7.15.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b25f0f0fa5260df9d7bb55d47c8bdc23fa3382c1a18f7c9cae122e6c320b1ad", size = 252628, upload-time = "2026-07-12T20:57:15.892Z" }, + { url = "https://files.pythonhosted.org/packages/1a/30/a5e1871e5d93416511f8e359d1ccebfe0cbb050a1bbf7dd20228533ec0cf/coverage-7.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a2effcbd93ae340a58db718fe4181d967f84d352c4cefeaab4ff82ce813901a", size = 254399, upload-time = "2026-07-12T20:57:17.703Z" }, + { url = "https://files.pythonhosted.org/packages/2d/26/c36fbffd549dadbdd1a75827528fb00a4c46aa3187b007b750b1e2cebbf2/coverage-7.15.1-cp313-cp313-win32.whl", hash = "sha256:895e65c96aef0cecea250f6e35e9a32f11375514e1a0cb5210e0fda128c04e8e", size = 223564, upload-time = "2026-07-12T20:57:19.253Z" }, + { url = "https://files.pythonhosted.org/packages/16/fc/becbb9d2c4206d242b9b1e1e8e24a42f7926c0200dd3c788b9fab4bb96d5/coverage-7.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6d0a28b63a0d75f9ed5118105d1154fc3aa40a8605a30d5d87e3d043ad90fe7", size = 224106, upload-time = "2026-07-12T20:57:21.108Z" }, + { url = "https://files.pythonhosted.org/packages/d3/30/1cfc641461369b6858799fca61c0a8b5edc490c519bf7c636ffa6bbf556f/coverage-7.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:b4ee9818e8bae3544379ad2c09b851c4fb886aaa8860d57a1c1316ddcb16db49", size = 223497, upload-time = "2026-07-12T20:57:22.734Z" }, + { url = "https://files.pythonhosted.org/packages/f0/46/81961952e7aebfb38ad0ae4264e8954cc607a7af9e7ac111f9fa986595cc/coverage-7.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a886af95f59edf67d5770fd3564d53f4a8af93f25f8c1d60d27e00d7f5674ee8", size = 221560, upload-time = "2026-07-12T20:57:24.282Z" }, + { url = "https://files.pythonhosted.org/packages/13/d2/ee14d715889f216baf47301d9f469e08fff6995552aaf67e897b282865f6/coverage-7.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:985657ebd707941de90d488d1cbb5efac20bdf81f7b91eba771624ccda4d36f4", size = 221894, upload-time = "2026-07-12T20:57:25.87Z" }, + { url = "https://files.pythonhosted.org/packages/f3/38/f830bc6e6c2c5f23f43847125e6c650d378872f7eeba8d49f1d42193e8a9/coverage-7.15.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5bbe2a06e0a5e1404d9ffbdb49b819bbd6a3bb198ebea4c8dfe7ad9f1e1c2e81", size = 252938, upload-time = "2026-07-12T20:57:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/0d3dd963631259d794c898735d5436e68d6a8d40749c419a07ff7c171469/coverage-7.15.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bde0fe24083d0b7b3dbafa7a09f0796410af1afa2523f28f5f208d8340a4aaca", size = 255445, upload-time = "2026-07-12T20:57:29.234Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fd/aabed228557565c958259251b89bab8c5669b31291fa63b3e2154ebb017a/coverage-7.15.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f89f7453d6d46db14cf233e2cd8edcd78de2b9c49d4f1dc109590b4e5dbfbb74", size = 256790, upload-time = "2026-07-12T20:57:30.826Z" }, + { url = "https://files.pythonhosted.org/packages/bc/aa/1cc888e5d3623e603c4e5399653cb25728bb2b40d7519188a3e293d24620/coverage-7.15.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc3656c9ecc27b36bd0907455b77f83c0069ca9ad4a66dec892b76c696eb6047", size = 259104, upload-time = "2026-07-12T20:57:32.63Z" }, + { url = "https://files.pythonhosted.org/packages/5f/61/fc16d5f5e53098dae41efa21e8ccc611a9b4fe922750dd03dc56db552182/coverage-7.15.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:24d8e85a2a45e44883b488c2659f51fa761dad5353fdb319b672a93facbd2ca9", size = 252956, upload-time = "2026-07-12T20:57:34.316Z" }, + { url = "https://files.pythonhosted.org/packages/a1/f3/52384668c3de4519ca770bf1975a89e4d6eb5aa2faf0da0577a14008cba4/coverage-7.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:68931b5fe746ed4fdaa8892989cab9e6c35781eeb3b0ab2ded893d561e1b3652", size = 254797, upload-time = "2026-07-12T20:57:35.947Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/54b807e7c1868178e902fd8360b5d4e559394462f97285c50edf1c4608db/coverage-7.15.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1ce6947e2a95534ecaa5a15e73c21e550514c980d80eda204d064d789a95f6a4", size = 252762, upload-time = "2026-07-12T20:57:37.856Z" }, + { url = "https://files.pythonhosted.org/packages/7b/48/dde8adf0338e3ace738757dccf1ce817e5fdcadfae77e1b48a77e5a3b265/coverage-7.15.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:841befdbc89b9c82435fc25b0f4f41858b6238693e45af758bec4cfc1968171c", size = 257037, upload-time = "2026-07-12T20:57:39.488Z" }, + { url = "https://files.pythonhosted.org/packages/07/f2/179dd88cf60a0aeeee16a970ffe250dccea8b80ed4beab4c5d3f6c41ad4b/coverage-7.15.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5d3de58b837375e7f4c0e1a088ccab5f655efb2fd7427b729df02c862a559633", size = 252577, upload-time = "2026-07-12T20:57:41.363Z" }, + { url = "https://files.pythonhosted.org/packages/2c/37/8a593d69ab521beb6a105a2017cac4ba94425ee0a8349e29c3c0b522d24f/coverage-7.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b1801963f9f44ae0c0f6d737bc7aeb2bbcde7d1fe7e3b43cddc1961af42d3b41", size = 254235, upload-time = "2026-07-12T20:57:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/6d/34/bc9b3bced66f2cdad4bf5e57ae51c54ea226e8aaaebfc9370a9a11877bf3/coverage-7.15.1-cp314-cp314-win32.whl", hash = "sha256:8c7953c4128ef53b6ffb5f90d87c87d4ce26731df294760bb2314eb0e069e44b", size = 223771, upload-time = "2026-07-12T20:57:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f3/4d20337bed61915d14349e62b88d5e4144d5a9872b64adbe90e9906db6db/coverage-7.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:6f0bab60a582d415f0fb535ccff13ba334a47a1538f98913330a525d23bd535a", size = 224257, upload-time = "2026-07-12T20:57:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/7b/df/bbfeae4948f3ded516f92b32f2d57952427fc5ecfc0924487bb6ee6a5f38/coverage-7.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:0f410ee8f0ac4ec7db71bc0b7632a8b9994e1cad2755bd1566c17e6a162caa74", size = 223683, upload-time = "2026-07-12T20:57:48.106Z" }, + { url = "https://files.pythonhosted.org/packages/35/65/0b431856064e387d1f5cf474625e4a0465e907024d42f35de6af19ced0be/coverage-7.15.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:fc868bab88e049d41fcd41766810d790a8b960053be2a45e060f5ce0d31d258b", size = 222298, upload-time = "2026-07-12T20:57:49.882Z" }, + { url = "https://files.pythonhosted.org/packages/a6/96/50eac9bd49df8a3df5f3d38746d1bf332299dffb554486c94ebd55c9dc49/coverage-7.15.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:206d4ec6028f2773b40932d09f074539d6bcdd8f6b318d40cb04bdbd68ed0b49", size = 222561, upload-time = "2026-07-12T20:57:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5b/6ba1c4a27e10b8816fd2622b98162c83d3bdf1185097360373611bf96364/coverage-7.15.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:620482ef1c9f4e61f962e159325fe77dea59d16e39d9c9470d069053b244d864", size = 263923, upload-time = "2026-07-12T20:57:53.392Z" }, + { url = "https://files.pythonhosted.org/packages/e4/59/fe03ade97a3ca2d890e98c572cf48a99fda9adba85757c34b823f41efe1e/coverage-7.15.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d385fc9b054e309ad3cecdc77b586d2af0c98aeec2fdb3773544586f366e817c", size = 266043, upload-time = "2026-07-12T20:57:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/16/e0/55c4b1217a572a43e13b39e1eb78d0da29fb23679003bd0cdf22c50b1978/coverage-7.15.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1198bca9c0dd7c188aae1f185b0c0b5fc4f0a2b6909000858c29550320bdb07", size = 268465, upload-time = "2026-07-12T20:57:57.017Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/ee47944f76afc03909119b036fe9e0da8cbd274a5141287de79791a0fb6d/coverage-7.15.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5d0297e6a070eadb49df7cddd0ab6f420b8b689dd8904c7dd815a323168fa57e", size = 269584, upload-time = "2026-07-12T20:57:58.958Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8a/6b4d9779c7b2e21c3d12c3425e3261aa7411399319e27aa402dfec4db5d0/coverage-7.15.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916fcf2214f56960e409561b37fc32a160a42b6e85483d0652d7b70fa55d707e", size = 263019, upload-time = "2026-07-12T20:58:00.979Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1e/db5c7fa0c8ba5ece390a1e1a3f30db71d440240a80589df28e66a7503c40/coverage-7.15.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f837bae572c7869ffaa502e604c87e182543012831cf87aae4586ad090ac6dcf", size = 265916, upload-time = "2026-07-12T20:58:03.005Z" }, + { url = "https://files.pythonhosted.org/packages/83/53/fe5176682b00709b13fab36addd26883139d0dea430816fea412e69255e2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3ea65e3ee6c7c32349fd00559927a9e577bdd72386087eeed1c42b62dfce9b82", size = 263520, upload-time = "2026-07-12T20:58:04.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/16f15127be93fbc70c667df5ec5dce934fc76c9b0888d84969a5d5341e2c/coverage-7.15.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:345034976f46a1c54bd17f4e43eb30bb92cb7082fcddff03250cff136cc4eb82", size = 267254, upload-time = "2026-07-12T20:58:06.824Z" }, + { url = "https://files.pythonhosted.org/packages/cd/73/e5119111f6f065376395a525f7ce6e9174d83f3db6d217ea0211a61cca4d/coverage-7.15.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4f051a64eb8f8addb4661c2b41d6eea5b7ebc68ad4b2baea8d9bc54e1956e5f7", size = 262366, upload-time = "2026-07-12T20:58:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d7/9c/6d0a81182df18a73b081e7a8630f0e2a52b12dfd7898c6ab839551a454d2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a7625770f7720b49bb30d194ad2f8d50fab3c5177874af3d2399676f95f9c594", size = 264680, upload-time = "2026-07-12T20:58:10.359Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a2/bac0cbd4450638f1be2041a464b1766c8cc94abf705a2df6f1c8d4be870d/coverage-7.15.1-cp314-cp314t-win32.whl", hash = "sha256:81e503d130a472ad1bd38199ecd35116b40d92bcd31e27a2cacde035381f2070", size = 224077, upload-time = "2026-07-12T20:58:12.065Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b2/d83c5403155172a43ba47c08641bad3f89822d8405102423a41339d2c857/coverage-7.15.1-cp314-cp314t-win_amd64.whl", hash = "sha256:724e878b213b302ad46e9f2fc872d386613f20ebfc492a211482d917ea76c14f", size = 224908, upload-time = "2026-07-12T20:58:13.956Z" }, + { url = "https://files.pythonhosted.org/packages/cc/41/442b74cad832cc77712080585455482e7cc4f4a9a13192f65731dcd18231/coverage-7.15.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ce2f05c14d077f406fefc4fa5e4f093ad0e0787549f6582535d6e28766f0361b", size = 224219, upload-time = "2026-07-12T20:58:16.315Z" }, + { url = "https://files.pythonhosted.org/packages/34/98/07a67cf1a26e795d617ed5c540c042b0ac87b72f810c30c07f076cf334f3/coverage-7.15.1-py3-none-any.whl", hash = "sha256:717d01e6e00bed56ad13306f19e0dd2f4f645ee8159d2c72c72301d6cfc7090c", size = 213284, upload-time = "2026-07-12T20:58:18.079Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "librt" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/2f/3908645ddddab7120b46295e541ead308109fa48dbec7d67d7a778870d60/librt-0.13.0.tar.gz", hash = "sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781", size = 211402, upload-time = "2026-07-08T12:26:29.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/25/a6498964cfeec270c468cffdc118f69c29b412593610d55fa1327ca51ff4/librt-0.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1b5a7bbff495baedbd9b916c367d66854008f8f3b575908ded477c499dc60082", size = 148029, upload-time = "2026-07-08T12:24:45.961Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/dc86d1bffd8e0c2818bace29d9f7783cfbb8e0673bf3673b5bbd5bbe0420/librt-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34bc7938b9fdf14fe32a406c19c71faf894c5cee7e7474bd0be2f17200b82d14", size = 153036, upload-time = "2026-07-08T12:24:47.257Z" }, + { url = "https://files.pythonhosted.org/packages/29/3f/b923826660f02f286186cd9303d52bb05ced0a13708edc104dc8480920e3/librt-0.13.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f40e56b61b41be5f7dec938cfeffd660668cf4b5e72c78e7bd671d66b7bc2c79", size = 493062, upload-time = "2026-07-08T12:24:48.483Z" }, + { url = "https://files.pythonhosted.org/packages/88/87/6c0980a9c9b1302cb68d108906697b89eceb55889bb1dcf77c109aa56ca5/librt-0.13.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:9c5d02b89de5acd0379a51ec44a89476fb03df6145442e1c8ecd6bee2f91b176", size = 485510, upload-time = "2026-07-08T12:24:49.727Z" }, + { url = "https://files.pythonhosted.org/packages/32/81/795ae3b9df5dd94079fb807e38191855e023e8c6249014ae6bc3f0d9a490/librt-0.13.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7db9a3ff32ef5f7d1703d93831a3316cdf0b537de6a1cc03cc8fdd09b9194e89", size = 515909, upload-time = "2026-07-08T12:24:51.135Z" }, + { url = "https://files.pythonhosted.org/packages/20/e5/182de15abce8907108a6fdb41487de65beb5099b74dc5841b19b099168db/librt-0.13.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3dbb2a31882456cadc7053378e81ad7ed7693db4ac9f98ab5f81ef034aa8ec9f", size = 508620, upload-time = "2026-07-08T12:24:52.358Z" }, + { url = "https://files.pythonhosted.org/packages/32/03/33978d32db76e1f66377e8f78e42a2ca3c162143331677d1f50bbad36cfb/librt-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c6014e3c80f9c1fe268ef8b0e0ef113bac672cc032f2f93866e7ddad4f3e663d", size = 530363, upload-time = "2026-07-08T12:24:53.503Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f5/b291fbd2d00f7d8287bcbf67b5aa0c6afed4bc26cef23e079629c47a2c04/librt-0.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:091b60a4d2174fc1ec5c34cdc0b72efb6224753d76b7da61ebeab7a191aec8bd", size = 534209, upload-time = "2026-07-08T12:24:55.138Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/6f41f17939d191bc21609f220da8509316bc62797f078545fe83be522e78/librt-0.13.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:66cb1138f384a191a6d75f986064841fcfdc0cea98f7bd9c9ab9b38049917588", size = 514254, upload-time = "2026-07-08T12:24:56.276Z" }, + { url = "https://files.pythonhosted.org/packages/af/c2/2e4befa5410a7443019c14abccc94ff619797171f6b72013635fb87f31d7/librt-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:17221a7569f8f292aa0014226e48aa25b8c2b08da18088cd230953d0ea0f9cd1", size = 557611, upload-time = "2026-07-08T12:24:57.561Z" }, + { url = "https://files.pythonhosted.org/packages/ab/54/8b69f81448417adbc040a2185f4e2eece1e1994b7dcfaeed4662b30f98a5/librt-0.13.0-cp311-cp311-win32.whl", hash = "sha256:fc67741da44c6eaa90e01eafb586bbba9b51eb5b6ed381ee6f5ae72eb3316d21", size = 104906, upload-time = "2026-07-08T12:24:58.806Z" }, + { url = "https://files.pythonhosted.org/packages/76/5a/f4aaf37b50f2fde12c8c663b83fdd499cdc24f957f19543d7414bfcc9e25/librt-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:cc99dfb62b23c9207c33d0be8a2e2af7a42e21e6ea388b380a0c948c7b88953b", size = 125852, upload-time = "2026-07-08T12:25:00.065Z" }, + { url = "https://files.pythonhosted.org/packages/f2/99/bf1820e6feeabc2f218c24450ec0c995d6a91e8ba0fd3caf042c9e8adb2a/librt-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:40ccd13c252d3fe473ffc8a57be7565abc8b64cf1b108344c859d5164f7f3e0c", size = 111832, upload-time = "2026-07-08T12:25:01.148Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/b2933ddae222dac338476abb872641169a5cfed2c2bb5444a5b07b32b0c3/librt-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0", size = 150990, upload-time = "2026-07-08T12:25:02.42Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/db98f744ca50e6efc9c95c70ee49b77aefac31f6a3fc7c83754a42d6a74f/librt-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5", size = 155238, upload-time = "2026-07-08T12:25:03.681Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/a197e7bc72baf2c61ce7fdc6906a5054dc05bd8da0819aa894e4857bf87e/librt-0.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9", size = 503073, upload-time = "2026-07-08T12:25:05.049Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e7/7887712e27da7c1ab80fcabb1de6eb24243964f6557cae530d4b70706dbd/librt-0.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b", size = 496528, upload-time = "2026-07-08T12:25:06.26Z" }, + { url = "https://files.pythonhosted.org/packages/94/f0/f2283385bb6b950b26a1410f4ce51ec27231e0b3a4b925c46366d218b198/librt-0.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03", size = 531786, upload-time = "2026-07-08T12:25:07.658Z" }, + { url = "https://files.pythonhosted.org/packages/36/11/69ac3b54766ffba5fd7e5acebfb048d66dbe1f9f2d14516c2b3edc59cf87/librt-0.13.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e", size = 524393, upload-time = "2026-07-08T12:25:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/61/5f/d72f95fd444a926a3c14b4e24979474116988dd57a45be242077c45d3c22/librt-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd", size = 543026, upload-time = "2026-07-08T12:25:10.459Z" }, + { url = "https://files.pythonhosted.org/packages/c4/08/dcd9993ad192737a004ba263d549f8ea605b326b952e7d6205c7d4170b76/librt-0.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348", size = 546829, upload-time = "2026-07-08T12:25:11.716Z" }, + { url = "https://files.pythonhosted.org/packages/96/d5/6d9bb2f54e4109a956b7128836529653eb9d740f784bc47ed10a02c1000e/librt-0.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7", size = 535700, upload-time = "2026-07-08T12:25:13.144Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f2/10946922503858a359492fa27f13e86228bde702116a740ac7b3cd185f24/librt-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82", size = 573566, upload-time = "2026-07-08T12:25:14.336Z" }, + { url = "https://files.pythonhosted.org/packages/48/a8/94f00e3c99479a18088af3685ea016c42f3c7d5d1964d8dbb40c08d7f1aa/librt-0.13.0-cp312-cp312-win32.whl", hash = "sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3", size = 106099, upload-time = "2026-07-08T12:25:16.159Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7b/2da9c74c1ed25a89cc4e1c8e007ea2eb4a0f1fafa3e70d757fe3242c5c5c/librt-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa", size = 126934, upload-time = "2026-07-08T12:25:17.275Z" }, + { url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" }, + { url = "https://files.pythonhosted.org/packages/67/3b/18e7b63255297a2bdc9c25c8d6d4ca8eca9f63aceb1252c0f7427ac7099e/librt-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3", size = 151027, upload-time = "2026-07-08T12:25:19.638Z" }, + { url = "https://files.pythonhosted.org/packages/4d/68/e2248452c00d1a03b45fee1752cdc8f790a476efd2402b75181da88a9e61/librt-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c", size = 155152, upload-time = "2026-07-08T12:25:20.851Z" }, + { url = "https://files.pythonhosted.org/packages/0e/16/52b1c99bf19057a062aac39c900cbb81499f6f75d6c537c14463d247ba78/librt-0.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c", size = 502499, upload-time = "2026-07-08T12:25:22.055Z" }, + { url = "https://files.pythonhosted.org/packages/9f/54/b811151805c795f55e0dedee6ec687b75f9982a8105d240ea3910737a77b/librt-0.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b", size = 496108, upload-time = "2026-07-08T12:25:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f8/094d6b2bd93f3fdaa54db54cc788c4a365333bddad65ab02e04da0b1d004/librt-0.13.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9", size = 531576, upload-time = "2026-07-08T12:25:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/2e/40/541733d5755824f968f7ec39d78ffbd75d145964157ae5e69a09ec6d7326/librt-0.13.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db", size = 524390, upload-time = "2026-07-08T12:25:25.898Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b5/255673cfdbf5ba663339d36cd863c897289ab4337577e19f9405ce059f36/librt-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6", size = 543053, upload-time = "2026-07-08T12:25:27.436Z" }, + { url = "https://files.pythonhosted.org/packages/9e/11/ab5005e9c9850710f21e354201bf090646349d3fabf5f951eaf70235729e/librt-0.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7", size = 546387, upload-time = "2026-07-08T12:25:28.65Z" }, + { url = "https://files.pythonhosted.org/packages/a2/04/a5d7ce1d1df1afd15ca283dcdf7530ac073e12d69ae8c40879dda96f7868/librt-0.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1", size = 535970, upload-time = "2026-07-08T12:25:30.171Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/927e267a6daa290174ac281b23c9804c8829b042ade9c6f24a065f540958/librt-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a", size = 573582, upload-time = "2026-07-08T12:25:31.507Z" }, + { url = "https://files.pythonhosted.org/packages/10/24/b6c5213efe39c19f9e13605644d0cf063b4ddaa33ac2e45b088e23a70e2e/librt-0.13.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628", size = 82189, upload-time = "2026-07-08T12:25:32.675Z" }, + { url = "https://files.pythonhosted.org/packages/4c/00/d29736be177a906ac0b84a5b04b4fbfa22c776dc2f366de4172b0f968c08/librt-0.13.0-cp313-cp313-win32.whl", hash = "sha256:79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927", size = 106193, upload-time = "2026-07-08T12:25:33.692Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ac/aff6fb45393cb8912f39dfb156ef6b2d1cadb207ff465fc8f66141054be8/librt-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650", size = 126962, upload-time = "2026-07-08T12:25:34.769Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3a/d68cb2b334d53fd30fac81d3a489ce4ba0d9506f4df43fcf676b68352b19/librt-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566", size = 112127, upload-time = "2026-07-08T12:25:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/7b/66/f49ae0d592bd45b6941e9a8bafcb6a87cddcd501ee7874707e767f01b585/librt-0.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71", size = 149818, upload-time = "2026-07-08T12:25:37.203Z" }, + { url = "https://files.pythonhosted.org/packages/3d/50/51c76d74014d04fb95b6506d286808984b78a2f7a41039094e6b2194ac48/librt-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6", size = 154071, upload-time = "2026-07-08T12:25:39.399Z" }, + { url = "https://files.pythonhosted.org/packages/b8/fe/f19b0f5f82d5a1f2da736586bc840abd00ce07d6388136ae80b7333883fc/librt-0.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f", size = 494168, upload-time = "2026-07-08T12:25:40.641Z" }, + { url = "https://files.pythonhosted.org/packages/94/bc/b8550c75775127fd31a5f20e8775997f7b527ad661fc8ddccd7497c064f7/librt-0.13.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f19e181de5b3a1148bb3420b8c4b0b0ea0fce6950099724ad151d6cea5acc180", size = 491054, upload-time = "2026-07-08T12:25:41.905Z" }, + { url = "https://files.pythonhosted.org/packages/30/14/4d0204867623df3f33f86efd3d3692ba5e01321443f4d6eab35a22697618/librt-0.13.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22034924f5b42d5a56371cf271771bfeaabf235a7a8b6264bef2d20013f786c6", size = 523006, upload-time = "2026-07-08T12:25:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/19/0a/c45fc9a260934696bace1ac5df1e148ac92bd71767aee3bf7cd7a4534f4c/librt-0.13.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7897db4e95e22468bdda33d8e012ceacd0182abf001e6389d763f0def6286b9", size = 515058, upload-time = "2026-07-08T12:25:44.541Z" }, + { url = "https://files.pythonhosted.org/packages/13/0a/50c5ce45b326854ef8fa6ae4c36cf5142e5c55315eaf9e51d0ae73ac4da3/librt-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ce61b3746545029d4f5c17d6bd74b676254ad98433086c846ffb5e8fa73f007", size = 534025, upload-time = "2026-07-08T12:25:45.825Z" }, + { url = "https://files.pythonhosted.org/packages/89/2d/08c413c8f93fc13b8103624fce38e5caa86cd08cbbc8465870ab287af54b/librt-0.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:46c330e82565962c761dbce7941be2cff7db674ee807455a8d0cadc5f9b759b0", size = 540557, upload-time = "2026-07-08T12:25:47.059Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c1/93af71fb4a364952210051811dd4e40174e79656b050c89cacac18af3330/librt-0.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:375f5af8f99cbaa99dd293af986e3d57caabc9ba81a5d3f021603764854197a1", size = 523201, upload-time = "2026-07-08T12:25:48.392Z" }, + { url = "https://files.pythonhosted.org/packages/c1/6e/9766f07b676a4889d9f8bc2864e9ba5fff165653143ef4dda7df6aa34d16/librt-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9320d34c3376ae204b2cd176e8d4883a013934e0aef822f1aed9c536490c275d", size = 565740, upload-time = "2026-07-08T12:25:49.678Z" }, + { url = "https://files.pythonhosted.org/packages/a2/1e/664e3472ce2b6e10e9b83f29d4a36eb982ff6b5a169ae7567bba3a4c4ff5/librt-0.13.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9af313c66157a69dc69ea0059a66961692250e0dc95af9c385a48ffb770a0d16", size = 81611, upload-time = "2026-07-08T12:25:50.857Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d4/8582a4d65e2234673685e07309d02c230b28a85724eb0acbf13f019b7f6e/librt-0.13.0-cp314-cp314-win32.whl", hash = "sha256:f2a7253458e34f33543551394ae4fe104b497ec2a65ac266074de64c1df82e37", size = 100106, upload-time = "2026-07-08T12:25:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/63/ce/0cb99efe6086b46cd985dc26672166fae312a239690e75871f7fafbd3fc5/librt-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:a3dfe4edf10e8ed7e55b026a8bfc2c2a8704218b659cd4bffdf604fab966dc39", size = 121209, upload-time = "2026-07-08T12:25:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/26/85/4f3ccb083a3c9b0d42e223acdb3c3f507953324a59cdcab4826e8e2e3b89/librt-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:68a5faee4bba381cb93b5961f684a514cf0053cb92308ff9c792c2fea0b174c6", size = 106404, upload-time = "2026-07-08T12:25:54.253Z" }, + { url = "https://files.pythonhosted.org/packages/b2/77/333191499538c8e8189de7a4cba8e6f49ee949fd6d6e6324b21fd1522466/librt-0.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a38fb81d8376dfa2f8963b265fec07637802b0d01e2a127c19c66cb070fb24f5", size = 159231, upload-time = "2026-07-08T12:25:55.432Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9e/2aa83758f22c278b837a1d8025898434ce2b8bff36678d5330ecaef56dff/librt-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d4c8d9bd5abce34b2e75edb3bf37ab0f34e49b1f915a40ae8468eb7c85bc5b46", size = 161300, upload-time = "2026-07-08T12:25:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c0/86791e936553ca763d6b3c2fb4d31d596cd00e14fa631c283a40ba01559a/librt-0.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:387e2f1d27e89bffe0d3f520f0da0662c973fd607ca16c1808f8a5085419485e", size = 582056, upload-time = "2026-07-08T12:25:58.144Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d3/a9ec15984a185e000c4d2a16ba28bd623124ad4c38a10974c7ff78e3a893/librt-0.13.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:4f6db193d2e5e0ed60359b9a5a682cd67205d0d3b1e459a867dd4b5c4e7eaa7a", size = 562758, upload-time = "2026-07-08T12:25:59.544Z" }, + { url = "https://files.pythonhosted.org/packages/3c/af/dbe36b78b19c06a55097f99305e4ea9458e2273e6ae16a3cbecaad7ee978/librt-0.13.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d38604854e8d22faadf683ec6c02bb0f886e2ba56ef981a1c36ee275f21ea22", size = 602095, upload-time = "2026-07-08T12:26:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a8/2966891b4dd2830f5203fbee92ac2c4947653a2390ba73dfa44244fad025/librt-0.13.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:371f7ce73026815dafd51c50ce38416e91428b28c4b2ec97cd39271164b0045c", size = 593452, upload-time = "2026-07-08T12:26:02.352Z" }, + { url = "https://files.pythonhosted.org/packages/61/f5/4df8bfc8405ecf8c0d525b4d69636f694bdd8620b313ec8b76e54a5926cc/librt-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3aaedf52171bee90860704c560bc798fe83b76247df47568e0197e9b13c735a0", size = 623729, upload-time = "2026-07-08T12:26:04.294Z" }, + { url = "https://files.pythonhosted.org/packages/d6/13/9ac202dffc8db06f75d06c08c2f9f6ff054be67d21272dcc078fa1cc0c57/librt-0.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:96bad8725a4f196a798366c25ce075d1f7543a4ec045ffc13e6a7ec095cdab04", size = 617077, upload-time = "2026-07-08T12:26:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/ebe38610716aee5cb28efd95089bb90192096179802779381e1c5dcf239c/librt-0.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6bf6a559ffe4a93bbea6cf31ddf01a7fd9ba342ef51f27beb178e318b74acd61", size = 599561, upload-time = "2026-07-08T12:26:07.21Z" }, + { url = "https://files.pythonhosted.org/packages/4f/5c/c2e72e236fff7abc716d5b1753b8b8cd3ea85ac46fe17d2e7c51d4e1c723/librt-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:301067672387902c55f94b51d5022304b36c966ea9fe1f21caab99a9bef487c9", size = 645511, upload-time = "2026-07-08T12:26:08.562Z" }, + { url = "https://files.pythonhosted.org/packages/0c/99/6203ce619dee940d6bfbe099ec3fe4be00a68e9d60f70abf906cf124fe66/librt-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:5fdcf34f86de8fb66d7dc7589f96ba91c4aa46671200d400e6fd6f109a483f18", size = 104357, upload-time = "2026-07-08T12:26:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/52/dd/843b6314087c41657c7036d7914d8f294bdf9b580aa8513ea0588c8e9a3d/librt-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:260c33e92263fa629b4f6d3c51967a1c2158fe6c33237aaa3ebeac586b085259", size = 126998, upload-time = "2026-07-08T12:26:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, +] + +[[package]] +name = "mypy" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/af/4e516a05d3ca2eb9283e9ec45b2c02225c1514dd6da49fd3c9eaa6639370/mypy-2.3.0.tar.gz", hash = "sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e", size = 3988104, upload-time = "2026-07-13T11:34:53.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/b9/d75b3082b05f1b3028828aeb18e74ae5ab0a0936051bbf1f32f59f654747/mypy-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3419d00717afbc5265b50dd14b1278f29ea4884dd398ab67873489ac093fd329", size = 14838725, upload-time = "2026-07-13T11:32:44.655Z" }, + { url = "https://files.pythonhosted.org/packages/a9/50/79a65c6ea6e115bc73296038a4543b2d5c91f07912b918a2c616a2514bba/mypy-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfca8ee88544090f86b6dcce05ec55d66eb48a762412ac2507810ba4bd793b6f", size = 13911128, upload-time = "2026-07-13T11:32:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/90/48/e11ed7716c26953ca321f726e452e374dbf81a6f2b8b212ec02af29b6b8f/mypy-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75cbb4b9ef04a0c84a957f07abc4504fbf64b8dcc145675101f2d3a78a4b1d6a", size = 14146742, upload-time = "2026-07-13T11:33:03.313Z" }, + { url = "https://files.pythonhosted.org/packages/06/72/6807565b1c4861ef66f7fdd98b51c61556356eab80235717b46c53bb8627/mypy-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:982e3d53dd23d0a4cef67dd66791fdbede0cf38f9eb617bf47663554c51e1e36", size = 15081418, upload-time = "2026-07-13T11:31:13.899Z" }, + { url = "https://files.pythonhosted.org/packages/00/80/1ea14c5d80e589e415973db3e47c78c2219a305b808b2b506395342c1d79/mypy-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85c5385b93012ffa3b31479ab579aef5415f4f3a32c6cf1ae07a984d2a0ff461", size = 15328164, upload-time = "2026-07-13T11:31:35.723Z" }, + { url = "https://files.pythonhosted.org/packages/37/28/8223157404a3d51920078459c37f80fbdc590e1d8ea049dc5ce48643022a/mypy-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:13b1b16e2fa39f3b2e33fb1c468abc7a69369fa2e886b4b87b5afc81472325cd", size = 11136472, upload-time = "2026-07-13T11:27:37.018Z" }, + { url = "https://files.pythonhosted.org/packages/6f/cc/ea27e5959c5f258585a756b252031f3b313583d81b5064b2bebc41d3706b/mypy-2.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:b5cd2f027a972a4a5f2278a11fac9747f5f81a53a30b714d74950b6807e55568", size = 10135800, upload-time = "2026-07-13T11:30:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/0e7e592619e2133596a47cdd642534b0456545c218430bd3b9d8fefdd1b1/mypy-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5", size = 15026523, upload-time = "2026-07-13T11:34:49.206Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/1e1731df090a857df2807177a4626863e5ac0f0256513c35780efe53986f/mypy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c", size = 14032189, upload-time = "2026-07-13T11:33:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/cab921f4a806e171f34113e6181dd23c55358ccf6a80741269ef594a410e/mypy-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491", size = 14198696, upload-time = "2026-07-13T11:32:12.767Z" }, + { url = "https://files.pythonhosted.org/packages/66/80/e6d008bb19fe446e3662d85e0e2717bf9f2d611a2164fb29d6e067dbf46c/mypy-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7", size = 15286904, upload-time = "2026-07-13T11:34:27.594Z" }, + { url = "https://files.pythonhosted.org/packages/db/83/94397c9293608a364aa03e8084fb34ede4ae976a260384b9b52929308135/mypy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3", size = 15528342, upload-time = "2026-07-13T11:34:07.819Z" }, + { url = "https://files.pythonhosted.org/packages/cf/96/d8b37d819adec6cfccfb1fd3afc1735d94717ddeafb45536db9c6943e09b/mypy-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c", size = 11218346, upload-time = "2026-07-13T11:28:27.745Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cd/cd9f725b19b19e5b530a154cf9bcf9e94279c5d55b3c34fb42b3aa48ea1b/mypy-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595", size = 10204525, upload-time = "2026-07-13T11:31:02.552Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ae/f7d056eb0294586a572d0d0d89580ec633c064db520f11d37d5a2fb833bd/mypy-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97", size = 14947298, upload-time = "2026-07-13T11:27:47.734Z" }, + { url = "https://files.pythonhosted.org/packages/32/d5/db3e7af01e7844d21662c6ddc1f7825ec7cb4053f0391ac02faf3638396f/mypy-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac", size = 13950768, upload-time = "2026-07-13T11:27:57.726Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fb/43c031f0190513d1ec248ed037eceb742ddd2a4d74bbf406658a28173837/mypy-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6", size = 14151586, upload-time = "2026-07-13T11:29:18.615Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c3/f8b2ffc60883084da91be51af58e88a7ffd4ff9795acb7d902ff88d31eb1/mypy-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b", size = 15227411, upload-time = "2026-07-13T11:30:29.904Z" }, + { url = "https://files.pythonhosted.org/packages/83/2e/16b917fc7adcf03f1aadddfc93aab804ffb234b1ab09c0ffd6d92a5d34a2/mypy-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2", size = 15478790, upload-time = "2026-07-13T11:33:14.686Z" }, + { url = "https://files.pythonhosted.org/packages/c0/88/aaa65a93c73d0cdae7e42f8adb302bf6885bb281302084f99d0290a35347/mypy-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757", size = 11234919, upload-time = "2026-07-13T11:33:39.28Z" }, + { url = "https://files.pythonhosted.org/packages/35/19/b40de63f1a80e63bc2d40f0679a6a8dbd34e95176c8122119bdf406aa552/mypy-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1", size = 10201510, upload-time = "2026-07-13T11:31:52.619Z" }, + { url = "https://files.pythonhosted.org/packages/a4/58/fa0ae047da911f540284009b4f44b96fe09d83c076d7c103e9d645f46303/mypy-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db", size = 14941909, upload-time = "2026-07-13T11:32:34.332Z" }, + { url = "https://files.pythonhosted.org/packages/15/14/2ba1d61452d7c2a7fe12741e8d374e52b183476b07aa7f9e2a0d02b0720a/mypy-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762", size = 13967581, upload-time = "2026-07-13T11:30:00.587Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5a/483fb9e5ffbbb1a28dccc7b0a13d141b17ac769b6c9f488c0a0c63698962/mypy-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef", size = 14168807, upload-time = "2026-07-13T11:28:48.6Z" }, + { url = "https://files.pythonhosted.org/packages/ae/77/70d7a10732063beb74ad713682cf871e88f5c5fa39bfc8beff8a524bf9cb/mypy-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37fa4de896a84e2dc9200d91e614c22563b43d1a266789d4bbac7b22ebe6192b", size = 15200144, upload-time = "2026-07-13T11:31:25.283Z" }, + { url = "https://files.pythonhosted.org/packages/56/72/766218ac783be4fdfcd699b90037b63017348a3e86fb2c1fbfb18302637d/mypy-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f1b3a98dfd21058bc759bb3337d5d1f61d0fdf9f3cf9c00f4291790fb5427bff", size = 15460389, upload-time = "2026-07-13T11:29:29.077Z" }, + { url = "https://files.pythonhosted.org/packages/38/4e/8a9db7411ecb8ec0cb1fd05dba432f28bafffcd38b4e887714a4a0506689/mypy-2.3.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:944c665d984157cb96a679dfb7a4a81dd1d36b24b9c284b699514e6e626b82d4", size = 7753664, upload-time = "2026-07-13T11:29:08.147Z" }, + { url = "https://files.pythonhosted.org/packages/65/4c/c3f8bfd6ed0e5e38b5a244403b27f821d433443df5a15a278417c10a3a3c/mypy-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:4359424140d985192c778c1ce2c114a10c1ca58a381ed79cfa70d37df94b299f", size = 11417237, upload-time = "2026-07-13T11:33:47.467Z" }, + { url = "https://files.pythonhosted.org/packages/3c/00/89a32eaf5ccf174bc4f90db0eaea5d70636c01b8d49f384bdab2e8834390/mypy-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:3dd0bed92c4bdec57c42505b96416fb9e6a5aa7be84d2809bcd5f2ecec2860d7", size = 10389252, upload-time = "2026-07-13T11:31:43.81Z" }, + { url = "https://files.pythonhosted.org/packages/31/56/104f93d69aa9f339b6b9d3b0a7faa699b8b466c942cf3ae86cc2a2ec0915/mypy-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:691fdc37132b1ae628d834f672e74de83462d9fb4aff621835767fb43a8dd373", size = 16385495, upload-time = "2026-07-13T11:29:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/f1d2123313f55efafdd27706960f43a771c62f1b68426c76043f3ab9ebf3/mypy-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aec15d465d477558fd842757b487849007311cf3897849cdda0e3162ac0ac556", size = 15098155, upload-time = "2026-07-13T11:30:40.301Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5d/d5f9200399b445e81726c4f23becee33f233aee81c72680b1ef3a258b641/mypy-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b352b7e49f5e6576009e8df730e1ff4f915cb565b851b396d2ffe2f5a6f5da88", size = 15514155, upload-time = "2026-07-13T11:34:38.569Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ce/69977c555f08faa3190cfde44189b89dbd56861b1ab97aa18fc5f3a2e4a3/mypy-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c6c6bf687b17f90dbfcad95b960d32eaa0154c00da45f03ab50bf8952e047fe", size = 16766351, upload-time = "2026-07-13T11:33:29.195Z" }, + { url = "https://files.pythonhosted.org/packages/bc/92/6648b6caa3ab9e00f9ac0c2a78307805f873dd48139b24a6f6f7c3667bbf/mypy-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f4ed18f111bfe2d599bca7468e7f9251042c1c2118f762c8de2766a56d773c60", size = 17043490, upload-time = "2026-07-13T11:30:53.927Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ab/0dc91d80f3f016634c68d451f294a97320fe903a9b6f90b9e57b3f7f1717/mypy-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0b025a93cffb9781d231f232be07a17912f35f10a313c24f301c81e842870654", size = 12146869, upload-time = "2026-07-13T11:29:38.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/b5/4c964d02634ba81f4d1c84838e5c5b18ab06d13ed568960f5d6318495ccc/mypy-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:adebc76aab4f3495a88b41d48aa4aff0c03f2822501da76625afcca5975f19e5", size = 10965113, upload-time = "2026-07-13T11:28:07.056Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/fdc54fe583ba3cafbcedfb70eeeaf03849f75b1827a07096c7bd996f582d/mypy-2.3.0-py3-none-any.whl", hash = "sha256:6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88", size = 2753292, upload-time = "2026-07-13T11:33:18.48Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, + { url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539, upload-time = "2026-07-09T20:00:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437, upload-time = "2026-07-09T20:01:00.006Z" }, + { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, + { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, + { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, + { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, + { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555, upload-time = "2026-07-09T20:01:26.941Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772, upload-time = "2026-07-09T20:01:29.497Z" }, + { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "wamprobe" +version = "0.1.0a0" +source = { editable = "." } + +[package.optional-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mypy", marker = "extra == 'dev'" }, + { name = "pytest", marker = "extra == 'dev'" }, + { name = "pytest-cov", marker = "extra == 'dev'" }, + { name = "ruff", marker = "extra == 'dev'" }, +] +provides-extras = ["dev"] From ff5a29f9238b5c4484f7ad15a75c608495646bae Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 07:35:38 +0000 Subject: [PATCH 02/20] Document WAM failure cases and adapter selection --- CHANGELOG.md | 2 + README.md | 5 +- docs/WAMProbe_PLAN.md | 10 + docs/research/ADAPTER_SELECTION.md | 98 ++++++++++ docs/research/WAM_VLA_FAILURE_CASES.md | 256 +++++++++++++++++++++++++ 5 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 docs/research/ADAPTER_SELECTION.md create mode 100644 docs/research/WAM_VLA_FAILURE_CASES.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 772c93e..e9604bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,3 +12,5 @@ All notable changes to WAMProbe will be documented here. - Action Dependence, Counterfactual Direction Accuracy, No-op Stability, state ADE, and Top-1 Regret metrics. - JSON/Markdown reports, schemas, CLI, CI, and contributor documentation. +- Phase 0 evidence map covering 15 reported WAM/VLA failure modes. +- Version-pinned audit selecting StarWAM as the first real adapter target. diff --git a/README.md b/README.md index 9ae69e0..aed796e 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,11 @@ The next milestones are: 2. a small image-rendered BlockPush benchmark; 3. the versioned intervention dataset loader; 4. LIBERO-CF-Mini paired simulator branches; -5. the first real WAM adapter, selected after checkpoint and license verification. +5. a pinned StarWAM inference adapter, followed by LingBot-VA as a published reference. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), -[quick-start notes](docs/QUICKSTART.md), and [design RFCs](docs/rfcs/). +[quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), +[adapter selection record](docs/research/ADAPTER_SELECTION.md), and [design RFCs](docs/rfcs/). ## Development diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 96d1389..5b1c465 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -728,6 +728,11 @@ v0.1 必须先实现无需外部大模型的 baseline，用于验证指标： 2. **FastWAM adapter**：用于 action-only/联合变体和效率评测，但要根据公开 checkpoint 实际能力启用指标； 3. **其他 WAM**：待 v0.1 协议稳定后通过独立 extras/plugin 添加。 +截至 2026-07-15 的代码、checkpoint、许可证与环境审计已完成。第一实现目标保留为 +StarWAM，LingBot-VA 调整为第二个论文对照，Fast-WAM 在权重许可证元数据明确前只作为 +action-only/效率设计参考。版本 pin、风险和第一实现切片见 +[`docs/research/ADAPTER_SELECTION.md`](research/ADAPTER_SELECTION.md)。 + ### 11.3 依赖隔离 v0.1： @@ -975,6 +980,11 @@ CLI 验收要求： ### Phase 0：问题收敛（第 1–2 周） +当前进度（2026-07-15）：15 个文献失败模式和首个 Adapter 审计已经完成，见 +[`docs/research/WAM_VLA_FAILURE_CASES.md`](research/WAM_VLA_FAILURE_CASES.md) 与 +[`docs/research/ADAPTER_SELECTION.md`](research/ADAPTER_SELECTION.md)。外部研究者 README +理解度检查仍需人工完成。 + 任务： - 阅读并整理 10–15 个 WAM/VLA 评测失败案例； diff --git a/docs/research/ADAPTER_SELECTION.md b/docs/research/ADAPTER_SELECTION.md new file mode 100644 index 0000000..aeb30a6 --- /dev/null +++ b/docs/research/ADAPTER_SELECTION.md @@ -0,0 +1,98 @@ +# First real WAM adapter selection + +- Decision date: 2026-07-15 +- Status: provisional implementation decision +- Target: first GPU-backed adapter after the CPU PointMass vertical slice + +## Decision + +Use **StarWAM** as the first implementation target, pinned to code commit +`f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7` and ModelScope model revision +`7d4bfe3ec76172ca17169fa959d21da099d386fe`. + +Use **LingBot-VA** as the second, published-reference adapter. Keep **Fast-WAM** as an +action-only/feature-conditioned comparison after its checkpoint license metadata is made +explicit. + +This decision selects an integration target; it does not claim that StarWAM is the best +policy. The first milestone is a reproducible inference probe, not training or leaderboard +comparison. + +## Hard gates + +An adapter is eligible only when all of the following can be checked: + +1. code license and model-weight terms are identifiable; +2. a public checkpoint is downloadable at a pin-able revision; +3. inference exposes actions, futures, or world features that can be declared honestly; +4. a supported benchmark and action normalization path exist; +5. Python, PyTorch, CUDA, and simulator dependencies can be isolated; +6. evaluation fits the available hardware or documents a concrete offload path. + +## Candidate audit + +| Candidate | Code / weights | Observable capability | Reproducibility and compute | Decision | +|---|---|---|---|---| +| StarWAM | Apache-2.0 / ModelScope reports Apache-2.0 | MoT, Shared-DiT, and feature-conditioned families; video/world and action paths; LIBERO recipes | Python 3.11 recipe, PyTorch 2.6/CUDA 12.4, single-GPU rollout command, released 5B checkpoints; training launchers default to 8 GPUs | **First target** | +| LingBot-VA | Apache-2.0 / Hugging Face model card tags Apache-2.0 | autoregressive video-action generation, action inference, LIBERO and RoboTwin evaluation | Python 3.10.16, PyTorch 2.9/CUDA 12.6; README reports about 18 GB VRAM for image-to-video-action and 24 GB for RoboTwin with offload | **Second reference** | +| Fast-WAM | MIT / public checkpoint has no explicit license tag in the Hugging Face API | direct action output at inference; video co-training features, but no decoded future in the primary inference path | Python 3.10, PyTorch 2.7.1/CUDA 12.8; benchmark manager defaults to 8 GPUs and can be reduced | **Defer full integration** | + +## Why StarWAM first + +- Its taxonomy-level model families map directly onto WAMProbe's capability protocol. +- The same codebase can exercise `future_pixels`, `world_features`, and action-oriented + variants without pretending they support identical metrics. +- It includes LIBERO recipes, rollout utilities, action statistics, and two released + checkpoint families. +- Its code and weight repository both identify Apache-2.0 terms. +- The local host has four RTX 4090 GPUs with 24 GB VRAM each, which makes a focused + inference spike plausible without committing to retraining. + +## Risks and controls + +| Risk | Control | +|---|---| +| StarWAM is an early research release without a technical report | Treat it as an API/protocol integration target; use LingBot-VA for paper-backed comparison | +| ModelScope model card is a default template | Pin the Git revision and checksum every downloaded file before inference | +| The model repository is roughly 32.1 GB before the separate backbone | Start with one checkpoint family and document disk/VRAM measurements | +| Training recipes use eight GPUs | Do not train in the first adapter milestone; validate one-context inference and cached outputs | +| Action and state normalization may silently mismatch | Store upstream stats hash and normalization metadata in every WAMProbe result | +| Pixel and feature variants expose different evidence | Declare capabilities explicitly; skip incompatible metrics with a reason | + +## Pinned upstream snapshot + +| Item | Pin or observation | +|---|---| +| StarWAM code | [`shaohua-pan/StarWAM@f6c771f`](https://github.com/shaohua-pan/StarWAM/tree/f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7) | +| StarWAM weights | [`panshaohua/starwam@7d4bfe3`](https://www.modelscope.cn/models/panshaohua/starwam) | +| LingBot-VA code | [`Robbyant/lingbot-va@7c6ffa9`](https://github.com/Robbyant/lingbot-va/tree/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb) | +| LingBot-VA LIBERO weights | Hugging Face revision `0e89d1e753019988aba484e8da2dc0810e264d9f` | +| Fast-WAM code | [`yuantianyuan01/FastWAM@45d8e14`](https://github.com/yuantianyuan01/FastWAM/tree/45d8e1458921d83f8ad6cf9ce993d371208dabd0) | +| Fast-WAM weights | Hugging Face revision `139eebb6d90cdd9bdbbe465f72c6edc9ad5a518a`; weight license metadata unresolved | + +Pins record what was audited. They should be updated only through a reviewed compatibility +change, not silently moved to an upstream default branch. + +## First adapter implementation slice + +The next coding slice is deliberately narrow: + +1. add a `starwam` optional dependency group or isolated environment specification; +2. implement metadata-only discovery and `wamprobe doctor` compatibility checks first; +3. download one checkpoint family and record immutable hashes outside Git; +4. run one LIBERO observation through upstream inference and save a typed, cached artifact; +5. expose only capabilities verified from that artifact; +6. add a mocked CPU contract test and an opt-in GPU smoke test; +7. defer counterfactual scoring until simulator snapshot/restore produces paired branches. + +## Sources checked + +- [StarWAM official repository](https://github.com/shaohua-pan/StarWAM) +- [StarWAM model repository](https://www.modelscope.cn/models/panshaohua/starwam) +- [LingBot-VA official repository](https://github.com/Robbyant/lingbot-va) +- [LingBot-VA LIBERO checkpoint](https://huggingface.co/robbyant/lingbot-va-posttrain-libero-long) +- [Fast-WAM official repository](https://github.com/yuantianyuan01/FastWAM) +- [Fast-WAM checkpoint](https://huggingface.co/yuanty/fastwam) + +Repository metadata, README requirements, checkpoint visibility, and license fields were +checked on 2026-07-15. No checkpoint was downloaded or executed during this audit. diff --git a/docs/research/WAM_VLA_FAILURE_CASES.md b/docs/research/WAM_VLA_FAILURE_CASES.md new file mode 100644 index 0000000..83bbe42 --- /dev/null +++ b/docs/research/WAM_VLA_FAILURE_CASES.md @@ -0,0 +1,256 @@ +# WAM/VLA failure-case evidence map + +- Status: Phase 0 literature screen +- Evidence cutoff: 2026-07-15 +- Scope: World Action Models (WAMs), video-action models, and closely related VLAs + +## Purpose + +This document turns reported WAM/VLA weaknesses into testable requirements for +WAMProbe. It is an evidence map, not an independent reproduction of every cited result. +The central question is whether a model's predicted future is controlled by the proposed +action and useful for control, rather than merely plausible or correlated with success. + +## Evidence labels + +- **R1 — reported empirical finding:** the cited paper reports a controlled comparison, + diagnostic, or quantified result. WAMProbe has not independently reproduced it yet. +- **R2 — author-stated limitation:** the cited paper states the problem as motivation; the + exact failure rate may depend on its setup. +- **H — WAMProbe hypothesis:** a proposed probe or expected result. It must not be quoted + as a literature finding until tested. + +Sources are version-pinned arXiv records or official repositories. Cases based only on an +abstract are kept at R2 unless the abstract explicitly describes a controlled empirical +study. Three central papers—dynamic consistency, behavioral diagnostics, and Fast-WAM— +were additionally checked against their full text. + +## Case index + +| ID | Reported failure or ambiguity | Evidence | WAMProbe test direction | Source | +|---|---|---|---|---| +| FC-01 | Final success hides target progress, distractor motion, action smoothness, and runtime differences | R1 | behavior profile beside success | [S1] | +| FC-02 | A future can look plausible while being dynamically incompatible with its action | R1 | paired action-state consistency | [S2] | +| FC-03 | Static/background collapse can make a failed rollout deceptively consistent | R1 | consistency plus a minimum-dynamics guard | [S2] | +| FC-04 | Fixed action chunks keep executing after the imagined future diverges from reality | R1 | prefix consistency and adaptive replanning signal | [S3] | +| FC-05 | One stochastic diffusion sample is brittle and errors compound over an episode | R1 | multi-seed calibration and candidate regret | [S4] | +| FC-06 | Pixel prediction entangles dynamics with lighting/texture and weakens under occlusion or off-screen motion | R1 | perturbation invariance plus track/state probes | [S5] | +| FC-07 | Robot-data fine-tuning can erase compositional priors inherited from video pretraining | R1 | ID/OOD compositional split and feature reliance | [S6] | +| FC-08 | Uniform action loss underweights slow, precision-critical interaction phases | R1 | phase-stratified metrics around contact | [S7] | +| FC-09 | Policies can exploit scene-level shortcuts instead of causally relevant objects | R1 | object/distractor counterfactual swaps | [S8] | +| FC-10 | Numerical actions can misalign with video priors; visual action encodings can miss temporal motion | R1 | representation-aware action intervention tests | [S9] | +| FC-11 | Dense video imagination spends compute on irrelevant details and long-horizon errors can mislead control | R1 | budgeted utility and horizon sweep | [S10] | +| FC-12 | Test-time future generation may add large latency without improving task success | R1 | matched-training inference ablation | [S11] | +| FC-13 | Strong semantic VLAs can still fail on unseen physical motions and environments | R1 | physical-motion OOD split | [S12] | +| FC-14 | Learned policies can fail when hidden dynamics such as mass or friction change | R1 | simulator parameter interventions | [S13] | +| FC-15 | Robustness depends on how video priors are integrated, not merely on the WAM/VLA label | R1 | capability- and architecture-stratified reporting | [S14] | + +## Detailed implications + +### FC-01: success-only evaluation hides behavior + +Mai et al. report that success rate alone conceals differences in motion consistency, +target-object progress, distractor disturbance, runtime, and future-oriented internal +representations. They also find that the pattern varies across sequential, joint, and +auxiliary WAM designs [S1]. + +**H:** WAMProbe should never replace its metric profile with one aggregate score. LIBERO +evaluation should pair success with target progress, distractor displacement, control +smoothness, and runtime. Results must be grouped by declared capability and architecture. + +### FC-02: plausible is not action-compatible + +Ruan et al. show that action-state consistency separates many successful and failed +rollouts and captures information beyond visual realism [S2]. Their full text reports that +a generated future may misrepresent contact or object motion while remaining visually +plausible. + +**H:** shared-state action branches are WAMProbe's primary unit. Counterfactual Direction +Accuracy, state ADE/FDE, and candidate regret should be reported together; video quality is +secondary and cannot establish action grounding. + +### FC-03: static futures can game consistency + +The same study identifies *background collapse*: a failed, low-motion trajectory can +receive a favorable consistency score because a static background is easy to predict +[S2]. This is a concrete warning that consistency itself can be gamed. + +**H:** any learned action-state consistency metric needs a transition-magnitude diagnostic. +Near-static non-noop branches should be flagged, while true no-op branches remain a +separate No-op Stability test. A model must not gain credit merely by predicting no change. + +### FC-04: fixed chunks ignore future-reality divergence + +Wang et al. identify a deployment failure in which a WAM executes a fixed action chunk +without checking whether reality still matches the predicted future. Their adaptive +verifier replans earlier in difficult or contact-rich phases and reports improved real-world +success [S3]. + +**H:** introduce Horizon Prefix Consistency before closed-loop utility. Measure the first +time at which predicted and realized state prefixes diverge, then correlate that event with +the action-chunk length that should have been executed. + +### FC-05: a single stochastic sample is brittle + +Dai et al. attribute brittleness to committing to one stochastic diffusion/flow action +sample per round and report improvements from parallel candidate sampling and geometric +consensus [S4]. + +**H:** adapters with stochastic generation must expose deterministic seeds. WAMProbe +should report per-context seed variance, candidate ranking stability, calibration, and +Top-1 Regret rather than evaluating one lucky rollout. + +### FC-06: pixels entangle dynamics and appearance + +Guan et al. report that pixel prediction entangles dynamics with lighting and texture. +Their point-track representation is intended to preserve long-horizon motion through +occlusion and partial out-of-frame movement, with the largest reported gains on those +settings [S5]. + +**H:** the pixel benchmark should include lighting/texture perturbations that preserve +dynamics. Where simulator state or tracks are available, WAMProbe should compare them +with pixel metrics instead of treating pixel similarity as ground truth. + +### FC-07: video priors can be lost during robot fine-tuning + +Mishra et al. systematically study a video-action generalization gap: compositional priors +present in a generative video model can weaken after fine-tuning on robot actions [S6]. +They report that reliance on future-predictive latents changes with task phase. + +**H:** evaluate identical models on ID and compositional-OOD intervention groups. Feature +reliance should be measured separately for planning, approach, contact, and precision +phases rather than averaged over an episode. + +### FC-08: not every action timestep is equally important + +Peng et al. identify a mismatch between uniform loss weighting and manipulation's +physical hierarchy: low-velocity interaction segments can decide success while fast +transit segments tolerate larger errors [S7]. + +**H:** state ADE must be accompanied by phase-conditioned ADE/FDE and control regret. +Errors around contact, release, and low-velocity alignment should be reported separately. + +### FC-09: scene shortcuts can replace causal grounding + +Sun et al. identify shortcut learning in robotic foundation models: policies exploit +predictive but non-causal scene correlations rather than task-relevant visual evidence +[S8]. Their experiments cover VLA and WAM-style pipelines. + +**H:** construct paired scenes that change background, distractor identity, or irrelevant +object placement while preserving task dynamics. A grounded model's action ranking and +relevant future transition should remain stable. + +### FC-10: action representation can be mismatched + +Chen et al. argue that numerical robot actions do not align naturally with pretrained video +generators, while earlier visual action representations can omit temporal motion structure. +Their optical-flow formulation reports gains in both policy and world-model modes [S9]. + +**H:** metric requirements must be representation-aware. State actions, end-effector +trajectories, point tracks, flow, and latent actions need explicit conversion metadata; an +unsupported comparison must be skipped rather than silently approximated. + +### FC-11: more imagined frames are not automatically more useful + +Zhang et al. identify three coupled limitations of video-based WAMs: dense multi-frame +tokens increase inference cost, video prediction spends capacity on action-irrelevant +details, and long-horizon errors may mislead action prediction [S10]. Their ImageWAM +results report lower cost with target-frame editing features. + +**H:** sweep prediction horizon and compute budget. Report control utility per unit latency +and memory, and check whether longer futures reduce Top-1 Regret instead of assuming that +more frames imply better world understanding. + +### FC-12: decoded future imagination may have little control value + +Fast-WAM provides a controlled comparison between video co-training and explicit future +generation at inference. Its full text reports similar benchmark success for Fast-WAM and +imagine-then-execute variants, while the latter have substantially higher latency; removing +video co-training causes a much larger performance drop [S11]. + +**H:** WAMProbe must distinguish `future_pixels`, `world_features`, and `action_only` +capabilities. Compare models with matched training while masking or bypassing decoded +future generation, then report both control utility and cost. + +### FC-13: semantic generalization is not physical generalization + +Ye et al. motivate DreamZero with the observation that strong VLAs can generalize +semantically yet struggle with unseen physical motions and novel environments [S12]. They +report improved real-robot generalization from joint video-action modeling. + +**H:** benchmark splits should independently vary language, object identity, geometry, +motion, and environment. A single OOD average cannot reveal which form of generalization +the model acquired. + +### FC-14: hidden dynamics shift + +Lyu et al. report that prior learning-based manipulation methods depend on multi-view/pose +tracking and fail to generalize across changes such as object mass and table friction [S13]. + +**H:** paired simulator snapshots should vary one hidden physical parameter at a time. +Measure whether the predicted action effect, uncertainty, and candidate ranking change in +the correct direction. + +### FC-15: architecture matters more than the family label + +Zhang et al. compare WAMs, VLAs, and hybrids under visual and language perturbations and +report that robustness varies with how video priors are integrated; hybrid methods occupy +an intermediate regime [S14]. This complements the architecture-dependent behavior found +in [S1]. + +**H:** WAMProbe reports no universal “WAM versus VLA” winner. Every result row must carry +a capability manifest and architecture tag, and comparisons must remain within compatible +output groups. + +## Resulting v0.1 priorities + +The literature screen changes implementation order in five ways: + +1. add context-block confidence intervals and seed variance before adding learned metrics; +2. add a low-dynamics/background-collapse guard to future consistency evaluation; +3. make task phase and contact state first-class benchmark metadata; +4. report budgeted control utility beside prediction quality; +5. preserve capability-stratified reports rather than creating a composite leaderboard. + +## Sources + +- **[S1]** Mai et al., “Beyond Task Success: Behavioral and Representational + Diagnostics for WAM and VLA,” [arXiv:2606.01095v1](https://arxiv.org/abs/2606.01095v1). +- **[S2]** Ruan et al., “Is the Future Compatible? Diagnosing Dynamic Consistency in + World Action Models,” [arXiv:2605.07514v1](https://arxiv.org/abs/2605.07514v1). +- **[S3]** Wang et al., “When to Trust Imagination: Adaptive Action Execution for World + Action Models,” [arXiv:2605.06222v2](https://arxiv.org/abs/2605.06222v2). +- **[S4]** Dai et al., “Geometry Guided Self-Consistency for Physical AI,” + [arXiv:2605.08638v1](https://arxiv.org/abs/2605.08638v1). +- **[S5]** Guan et al., “Point Tracking Improves World Action Models,” + [arXiv:2605.23856v1](https://arxiv.org/abs/2605.23856v1). +- **[S6]** Mishra et al., “Understanding and Mitigating the Video-Action Generalization + Gap via Temporal Ratio,” [arXiv:2607.08127v1](https://arxiv.org/abs/2607.08127v1). +- **[S7]** Peng et al., “AttenA+: Rectifying Action Inequality in Robotic Foundation + Models,” [arXiv:2605.13548v3](https://arxiv.org/abs/2605.13548v3). +- **[S8]** Sun et al., “Artificial Foveated Perception for Mitigating Shortcut Learning in + Robotic Foundation Models,” [arXiv:2607.10655v1](https://arxiv.org/abs/2607.10655v1). +- **[S9]** Chen et al., “FlowWAM: Optical Flow as a Unified Action Representation for + World Action Models,” [arXiv:2607.13017v1](https://arxiv.org/abs/2607.13017v1). +- **[S10]** Zhang et al., “ImageWAM: Do World Action Models Really Need Video + Generation, or Just Image Editing?,” + [arXiv:2606.19531v1](https://arxiv.org/abs/2606.19531v1). +- **[S11]** Yuan et al., “Fast-WAM: Do World Action Models Need Test-time Future + Imagination?,” [arXiv:2603.16666v2](https://arxiv.org/abs/2603.16666v2). +- **[S12]** Ye et al., “World Action Models are Zero-shot Policies,” + [arXiv:2602.15922v1](https://arxiv.org/abs/2602.15922v1). +- **[S13]** Lyu et al., “DyWA: Dynamics-adaptive World Action Model for Generalizable + Non-prehensile Manipulation,” + [arXiv:2503.16806v2](https://arxiv.org/abs/2503.16806v2). +- **[S14]** Zhang et al., “Do World Action Models Generalize Better than VLAs? A + Robustness Study,” [arXiv:2603.22078v3](https://arxiv.org/abs/2603.22078v3). + +## Known limits of this screen + +- It records claims made by the cited authors; it does not certify their experimental + validity or compare incompatible benchmark versions. +- Several sources are recent preprints and may change after the pinned arXiv version. +- The external-reader comprehension check in Phase 0 still requires a human reviewer and + cannot be completed by repository automation. +- Each accepted metric still needs a dedicated RFC, reference baseline behavior, and a + reproducible sanity test before implementation. From 114606e52b99762b62fcdfd020afe265086bf1ab Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 08:15:14 +0000 Subject: [PATCH 03/20] Define pinned upstream model store --- .gitignore | 3 +- CHANGELOG.md | 2 + README.md | 4 + checkpoints/README.md | 129 ++++++++++++++++++++++++++++ configs/models/.gitkeep | 1 - configs/models/upstream_models.json | 88 +++++++++++++++++++ pyproject.toml | 3 + 7 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 checkpoints/README.md delete mode 100644 configs/models/.gitkeep create mode 100644 configs/models/upstream_models.json diff --git a/.gitignore b/.gitignore index 14b00e9..2747230 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,8 @@ dist/ # Local data, model weights, generated predictions, and reports data/ -checkpoints/ +checkpoints/* +!checkpoints/README.md runs/ *.ckpt *.pt diff --git a/CHANGELOG.md b/CHANGELOG.md index e9604bc..42098b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,3 +14,5 @@ All notable changes to WAMProbe will be documented here. - JSON/Markdown reports, schemas, CLI, CI, and contributor documentation. - Phase 0 evidence map covering 15 reported WAM/VLA failure modes. - Version-pinned audit selecting StarWAM as the first real adapter target. +- Provider-scoped local model store with pinned revisions, expected paths, sizes, and + checksums for the first StarWAM integration. diff --git a/README.md b/README.md index aed796e..9924ad5 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,10 @@ runs/pointmass-demo/ See the committed [example report](examples/pointmass-demo/report.md) for the expected baseline ordering. +Real-model weights are never committed to Git. Before the StarWAM integration, follow the +[model-store layout and download rules](checkpoints/README.md); the first spike requires +approximately 46.3 GB of pinned StarWAM and Wan2.2 artifacts. + You can also run the module directly: ```bash diff --git a/checkpoints/README.md b/checkpoints/README.md new file mode 100644 index 0000000..b3322df --- /dev/null +++ b/checkpoints/README.md @@ -0,0 +1,129 @@ +# Model store + +Model weights are local artifacts and must never be committed to Git. WAMProbe mirrors +each upstream model repository under: + +```text +checkpoints/upstream//// +``` + +Repository names and capitalization are preserved so that a path identifies exactly one +upstream source. The machine-readable pins live in +[`configs/models/upstream_models.json`](../configs/models/upstream_models.json). + +## Download now + +Only two sources are required for the first StarWAM adapter spike: + +| Source | Download | Size | Target directory | +|---|---|---:|---| +| Wan2.2-TI2V-5B backbone | complete Hugging Face snapshot | 34.2 GB | `upstream/huggingface/Wan-AI/Wan2.2-TI2V-5B/` | +| StarWAM LIBERO MoT | MoT checkpoint plus `action_stats.json` | 12.0 GB | `upstream/modelscope/panshaohua/starwam/` | + +Minimum model storage is approximately **46.3 GB**. Reserve at least **100 GB** for +download caches, simulator assets, and generated predictions. + +### 1. Wan2.2-TI2V-5B + +Source: [Wan-AI/Wan2.2-TI2V-5B](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) + +Pinned revision: + +```text +921dbaf3f1674a56f47e83fb80a34bac8a8f203e +``` + +Download the complete repository into: + +```text +/data/Project/MY/WAMProbe/checkpoints/upstream/huggingface/Wan-AI/Wan2.2-TI2V-5B/ +``` + +Optional CLI command: + +```bash +hf download Wan-AI/Wan2.2-TI2V-5B \ + --revision 921dbaf3f1674a56f47e83fb80a34bac8a8f203e \ + --local-dir checkpoints/upstream/huggingface/Wan-AI/Wan2.2-TI2V-5B +``` + +Do not rename the three `diffusion_pytorch_model-*.safetensors` shards, their index, +`Wan2.2_VAE.pth`, the T5 encoder, or tokenizer directories. + +### 2. StarWAM LIBERO MoT + +Source: [panshaohua/starwam](https://www.modelscope.cn/models/panshaohua/starwam) + +Pinned Git revision: + +```text +7d4bfe3ec76172ca17169fa959d21da099d386fe +``` + +The two required files, preserving their upstream relative paths, are: + +```text +upstream/modelscope/panshaohua/starwam/ +└── starwam-libero/ + ├── action_stats.json + └── mot/ + └── starwam_wan225b_mot.pt +``` + +Expected checksums: + +```text +d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378 starwam-libero/mot/starwam_wan225b_mot.pt +9f65fb518ca446e0d5ca9e8127e960fe3d11e6466e4f48ba9bb1135b1e0fb4f0 starwam-libero/action_stats.json +``` + +You may download those two files manually. To use Git LFS while avoiding the optional +20 GB Shared-DiT payload: + +```bash +GIT_LFS_SKIP_SMUDGE=1 git clone \ + https://www.modelscope.cn/panshaohua/starwam.git \ + checkpoints/upstream/modelscope/panshaohua/starwam + +git -C checkpoints/upstream/modelscope/panshaohua/starwam \ + checkout 7d4bfe3ec76172ca17169fa959d21da099d386fe + +git -C checkpoints/upstream/modelscope/panshaohua/starwam lfs pull \ + --include="starwam-libero/mot/starwam_wan225b_mot.pt" +``` + +## Optional later downloads + +Do not download these for the first adapter spike: + +| Model | Size | Target | Status | +|---|---:|---|---| +| StarWAM Shared-DiT checkpoint | 20.1 GB | `upstream/modelscope/panshaohua/starwam/starwam-libero/sharedit/` | optional second StarWAM family | +| LingBot-VA LIBERO-Long | 24.4 GB | `upstream/huggingface/robbyant/lingbot-va-posttrain-libero-long/` | second published-reference adapter | +| Fast-WAM LIBERO checkpoint | 12.0 GB | `upstream/huggingface/yuanty/fastwam/` | blocked until weight license metadata is explicit | + +The LingBot-VA snapshot, when needed, is pinned to revision +`0e89d1e753019988aba484e8da2dc0810e264d9f`. Fast-WAM is listed in the manifest +for auditability but should not be downloaded yet. + +## Placement rules + +1. Keep the upstream directory layout and filenames unchanged. +2. Never mix files from two revisions in one repository directory. +3. Do not put source code, datasets, simulator assets, or experiment outputs here. +4. Do not commit model files, Hugging Face caches, Git LFS objects, or symlink targets. +5. A directory may be a symlink to another disk, but its path inside this repository must + still follow the provider/owner/repository convention. +6. Record any approved revision change in `upstream_models.json` before running it. +7. Keep normalization statistics beside the checkpoint from which they were released. + +After the required files are in place, run: + +```bash +sha256sum \ + checkpoints/upstream/modelscope/panshaohua/starwam/starwam-libero/mot/starwam_wan225b_mot.pt \ + checkpoints/upstream/modelscope/panshaohua/starwam/starwam-libero/action_stats.json +``` + +WAMProbe will add `wamprobe doctor` as the next implementation slice to validate this +layout, revisions, required files, and checkpoint metadata without loading the GPU model. diff --git a/configs/models/.gitkeep b/configs/models/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/configs/models/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/configs/models/upstream_models.json b/configs/models/upstream_models.json new file mode 100644 index 0000000..5d3eadd --- /dev/null +++ b/configs/models/upstream_models.json @@ -0,0 +1,88 @@ +{ + "schema_version": "0.1", + "store_root": "checkpoints/upstream", + "required_now": [ + { + "name": "Wan2.2-TI2V-5B", + "provider": "huggingface", + "repository": "Wan-AI/Wan2.2-TI2V-5B", + "revision": "921dbaf3f1674a56f47e83fb80a34bac8a8f203e", + "license": "Apache-2.0", + "target": "huggingface/Wan-AI/Wan2.2-TI2V-5B", + "download_mode": "full_snapshot", + "expected_bytes": 34203123497, + "required_paths": [ + "config.json", + "Wan2.2_VAE.pth", + "diffusion_pytorch_model-00001-of-00003.safetensors", + "diffusion_pytorch_model-00002-of-00003.safetensors", + "diffusion_pytorch_model-00003-of-00003.safetensors", + "diffusion_pytorch_model.safetensors.index.json", + "models_t5_umt5-xxl-enc-bf16.pth", + "google/umt5-xxl/tokenizer.json" + ] + }, + { + "name": "StarWAM LIBERO MoT", + "provider": "modelscope", + "repository": "panshaohua/starwam", + "revision": "7d4bfe3ec76172ca17169fa959d21da099d386fe", + "license": "Apache-2.0", + "target": "modelscope/panshaohua/starwam", + "download_mode": "selected_files", + "expected_bytes": 12042006874, + "required_files": [ + { + "path": "starwam-libero/mot/starwam_wan225b_mot.pt", + "bytes": 12042005132, + "sha256": "d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378" + }, + { + "path": "starwam-libero/action_stats.json", + "bytes": 1742, + "sha256": "9f65fb518ca446e0d5ca9e8127e960fe3d11e6466e4f48ba9bb1135b1e0fb4f0" + } + ] + } + ], + "optional_later": [ + { + "name": "StarWAM LIBERO Shared-DiT", + "provider": "modelscope", + "repository": "panshaohua/starwam", + "revision": "7d4bfe3ec76172ca17169fa959d21da099d386fe", + "license": "Apache-2.0", + "target": "modelscope/panshaohua/starwam", + "path": "starwam-libero/sharedit/starwam_wan225b_shareddit.pt", + "bytes": 20076857590, + "sha256": "76effa606b4954ba79ae53755e1a3baf412b6d88ed64db50ba9d896bd0ac5c40", + "status": "optional" + }, + { + "name": "LingBot-VA LIBERO-Long", + "provider": "huggingface", + "repository": "robbyant/lingbot-va-posttrain-libero-long", + "revision": "0e89d1e753019988aba484e8da2dc0810e264d9f", + "license": "Apache-2.0", + "target": "huggingface/robbyant/lingbot-va-posttrain-libero-long", + "download_mode": "full_snapshot", + "expected_bytes": 24382314409, + "status": "optional" + }, + { + "name": "Fast-WAM LIBERO", + "provider": "huggingface", + "repository": "yuanty/fastwam", + "revision": "139eebb6d90cdd9bdbbe465f72c6edc9ad5a518a", + "license": "UNVERIFIED", + "target": "huggingface/yuanty/fastwam", + "download_mode": "selected_files", + "expected_bytes": 12041776079, + "required_paths": [ + "libero_uncond_2cam224.pt", + "libero_uncond_2cam224_dataset_stats.json" + ], + "status": "blocked" + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index c6b2c45..8f661d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,10 @@ exclude = [ "/.pytest_cache", "/.ruff_cache", "/.venv", + "/checkpoints", + "/data", "/dist", + "/runs", ] [tool.pytest.ini_options] From a47ee979a097dbefda7908076c0917ec69678504 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 08:35:45 +0000 Subject: [PATCH 04/20] Add model artifact doctor --- .gitignore | 1 + CHANGELOG.md | 2 + README.md | 8 + checkpoints/README.md | 12 +- docs/QUICKSTART.md | 14 ++ environments/starwam/README.md | 54 +++++ pyproject.toml | 1 + src/wamprobe/cli.py | 54 +++++ src/wamprobe/doctor.py | 410 +++++++++++++++++++++++++++++++++ tests/test_doctor.py | 305 ++++++++++++++++++++++++ 10 files changed, 859 insertions(+), 2 deletions(-) create mode 100644 environments/starwam/README.md create mode 100644 src/wamprobe/doctor.py create mode 100644 tests/test_doctor.py diff --git a/.gitignore b/.gitignore index 2747230..ce3af87 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ data/ checkpoints/* !checkpoints/README.md runs/ +vendor/ *.ckpt *.pt *.pth diff --git a/CHANGELOG.md b/CHANGELOG.md index 42098b2..a752b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,3 +16,5 @@ All notable changes to WAMProbe will be documented here. - Version-pinned audit selecting StarWAM as the first real adapter target. - Provider-scoped local model store with pinned revisions, expected paths, sizes, and checksums for the first StarWAM integration. +- `wamprobe doctor` with safe manifest parsing, revision/size/hash validation, and JSON + output for local model artifacts. diff --git a/README.md b/README.md index 9924ad5..30cc2c0 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,13 @@ Real-model weights are never committed to Git. Before the StarWAM integration, f [model-store layout and download rules](checkpoints/README.md); the first spike requires approximately 46.3 GB of pinned StarWAM and Wan2.2 artifacts. +Validate local artifacts without importing PyTorch or upstream model code: + +```bash +wamprobe doctor +wamprobe doctor --verify-hashes # streams the 12 GB StarWAM SHA256 check +``` + You can also run the module directly: ```bash @@ -79,6 +86,7 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - Action Dependence, Counterfactual Direction Accuracy, No-op Stability, state ADE, and Top-1 Regret; - JSON and Markdown reports; +- `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap diff --git a/checkpoints/README.md b/checkpoints/README.md index b3322df..7a25279 100644 --- a/checkpoints/README.md +++ b/checkpoints/README.md @@ -125,5 +125,13 @@ sha256sum \ checkpoints/upstream/modelscope/panshaohua/starwam/starwam-libero/action_stats.json ``` -WAMProbe will add `wamprobe doctor` as the next implementation slice to validate this -layout, revisions, required files, and checkpoint metadata without loading the GPU model. +Validate the completed store without importing the GPU model: + +```bash +uv run wamprobe doctor +uv run wamprobe doctor --verify-hashes +``` + +The fast check validates required paths, exact selected-file sizes, Git/Hugging Face +revision metadata, and unresolved Git LFS pointers. `--verify-hashes` additionally streams +SHA256 checks for selected files and is expected to take longer for the 12 GB checkpoint. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 83119bd..ee6ac38 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -40,6 +40,20 @@ WAMProbe intentionally reports a metric profile, not a composite score. A model action-dependent but physically wrong, or visually accurate but useless for candidate selection. +## Validate real-model artifacts + +The first real adapter uses pinned StarWAM and Wan2.2 files stored outside Git. Follow +[`checkpoints/README.md`](../checkpoints/README.md), then run: + +```bash +wamprobe doctor +wamprobe doctor --verify-hashes +``` + +Exit code `0` means every required model passed. Exit code `1` means an artifact is +missing, incomplete, the wrong size/hash, or from a conflicting recorded revision. Exit +code `2` means the manifest itself could not be read safely. Use `--json` for automation. + ## What this demo does not prove PointMass-2D validates the evaluator and guards against metric bugs. It is not evidence diff --git a/environments/starwam/README.md b/environments/starwam/README.md new file mode 100644 index 0000000..0cb0c28 --- /dev/null +++ b/environments/starwam/README.md @@ -0,0 +1,54 @@ +# Isolated StarWAM environment + +StarWAM is intentionally isolated from WAMProbe's dependency-free core package. The first +adapter is pinned to upstream commit: + +```text +f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7 +``` + +Local upstream code belongs under `vendor/upstream/starwam/` and is ignored by Git and +package builds. Model files belong under `checkpoints/upstream/`; do not copy weights into +the source checkout. + +## Reproduce the environment + +```bash +git clone https://github.com/shaohua-pan/StarWAM.git vendor/upstream/starwam +git -C vendor/upstream/starwam checkout f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7 + +uv venv --python 3.11 environments/starwam/.venv + +uv pip install --python environments/starwam/.venv/bin/python \ + torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 \ + --index-url https://download.pytorch.org/whl/cu124 + +uv pip install --python environments/starwam/.venv/bin/python \ + -r vendor/upstream/starwam/examples/libero/requirements.txt + +uv pip install --python environments/starwam/.venv/bin/python \ + -e vendor/upstream/starwam +``` + +`flash-attn` is an upstream requirement for some attention paths, but it should be built +only after the pinned PyTorch/CUDA environment succeeds: + +```bash +environments/starwam/.venv/bin/python -m pip install flash-attn --no-build-isolation +``` + +The first proving step is metadata-only and must not deserialize an untrusted pickle. A +later checkpoint load must use upstream trusted code, `map_location="cpu"` or an explicit +device map, inference mode, deterministic seeds, and a recorded peak-memory measurement. + +## Local paths + +```text +vendor/upstream/starwam/ +checkpoints/upstream/huggingface/Wan-AI/Wan2.2-TI2V-5B/ +checkpoints/upstream/modelscope/panshaohua/starwam/ +environments/starwam/.venv/ +``` + +Only this environment documentation is tracked. The virtual environment, upstream source, +weights, caches, LIBERO assets, and predictions stay local. diff --git a/pyproject.toml b/pyproject.toml index 8f661d5..dce7837 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ exclude = [ "/data", "/dist", "/runs", + "/vendor", ] [tool.pytest.ini_options] diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index ca84bc1..29e2c29 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import json +import sys from collections.abc import Sequence from pathlib import Path @@ -14,6 +16,7 @@ ) from wamprobe.api.model import WAMAdapter from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.doctor import ModelManifestError, check_model_store, load_manifest from wamprobe.evaluation import evaluate from wamprobe.reporting import write_reports @@ -34,6 +37,25 @@ def _build_parser() -> argparse.ArgumentParser: default=Path("runs/pointmass-demo"), help="directory for summary.json and report.md", ) + doctor = subparsers.add_parser("doctor", help="validate pinned local model artifacts") + doctor.add_argument( + "--manifest", + type=Path, + default=Path("configs/models/upstream_models.json"), + help="pinned upstream model manifest", + ) + doctor.add_argument( + "--store-root", + type=Path, + default=None, + help="override the manifest's local model-store root", + ) + doctor.add_argument( + "--verify-hashes", + action="store_true", + help="stream SHA256 verification for files that declare a checksum", + ) + doctor.add_argument("--json", action="store_true", help="emit a machine-readable report") return parser @@ -58,6 +80,36 @@ def _run_demo(args: argparse.Namespace) -> int: return 0 +def _run_doctor(args: argparse.Namespace) -> int: + try: + manifest = load_manifest(args.manifest) + report = check_model_store( + manifest, + store_root=args.store_root, + verify_hashes=args.verify_hashes, + ) + except (OSError, ModelManifestError) as error: + print(f"WAMProbe doctor error: {error}", file=sys.stderr) + return 2 + if args.json: + print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) + else: + print("WAMProbe doctor") + for model in report.models: + status = "PASS" if model.passed else "FAIL" + print(f"[{status}] {model.name} ({model.target})") + for file_check in model.files: + file_status = "PASS" if file_check.passed else "FAIL" + print(f" [{file_status}] {file_check.path}: {file_check.detail}") + if model.revision_ok is False: + print(f" [FAIL] Local revision metadata does not match {model.revision}") + passed_count = sum(model.passed for model in report.models) + print(f"Summary: {passed_count}/{len(report.models)} required models passed") + if not report.hashes_verified: + print("SHA256: skipped (use --verify-hashes for full verification)") + return 0 if report.passed else 1 + + def main(argv: Sequence[str] | None = None) -> int: """Run WAMProbe and return a process exit code.""" @@ -65,5 +117,7 @@ def main(argv: Sequence[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "demo": return _run_demo(args) + if args.command == "doctor": + return _run_doctor(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/doctor.py b/src/wamprobe/doctor.py new file mode 100644 index 0000000..e21e6af --- /dev/null +++ b/src/wamprobe/doctor.py @@ -0,0 +1,410 @@ +"""Validate pinned local model artifacts without importing model frameworks.""" + +from __future__ import annotations + +import hashlib +import json +import string +from dataclasses import dataclass +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import cast + + +class ModelManifestError(ValueError): + """Raised when a model manifest is malformed or unsafe.""" + + +@dataclass(frozen=True, slots=True) +class FileRequirement: + """One file that must be present under a model repository snapshot.""" + + path: str + expected_bytes: int | None = None + sha256: str | None = None + + +@dataclass(frozen=True, slots=True) +class ModelRequirement: + """Pinned model repository and the files WAMProbe needs from it.""" + + name: str + provider: str + repository: str + revision: str + license: str + target: str + download_mode: str + expected_bytes: int + files: tuple[FileRequirement, ...] + + +@dataclass(frozen=True, slots=True) +class ModelManifest: + """Required model artifacts for one WAMProbe development milestone.""" + + schema_version: str + store_root: str + required_models: tuple[ModelRequirement, ...] + + +@dataclass(frozen=True, slots=True) +class FileCheck: + """Validation result for one required local artifact.""" + + path: str + exists: bool + size_ok: bool | None + sha256_ok: bool | None + detail: str + + @property + def passed(self) -> bool: + """Return whether the file satisfies every requested check.""" + + return ( + self.exists + and self.size_ok is not False + and self.sha256_ok is not False + and self.detail != "Git LFS pointer; payload not downloaded" + ) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible representation.""" + + return { + "path": self.path, + "exists": self.exists, + "size_ok": self.size_ok, + "sha256_ok": self.sha256_ok, + "detail": self.detail, + "passed": self.passed, + } + + +@dataclass(frozen=True, slots=True) +class ModelCheck: + """Validation result for one pinned model repository.""" + + name: str + target: str + revision: str + revision_ok: bool | None + expected_bytes: int + files: tuple[FileCheck, ...] + + @property + def passed(self) -> bool: + """Return whether the repository satisfies all required checks.""" + + return self.revision_ok is not False and all(item.passed for item in self.files) + + @property + def missing_paths(self) -> tuple[str, ...]: + """Return required paths that are not regular files.""" + + return tuple(item.path for item in self.files if not item.exists) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible representation.""" + + return { + "name": self.name, + "target": self.target, + "revision": self.revision, + "revision_ok": self.revision_ok, + "expected_bytes": self.expected_bytes, + "passed": self.passed, + "files": [item.to_dict() for item in self.files], + } + + +@dataclass(frozen=True, slots=True) +class DoctorReport: + """Complete local model-store validation report.""" + + schema_version: str + store_root: Path + hashes_verified: bool + models: tuple[ModelCheck, ...] + + @property + def passed(self) -> bool: + """Return whether every required model passed.""" + + return all(model.passed for model in self.models) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible representation.""" + + return { + "schema_version": self.schema_version, + "store_root": str(self.store_root), + "hashes_verified": self.hashes_verified, + "passed": self.passed, + "models": [model.to_dict() for model in self.models], + } + + +def _as_mapping(value: object, location: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ModelManifestError(f"{location} must be a JSON object") + return cast(dict[str, object], value) + + +def _as_list(value: object, location: str) -> list[object]: + if not isinstance(value, list): + raise ModelManifestError(f"{location} must be a JSON array") + return cast(list[object], value) + + +def _as_string(value: object, location: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ModelManifestError(f"{location} must be a non-empty string") + return value + + +def _as_positive_int(value: object, location: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ModelManifestError(f"{location} must be a positive integer") + return value + + +def _safe_relative_path(value: object, location: str) -> str: + path_text = _as_string(value, location) + posix_path = PurePosixPath(path_text) + windows_path = PureWindowsPath(path_text) + if ( + posix_path.is_absolute() + or windows_path.is_absolute() + or windows_path.drive + or ".." in posix_path.parts + or "\\" in path_text + ): + raise ModelManifestError(f"{location} must be a safe relative path") + return posix_path.as_posix() + + +def _hex_digest(value: object, length: int, location: str) -> str: + digest = _as_string(value, location).lower() + if len(digest) != length or any(character not in string.hexdigits for character in digest): + raise ModelManifestError(f"{location} must be a {length}-character hexadecimal digest") + return digest + + +def _parse_file(value: object, location: str) -> FileRequirement: + data = _as_mapping(value, location) + return FileRequirement( + path=_safe_relative_path(data.get("path"), f"{location}.path"), + expected_bytes=_as_positive_int(data.get("bytes"), f"{location}.bytes"), + sha256=_hex_digest(data.get("sha256"), 64, f"{location}.sha256"), + ) + + +def _parse_model(value: object, index: int) -> ModelRequirement: + location = f"required_now[{index}]" + data = _as_mapping(value, location) + mode = _as_string(data.get("download_mode"), f"{location}.download_mode") + if mode == "full_snapshot": + paths = _as_list(data.get("required_paths"), f"{location}.required_paths") + files = tuple( + FileRequirement(path=_safe_relative_path(path, f"{location}.required_paths[{i}]")) + for i, path in enumerate(paths) + ) + elif mode == "selected_files": + raw_files = _as_list(data.get("required_files"), f"{location}.required_files") + files = tuple( + _parse_file(item, f"{location}.required_files[{i}]") for i, item in enumerate(raw_files) + ) + else: + raise ModelManifestError( + f"{location}.download_mode must be 'full_snapshot' or 'selected_files'" + ) + if not files: + raise ModelManifestError(f"{location} must declare at least one required file") + file_paths = [item.path for item in files] + if len(file_paths) != len(set(file_paths)): + raise ModelManifestError(f"{location} contains duplicate required file paths") + return ModelRequirement( + name=_as_string(data.get("name"), f"{location}.name"), + provider=_as_string(data.get("provider"), f"{location}.provider"), + repository=_as_string(data.get("repository"), f"{location}.repository"), + revision=_hex_digest(data.get("revision"), 40, f"{location}.revision"), + license=_as_string(data.get("license"), f"{location}.license"), + target=_safe_relative_path(data.get("target"), f"{location}.target"), + download_mode=mode, + expected_bytes=_as_positive_int(data.get("expected_bytes"), f"{location}.expected_bytes"), + files=files, + ) + + +def load_manifest(path: Path) -> ModelManifest: + """Load and validate a pinned upstream-model manifest.""" + + try: + raw = cast(object, json.loads(path.read_text(encoding="utf-8"))) + except json.JSONDecodeError as error: + raise ModelManifestError(f"invalid JSON in model manifest: {path}") from error + data = _as_mapping(raw, "manifest") + version = _as_string(data.get("schema_version"), "schema_version") + if version != "0.1": + raise ModelManifestError(f"unsupported model manifest schema: {version}") + raw_models = _as_list(data.get("required_now"), "required_now") + if not raw_models: + raise ModelManifestError("required_now must contain at least one model") + models = tuple(_parse_model(value, index) for index, value in enumerate(raw_models)) + targets = [model.target for model in models] + if len(targets) != len(set(targets)): + raise ModelManifestError("required_now contains duplicate target directories") + return ModelManifest( + schema_version=version, + store_root=_safe_relative_path(data.get("store_root"), "store_root"), + required_models=models, + ) + + +def _local_path(root: Path, relative_path: str) -> Path: + return root.joinpath(*PurePosixPath(relative_path).parts) + + +def _is_lfs_pointer(path: Path) -> bool: + if path.stat().st_size > 512: + return False + try: + first_line = path.read_text(encoding="utf-8").splitlines()[0] + except (UnicodeDecodeError, IndexError): + return False + return first_line == "version https://git-lfs.github.com/spec/v1" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _git_head(repository_root: Path) -> str | None: + head_path = repository_root / ".git" / "HEAD" + try: + head = head_path.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return None + if not head.startswith("ref: "): + return head + reference = repository_root / ".git" / head.removeprefix("ref: ") + try: + return reference.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return None + + +def _huggingface_revisions( + repository_root: Path, + requirements: tuple[FileRequirement, ...], +) -> set[str]: + revisions: set[str] = set() + metadata_root = repository_root / ".cache" / "huggingface" / "download" + for requirement in requirements: + metadata = _local_path(metadata_root, f"{requirement.path}.metadata") + try: + with metadata.open(encoding="utf-8") as stream: + revision = stream.readline(256).strip() + except FileNotFoundError: + continue + if revision: + revisions.add(revision) + return revisions + + +def _check_file( + model_root: Path, + requirement: FileRequirement, + *, + verify_hashes: bool, +) -> FileCheck: + path = _local_path(model_root, requirement.path) + if not path.is_file(): + return FileCheck( + path=requirement.path, + exists=False, + size_ok=None, + sha256_ok=None, + detail="missing required file", + ) + if _is_lfs_pointer(path): + return FileCheck( + path=requirement.path, + exists=True, + size_ok=False if requirement.expected_bytes is not None else None, + sha256_ok=None, + detail="Git LFS pointer; payload not downloaded", + ) + size_ok = ( + None + if requirement.expected_bytes is None + else path.stat().st_size == requirement.expected_bytes + ) + sha256_ok = ( + _sha256(path) == requirement.sha256 + if verify_hashes and requirement.sha256 is not None + else None + ) + details: list[str] = [] + if size_ok is False: + details.append( + f"size mismatch: expected {requirement.expected_bytes}, got {path.stat().st_size}" + ) + if sha256_ok is False: + details.append("SHA256 mismatch") + if not details: + details.append("present") + return FileCheck( + path=requirement.path, + exists=True, + size_ok=size_ok, + sha256_ok=sha256_ok, + detail="; ".join(details), + ) + + +def check_model_store( + manifest: ModelManifest, + *, + store_root: Path | None = None, + verify_hashes: bool = False, +) -> DoctorReport: + """Check required local artifacts without loading model code or pickle data.""" + + root = store_root if store_root is not None else Path(manifest.store_root) + model_checks: list[ModelCheck] = [] + for model in manifest.required_models: + model_root = _local_path(root, model.target) + git_head = _git_head(model_root) + huggingface_revisions = _huggingface_revisions(model_root, model.files) + if git_head is not None: + revision_ok = git_head == model.revision + elif huggingface_revisions: + revision_ok = huggingface_revisions == {model.revision} + else: + revision_ok = None + files = tuple( + _check_file(model_root, requirement, verify_hashes=verify_hashes) + for requirement in model.files + ) + model_checks.append( + ModelCheck( + name=model.name, + target=model.target, + revision=model.revision, + revision_ok=revision_ok, + expected_bytes=model.expected_bytes, + files=files, + ) + ) + return DoctorReport( + schema_version=manifest.schema_version, + store_root=root, + hashes_verified=verify_hashes, + models=tuple(model_checks), + ) diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..8b9bc31 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,305 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from wamprobe.cli import main +from wamprobe.doctor import ModelManifestError, check_model_store, load_manifest + + +def _write_manifest(path: Path, required_now: list[dict[str, object]]) -> Path: + manifest = { + "schema_version": "0.1", + "store_root": "checkpoints/upstream", + "required_now": required_now, + "optional_later": [], + } + path.write_text(json.dumps(manifest), encoding="utf-8") + return path + + +def test_doctor_reports_missing_required_snapshot_file(tmp_path: Path) -> None: + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "example-model", + "provider": "huggingface", + "repository": "owner/example-model", + "revision": "a" * 40, + "license": "Apache-2.0", + "target": "huggingface/owner/example-model", + "download_mode": "full_snapshot", + "expected_bytes": 100, + "required_paths": ["config.json", "weights/model.safetensors"], + } + ], + ) + + report = check_model_store(load_manifest(manifest_path), store_root=tmp_path / "models") + + assert not report.passed + assert report.models[0].missing_paths == ( + "config.json", + "weights/model.safetensors", + ) + + +def test_doctor_verifies_selected_file_size_and_hash(tmp_path: Path) -> None: + store_root = tmp_path / "models" + model_root = store_root / "modelscope" / "owner" / "model" + model_root.mkdir(parents=True) + payload = b"verified checkpoint bytes" + checkpoint = model_root / "weights.bin" + checkpoint.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "selected-model", + "provider": "modelscope", + "repository": "owner/model", + "revision": "b" * 40, + "license": "Apache-2.0", + "target": "modelscope/owner/model", + "download_mode": "selected_files", + "expected_bytes": len(payload), + "required_files": [ + { + "path": "weights.bin", + "bytes": len(payload), + "sha256": digest, + } + ], + } + ], + ) + + report = check_model_store( + load_manifest(manifest_path), + store_root=store_root, + verify_hashes=True, + ) + + assert report.passed + file_check = report.models[0].files[0] + assert file_check.size_ok is True + assert file_check.sha256_ok is True + + +def test_manifest_rejects_paths_that_escape_the_model_store(tmp_path: Path) -> None: + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "unsafe-model", + "provider": "huggingface", + "repository": "owner/model", + "revision": "c" * 40, + "license": "Apache-2.0", + "target": "../outside", + "download_mode": "full_snapshot", + "expected_bytes": 1, + "required_paths": ["weights.bin"], + } + ], + ) + + with pytest.raises(ModelManifestError, match="safe relative path"): + load_manifest(manifest_path) + + +def test_manifest_rejects_unsafe_required_file_path(tmp_path: Path) -> None: + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "unsafe-file-model", + "provider": "modelscope", + "repository": "owner/model", + "revision": "e" * 40, + "license": "Apache-2.0", + "target": "modelscope/owner/model", + "download_mode": "selected_files", + "expected_bytes": 1, + "required_files": [ + { + "path": "../../outside.bin", + "bytes": 1, + "sha256": "f" * 64, + } + ], + } + ], + ) + + with pytest.raises(ModelManifestError, match="safe relative path"): + load_manifest(manifest_path) + + +def test_doctor_rejects_lfs_pointer_and_wrong_git_revision(tmp_path: Path) -> None: + store_root = tmp_path / "models" + model_root = store_root / "modelscope" / "owner" / "model" + (model_root / ".git").mkdir(parents=True) + (model_root / ".git" / "HEAD").write_text("0" * 40, encoding="utf-8") + pointer = model_root / "weights.bin" + pointer.write_text( + f"version https://git-lfs.github.com/spec/v1\noid sha256:{'1' * 64}\nsize 123\n", + encoding="utf-8", + ) + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "pointer-model", + "provider": "modelscope", + "repository": "owner/model", + "revision": "1" * 40, + "license": "Apache-2.0", + "target": "modelscope/owner/model", + "download_mode": "selected_files", + "expected_bytes": 123, + "required_files": [ + { + "path": "weights.bin", + "bytes": 123, + "sha256": "1" * 64, + } + ], + } + ], + ) + + report = check_model_store(load_manifest(manifest_path), store_root=store_root) + + assert not report.passed + assert report.models[0].revision_ok is False + assert report.models[0].files[0].detail == "Git LFS pointer; payload not downloaded" + + +def test_doctor_reports_checksum_mismatch(tmp_path: Path) -> None: + store_root = tmp_path / "models" + model_root = store_root / "modelscope" / "owner" / "model" + model_root.mkdir(parents=True) + payload = b"wrong" + (model_root / "weights.bin").write_bytes(payload) + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "bad-checksum-model", + "provider": "modelscope", + "repository": "owner/model", + "revision": "2" * 40, + "license": "Apache-2.0", + "target": "modelscope/owner/model", + "download_mode": "selected_files", + "expected_bytes": len(payload), + "required_files": [ + { + "path": "weights.bin", + "bytes": len(payload), + "sha256": hashlib.sha256(b"right").hexdigest(), + } + ], + } + ], + ) + + report = check_model_store( + load_manifest(manifest_path), + store_root=store_root, + verify_hashes=True, + ) + + assert not report.passed + assert report.models[0].files[0].size_ok is True + assert report.models[0].files[0].sha256_ok is False + assert report.models[0].files[0].detail == "SHA256 mismatch" + + +def test_doctor_checks_huggingface_download_revision(tmp_path: Path) -> None: + store_root = tmp_path / "models" + model_root = store_root / "huggingface" / "owner" / "model" + metadata = model_root / ".cache" / "huggingface" / "download" / "config.json.metadata" + metadata.parent.mkdir(parents=True) + metadata.write_text(f"{'3' * 40}\netag\ntimestamp\n", encoding="utf-8") + (model_root / "config.json").write_text("{}", encoding="utf-8") + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "wrong-revision-model", + "provider": "huggingface", + "repository": "owner/model", + "revision": "4" * 40, + "license": "Apache-2.0", + "target": "huggingface/owner/model", + "download_mode": "full_snapshot", + "expected_bytes": 2, + "required_paths": ["config.json"], + } + ], + ) + + report = check_model_store(load_manifest(manifest_path), store_root=store_root) + + assert not report.passed + assert report.models[0].revision_ok is False + + +def test_doctor_cli_reports_invalid_manifest( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + manifest_path = tmp_path / "models.json" + manifest_path.write_text("not JSON", encoding="utf-8") + + exit_code = main(["doctor", "--manifest", str(manifest_path)]) + + assert exit_code == 2 + assert "invalid JSON" in capsys.readouterr().err + + +def test_doctor_cli_emits_machine_readable_report( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + store_root = tmp_path / "models" + model_root = store_root / "huggingface" / "owner" / "model" + model_root.mkdir(parents=True) + (model_root / "config.json").write_text("{}", encoding="utf-8") + manifest_path = _write_manifest( + tmp_path / "models.json", + [ + { + "name": "ready-model", + "provider": "huggingface", + "repository": "owner/model", + "revision": "d" * 40, + "license": "Apache-2.0", + "target": "huggingface/owner/model", + "download_mode": "full_snapshot", + "expected_bytes": 2, + "required_paths": ["config.json"], + } + ], + ) + + exit_code = main( + [ + "doctor", + "--manifest", + str(manifest_path), + "--store-root", + str(store_root), + "--json", + ] + ) + + assert exit_code == 0 + output = json.loads(capsys.readouterr().out) + assert output["passed"] is True + assert output["models"][0]["name"] == "ready-model" From b7c21d06728b89647e99e5c92a860a7a8f824db7 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 09:12:22 +0000 Subject: [PATCH 05/20] Document StarWAM preflight --- .gitattributes | 1 + CHANGELOG.md | 2 + docs/research/ADAPTER_SELECTION.md | 19 +++++++- environments/starwam/README.md | 32 +++++++++++--- .../patches/0001-safe-inference-loads.patch | 44 +++++++++++++++++++ 5 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 .gitattributes create mode 100644 environments/starwam/patches/0001-safe-inference-loads.patch diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..51dabdd --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +environments/starwam/patches/*.patch -whitespace diff --git a/CHANGELOG.md b/CHANGELOG.md index a752b5f..ff0ffca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,3 +18,5 @@ All notable changes to WAMProbe will be documented here. checksums for the first StarWAM integration. - `wamprobe doctor` with safe manifest parsing, revision/size/hash validation, and JSON output for local model artifacts. +- Reproducible StarWAM environment preflight and a narrow patch that restricts verified + inference checkpoint loads to PyTorch's weights-only deserializer. diff --git a/docs/research/ADAPTER_SELECTION.md b/docs/research/ADAPTER_SELECTION.md index aeb30a6..abbe77c 100644 --- a/docs/research/ADAPTER_SELECTION.md +++ b/docs/research/ADAPTER_SELECTION.md @@ -1,7 +1,7 @@ # First real WAM adapter selection - Decision date: 2026-07-15 -- Status: provisional implementation decision +- Status: implementation preflight passed - Target: first GPU-backed adapter after the CPU PointMass vertical slice ## Decision @@ -73,6 +73,23 @@ An adapter is eligible only when all of the following can be checked: Pins record what was audited. They should be updated only through a reviewed compatibility change, not silently moved to an upstream default branch. +## Implementation preflight result + +The pinned Wan2.2 and StarWAM artifacts were downloaded and verified on 2026-07-15. The +required files total 46.25 GB, both model groups pass `wamprobe doctor --verify-hashes`, +and no incomplete fragments remain. An isolated Python 3.11 / PyTorch 2.6.0+cu124 +environment imports the pinned StarWAM source successfully. + +Restricted checkpoint inspection found only tensor mappings in the StarWAM, Wan VAE, and +Wan T5 payloads. A meta-device architecture build matched all 3,300 checkpoint keys with +zero missing or unexpected entries. A load-only test on one RTX 4090 peaked at 12.16 GB +allocated and 12.38 GB reserved, used deterministic seed 42 and evaluation mode, and +released the allocation afterward. + +These results clear artifact, environment, architecture, and loadability gates only. No +LIBERO observation was executed, so action correctness, preprocessing correctness, runtime +latency, and rollout success remain unverified. + ## First adapter implementation slice The next coding slice is deliberately narrow: diff --git a/environments/starwam/README.md b/environments/starwam/README.md index 0cb0c28..c5c3783 100644 --- a/environments/starwam/README.md +++ b/environments/starwam/README.md @@ -16,6 +16,8 @@ the source checkout. ```bash git clone https://github.com/shaohua-pan/StarWAM.git vendor/upstream/starwam git -C vendor/upstream/starwam checkout f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7 +git -C vendor/upstream/starwam apply \ + "$(git rev-parse --show-toplevel)/environments/starwam/patches/0001-safe-inference-loads.patch" uv venv --python 3.11 environments/starwam/.venv @@ -30,16 +32,36 @@ uv pip install --python environments/starwam/.venv/bin/python \ -e vendor/upstream/starwam ``` -`flash-attn` is an upstream requirement for some attention paths, but it should be built -only after the pinned PyTorch/CUDA environment succeeds: +`flash-attn` is optional for some attention paths. Build it only if the selected path +actually imports it and after the pinned PyTorch/CUDA environment succeeds: ```bash environments/starwam/.venv/bin/python -m pip install flash-attn --no-build-isolation ``` -The first proving step is metadata-only and must not deserialize an untrusted pickle. A -later checkpoint load must use upstream trusted code, `map_location="cpu"` or an explicit -device map, inference mode, deterministic seeds, and a recorded peak-memory measurement. +The tracked patch changes only model-inference checkpoint reads that were verified to be +pure tensor mappings. It keeps LIBERO's separate legacy init-state compatibility path +unchanged. Do not generalize `weights_only=True` changes to arbitrary data or optimizer +checkpoints without inspecting their payloads first. + +## Verified preflight + +The pinned artifacts and environment passed the following checks on 2026-07-15: + +- `wamprobe doctor --verify-hashes`: 2/2 required model groups passed, with no incomplete + download fragments; +- environment: Python 3.11.15, PyTorch 2.6.0+cu124, four RTX 4090 GPUs, and 136 compatible + installed packages; +- restricted metadata load: `weights_only=True`, `mmap=True`, and `map_location="meta"` + for the StarWAM, Wan VAE, and Wan T5 checkpoints; +- StarWAM architecture contract: 3,300/3,300 state keys matched, with zero missing or + unexpected keys; +- GPU 2 load-only smoke check: deterministic seed 42, `model.eval()`, 12.16 GB peak + allocated and 12.38 GB peak reserved, followed by successful memory release. + +This is a loadability check, not a successful LIBERO rollout or evidence that generated +actions are correct. The next gate is one typed observation-to-action artifact with its +resolved preprocessing and normalization metadata. ## Local paths diff --git a/environments/starwam/patches/0001-safe-inference-loads.patch b/environments/starwam/patches/0001-safe-inference-loads.patch new file mode 100644 index 0000000..d424bdf --- /dev/null +++ b/environments/starwam/patches/0001-safe-inference-loads.patch @@ -0,0 +1,44 @@ +diff --git a/examples/libero/rollout.py b/examples/libero/rollout.py +index ad0574f..22ea9a2 100644 +--- a/examples/libero/rollout.py ++++ b/examples/libero/rollout.py +@@ -116,7 +116,7 @@ def _load_checkpoint(model: torch.nn.Module, checkpoint: str | Path) -> dict[str + path = checkpoint / name + if not path.is_file(): + continue +- payload = torch.load(path, map_location="cpu", weights_only=False) ++ payload = torch.load(path, map_location="cpu", weights_only=True) + state = _extract_checkpoint_state(payload) + _load_model_state(model, state) + return _checkpoint_meta(payload, path) +@@ -135,7 +135,7 @@ def _load_checkpoint(model: torch.nn.Module, checkpoint: str | Path) -> dict[str + f"Unsupported checkpoint directory {checkpoint}. Expected model.pt, pytorch_model.bin, or *.safetensors." + ) + +- payload = torch.load(checkpoint, map_location="cpu", weights_only=False) ++ payload = torch.load(checkpoint, map_location="cpu", weights_only=True) + state = _extract_checkpoint_state(payload) + _load_model_state(model, state) + return _checkpoint_meta(payload, checkpoint) +diff --git a/starwam/backbone/wan22.py b/starwam/backbone/wan22.py +index 9a14d12..056b570 100644 +--- a/starwam/backbone/wan22.py ++++ b/starwam/backbone/wan22.py +@@ -602,7 +602,7 @@ class T5EncoderModel: + dtype=dtype, + device=device).eval().requires_grad_(False) + logging.info(f'loading {checkpoint_path}') +- model.load_state_dict(torch.load(checkpoint_path, map_location='cpu', weights_only=False)) ++ model.load_state_dict(torch.load(checkpoint_path, map_location='cpu', weights_only=True)) + self.model = model + if shard_fn is not None: + self.model = shard_fn(self.model, sync_module_states=False) +@@ -1490,7 +1490,7 @@ def _video_vae(pretrained_path=None, z_dim=16, dim=160, device="cpu", **kwargs): + # load checkpoint + logging.info(f"loading {pretrained_path}") + model.load_state_dict( +- torch.load(pretrained_path, map_location=device, weights_only=False), assign=True) ++ torch.load(pretrained_path, map_location=device, weights_only=True), assign=True) + + return model + From 2aa674adc35ce50ee5187d4317842e881e267cde Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 09:59:58 +0000 Subject: [PATCH 06/20] add typed StarWAM LIBERO inference slice --- README.md | 8 +- docs/QUICKSTART.md | 21 + docs/research/ADAPTER_SELECTION.md | 28 +- environments/starwam/README.md | 42 +- .../patches/0001-safe-inference-loads.patch | 25 +- environments/starwam/run_libero_smoke.py | 832 ++++++++++++++++++ src/wamprobe/adapters/__init__.py | 4 + src/wamprobe/adapters/starwam.py | 132 +++ src/wamprobe/api/__init__.py | 8 +- src/wamprobe/api/model.py | 27 + src/wamprobe/api/robotics.py | 191 ++++ src/wamprobe/artifacts.py | 261 ++++++ tests/test_artifacts.py | 124 +++ tests/test_robotics_api.py | 75 ++ tests/test_starwam_adapter.py | 82 ++ 15 files changed, 1836 insertions(+), 24 deletions(-) create mode 100644 environments/starwam/run_libero_smoke.py create mode 100644 src/wamprobe/adapters/starwam.py create mode 100644 src/wamprobe/api/robotics.py create mode 100644 src/wamprobe/artifacts.py create mode 100644 tests/test_artifacts.py create mode 100644 tests/test_robotics_api.py create mode 100644 tests/test_starwam_adapter.py diff --git a/README.md b/README.md index 30cc2c0..355fe6e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ runs/pointmass-demo/ See the committed [example report](examples/pointmass-demo/report.md) for the expected baseline ordering. -Real-model weights are never committed to Git. Before the StarWAM integration, follow the +Real-model weights are never committed to Git. Before running the StarWAM integration, follow the [model-store layout and download rules](checkpoints/README.md); the first spike requires approximately 46.3 GB of pinned StarWAM and Wan2.2 artifacts. @@ -79,7 +79,7 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo ## Current runnable scope -- typed, model-agnostic `WAMAdapter` protocol; +- typed, model-agnostic `WAMAdapter` and `ActionPredictorAdapter` protocols; - capability manifest data model and JSON Schema; - paired PointMass-2D counterfactual interventions; - oracle, copy-last-frame, wrong-direction, and action-agnostic baselines; @@ -87,6 +87,8 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo and Top-1 Regret; - JSON and Markdown reports; - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; +- typed robot observations, action predictions, deterministic prediction artifacts, and a + pinned StarWAM/LIBERO smoke runner; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap @@ -97,7 +99,7 @@ The next milestones are: 2. a small image-rendered BlockPush benchmark; 3. the versioned intervention dataset loader; 4. LIBERO-CF-Mini paired simulator branches; -5. a pinned StarWAM inference adapter, followed by LingBot-VA as a published reference. +5. paired LIBERO simulator branches, then LingBot-VA as a published reference adapter. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index ee6ac38..45adaa2 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -54,6 +54,27 @@ Exit code `0` means every required model passed. Exit code `1` means an artifact missing, incomplete, the wrong size/hash, or from a conflicting recorded revision. Exit code `2` means the manifest itself could not be read safely. Use `--json` for automation. +## Run the opt-in StarWAM smoke probe + +After reproducing the isolated environment in +[`environments/starwam/README.md`](../environments/starwam/README.md), select a physical +GPU with enough free memory and run the text encoder and action model in separate +processes: + +```bash +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode text-cache --gpu-index 0 --minimum-free-gib 13 + +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode infer --gpu-index 0 --minimum-free-gib 15 \ + --num-inference-steps 1 --vae-device cpu +``` + +The runner fixes LIBERO suite/task/init state/wait steps and seed, verifies upstream +revisions and input hashes, records both camera transformations and proprio ordering, and +writes a cache-keyed prediction JSON below `runs/starwam-libero-smoke/`. Use the one-step +setting only as an integration smoke test; the published StarWAM recipe uses eight steps. + ## What this demo does not prove PointMass-2D validates the evaluator and guards against metric bugs. It is not evidence diff --git a/docs/research/ADAPTER_SELECTION.md b/docs/research/ADAPTER_SELECTION.md index abbe77c..5a3eddc 100644 --- a/docs/research/ADAPTER_SELECTION.md +++ b/docs/research/ADAPTER_SELECTION.md @@ -86,21 +86,29 @@ zero missing or unexpected entries. A load-only test on one RTX 4090 peaked at 1 allocated and 12.38 GB reserved, used deterministic seed 42 and evaluation mode, and released the allocation afterward. -These results clear artifact, environment, architecture, and loadability gates only. No -LIBERO observation was executed, so action correctness, preprocessing correctness, runtime -latency, and rollout success remain unverified. +The first typed inference slice was subsequently completed on the same date. A fixed +`libero_spatial` task-0 observation was generated from init state 0 after 30 dummy wait +steps, with two raw RGB frames and the exact eight-dimensional proprio ordering. The +pinned released model produced finite, denormalized `[32, 7]` action chunks with both the +one-step smoke setting and the released eight-step setting. With the FP32 VAE offloaded to +CPU and the BF16 action/DiT model on an RTX 4090, the eight-step model call took 3.38 +seconds on the first run and 2.81 seconds in the final recorded artifact, peaking at +11.386 GiB allocated / 11.529 GiB reserved GPU memory. + +This clears the observation-to-action integration gate only. No action was executed in +the simulator, and no rollout success or policy-quality claim is made. ## First adapter implementation slice The next coding slice is deliberately narrow: -1. add a `starwam` optional dependency group or isolated environment specification; -2. implement metadata-only discovery and `wamprobe doctor` compatibility checks first; -3. download one checkpoint family and record immutable hashes outside Git; -4. run one LIBERO observation through upstream inference and save a typed, cached artifact; -5. expose only capabilities verified from that artifact; -6. add a mocked CPU contract test and an opt-in GPU smoke test; -7. defer counterfactual scoring until simulator snapshot/restore produces paired branches. +1. isolated StarWAM environment and pinned local upstream snapshots: **complete**; +2. metadata-only discovery and `wamprobe doctor` checks: **complete**; +3. immutable model pins and required hashes: **complete**; +4. typed, cached observation-to-action artifact: **complete**; +5. capability declaration and mocked CPU contract tests: **complete**; +6. opt-in fixed LIBERO GPU smoke runner: **complete**; +7. paired simulator snapshot/restore and counterfactual scoring: **next**. ## Sources checked diff --git a/environments/starwam/README.md b/environments/starwam/README.md index c5c3783..d933f19 100644 --- a/environments/starwam/README.md +++ b/environments/starwam/README.md @@ -40,8 +40,10 @@ environments/starwam/.venv/bin/python -m pip install flash-attn --no-build-isola ``` The tracked patch changes only model-inference checkpoint reads that were verified to be -pure tensor mappings. It keeps LIBERO's separate legacy init-state compatibility path -unchanged. Do not generalize `weights_only=True` changes to arbitrary data or optimizer +pure tensor mappings. It also constructs UMT5 on the meta device before assigning its +memory-mapped weights, avoiding a duplicate random model on the GPU. The WAMProbe smoke +runner loads the pinned LIBERO NumPy init state with `weights_only=True` and a minimal +NumPy allowlist. Do not generalize these changes to arbitrary data or optimizer checkpoints without inspecting their payloads first. ## Verified preflight @@ -58,10 +60,40 @@ The pinned artifacts and environment passed the following checks on 2026-07-15: unexpected keys; - GPU 2 load-only smoke check: deterministic seed 42, `model.eval()`, 12.16 GB peak allocated and 12.38 GB peak reserved, followed by successful memory release. +- fixed LIBERO observation: `libero_spatial` task 0, init state 0, 30 wait steps, two + 256x256 RGB frames, eight ordered proprio values, and observation content SHA256 + `051ac984bcbfad0ecd0184f86e0c3b1d148502a849337132b438fbf62f22a29b`; +- real one-step action smoke: a denormalized `[32, 7]` chunk on an RTX 4090, 2.95 seconds + model-call latency, 11.386 GiB peak allocated, and 11.529 GiB peak reserved. The VAE + ran in FP32 on CPU while the released StarWAM action/DiT model ran in BF16 on GPU. +- released eight-step setting: the same typed `[32, 7]` path completed in 2.81 seconds + with the same measured GPU peak and cache key + `c6c46faab2a33940fe40d20aba3fab311f8e17885ed753f963c8b51161d939e1`. -This is a loadability check, not a successful LIBERO rollout or evidence that generated -actions are correct. The next gate is one typed observation-to-action artifact with its -resolved preprocessing and normalization metadata. +These runs prove the typed observation-to-action integration path, not policy quality or +rollout success. No predicted action was executed in the simulator, so neither artifact +may be reported as a LIBERO benchmark result. + +## Reproduce the fixed smoke run + +Choose one physical GPU with sufficient free memory. Run T5 caching in a separate process +so its memory is fully released before the action model is loaded: + +```bash +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode observe --gpu-index 0 + +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode text-cache --gpu-index 0 --minimum-free-gib 13 + +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode infer --gpu-index 0 --minimum-free-gib 15 \ + --num-inference-steps 8 --vae-device cpu +``` + +Add `--verify-large-hashes` to rehash the checkpoint, VAE, and T5 before a run. Generated +PNGs, text caches, observation metadata, and prediction JSON remain below +`runs/starwam-libero-smoke/` and are ignored by Git. ## Local paths diff --git a/environments/starwam/patches/0001-safe-inference-loads.patch b/environments/starwam/patches/0001-safe-inference-loads.patch index d424bdf..74f894f 100644 --- a/environments/starwam/patches/0001-safe-inference-loads.patch +++ b/environments/starwam/patches/0001-safe-inference-loads.patch @@ -21,19 +21,34 @@ index ad0574f..22ea9a2 100644 _load_model_state(model, state) return _checkpoint_meta(payload, checkpoint) diff --git a/starwam/backbone/wan22.py b/starwam/backbone/wan22.py -index 9a14d12..056b570 100644 +index 9a14d12..5b79254 100644 --- a/starwam/backbone/wan22.py +++ b/starwam/backbone/wan22.py -@@ -602,7 +602,7 @@ class T5EncoderModel: +@@ -600,14 +600,21 @@ class T5EncoderModel: + encoder_only=True, + return_tokenizer=False, dtype=dtype, - device=device).eval().requires_grad_(False) +- device=device).eval().requires_grad_(False) ++ device="meta").eval().requires_grad_(False) logging.info(f'loading {checkpoint_path}') - model.load_state_dict(torch.load(checkpoint_path, map_location='cpu', weights_only=False)) -+ model.load_state_dict(torch.load(checkpoint_path, map_location='cpu', weights_only=True)) ++ state = torch.load( ++ checkpoint_path, ++ map_location='cpu', ++ weights_only=True, ++ mmap=True, ++ ) ++ model.load_state_dict(state, assign=True) self.model = model if shard_fn is not None: self.model = shard_fn(self.model, sync_module_states=False) -@@ -1490,7 +1490,7 @@ def _video_vae(pretrained_path=None, z_dim=16, dim=160, device="cpu", **kwargs): + else: + self.model.to(self.device) ++ del state + # init tokenizer + self.tokenizer = HuggingfaceTokenizer( + name=tokenizer_path, seq_len=text_len, clean='whitespace') +@@ -1490,7 +1497,7 @@ def _video_vae(pretrained_path=None, z_dim=16, dim=160, device="cpu", **kwargs): # load checkpoint logging.info(f"loading {pretrained_path}") model.load_state_dict( diff --git a/environments/starwam/run_libero_smoke.py b/environments/starwam/run_libero_smoke.py new file mode 100644 index 0000000..cee8b3d --- /dev/null +++ b/environments/starwam/run_libero_smoke.py @@ -0,0 +1,832 @@ +#!/usr/bin/env python +"""Run one pinned LIBERO observation through the released StarWAM MoT policy. + +The script is intentionally kept in the isolated StarWAM environment. It writes raw +observations, text caches, and prediction artifacts below ``runs/``, which is ignored by +Git. Run text-cache generation in a separate process before inference so its 11 GB T5 +encoder is fully released before the action model is loaded. +""" + +from __future__ import annotations + +import argparse +import copy +import gc +import hashlib +import json +import math +import os +import random +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +SOURCE_ROOT = REPOSITORY_ROOT / "src" +STARWAM_ROOT = REPOSITORY_ROOT / "vendor" / "upstream" / "starwam" +LIBERO_ROOT = REPOSITORY_ROOT / "vendor" / "upstream" / "libero" +BACKBONE_ROOT = ( + REPOSITORY_ROOT / "checkpoints" / "upstream" / "huggingface" / "Wan-AI" / "Wan2.2-TI2V-5B" +) +MODEL_ROOT = REPOSITORY_ROOT / "checkpoints" / "upstream" / "modelscope" / "panshaohua" / "starwam" +CHECKPOINT = MODEL_ROOT / "starwam-libero" / "mot" / "starwam_wan225b_mot.pt" +ACTION_STATS = MODEL_ROOT / "starwam-libero" / "action_stats.json" +VAE_CHECKPOINT = BACKBONE_ROOT / "Wan2.2_VAE.pth" +TEXT_ENCODER_CHECKPOINT = BACKBONE_ROOT / "models_t5_umt5-xxl-enc-bf16.pth" +RECIPE = ( + STARWAM_ROOT + / "examples" + / "libero" + / "configs" + / "recipes" + / "starwam_libero_mot_wan22_5b.yaml" +) + +STARWAM_REVISION = "f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7" +LIBERO_REVISION = "8f1084e3132a39270c3a13ebe37270a43ece2a01" +MODEL_REVISION = "7d4bfe3ec76172ca17169fa959d21da099d386fe" +BACKBONE_REVISION = "921dbaf3f1674a56f47e83fb80a34bac8a8f203e" +CHECKPOINT_SHA256 = "d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378" +ACTION_STATS_SHA256 = "9f65fb518ca446e0d5ca9e8127e960fe3d11e6466e4f48ba9bb1135b1e0fb4f0" +VAE_SHA256 = "20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36" +TEXT_ENCODER_SHA256 = "7cace0da2b446bbbbc57d031ab6cf163a3d59b366da94e5afe36745b746fd81d" + +TASK_SUITE = "libero_spatial" +TASK_ID = 0 +INIT_STATE_INDEX = 0 +WAIT_STEPS = 30 +SEED = 42 +TASK = "pick up the black bowl between the plate and the ramekin and place it on the plate" +CONTEXT_ID = "libero-spatial-task0-init0-wait30" +BDDL_SHA256 = "9b59eb1287802868ad9bc78d58e6d36d4ba31134e679cfdbdf4b0feb660c959b" +INIT_STATES_SHA256 = "cbbc73792ce546c9bec181fd328a411d3183074840b282671dee481511381d0a" +TEXT_PROMPT_TEMPLATE = ( + "A video recorded from a robot's point of view executing the following instruction: {task}" +) +CAMERA_ORDER = ("agentview", "wrist") +PROPRIO_FIELDS = ( + "eef_x", + "eef_y", + "eef_z", + "axis_angle_x", + "axis_angle_y", + "axis_angle_z", + "gripper_qpos_0", + "gripper_qpos_1", +) +LIBERO_DUMMY_ACTION = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0] + +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from wamprobe.adapters.starwam import ( # noqa: E402 + StarWAMAdapter, + StarWAMBackendResult, + StarWAMRelease, +) +from wamprobe.api.robotics import InferenceRuntime, RGBFrame, RobotObservation # noqa: E402 +from wamprobe.artifacts import ( # noqa: E402 + ActionPredictionArtifact, + ModelProvenance, + ResolvedInference, + ResolvedPreprocessing, +) + + +@dataclass(frozen=True, slots=True) +class CapturedObservation: + """One fixed simulator observation plus benchmark provenance.""" + + observation: RobotObservation + done_during_wait: bool + bddl_path: Path + init_states_path: Path + init_states_shape: tuple[int, ...] + init_states_dtype: str + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(8 * 1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _git_revision(path: Path) -> str: + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _verify_revision(path: Path, expected: str, label: str) -> None: + actual = _git_revision(path) + if actual != expected: + raise RuntimeError(f"{label} revision mismatch: expected {expected}, got {actual}") + + +def _verify_huggingface_revision(path: Path, expected: str) -> None: + metadata = path / ".cache" / "huggingface" / "download" / "Wan2.2_VAE.pth.metadata" + if not metadata.is_file(): + raise FileNotFoundError(f"missing Hugging Face revision metadata: {metadata}") + actual = metadata.read_text(encoding="utf-8").splitlines()[0] + if actual != expected: + raise RuntimeError(f"Wan2.2 revision mismatch: expected {expected}, got {actual}") + + +def _verify_inputs(*, verify_large_hashes: bool) -> None: + required = (CHECKPOINT, ACTION_STATS, VAE_CHECKPOINT, TEXT_ENCODER_CHECKPOINT, RECIPE) + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise FileNotFoundError(f"missing required files: {missing}") + _verify_revision(STARWAM_ROOT, STARWAM_REVISION, "StarWAM") + _verify_revision(LIBERO_ROOT, LIBERO_REVISION, "LIBERO") + _verify_revision(MODEL_ROOT, MODEL_REVISION, "StarWAM model") + _verify_huggingface_revision(BACKBONE_ROOT, BACKBONE_REVISION) + if _sha256(ACTION_STATS) != ACTION_STATS_SHA256: + raise RuntimeError("released action/state statistics hash mismatch") + if verify_large_hashes: + expected_hashes = { + CHECKPOINT: CHECKPOINT_SHA256, + VAE_CHECKPOINT: VAE_SHA256, + TEXT_ENCODER_CHECKPOINT: TEXT_ENCODER_SHA256, + } + for path, expected in expected_hashes.items(): + actual = _sha256(path) + if actual != expected: + raise RuntimeError(f"hash mismatch for {path}: expected {expected}, got {actual}") + + +def _configure_process(gpu_index: int, run_dir: Path) -> None: + os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_index) + os.environ["MUJOCO_EGL_DEVICE_ID"] = str(gpu_index) + os.environ.setdefault("MUJOCO_GL", "egl") + os.environ.setdefault("PYOPENGL_PLATFORM", "egl") + config_dir = run_dir / "libero_config" + config_dir.mkdir(parents=True, exist_ok=True) + os.environ["LIBERO_CONFIG_PATH"] = str(config_dir) + + benchmark_root = LIBERO_ROOT / "libero" / "libero" + config = { + "benchmark_root": str(benchmark_root), + "bddl_files": str(benchmark_root / "bddl_files"), + "init_states": str(benchmark_root / "init_files"), + "datasets": str(LIBERO_ROOT / "libero" / "datasets"), + "assets": str(benchmark_root / "assets"), + } + config_path = config_dir / "config.yaml" + config_path.write_text( + "".join(f"{key}: {json.dumps(value)}\n" for key, value in config.items()), + encoding="utf-8", + ) + for path in (STARWAM_ROOT, LIBERO_ROOT): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + + +def _set_seed(seed: int) -> None: + import numpy as np + import torch + + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + +def _require_cuda_memory(minimum_free_gib: float) -> Any: + import torch + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required for the released 5B StarWAM smoke test") + device = torch.device("cuda:0") + torch.cuda.set_device(device) + free_bytes, total_bytes = torch.cuda.mem_get_info(device) + minimum_bytes = int(minimum_free_gib * 1024**3) + if free_bytes < minimum_bytes: + raise RuntimeError( + f"insufficient free VRAM: need {minimum_free_gib:.1f} GiB, " + f"have {free_bytes / 1024**3:.1f} GiB" + ) + print( + json.dumps( + { + "cuda_device": torch.cuda.get_device_name(device), + "free_gib": round(free_bytes / 1024**3, 2), + "total_gib": round(total_bytes / 1024**3, 2), + }, + sort_keys=True, + ) + ) + return device + + +def _quat_to_axis_angle(quaternion: Any) -> Any: + import numpy as np + + quat = np.asarray(quaternion, dtype=np.float32).copy() + quat[3] = np.clip(quat[3], -1.0, 1.0) + denominator = math.sqrt(max(0.0, 1.0 - float(quat[3]) ** 2)) + if math.isclose(denominator, 0.0): + return np.zeros(3, dtype=np.float32) + angle = 2.0 * math.acos(float(quat[3])) + return (quat[:3] * angle / denominator).astype(np.float32) + + +def _load_init_states(path: Path) -> Any: + """Load the pinned NumPy array with a minimal weights-only allowlist.""" + + import numpy as np + import torch + from numpy.core.multiarray import _reconstruct # type: ignore[attr-defined] + + safe_globals = [_reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType] + with torch.serialization.safe_globals(safe_globals): + states = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(states, np.ndarray) or states.dtype != np.float64: + raise TypeError("LIBERO init states must be a float64 NumPy array") + return states + + +def _capture_observation() -> CapturedObservation: + import numpy as np + from libero.libero import benchmark, get_libero_path + from libero.libero.envs import OffScreenRenderEnv + + suite = benchmark.get_benchmark_dict()[TASK_SUITE]() + task = suite.get_task(TASK_ID) + if task.language != TASK: + raise RuntimeError(f"pinned LIBERO task changed: expected {TASK!r}, got {task.language!r}") + + bddl_path = Path(suite.get_task_bddl_file_path(TASK_ID)) + init_states_path = ( + Path(get_libero_path("init_states")) / task.problem_folder / task.init_states_file + ) + if _sha256(bddl_path) != BDDL_SHA256: + raise RuntimeError("LIBERO BDDL hash mismatch") + if _sha256(init_states_path) != INIT_STATES_SHA256: + raise RuntimeError("LIBERO init-state hash mismatch") + init_states = _load_init_states(init_states_path) + if init_states.shape != (50, 92): + raise RuntimeError(f"unexpected LIBERO init-state shape: {init_states.shape}") + + environment = OffScreenRenderEnv( + bddl_file_name=str(bddl_path), + camera_heights=256, + camera_widths=256, + ) + environment.seed(SEED) + done = False + try: + environment.reset() + observation = environment.set_init_state(init_states[INIT_STATE_INDEX]) + for _ in range(WAIT_STEPS): + observation, _, done, _ = environment.step(LIBERO_DUMMY_ACTION) + finally: + environment.close() + + agentview = np.ascontiguousarray(observation["agentview_image"], dtype=np.uint8) + wrist = np.ascontiguousarray(observation["robot0_eye_in_hand_image"], dtype=np.uint8) + proprio = np.concatenate( + [ + np.asarray(observation["robot0_eef_pos"], dtype=np.float32), + _quat_to_axis_angle(observation["robot0_eef_quat"]), + np.asarray(observation["robot0_gripper_qpos"], dtype=np.float32), + ] + ) + if proprio.shape != (8,): + raise RuntimeError(f"unexpected LIBERO proprio shape: {proprio.shape}") + + typed_observation = RobotObservation( + context_id=CONTEXT_ID, + task=TASK, + frames=( + RGBFrame("agentview", 256, 256, agentview.tobytes()), + RGBFrame("wrist", 256, 256, wrist.tobytes()), + ), + proprio=tuple(float(value) for value in proprio), + proprio_fields=PROPRIO_FIELDS, + ) + return CapturedObservation( + observation=typed_observation, + done_during_wait=bool(done), + bddl_path=bddl_path, + init_states_path=init_states_path, + init_states_shape=tuple(int(value) for value in init_states.shape), + init_states_dtype=str(init_states.dtype), + ) + + +def _resolved_config(run_dir: Path) -> Any: + from starwam.config import load_config + + config = load_config(str(RECIPE)) + config.backbone.pretrained_model_id = str(BACKBONE_ROOT) + config.backbone.load_text_encoder = False + config.framework.action_expert_init_from = None + config.data.action_stats_path = str(ACTION_STATS) + config.data.state_stats_path = str(ACTION_STATS) + config.data.text_embedding_cache_dir = str(run_dir / "text_embedding_cache") + return config + + +def _text_cache_path(config: Any) -> Path: + from starwam.data.lerobot import text_cache_path + + return text_cache_path( + config.data.text_embedding_cache_dir, + TASK, + int(config.data.text_len), + TEXT_PROMPT_TEMPLATE, + str(config.data.text_cache_encoder_id), + ) + + +def _load_text_cache(path: Path, config: Any) -> tuple[Any, Any]: + import torch + + payload = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(payload, dict): + raise TypeError("text cache must contain a mapping") + if payload.get("task") != TASK or payload.get("prompt") != TEXT_PROMPT_TEMPLATE.format( + task=TASK + ): + raise RuntimeError("text cache task or prompt mismatch") + context = payload.get("context") + mask = payload.get("mask") + if not isinstance(context, torch.Tensor) or not isinstance(mask, torch.Tensor): + raise TypeError("text cache context and mask must be tensors") + expected_shape = (int(config.data.text_len), 4096) + if tuple(context.shape) != expected_shape or tuple(mask.shape) != (expected_shape[0],): + raise RuntimeError( + f"text cache shape mismatch: context={tuple(context.shape)} mask={tuple(mask.shape)}" + ) + return context, mask + + +def _text_condition_sha256(context: Any, mask: Any) -> str: + import torch + + metadata = { + "task": TASK, + "prompt": TEXT_PROMPT_TEMPLATE.format(task=TASK), + "context_shape": list(context.shape), + "context_dtype": str(context.dtype), + "mask_shape": list(mask.shape), + "mask_dtype": str(mask.dtype), + } + digest = hashlib.sha256( + json.dumps(metadata, separators=(",", ":"), sort_keys=True).encode("utf-8") + ) + digest.update(context.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes()) + digest.update(mask.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + +def _prepare_text_cache(config: Any, minimum_free_gib: float) -> Path: + import torch + from starwam.backbone.wan22 import Wan22TextEncoder + from starwam.data.lerobot import save_text_cache + + cache_path = _text_cache_path(config) + if cache_path.is_file(): + _load_text_cache(cache_path, config) + print(f"text cache already valid: {cache_path}") + return cache_path + + device = _require_cuda_memory(minimum_free_gib) + _set_seed(SEED) + encoder = Wan22TextEncoder( + ckpt_path=str(TEXT_ENCODER_CHECKPOINT), + tokenizer_path=str(BACKBONE_ROOT / "google" / "umt5-xxl"), + text_len=int(config.data.text_len), + device=str(device), + dtype=torch.bfloat16, + ) + prompt = TEXT_PROMPT_TEMPLATE.format(task=TASK) + try: + with torch.inference_mode(): + context, mask = encoder.encode([prompt]) + save_text_cache(cache_path, context[0], mask[0], prompt, TASK) + _load_text_cache(cache_path, config) + finally: + del encoder + gc.collect() + torch.cuda.empty_cache() + print(f"wrote text cache: {cache_path}") + return cache_path + + +def _preprocessing(text_cache_sha256: str) -> ResolvedPreprocessing: + return ResolvedPreprocessing( + camera_order=CAMERA_ORDER, + camera_transforms=("rotate_180", "rotate_180"), + source_size=(256, 256), + resized_camera_size=(224, 224), + model_size=(224, 448), + resize_interpolation="PIL_BILINEAR", + channel_order="RGB", + tensor_layout="BCHW", + camera_concatenation="horizontal", + input_value_range=(-1.0, 1.0), + text_prompt=TEXT_PROMPT_TEMPLATE.format(task=TASK), + text_cache_sha256=text_cache_sha256, + proprio_fields=PROPRIO_FIELDS, + action_normalization="minmax_then_libero_gripper_binary_inversion", + action_stats_sha256=ACTION_STATS_SHA256, + ) + + +def _preprocess_frames(observation: RobotObservation, preprocessing: ResolvedPreprocessing) -> Any: + import numpy as np + from PIL import Image + + frames = {frame.camera_name: frame for frame in observation.frames} + resized = [] + target_height, target_width = preprocessing.resized_camera_size + for camera_name, transform in zip( + preprocessing.camera_order, preprocessing.camera_transforms, strict=True + ): + frame = frames[camera_name] + image = np.frombuffer(frame.data, dtype=np.uint8).reshape(frame.height, frame.width, 3) + if transform == "rotate_180": + image = np.ascontiguousarray(image[::-1, ::-1]) + resized.append( + np.asarray( + Image.fromarray(image).resize( + (target_width, target_height), resample=Image.Resampling.BILINEAR + ), + dtype=np.uint8, + ) + ) + axis = 1 if preprocessing.camera_concatenation == "horizontal" else 0 + model_image = np.ascontiguousarray(np.concatenate(resized, axis=axis)) + if tuple(model_image.shape) != (*preprocessing.model_size, 3): + raise RuntimeError(f"preprocessed image has unexpected shape {model_image.shape}") + return model_image + + +def _write_observation( + captured: CapturedObservation, + preprocessing: ResolvedPreprocessing, + run_dir: Path, +) -> Path: + import numpy as np + from PIL import Image + + output_dir = run_dir / "observations" / captured.observation.context_id + output_dir.mkdir(parents=True, exist_ok=True) + for frame in captured.observation.frames: + image = np.frombuffer(frame.data, dtype=np.uint8).reshape(frame.height, frame.width, 3) + Image.fromarray(image).save(output_dir / f"{frame.camera_name}.png") + Image.fromarray(_preprocess_frames(captured.observation, preprocessing)).save( + output_dir / "model_input.png" + ) + metadata = { + "observation": captured.observation.descriptor(), + "preprocessing": preprocessing.to_dict(), + "libero": { + "revision": LIBERO_REVISION, + "task_suite": TASK_SUITE, + "task_id": TASK_ID, + "init_state_index": INIT_STATE_INDEX, + "wait_steps": WAIT_STEPS, + "seed": SEED, + "done_during_wait": captured.done_during_wait, + "bddl_path": str(captured.bddl_path), + "bddl_sha256": BDDL_SHA256, + "init_states_path": str(captured.init_states_path), + "init_states_sha256": INIT_STATES_SHA256, + "init_states_shape": list(captured.init_states_shape), + "init_states_dtype": captured.init_states_dtype, + }, + } + metadata_path = output_dir / "observation.json" + metadata_path.write_text( + json.dumps(metadata, allow_nan=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(f"wrote observation: {metadata_path}") + return metadata_path + + +def _load_stats() -> dict[str, dict[str, Any]]: + import torch + + payload = json.loads(ACTION_STATS.read_text(encoding="utf-8")) + output: dict[str, dict[str, Any]] = {} + for group in ("action", "state"): + values = payload.get(group) + if not isinstance(values, dict): + raise TypeError(f"missing statistics group: {group}") + output[group] = { + key: torch.tensor(value, dtype=torch.float32) + for key, value in values.items() + if key in {"min", "max", "mean", "std"} + } + return output + + +def _normalize_proprio(proprio: Any, stats: dict[str, Any]) -> Any: + state_min = stats["min"][: proprio.shape[-1]].to(dtype=proprio.dtype) + state_max = stats["max"][: proprio.shape[-1]].to(dtype=proprio.dtype) + normalized = 2.0 * (proprio - state_min) / (state_max - state_min).clamp_min(1e-6) - 1.0 + return normalized.clamp(-5.0, 5.0) + + +def _denormalize_actions(actions: Any, stats: dict[str, Any]) -> Any: + import numpy as np + + action = actions.detach().float().cpu() + action_min = stats["min"] + action_max = stats["max"] + denormalized = (action.clamp(-1.0, 1.0) + 1.0) * 0.5 * (action_max - action_min).clamp_min( + 1e-6 + ) + action_min + output = denormalized.numpy() + if output.ndim == 3: + output = output[0] + if output.shape != (32, 7): + raise RuntimeError(f"StarWAM returned unexpected action shape: {output.shape}") + output[..., -1] = np.where(output[..., -1] > 0.5, -1.0, 1.0) + return output.astype(np.float32) + + +def _build_model(config: Any, device: Any, vae_device: str) -> Any: + import torch + from starwam import build_framework + from starwam.backbone.wan22 import Wan22VAE + from starwam.modules.scheduler import FlowMatchScheduler + + model_config = copy.deepcopy(config) + model_config.backbone.pretrained_model_id = str( + REPOSITORY_ROOT / "runs" / "starwam-libero-smoke" / "no-eager-backbone-load" + ) + model_config.framework.action_expert_init_from = None + + original_stats = FlowMatchScheduler._precompute_training_weight_stats + + def cpu_scheduler_stats(scheduler: Any) -> tuple[float, float]: + with torch.device("cpu"): + return original_stats(scheduler) + + if hasattr(torch.__future__, "set_swap_module_params_on_conversion"): + torch.__future__.set_swap_module_params_on_conversion(True) + FlowMatchScheduler._precompute_training_weight_stats = cpu_scheduler_stats + try: + with torch.device("meta"): + model = build_framework(model_config, device="meta", dtype=torch.bfloat16) + finally: + FlowMatchScheduler._precompute_training_weight_stats = original_stats + + state = torch.load(CHECKPOINT, map_location="cpu", weights_only=True, mmap=True) + result = model.load_state_dict(state, strict=True, assign=True) + if result.missing_keys or result.unexpected_keys: + raise RuntimeError( + f"checkpoint mismatch: missing={result.missing_keys} " + f"unexpected={result.unexpected_keys}" + ) + model.to(device=device, dtype=torch.bfloat16) + del state + gc.collect() + + model.backbone.vae = Wan22VAE( + vae_pth=str(VAE_CHECKPOINT), + device=vae_device, + dtype=torch.float32, + in_channels=48, + ) + model.eval() + return model + + +class LocalStarWAMBackend: + """Heavy runtime seam consumed by the dependency-free StarWAM adapter.""" + + def __init__( + self, + model: Any, + config: Any, + context: Any, + context_mask: Any, + stats: dict[str, dict[str, Any]], + preprocessing: ResolvedPreprocessing, + device: Any, + runtime_device: str, + num_inference_steps: int, + ) -> None: + self.model = model + self.config = config + self.context = context + self.context_mask = context_mask + self.stats = stats + self.preprocessing = preprocessing + self.device = device + self.runtime_device = runtime_device + self.num_inference_steps = num_inference_steps + + def infer_action(self, observation: RobotObservation, *, seed: int) -> StarWAMBackendResult: + import torch + + image = _preprocess_frames(observation, self.preprocessing) + input_image = ( + torch.as_tensor(image, dtype=torch.float32).permute(2, 0, 1).unsqueeze(0) + * (2.0 / 255.0) + - 1.0 + ).to(device=self.device, dtype=torch.bfloat16) + proprio = torch.tensor(observation.proprio, dtype=torch.float32).view(1, 8) + proprio = _normalize_proprio(proprio, self.stats["state"]).to( + device=self.device, dtype=torch.bfloat16 + ) + + torch.cuda.reset_peak_memory_stats(self.device) + torch.cuda.synchronize(self.device) + start = time.perf_counter() + with torch.inference_mode(): + prediction = self.model.infer_action( + input_image=input_image, + context=self.context, + context_mask=self.context_mask, + action_horizon=int(self.config.framework.chunk_size), + num_inference_steps=self.num_inference_steps, + seed=seed, + proprio=proprio, + num_video_frames=9, + ) + torch.cuda.synchronize(self.device) + latency = time.perf_counter() - start + denormalized = _denormalize_actions(prediction, self.stats["action"]) + runtime = InferenceRuntime( + device=self.runtime_device, + dtype="bfloat16", + latency_seconds=latency, + peak_allocated_bytes=torch.cuda.max_memory_allocated(self.device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(self.device), + ) + return StarWAMBackendResult( + actions=tuple(tuple(float(value) for value in row) for row in denormalized), + runtime=runtime, + ) + + def close(self) -> None: + import torch + + self.model = None + self.context = None + self.context_mask = None + gc.collect() + torch.cuda.empty_cache() + + +def _run_inference( + config: Any, + run_dir: Path, + minimum_free_gib: float, + num_inference_steps: int, + vae_device: str, + physical_gpu_index: int, +) -> Path: + import torch + + text_cache_path = _text_cache_path(config) + if not text_cache_path.is_file(): + raise FileNotFoundError( + f"missing text cache {text_cache_path}; run --mode text-cache first" + ) + context, context_mask = _load_text_cache(text_cache_path, config) + preprocessing = _preprocessing(_text_condition_sha256(context, context_mask)) + captured = _capture_observation() + _write_observation(captured, preprocessing, run_dir) + + device = _require_cuda_memory(minimum_free_gib) + _set_seed(SEED) + context = context.unsqueeze(0).to(device=device, dtype=torch.bfloat16) + context_mask = context_mask.unsqueeze(0).to(device=device, dtype=torch.bool) + model = _build_model(config, device, vae_device) + backend = LocalStarWAMBackend( + model=model, + config=config, + context=context, + context_mask=context_mask, + stats=_load_stats(), + preprocessing=preprocessing, + device=device, + runtime_device=( + f"cuda:0 (physical GPU {physical_gpu_index}, {torch.cuda.get_device_name(device)})" + ), + num_inference_steps=num_inference_steps, + ) + adapter = StarWAMAdapter(backend, release=StarWAMRelease()) + try: + prediction = adapter.predict_action(captured.observation, seed=SEED) + artifact = ActionPredictionArtifact( + observation=captured.observation, + prediction=prediction, + provenance=ModelProvenance( + adapter_version="0.1.0a0", + upstream_code_revision=STARWAM_REVISION, + model_revision=MODEL_REVISION, + backbone_revision=BACKBONE_REVISION, + checkpoint_sha256=CHECKPOINT_SHA256, + vae_sha256=VAE_SHA256, + text_encoder_sha256=TEXT_ENCODER_SHA256, + ), + preprocessing=preprocessing, + inference=ResolvedInference( + engine="starwam_mot_infer_action", + num_inference_steps=num_inference_steps, + scheduler="continuous_flow_match_euler", + scheduler_shift=float(config.framework.action_scheduler.infer_shift), + video_conditioning=str(config.framework.action_video_conditioning), + vae_device=vae_device, + sampled_video_frames=9, + ), + ) + output = run_dir / "predictions" / f"{artifact.cache_key}.json" + artifact.write_json(output) + print( + json.dumps( + { + "prediction_artifact": str(output), + "cache_key": artifact.cache_key, + "action_shape": [prediction.horizon, prediction.action_dim], + "latency_seconds": prediction.runtime.latency_seconds, + "peak_allocated_gib": round( + prediction.runtime.peak_allocated_bytes / 1024**3, 3 + ), + "peak_reserved_gib": round(prediction.runtime.peak_reserved_bytes / 1024**3, 3), + }, + indent=2, + sort_keys=True, + ) + ) + return output + finally: + adapter.close() + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("observe", "text-cache", "infer"), default="infer") + parser.add_argument("--gpu-index", type=int, default=0, help="physical CUDA/EGL GPU") + parser.add_argument( + "--run-dir", type=Path, default=REPOSITORY_ROOT / "runs" / "starwam-libero-smoke" + ) + parser.add_argument("--minimum-free-gib", type=float, default=18.0) + parser.add_argument("--num-inference-steps", type=int, default=1) + parser.add_argument("--vae-device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument( + "--verify-large-hashes", + action="store_true", + help="rehash the 12 GB checkpoint, 2.8 GB VAE, and 11 GB T5 before running", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.gpu_index < 0: + raise ValueError("gpu-index must be non-negative") + if args.minimum_free_gib <= 0: + raise ValueError("minimum-free-gib must be positive") + if args.num_inference_steps <= 0: + raise ValueError("num-inference-steps must be positive") + run_dir = args.run_dir.expanduser().resolve() + _configure_process(args.gpu_index, run_dir) + _verify_inputs(verify_large_hashes=args.verify_large_hashes) + config = _resolved_config(run_dir) + + if args.mode == "text-cache": + _prepare_text_cache(config, args.minimum_free_gib) + return + if args.mode == "observe": + text_cache_path = _text_cache_path(config) + if text_cache_path.is_file(): + context, context_mask = _load_text_cache(text_cache_path, config) + text_cache_hash = _text_condition_sha256(context, context_mask) + else: + text_cache_hash = "0" * 64 + captured = _capture_observation() + _write_observation(captured, _preprocessing(text_cache_hash), run_dir) + return + _run_inference( + config, + run_dir, + minimum_free_gib=args.minimum_free_gib, + num_inference_steps=args.num_inference_steps, + vae_device="cuda:0" if args.vae_device == "cuda" else "cpu", + physical_gpu_index=args.gpu_index, + ) + + +if __name__ == "__main__": + main() diff --git a/src/wamprobe/adapters/__init__.py b/src/wamprobe/adapters/__init__.py index 829dde7..042a8e2 100644 --- a/src/wamprobe/adapters/__init__.py +++ b/src/wamprobe/adapters/__init__.py @@ -6,10 +6,14 @@ OraclePointMassAdapter, WrongDirectionAdapter, ) +from wamprobe.adapters.starwam import StarWAMAdapter, StarWAMBackendResult, StarWAMRelease __all__ = [ "ActionAgnosticAdapter", "CopyLastFrameAdapter", "OraclePointMassAdapter", + "StarWAMAdapter", + "StarWAMBackendResult", + "StarWAMRelease", "WrongDirectionAdapter", ] diff --git a/src/wamprobe/adapters/starwam.py b/src/wamprobe/adapters/starwam.py new file mode 100644 index 0000000..e6ddac0 --- /dev/null +++ b/src/wamprobe/adapters/starwam.py @@ -0,0 +1,132 @@ +"""Dependency-isolated contract wrapper for the pinned StarWAM release.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.errors import ValidationError +from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RobotObservation + + +@dataclass(frozen=True, slots=True) +class StarWAMRelease: + """Verified public contract for the first StarWAM checkpoint.""" + + model_id: str = "starwam-wan22-5b-mot-libero" + action_space: str = "libero-eef-delta-pose-gripper-v1" + action_horizon: int = 32 + action_dim: int = 7 + upstream_code_revision: str = "f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7" + model_revision: str = "7d4bfe3ec76172ca17169fa959d21da099d386fe" + checkpoint_sha256: str = "d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378" + action_stats_sha256: str = "9f65fb518ca446e0d5ca9e8127e960fe3d11e6466e4f48ba9bb1135b1e0fb4f0" + + def __post_init__(self) -> None: + if self.action_horizon <= 0 or self.action_dim <= 0: + raise ValueError("StarWAM action horizon and dimension must be positive") + + @staticmethod + def proprio_fields() -> tuple[str, ...]: + """Return the exact eight-dimensional upstream proprio ordering.""" + + return ( + "eef_x", + "eef_y", + "eef_z", + "axis_angle_x", + "axis_angle_y", + "axis_angle_z", + "gripper_qpos_0", + "gripper_qpos_1", + ) + + +@dataclass(frozen=True, slots=True) +class StarWAMBackendResult: + """Raw result returned by an isolated StarWAM runtime backend.""" + + actions: tuple[tuple[float, ...], ...] + runtime: InferenceRuntime + + +class StarWAMBackend(Protocol): + """Small seam implemented inside the optional PyTorch environment.""" + + def infer_action( + self, + observation: RobotObservation, + *, + seed: int, + ) -> StarWAMBackendResult: + """Run one denormalized action-chunk prediction.""" + + ... + + def close(self) -> None: + """Release model and accelerator resources.""" + + ... + + +class StarWAMAdapter: + """Validate StarWAM inputs/outputs while keeping heavy dependencies isolated.""" + + def __init__(self, backend: StarWAMBackend, *, release: StarWAMRelease | None = None) -> None: + self._backend = backend + self._release = release or StarWAMRelease() + + @property + def release(self) -> StarWAMRelease: + """Return immutable upstream and action-space provenance.""" + + return self._release + + @property + def capabilities(self) -> ModelCapabilities: + """Declare only behavior verified for the released MoT checkpoint.""" + + return ModelCapabilities( + model_id=self._release.model_id, + future_representation=FutureRepresentation.NONE, + predicts_actions=True, + scores_actions=False, + exposes_world_features=False, + stochastic=True, + deterministic_seed=True, + supports_batching=False, + ) + + def predict_action(self, observation: RobotObservation, *, seed: int) -> ActionPrediction: + """Predict and validate one denormalized LIBERO action chunk.""" + + if observation.proprio_fields != self._release.proprio_fields(): + raise ValidationError("StarWAM observation proprio fields do not match the release") + result = self._backend.infer_action(observation, seed=seed) + if len(result.actions) != self._release.action_horizon: + raise ValidationError( + "StarWAM backend action horizon does not match the release: " + f"expected {self._release.action_horizon}, got {len(result.actions)}" + ) + if any(len(action) != self._release.action_dim for action in result.actions): + raise ValidationError( + "StarWAM backend action dimension does not match the release: " + f"expected {self._release.action_dim}" + ) + try: + return ActionPrediction( + model_id=self._release.model_id, + context_id=observation.context_id, + action_space=self._release.action_space, + actions=result.actions, + seed=seed, + runtime=result.runtime, + ) + except ValueError as error: + raise ValidationError(f"invalid StarWAM backend prediction: {error}") from error + + def close(self) -> None: + """Release resources owned by the isolated backend.""" + + self._backend.close() diff --git a/src/wamprobe/api/__init__.py b/src/wamprobe/api/__init__.py index d9cbbc1..b303c9f 100644 --- a/src/wamprobe/api/__init__.py +++ b/src/wamprobe/api/__init__.py @@ -1,7 +1,8 @@ """Public, model-agnostic WAMProbe API.""" from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities -from wamprobe.api.model import WAMAdapter +from wamprobe.api.model import ActionPredictorAdapter, WAMAdapter +from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RGBFrame, RobotObservation from wamprobe.api.types import ( Action2D, Context2D, @@ -13,11 +14,16 @@ __all__ = [ "Action2D", + "ActionPrediction", + "ActionPredictorAdapter", "Context2D", "FutureRepresentation", "InterventionGroup", "InterventionSuite", + "InferenceRuntime", "ModelCapabilities", + "RGBFrame", + "RobotObservation", "Trajectory2D", "Vec2", "WAMAdapter", diff --git a/src/wamprobe/api/model.py b/src/wamprobe/api/model.py index aefc26e..d5cdf90 100644 --- a/src/wamprobe/api/model.py +++ b/src/wamprobe/api/model.py @@ -5,6 +5,7 @@ from typing import Protocol, runtime_checkable from wamprobe.api.capabilities import ModelCapabilities +from wamprobe.api.robotics import ActionPrediction, RobotObservation from wamprobe.api.types import Action2D, Context2D, Trajectory2D @@ -34,3 +35,29 @@ def close(self) -> None: """Release resources owned by the adapter.""" ... + + +@runtime_checkable +class ActionPredictorAdapter(Protocol): + """Protocol for WAMs that directly emit robot action chunks.""" + + @property + def capabilities(self) -> ModelCapabilities: + """Describe the action capabilities exposed by the adapter.""" + + ... + + def predict_action( + self, + observation: RobotObservation, + *, + seed: int, + ) -> ActionPrediction: + """Predict one semantically typed action chunk.""" + + ... + + def close(self) -> None: + """Release resources owned by the adapter.""" + + ... diff --git a/src/wamprobe/api/robotics.py b/src/wamprobe/api/robotics.py new file mode 100644 index 0000000..9a6aa52 --- /dev/null +++ b/src/wamprobe/api/robotics.py @@ -0,0 +1,191 @@ +"""Dependency-free robot observations and action predictions.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass + + +def _require_non_empty(value: str, field_name: str) -> None: + if not value.strip(): + raise ValueError(f"{field_name} must not be empty") + + +@dataclass(frozen=True, slots=True) +class RGBFrame: + """One tightly packed uint8 RGB observation frame.""" + + camera_name: str + height: int + width: int + data: bytes + + def __post_init__(self) -> None: + _require_non_empty(self.camera_name, "camera_name") + if self.height <= 0 or self.width <= 0: + raise ValueError("RGB frame dimensions must be positive") + expected_bytes = self.height * self.width * 3 + if len(self.data) != expected_bytes: + raise ValueError( + f"RGB payload size must be {expected_bytes} bytes, got {len(self.data)}" + ) + + @property + def sha256(self) -> str: + """Return a content checksum for cache identity and provenance.""" + + return hashlib.sha256(self.data).hexdigest() + + def descriptor(self) -> dict[str, object]: + """Return JSON metadata without embedding raw image bytes.""" + + return { + "camera_name": self.camera_name, + "shape": [self.height, self.width, 3], + "dtype": "uint8", + "sha256": self.sha256, + } + + +@dataclass(frozen=True, slots=True) +class RobotObservation: + """A named robot context with explicit RGB and proprio semantics.""" + + context_id: str + task: str + frames: tuple[RGBFrame, ...] + proprio: tuple[float, ...] + proprio_fields: tuple[str, ...] + + def __post_init__(self) -> None: + _require_non_empty(self.context_id, "context_id") + _require_non_empty(self.task, "task") + if not self.frames: + raise ValueError("robot observation requires at least one RGB frame") + camera_names = [frame.camera_name for frame in self.frames] + if len(camera_names) != len(set(camera_names)): + raise ValueError("camera names must be unique") + if len(self.proprio) != len(self.proprio_fields): + raise ValueError("proprio values and fields must have the same length") + if len(self.proprio_fields) != len(set(self.proprio_fields)): + raise ValueError("proprio fields must be unique") + if any(not field.strip() for field in self.proprio_fields): + raise ValueError("proprio fields must not be empty") + if any(not math.isfinite(value) for value in self.proprio): + raise ValueError("proprio values must be finite") + + @property + def content_sha256(self) -> str: + """Return a stable digest over every model-visible observation input.""" + + payload = { + "context_id": self.context_id, + "task": self.task, + "frames": [frame.descriptor() for frame in self.frames], + "proprio": list(self.proprio), + "proprio_fields": list(self.proprio_fields), + } + encoded = json.dumps( + payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def descriptor(self) -> dict[str, object]: + """Return JSON metadata suitable for a cached prediction artifact.""" + + return { + "context_id": self.context_id, + "task": self.task, + "content_sha256": self.content_sha256, + "frames": [frame.descriptor() for frame in self.frames], + "proprio": list(self.proprio), + "proprio_fields": list(self.proprio_fields), + } + + +@dataclass(frozen=True, slots=True) +class InferenceRuntime: + """Measured runtime and device data for one model call.""" + + device: str + dtype: str + latency_seconds: float + peak_allocated_bytes: int + peak_reserved_bytes: int + + def __post_init__(self) -> None: + _require_non_empty(self.device, "runtime device") + _require_non_empty(self.dtype, "runtime dtype") + if not math.isfinite(self.latency_seconds) or self.latency_seconds < 0: + raise ValueError("latency_seconds must be finite and non-negative") + if self.peak_allocated_bytes < 0 or self.peak_reserved_bytes < 0: + raise ValueError("peak memory measurements must be non-negative") + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible runtime record.""" + + return { + "device": self.device, + "dtype": self.dtype, + "latency_seconds": self.latency_seconds, + "peak_allocated_bytes": self.peak_allocated_bytes, + "peak_reserved_bytes": self.peak_reserved_bytes, + } + + +@dataclass(frozen=True, slots=True) +class ActionPrediction: + """A semantically named, denormalized robot action chunk.""" + + model_id: str + context_id: str + action_space: str + actions: tuple[tuple[float, ...], ...] + seed: int + runtime: InferenceRuntime + + def __post_init__(self) -> None: + _require_non_empty(self.model_id, "model_id") + _require_non_empty(self.context_id, "context_id") + _require_non_empty(self.action_space, "action_space") + if isinstance(self.seed, bool) or not isinstance(self.seed, int) or self.seed < 0: + raise ValueError("seed must be a non-negative integer") + if not self.actions or not self.actions[0]: + raise ValueError("action prediction must contain a non-empty action chunk") + action_dim = len(self.actions[0]) + if any(len(action) != action_dim for action in self.actions): + raise ValueError("all predicted actions must have the same dimension") + if any(not math.isfinite(value) for action in self.actions for value in action): + raise ValueError("predicted actions must be finite") + + @property + def horizon(self) -> int: + """Return the number of predicted action steps.""" + + return len(self.actions) + + @property + def action_dim(self) -> int: + """Return the action-vector width.""" + + return len(self.actions[0]) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible prediction record.""" + + return { + "model_id": self.model_id, + "context_id": self.context_id, + "action_space": self.action_space, + "actions": [list(action) for action in self.actions], + "horizon": self.horizon, + "action_dim": self.action_dim, + "seed": self.seed, + "runtime": self.runtime.to_dict(), + } diff --git a/src/wamprobe/artifacts.py b/src/wamprobe/artifacts.py new file mode 100644 index 0000000..851b877 --- /dev/null +++ b/src/wamprobe/artifacts.py @@ -0,0 +1,261 @@ +"""Versioned, deterministic prediction-cache artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import math +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from wamprobe.api.errors import ValidationError +from wamprobe.api.robotics import ActionPrediction, RobotObservation + + +def _validate_hex_digest(value: str, length: int, field_name: str) -> None: + if len(value) != length or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{field_name} must be a lowercase {length}-character hex digest") + + +@dataclass(frozen=True, slots=True) +class ModelProvenance: + """Immutable code and checkpoint identity for a prediction.""" + + adapter_version: str + upstream_code_revision: str + model_revision: str + backbone_revision: str + checkpoint_sha256: str + vae_sha256: str + text_encoder_sha256: str + + def __post_init__(self) -> None: + if not self.adapter_version.strip(): + raise ValueError("adapter_version must not be empty") + _validate_hex_digest(self.upstream_code_revision, 40, "upstream_code_revision") + _validate_hex_digest(self.model_revision, 40, "model_revision") + _validate_hex_digest(self.backbone_revision, 40, "backbone_revision") + _validate_hex_digest(self.checkpoint_sha256, 64, "checkpoint_sha256") + _validate_hex_digest(self.vae_sha256, 64, "vae_sha256") + _validate_hex_digest(self.text_encoder_sha256, 64, "text_encoder_sha256") + + def to_dict(self) -> dict[str, object]: + """Return JSON-compatible model provenance.""" + + return { + "adapter_version": self.adapter_version, + "upstream_code_revision": self.upstream_code_revision, + "model_revision": self.model_revision, + "backbone_revision": self.backbone_revision, + "checkpoint_sha256": self.checkpoint_sha256, + "vae_sha256": self.vae_sha256, + "text_encoder_sha256": self.text_encoder_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ResolvedPreprocessing: + """Every model-visible preprocessing and normalization choice.""" + + camera_order: tuple[str, ...] + camera_transforms: tuple[str, ...] + source_size: tuple[int, int] + resized_camera_size: tuple[int, int] + model_size: tuple[int, int] + resize_interpolation: str + channel_order: str + tensor_layout: str + camera_concatenation: Literal["horizontal", "vertical"] + input_value_range: tuple[float, float] + text_prompt: str + text_cache_sha256: str + proprio_fields: tuple[str, ...] + action_normalization: str + action_stats_sha256: str + + def __post_init__(self) -> None: + if not self.camera_order or any(not item.strip() for item in self.camera_order): + raise ValueError("camera_order must contain non-empty names") + if len(self.camera_order) != len(set(self.camera_order)): + raise ValueError("camera_order names must be unique") + if len(self.camera_transforms) != len(self.camera_order): + raise ValueError("camera_transforms must align with camera_order") + if any(item not in {"identity", "rotate_180"} for item in self.camera_transforms): + raise ValueError("camera transforms must be identity or rotate_180") + if any( + value <= 0 for value in (*self.source_size, *self.resized_camera_size, *self.model_size) + ): + raise ValueError("source, resized camera, and model image sizes must be positive") + if self.resize_interpolation != "PIL_BILINEAR": + raise ValueError("resize_interpolation must be PIL_BILINEAR") + if self.channel_order != "RGB": + raise ValueError("channel_order must be RGB") + if self.tensor_layout != "BCHW": + raise ValueError("tensor_layout must be BCHW") + if self.camera_concatenation not in {"horizontal", "vertical"}: + raise ValueError("camera_concatenation must be horizontal or vertical") + resized_height, resized_width = self.resized_camera_size + camera_count = len(self.camera_order) + expected_model_size = ( + (resized_height, resized_width * camera_count) + if self.camera_concatenation == "horizontal" + else (resized_height * camera_count, resized_width) + ) + if self.model_size != expected_model_size: + raise ValueError( + "model_size does not match resized cameras and concatenation: " + f"expected {expected_model_size}" + ) + low, high = self.input_value_range + if not math.isfinite(low) or not math.isfinite(high) or low >= high: + raise ValueError("input_value_range must contain finite increasing values") + if not self.text_prompt.strip(): + raise ValueError("text_prompt must not be empty") + _validate_hex_digest(self.text_cache_sha256, 64, "text_cache_sha256") + if any(not field.strip() for field in self.proprio_fields): + raise ValueError("proprio_fields must not contain empty names") + if not self.action_normalization.strip(): + raise ValueError("action_normalization must not be empty") + _validate_hex_digest(self.action_stats_sha256, 64, "action_stats_sha256") + + def to_dict(self) -> dict[str, object]: + """Return JSON-compatible preprocessing metadata.""" + + return { + "camera_order": list(self.camera_order), + "camera_transforms": list(self.camera_transforms), + "source_size": list(self.source_size), + "resized_camera_size": list(self.resized_camera_size), + "model_size": list(self.model_size), + "resize_interpolation": self.resize_interpolation, + "channel_order": self.channel_order, + "tensor_layout": self.tensor_layout, + "camera_concatenation": self.camera_concatenation, + "input_value_range": list(self.input_value_range), + "text_prompt": self.text_prompt, + "text_cache_sha256": self.text_cache_sha256, + "proprio_fields": list(self.proprio_fields), + "action_normalization": self.action_normalization, + "action_stats_sha256": self.action_stats_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ResolvedInference: + """Every output-affecting StarWAM action-sampling choice.""" + + engine: str + num_inference_steps: int + scheduler: str + scheduler_shift: float + video_conditioning: str + vae_device: str + sampled_video_frames: int + + def __post_init__(self) -> None: + if not self.engine.strip() or not self.scheduler.strip(): + raise ValueError("inference engine and scheduler must not be empty") + if self.num_inference_steps <= 0: + raise ValueError("num_inference_steps must be positive") + if not math.isfinite(self.scheduler_shift) or self.scheduler_shift <= 0: + raise ValueError("scheduler_shift must be finite and positive") + if self.video_conditioning not in {"first_frame", "full_video"}: + raise ValueError("video_conditioning must be first_frame or full_video") + if not self.vae_device.strip(): + raise ValueError("vae_device must not be empty") + if self.sampled_video_frames <= 0: + raise ValueError("sampled_video_frames must be positive") + + def to_dict(self) -> dict[str, object]: + """Return JSON-compatible inference settings.""" + + return { + "engine": self.engine, + "num_inference_steps": self.num_inference_steps, + "scheduler": self.scheduler, + "scheduler_shift": self.scheduler_shift, + "video_conditioning": self.video_conditioning, + "vae_device": self.vae_device, + "sampled_video_frames": self.sampled_video_frames, + } + + +@dataclass(frozen=True, slots=True) +class ActionPredictionArtifact: + """One cacheable observation-to-action result with full provenance.""" + + observation: RobotObservation + prediction: ActionPrediction + provenance: ModelProvenance + preprocessing: ResolvedPreprocessing + inference: ResolvedInference + schema_version: str = "0.1" + + def __post_init__(self) -> None: + if self.schema_version != "0.1": + raise ValueError(f"unsupported action artifact schema: {self.schema_version}") + if self.observation.context_id != self.prediction.context_id: + raise ValidationError("observation and prediction context_id values do not match") + frame_names = tuple(frame.camera_name for frame in self.observation.frames) + if frame_names != self.preprocessing.camera_order: + raise ValidationError("observation frame order does not match resolved preprocessing") + if self.observation.proprio_fields != self.preprocessing.proprio_fields: + raise ValidationError("observation proprio fields do not match resolved preprocessing") + + @property + def cache_key(self) -> str: + """Return a deterministic key over inputs, seed, model, and preprocessing.""" + + key_payload = { + "schema_version": self.schema_version, + "observation": self.observation.descriptor(), + "model_id": self.prediction.model_id, + "seed": self.prediction.seed, + "provenance": self.provenance.to_dict(), + "preprocessing": self.preprocessing.to_dict(), + "inference": self.inference.to_dict(), + } + encoded = json.dumps( + key_payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def to_dict(self) -> dict[str, object]: + """Return the complete JSON-compatible cache artifact.""" + + return { + "schema_version": self.schema_version, + "cache_key": self.cache_key, + "provenance": self.provenance.to_dict(), + "observation": self.observation.descriptor(), + "preprocessing": self.preprocessing.to_dict(), + "inference": self.inference.to_dict(), + "prediction": self.prediction.to_dict(), + } + + def write_json(self, path: Path) -> None: + """Atomically write the artifact without persisting raw RGB bytes.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps( + self.to_dict(), + allow_nan=False, + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py new file mode 100644 index 0000000..e56450a --- /dev/null +++ b/tests/test_artifacts.py @@ -0,0 +1,124 @@ +import json +from pathlib import Path + +import pytest + +from wamprobe.api.errors import ValidationError +from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RGBFrame, RobotObservation +from wamprobe.artifacts import ( + ActionPredictionArtifact, + ModelProvenance, + ResolvedInference, + ResolvedPreprocessing, +) + + +def _inputs(seed: int = 42) -> tuple[RobotObservation, ActionPrediction]: + observation = RobotObservation( + context_id="libero-spatial-task0-init0-wait30", + task="pick up the black bowl and place it on the plate", + frames=( + RGBFrame("agentview", 1, 1, b"\x01\x02\x03"), + RGBFrame("wrist", 1, 1, b"\x04\x05\x06"), + ), + proprio=(0.0, 0.1), + proprio_fields=("eef_x", "eef_y"), + ) + prediction = ActionPrediction( + model_id="starwam-wan22-5b-mot-libero", + context_id=observation.context_id, + action_space="libero-eef-delta-pose-gripper-v1", + actions=((0.0,) * 7, (0.1,) * 7), + seed=seed, + runtime=InferenceRuntime("cuda:0", "bfloat16", 1.0, 100, 120), + ) + return observation, prediction + + +def _artifact(seed: int = 42) -> ActionPredictionArtifact: + observation, prediction = _inputs(seed) + return ActionPredictionArtifact( + observation=observation, + prediction=prediction, + provenance=ModelProvenance( + adapter_version="0.1", + upstream_code_revision="f" * 40, + model_revision="7" * 40, + backbone_revision="9" * 40, + checkpoint_sha256="d" * 64, + vae_sha256="2" * 64, + text_encoder_sha256="c" * 64, + ), + preprocessing=ResolvedPreprocessing( + camera_order=("agentview", "wrist"), + camera_transforms=("rotate_180", "rotate_180"), + source_size=(256, 256), + resized_camera_size=(224, 224), + model_size=(224, 448), + resize_interpolation="PIL_BILINEAR", + channel_order="RGB", + tensor_layout="BCHW", + camera_concatenation="horizontal", + input_value_range=(-1.0, 1.0), + text_prompt=( + "A video recorded from a robot's point of view executing the following " + "instruction: pick up the black bowl and place it on the plate" + ), + text_cache_sha256="a" * 64, + proprio_fields=("eef_x", "eef_y"), + action_normalization="minmax", + action_stats_sha256="9" * 64, + ), + inference=ResolvedInference( + engine="starwam_mot_infer_action", + num_inference_steps=1, + scheduler="continuous_flow_match_euler", + scheduler_shift=5.0, + video_conditioning="first_frame", + vae_device="cpu", + sampled_video_frames=9, + ), + ) + + +def test_action_artifact_cache_key_is_stable_and_seed_sensitive() -> None: + assert _artifact().cache_key == _artifact().cache_key + assert _artifact(seed=42).cache_key != _artifact(seed=43).cache_key + + +def test_action_artifact_writes_atomic_json_without_raw_rgb(tmp_path: Path) -> None: + artifact = _artifact() + output = tmp_path / "prediction.json" + + artifact.write_json(output) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == "0.1" + assert payload["cache_key"] == artifact.cache_key + assert payload["observation"]["frames"][0]["sha256"] + assert "data" not in payload["observation"]["frames"][0] + assert payload["preprocessing"]["camera_transforms"] == ["rotate_180"] * 2 + assert payload["provenance"]["vae_sha256"] == "2" * 64 + assert payload["inference"]["num_inference_steps"] == 1 + assert payload["prediction"]["actions"][1] == [0.1] * 7 + + +def test_action_artifact_rejects_mismatched_context() -> None: + observation, prediction = _inputs() + prediction = ActionPrediction( + model_id=prediction.model_id, + context_id="different-context", + action_space=prediction.action_space, + actions=prediction.actions, + seed=prediction.seed, + runtime=prediction.runtime, + ) + + with pytest.raises(ValidationError, match="context_id"): + ActionPredictionArtifact( + observation=observation, + prediction=prediction, + provenance=_artifact().provenance, + preprocessing=_artifact().preprocessing, + inference=_artifact().inference, + ) diff --git a/tests/test_robotics_api.py b/tests/test_robotics_api.py new file mode 100644 index 0000000..f8ccc44 --- /dev/null +++ b/tests/test_robotics_api.py @@ -0,0 +1,75 @@ +import math + +import pytest + +from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RGBFrame, RobotObservation + + +def _observation() -> RobotObservation: + return RobotObservation( + context_id="libero-spatial-task0-init0-wait30", + task="pick up the black bowl and place it on the plate", + frames=( + RGBFrame("agentview", height=2, width=2, data=bytes(range(12))), + RGBFrame("wrist", height=2, width=2, data=bytes(reversed(range(12)))), + ), + proprio=(0.1, 0.2, 0.3), + proprio_fields=("eef_x", "eef_y", "eef_z"), + ) + + +def test_robot_observation_has_stable_content_digest() -> None: + first = _observation() + second = _observation() + + assert first.content_sha256 == second.content_sha256 + assert len(first.content_sha256) == 64 + assert first.frames[0].sha256 != first.frames[1].sha256 + + +def test_rgb_frame_rejects_wrong_payload_size() -> None: + with pytest.raises(ValueError, match="RGB payload size"): + RGBFrame("agentview", height=2, width=2, data=b"too short") + + +def test_robot_observation_rejects_non_finite_proprio() -> None: + with pytest.raises(ValueError, match="finite"): + RobotObservation( + context_id="context-0", + task="task", + frames=(RGBFrame("agentview", height=1, width=1, data=b"\x00\x00\x00"),), + proprio=(math.nan,), + proprio_fields=("eef_x",), + ) + + +def test_action_prediction_validates_shape_and_runtime() -> None: + runtime = InferenceRuntime( + device="cuda:0", + dtype="bfloat16", + latency_seconds=1.25, + peak_allocated_bytes=100, + peak_reserved_bytes=120, + ) + prediction = ActionPrediction( + model_id="starwam-wan22-5b-mot-libero", + context_id="context-0", + action_space="libero-eef-delta-pose-gripper-v1", + actions=((0.0, 0.1, 0.2), (0.3, 0.4, 0.5)), + seed=42, + runtime=runtime, + ) + + assert prediction.horizon == 2 + assert prediction.action_dim == 3 + assert prediction.to_dict()["runtime"] == runtime.to_dict() + + with pytest.raises(ValueError, match="same dimension"): + ActionPrediction( + model_id="model", + context_id="context-0", + action_space="space", + actions=((0.0, 0.1), (0.2,)), + seed=42, + runtime=runtime, + ) diff --git a/tests/test_starwam_adapter.py b/tests/test_starwam_adapter.py new file mode 100644 index 0000000..9d5e92f --- /dev/null +++ b/tests/test_starwam_adapter.py @@ -0,0 +1,82 @@ +from dataclasses import replace + +import pytest + +from wamprobe.adapters.starwam import ( + StarWAMAdapter, + StarWAMBackendResult, + StarWAMRelease, +) +from wamprobe.api.capabilities import FutureRepresentation +from wamprobe.api.errors import ValidationError +from wamprobe.api.robotics import InferenceRuntime, RGBFrame, RobotObservation + + +class FakeStarWAMBackend: + def __init__(self) -> None: + self.calls: list[tuple[str, int]] = [] + self.closed = False + + def infer_action(self, observation: RobotObservation, *, seed: int) -> StarWAMBackendResult: + self.calls.append((observation.context_id, seed)) + return StarWAMBackendResult( + actions=((0.1,) * 7, (0.2,) * 7), + runtime=InferenceRuntime( + device="cpu", + dtype="float32", + latency_seconds=0.01, + peak_allocated_bytes=0, + peak_reserved_bytes=0, + ), + ) + + def close(self) -> None: + self.closed = True + + +def _observation() -> RobotObservation: + return RobotObservation( + context_id="libero-context-0", + task="pick up the bowl", + frames=(RGBFrame("agentview", 1, 1, b"\x00\x01\x02"),), + proprio=(0.0,) * 8, + proprio_fields=StarWAMRelease.proprio_fields(), + ) + + +def test_starwam_adapter_declares_only_verified_capabilities() -> None: + adapter = StarWAMAdapter(FakeStarWAMBackend(), release=StarWAMRelease(action_horizon=2)) + + capabilities = adapter.capabilities + + assert capabilities.model_id == "starwam-wan22-5b-mot-libero" + assert capabilities.future_representation is FutureRepresentation.NONE + assert capabilities.predicts_actions is True + assert capabilities.scores_actions is False + assert capabilities.exposes_world_features is False + assert capabilities.stochastic is True + assert capabilities.deterministic_seed is True + + +def test_starwam_adapter_returns_typed_action_prediction_and_closes_backend() -> None: + backend = FakeStarWAMBackend() + adapter = StarWAMAdapter(backend, release=StarWAMRelease(action_horizon=2)) + + prediction = adapter.predict_action(_observation(), seed=42) + adapter.close() + + assert prediction.context_id == "libero-context-0" + assert prediction.action_space == "libero-eef-delta-pose-gripper-v1" + assert prediction.horizon == 2 + assert prediction.action_dim == 7 + assert backend.calls == [("libero-context-0", 42)] + assert backend.closed is True + + +def test_starwam_adapter_rejects_backend_horizon_mismatch() -> None: + backend = FakeStarWAMBackend() + release = replace(StarWAMRelease(action_horizon=2), action_horizon=3) + adapter = StarWAMAdapter(backend, release=release) + + with pytest.raises(ValidationError, match="action horizon"): + adapter.predict_action(_observation(), seed=42) From 135ce7c1f3cd9988cf61953df290d27685fc7c00 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 11:32:15 +0000 Subject: [PATCH 07/20] add exact LIBERO counterfactual branches --- README.md | 13 +- docs/QUICKSTART.md | 26 +- docs/WAMProbe_PLAN.md | 7 + docs/research/ADAPTER_SELECTION.md | 16 +- docs/rfcs/0002-counterfactual-metrics.md | 6 + environments/libero/README.md | 55 ++ environments/libero/generate_cf_pilot.py | 847 +++++++++++++++++++++++ src/wamprobe/api/__init__.py | 18 + src/wamprobe/api/counterfactual.py | 502 ++++++++++++++ src/wamprobe/counterfactual.py | 141 ++++ tests/test_counterfactual_robotics.py | 265 +++++++ 11 files changed, 1885 insertions(+), 11 deletions(-) create mode 100644 environments/libero/README.md create mode 100644 environments/libero/generate_cf_pilot.py create mode 100644 src/wamprobe/api/counterfactual.py create mode 100644 src/wamprobe/counterfactual.py create mode 100644 tests/test_counterfactual_robotics.py diff --git a/README.md b/README.md index 355fe6e..ec821ed 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,9 @@ the same initial state └── expert act → predicted future / true future ``` -The project is in an early alpha. Its first runnable slice is deliberately small and -CPU-only: an analytic PointMass-2D benchmark verifies the public API and metrics against -an oracle and several intentionally broken baselines before expensive robot models are -integrated. +The project is in an early alpha. Its dependency-free core includes an analytic +PointMass-2D benchmark, while opt-in isolated runners now cover one released StarWAM +observation-to-action path and one real paired LIBERO simulator pilot. ## Why another evaluation project? @@ -89,6 +88,8 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - typed robot observations, action predictions, deterministic prediction artifacts, and a pinned StarWAM/LIBERO smoke runner; +- dependency-free robot action-branch/future contracts plus a four-branch, eight-step + LIBERO snapshot/restore pilot with exact branch-order validation; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap @@ -98,8 +99,8 @@ The next milestones are: 1. context-block bootstrap confidence intervals; 2. a small image-rendered BlockPush benchmark; 3. the versioned intervention dataset loader; -4. LIBERO-CF-Mini paired simulator branches; -5. paired LIBERO simulator branches, then LingBot-VA as a published reference adapter. +4. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; +5. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 45adaa2..75ad2ea 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -75,9 +75,27 @@ revisions and input hashes, records both camera transformations and proprio orde writes a cache-keyed prediction JSON below `runs/starwam-libero-smoke/`. Use the one-step setting only as an integration smoke test; the published StarWAM recipe uses eight steps. +## Generate the paired LIBERO counterfactual pilot + +This step uses the same isolated simulator environment but loads no model weights: + +```bash +environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ + --gpu-index 0 --horizon 8 +``` + +The runner restores one fixed snapshot before `no-op`, positive/negative end-effector X, +and gripper-close branches. It stores two camera views and state descriptors for all eight +future steps, repeats no-op, and executes all branches in reverse order. A successful run +requires byte-identical MuJoCo integration states, force-refreshed initial observations, +repeated rollouts, and order-independent branch results. See +[`environments/libero/README.md`](../environments/libero/README.md) for the output layout +and the required Python-side gripper-state restoration. + ## What this demo does not prove -PointMass-2D validates the evaluator and guards against metric bugs. It is not evidence -that the metrics transfer to manipulation videos. That requires paired simulator branches, -perception-based state extraction, stochastic generation analysis, and correlation with -closed-loop return; these are explicit later milestones. +PointMass-2D validates the evaluator and guards against metric bugs. The current LIBERO +pilot validates paired data generation for one context, not policy quality: its diagnostic +actions do not solve the task. Transfer claims still require multiple manipulation tasks, +WAM future predictions, stochastic generation analysis, and correlation with closed-loop +return. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 5b1c465..4c13f90 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -507,6 +507,13 @@ oracle > noisy action-aware > action-agnostic > wrong-direction/copy-last - 记录渲染 FPS、控制 FPS 和 action repeat； - 对每个任务写 benchmark card，说明哪些物理属性可以可靠评测。 +2026-07-15 pilot 进展：已在固定 `libero_spatial` task 0/context 上生成 4 个分支、 +每分支 8 步和 2 个相机视角。恢复契约采用 MuJoCo `mjSTATE_INTEGRATION`，并额外恢复 +robosuite 时钟、controller、observable、Python/NumPy RNG 与 Panda gripper 的 +`current_action`。两次 restore、重复 no-op、正序/逆序执行均达到逐状态完全一致， +最大状态误差为 0。该 pilot 的诊断动作 return/success 均为 0，因此只证明配对数据 +生成和分支分离，不是 LIBERO 策略成功率结果。 + ### 8.3 Tier 2：Closed-Loop Utility 目标：验证离线指标是否与控制收益相关。 diff --git a/docs/research/ADAPTER_SELECTION.md b/docs/research/ADAPTER_SELECTION.md index 5a3eddc..2675de4 100644 --- a/docs/research/ADAPTER_SELECTION.md +++ b/docs/research/ADAPTER_SELECTION.md @@ -98,6 +98,19 @@ seconds on the first run and 2.81 seconds in the final recorded artifact, peakin This clears the observation-to-action integration gate only. No action was executed in the simulator, and no rollout success or policy-quality claim is made. +The paired simulator slice was then completed on the same pinned LIBERO context. Four +eight-step action branches (`no-op`, positive/negative end-effector X, and gripper close) +were generated from one content-addressed MuJoCo `mjSTATE_INTEGRATION` snapshot. The +runner also restores robosuite clocks/controllers/observables, Python and NumPy RNG state, +and Panda gripper `current_action`. Without that last Python-side value, two branches were +experimentally shown to depend on execution order despite restoring MuJoCo state. + +The final pilot passed two independent snapshot restores, an exact repeated no-op rollout, +and forward-versus-reverse branch execution with zero maximum state error. It produced +four real futures and branch-separation metrics, but all returns and success values were +zero because these are diagnostic interventions rather than a task-solving policy. This +is not a LIBERO benchmark score. + ## First adapter implementation slice The next coding slice is deliberately narrow: @@ -108,7 +121,8 @@ The next coding slice is deliberately narrow: 4. typed, cached observation-to-action artifact: **complete**; 5. capability declaration and mocked CPU contract tests: **complete**; 6. opt-in fixed LIBERO GPU smoke runner: **complete**; -7. paired simulator snapshot/restore and counterfactual scoring: **next**. +7. paired simulator snapshot/restore and counterfactual scoring: **complete**; +8. execute cached StarWAM candidate branches and expand LIBERO-CF-Mini tasks: **next**. ## Sources checked diff --git a/docs/rfcs/0002-counterfactual-metrics.md b/docs/rfcs/0002-counterfactual-metrics.md index b68c39c..ae4e7c0 100644 --- a/docs/rfcs/0002-counterfactual-metrics.md +++ b/docs/rfcs/0002-counterfactual-metrics.md @@ -14,6 +14,12 @@ The primary evaluation unit is an intervention group: one shared context paired least two uniquely named actions. Ground-truth futures are generated by restoring the same simulator state before each action branch. +For the LIBERO alpha, “same simulator state” includes MuJoCo `mjSTATE_INTEGRATION` plus +Python-side runtime state that can affect controls: robosuite clocks/controllers and +observable reset policy, global RNG state, and stateful gripper commands. The generator +must prove repeated-rollout and branch-order invariance; equality of LIBERO's flattened +`time/qpos/qvel` wrapper state alone is insufficient. + The alpha validates five complementary measurements: 1. action dependence; diff --git a/environments/libero/README.md b/environments/libero/README.md new file mode 100644 index 0000000..53c2083 --- /dev/null +++ b/environments/libero/README.md @@ -0,0 +1,55 @@ +# LIBERO counterfactual pilot + +This isolated runner generates four real action branches from one pinned LIBERO state. It +does not load StarWAM or any other model weights; it reuses the existing Python 3.11 +environment because that environment already pins LIBERO, robosuite, MuJoCo, and Pillow. + +```bash +environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ + --gpu-index 0 --horizon 8 +``` + +The default output is ignored by Git and written below `runs/libero-cf-pilot/`: + +```text +benchmark_provenance.json +snapshots/integration_state.npy +snapshots/runtime_state.json +images/initial/*.png +images//*.png +branches/.json +intervention_group.json +validation.json +``` + +The state file uses MuJoCo `mjSTATE_INTEGRATION`, which includes more integration state +than LIBERO's `time/qpos/qvel` wrapper snapshot. The JSON sidecar also records robosuite +clock state, controller state, Python and NumPy RNG state, and each gripper's stateful +`current_action`. Omitting `current_action` makes branch results depend on execution order +even when MuJoCo state restoration is exact. + +The runner executes the branch set in forward and reverse order and repeats the no-op +branch. It fails after writing `validation.json` if restored state/observations, repeated +rollouts, or branch-order invariance are not exact. + +The canonical initial observation is force-refreshed from the restored end-of-control +state. It is intentionally compared against a second independent restore, not against the +camera value returned by the preceding `step()`: robosuite can sample camera observables +inside the control interval before the final simulator state is reached. + +## Verified fixed pilot + +The pinned 4-branch × 8-step run was repeated twice on 2026-07-15 with identical hashes: + +```text +snapshot semantic SHA256: 64560e918e964f8c772199dbc6b02e25f55ac7d8f2515649b69a4e25c1a9b53d +group SHA256: cd7b68d5eb096b149a5ef927977455f7866ad76f4cabb38a8cd3f045d7e54654 +artifact SHA256: 75552c33cc1dba35ed4a5dc1202db0de9f0ce74235fd85cee2b9c45ebd3c7648 +maximum restore error: 0.0 +``` + +Mean final end-effector pairwise distance was `0.0395948672` m, mean final raw +`object-state` pairwise distance was `0.0979067195`, and no-op end-effector drift was +`0.0000246710` m. Return spread and success rate were both zero. These diagnostic actions +validate paired branch generation and separation; they are not a task-solving policy or a +LIBERO success-rate result. diff --git a/environments/libero/generate_cf_pilot.py b/environments/libero/generate_cf_pilot.py new file mode 100644 index 0000000..3f2e033 --- /dev/null +++ b/environments/libero/generate_cf_pilot.py @@ -0,0 +1,847 @@ +#!/usr/bin/env python +"""Generate one pinned LIBERO counterfactual intervention group. + +This runner needs the isolated StarWAM virtual environment only for its already-pinned +LIBERO, robosuite, MuJoCo, PyTorch, and image dependencies. It does not load a WAM or any +model weights. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import os +import random +import subprocess +import sys +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +SOURCE_ROOT = REPOSITORY_ROOT / "src" +LIBERO_ROOT = REPOSITORY_ROOT / "vendor" / "upstream" / "libero" + +LIBERO_REVISION = "8f1084e3132a39270c3a13ebe37270a43ece2a01" +TASK_SUITE = "libero_spatial" +TASK_ID = 0 +INIT_STATE_INDEX = 0 +WAIT_STEPS = 30 +SEED = 42 +TASK = "pick up the black bowl between the plate and the ramekin and place it on the plate" +CONTEXT_ID = "libero-spatial-task0-init0-wait30" +BDDL_SHA256 = "9b59eb1287802868ad9bc78d58e6d36d4ba31134e679cfdbdf4b0feb660c959b" +INIT_STATES_SHA256 = "cbbc73792ce546c9bec181fd328a411d3183074840b282671dee481511381d0a" +ACTION_SPACE = "libero-eef-delta-pose-gripper-v1" +RESTORE_POLICY = "libero-integration-and-python-side-state-v1" +CAMERAS = ( + ("agentview", "agentview_image"), + ("wrist", "robot0_eye_in_hand_image"), +) +DUMMY_ACTION = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0) +BRANCH_ACTIONS = ( + ("noop", "no-op", DUMMY_ACTION), + ("move-x-positive", "directional-positive-x", (0.5, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0)), + ("move-x-negative", "directional-negative-x", (-0.5, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0)), + ("gripper-close", "gripper-close", (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)), +) + +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from wamprobe.api.counterfactual import ( # noqa: E402 + ActionBranch, + CounterfactualValidation, + RGBFrameReference, + RobotAction, + RobotFuture, + RobotInterventionGroup, + RobotStateFrame, + SimulatorSnapshotDescriptor, +) +from wamprobe.counterfactual import CounterfactualArtifact # noqa: E402 + + +@dataclass(slots=True) +class RuntimeSnapshot: + """In-memory state needed to restore one exact branch origin.""" + + integration_state: Any + libero_state: Any + timestep: int + simulation_time_seconds: float + done: bool + python_rng_state: object + numpy_rng_state: tuple[Any, ...] + controller_states: tuple[dict[str, object], ...] + gripper_actions: tuple[Any, ...] + + +@dataclass(frozen=True, slots=True) +class TraceStep: + """Exact comparison data retained only while validating the generator.""" + + integration_state: Any + observation_sha256: str + reward: float + success: bool + done: bool + + +@dataclass(frozen=True, slots=True) +class BranchTrace: + """One branch's exact per-step runtime trace.""" + + initial_state: Any + initial_observation_sha256: str + steps: tuple[TraceStep, ...] + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(8 * 1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _array_sha256(array: Any) -> str: + import numpy as np + + contiguous = np.ascontiguousarray(array) + metadata = json.dumps( + {"dtype": str(contiguous.dtype), "shape": list(contiguous.shape)}, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + digest = hashlib.sha256(metadata) + digest.update(contiguous.view(np.uint8).tobytes()) + return digest.hexdigest() + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps( + payload, + allow_nan=False, + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _git_revision(path: Path) -> str: + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _configure_process(gpu_index: int, run_dir: Path) -> None: + os.environ["MUJOCO_EGL_DEVICE_ID"] = str(gpu_index) + os.environ.setdefault("MUJOCO_GL", "egl") + os.environ.setdefault("PYOPENGL_PLATFORM", "egl") + config_dir = run_dir / "libero_config" + config_dir.mkdir(parents=True, exist_ok=True) + os.environ["LIBERO_CONFIG_PATH"] = str(config_dir) + + benchmark_root = LIBERO_ROOT / "libero" / "libero" + config = { + "benchmark_root": str(benchmark_root), + "bddl_files": str(benchmark_root / "bddl_files"), + "init_states": str(benchmark_root / "init_files"), + "datasets": str(LIBERO_ROOT / "libero" / "datasets"), + "assets": str(benchmark_root / "assets"), + } + config_path = config_dir / "config.yaml" + config_path.write_text( + "".join(f"{key}: {json.dumps(value)}\n" for key, value in config.items()), + encoding="utf-8", + ) + if str(LIBERO_ROOT) not in sys.path: + sys.path.insert(0, str(LIBERO_ROOT)) + + +def _set_seed(seed: int) -> None: + import numpy as np + + random.seed(seed) + np.random.seed(seed) + + +def _load_init_states(path: Path) -> Any: + """Load the pinned NumPy array with a minimal PyTorch weights-only allowlist.""" + + import numpy as np + import torch + from numpy.core.multiarray import _reconstruct # type: ignore[attr-defined] + + safe_globals = [_reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType] + with torch.serialization.safe_globals(safe_globals): + states = torch.load(path, map_location="cpu", weights_only=True) + if not isinstance(states, np.ndarray) or states.dtype != np.float64: + raise TypeError("LIBERO init states must be a float64 NumPy array") + return states + + +def _integration_state(environment: Any) -> Any: + import mujoco + import numpy as np + + specification = mujoco.mjtState.mjSTATE_INTEGRATION + state = np.empty( + mujoco.mj_stateSize(environment.sim.model._model, specification), + dtype=np.float64, + ) + mujoco.mj_getState( + environment.sim.model._model, + environment.sim.data._data, + state, + specification, + ) + return state + + +def _set_integration_state(environment: Any, state: Any) -> None: + import mujoco + + mujoco.mj_setState( + environment.sim.model._model, + environment.sim.data._data, + state, + mujoco.mjtState.mjSTATE_INTEGRATION, + ) + + +def _copy_controller_state(controller: Any) -> dict[str, object]: + """Copy mutable controller values while preserving its simulator reference.""" + + import numpy as np + + state: dict[str, object] = {} + for name, value in vars(controller).items(): + if name == "sim": + continue + if isinstance(value, np.ndarray): + state[name] = value.copy() + elif isinstance(value, (bool, int, float, str)) or value is None: + state[name] = value + return state + + +def _restore_controller_state(controller: Any, state: dict[str, object]) -> None: + import numpy as np + + for name, value in state.items(): + restored = value.copy() if isinstance(value, np.ndarray) else value + setattr(controller, name, restored) + + +def _json_value(value: object) -> object: + import numpy as np + + if isinstance(value, np.ndarray): + return { + "dtype": str(value.dtype), + "shape": list(value.shape), + "values": value.tolist(), + } + if isinstance(value, np.generic): + return value.item() + return value + + +def _numpy_rng_payload(state: tuple[Any, ...]) -> dict[str, object]: + return { + "bit_generator": str(state[0]), + "keys": [int(value) for value in state[1]], + "position": int(state[2]), + "has_gauss": int(state[3]), + "cached_gaussian": float(state[4]), + } + + +def _runtime_sidecar(snapshot: RuntimeSnapshot, state_uri: str) -> dict[str, object]: + return { + "schema_version": "0.1", + "context_id": CONTEXT_ID, + "state_uri": state_uri, + "libero_wrapper_state": [float(value) for value in snapshot.libero_state], + "timestep": snapshot.timestep, + "simulation_time_seconds": snapshot.simulation_time_seconds, + "done": snapshot.done, + "python_rng_state": snapshot.python_rng_state, + "numpy_rng_state": _numpy_rng_payload(snapshot.numpy_rng_state), + "controllers": [ + {name: _json_value(value) for name, value in state.items()} + for state in snapshot.controller_states + ], + "gripper_current_actions": [action.tolist() for action in snapshot.gripper_actions], + "restore_policy": { + "mujoco": "mjSTATE_INTEGRATION", + "controller": "restore copied mutable controller values", + "gripper": "restore GripperModel.current_action", + "observables": "clear cache, reset observables, force one update", + "observation_semantics": ( + "canonical observation is force-refreshed from the restored end-of-control state; " + "it is not the potentially earlier camera sample returned by the preceding step" + ), + "clock": "restore timestep, cur_time, and done", + "rng": "restore Python and NumPy global RNG states before observable reset", + }, + } + + +def _capture_runtime_snapshot(environment: Any) -> RuntimeSnapshot: + import numpy as np + + return RuntimeSnapshot( + integration_state=_integration_state(environment).copy(), + libero_state=np.asarray(environment.get_sim_state(), dtype=np.float64).copy(), + timestep=int(environment.env.timestep), + simulation_time_seconds=float(environment.env.cur_time), + done=bool(environment.env.done), + python_rng_state=random.getstate(), + numpy_rng_state=copy.deepcopy(np.random.get_state()), + controller_states=tuple( + _copy_controller_state(robot.controller) for robot in environment.robots + ), + gripper_actions=tuple( + np.asarray(robot.gripper.current_action, dtype=np.float64).copy() + for robot in environment.robots + ), + ) + + +def _restore_runtime_snapshot(environment: Any, snapshot: RuntimeSnapshot) -> Any: + import numpy as np + + random.setstate(snapshot.python_rng_state) + np.random.set_state(snapshot.numpy_rng_state) + _set_integration_state(environment, snapshot.integration_state) + environment.env.timestep = snapshot.timestep + environment.env.cur_time = snapshot.simulation_time_seconds + environment.env.done = snapshot.done + environment.sim.forward() + environment._post_process() + + for robot, controller_state, gripper_action in zip( + environment.robots, + snapshot.controller_states, + snapshot.gripper_actions, + strict=True, + ): + _restore_controller_state(robot.controller, controller_state) + robot.gripper.current_action = gripper_action.copy() + + environment.env._obs_cache = {} + for observable in environment.env._observables.values(): + observable.reset() + environment._update_observables(force=True) + observation = environment.env._get_observations() + + # forward() is required to refresh derived quantities, but it can change solver + # warm-start values. Restore the complete integration vector once more without a + # second forward so the branch begins byte-identically to the recorded snapshot. + _set_integration_state(environment, snapshot.integration_state) + return observation + + +def _quat_to_axis_angle(quaternion: Any) -> Any: + import numpy as np + + quat = np.asarray(quaternion, dtype=np.float64).copy() + quat[3] = np.clip(quat[3], -1.0, 1.0) + denominator = math.sqrt(max(0.0, 1.0 - float(quat[3]) ** 2)) + if math.isclose(denominator, 0.0): + return np.zeros(3, dtype=np.float64) + angle = 2.0 * math.acos(float(quat[3])) + return quat[:3] * angle / denominator + + +def _observation_sha256(observation: Any) -> str: + payload = { + key: _array_sha256(observation[key]) + for key in ( + "agentview_image", + "robot0_eye_in_hand_image", + "robot0_eef_pos", + "robot0_eef_quat", + "robot0_gripper_qpos", + "object-state", + ) + } + return hashlib.sha256( + json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + ).hexdigest() + + +def _write_rgb_references( + observation: Any, + *, + run_dir: Path, + relative_directory: Path, + step_index: int, +) -> tuple[RGBFrameReference, ...]: + import numpy as np + from PIL import Image + + references: list[RGBFrameReference] = [] + for camera_name, observation_key in CAMERAS: + image = np.ascontiguousarray(observation[observation_key], dtype=np.uint8) + height, width, channels = image.shape + if channels != 3: + raise RuntimeError(f"{camera_name} returned a non-RGB image: {image.shape}") + path = run_dir / relative_directory / f"step-{step_index:03d}-{camera_name}.png" + path.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(image).save(path) + references.append( + RGBFrameReference( + camera_name=camera_name, + height=int(height), + width=int(width), + uri=path.relative_to(run_dir).as_posix(), + sha256=_sha256_file(path), + size_bytes=path.stat().st_size, + ) + ) + return tuple(references) + + +def _typed_frame( + environment: Any, + observation: Any, + *, + run_dir: Path, + relative_directory: Path, + step_index: int, + reward: float, + success: bool, + done: bool, +) -> RobotStateFrame: + return RobotStateFrame( + step_index=step_index, + simulation_time_seconds=float(environment.env.cur_time), + eef_position=tuple(float(value) for value in observation["robot0_eef_pos"]), + eef_axis_angle=tuple( + float(value) for value in _quat_to_axis_angle(observation["robot0_eef_quat"]) + ), + gripper_qpos=tuple(float(value) for value in observation["robot0_gripper_qpos"]), + object_state=tuple(float(value) for value in observation["object-state"]), + rgb_frames=_write_rgb_references( + observation, + run_dir=run_dir, + relative_directory=relative_directory, + step_index=step_index, + ), + simulator_state_sha256=_array_sha256(_integration_state(environment)), + reward=float(reward), + success=bool(success), + done=bool(done), + ) + + +def _trace_step( + environment: Any, + observation: Any, + *, + reward: float, + success: bool, + done: bool, +) -> TraceStep: + return TraceStep( + integration_state=_integration_state(environment).copy(), + observation_sha256=_observation_sha256(observation), + reward=float(reward), + success=bool(success), + done=bool(done), + ) + + +def _rollout( + environment: Any, + snapshot: RuntimeSnapshot, + *, + branch_id: str, + action: tuple[float, ...], + horizon: int, + persist: bool, + run_dir: Path, + initial_frame: RobotStateFrame, +) -> tuple[RobotFuture | None, BranchTrace]: + import numpy as np + + initial_observation = _restore_runtime_snapshot(environment, snapshot) + initial_state = _integration_state(environment).copy() + typed_frames: list[RobotStateFrame] = [] + trace_steps: list[TraceStep] = [] + cumulative_return = 0.0 + succeeded = False + last_done = False + action_array = np.asarray(action, dtype=np.float64) + + for step_index in range(1, horizon + 1): + observation, reward, last_done, _ = environment.step(action_array) + success = bool(environment.check_success()) + succeeded = succeeded or success + cumulative_return += float(reward) + trace_steps.append( + _trace_step( + environment, + observation, + reward=float(reward), + success=success, + done=bool(last_done), + ) + ) + if persist: + typed_frames.append( + _typed_frame( + environment, + observation, + run_dir=run_dir, + relative_directory=Path("images") / branch_id, + step_index=step_index, + reward=float(reward), + success=success, + done=bool(last_done), + ) + ) + + future = ( + RobotFuture( + context_id=CONTEXT_ID, + branch_id=branch_id, + initial_frame=initial_frame, + frames=tuple(typed_frames), + cumulative_return=cumulative_return, + success=succeeded, + termination_reason="success" if succeeded else "horizon", + ) + if persist + else None + ) + trace = BranchTrace( + initial_state=initial_state, + initial_observation_sha256=_observation_sha256(initial_observation), + steps=tuple(trace_steps), + ) + return future, trace + + +def _trace_exact(left: BranchTrace, right: BranchTrace) -> bool: + import numpy as np + + return ( + np.array_equal(left.initial_state, right.initial_state) + and left.initial_observation_sha256 == right.initial_observation_sha256 + and len(left.steps) == len(right.steps) + and all( + np.array_equal(left_step.integration_state, right_step.integration_state) + and left_step.observation_sha256 == right_step.observation_sha256 + and left_step.reward == right_step.reward + and left_step.success == right_step.success + and left_step.done == right_step.done + for left_step, right_step in zip(left.steps, right.steps, strict=True) + ) + ) + + +def _max_state_error(comparisons: list[tuple[BranchTrace, BranchTrace]]) -> float: + import numpy as np + + errors: list[float] = [] + for left, right in comparisons: + errors.append(float(np.max(np.abs(left.initial_state - right.initial_state)))) + errors.extend( + float(np.max(np.abs(left_step.integration_state - right_step.integration_state))) + for left_step, right_step in zip(left.steps, right.steps, strict=True) + ) + return max(errors, default=0.0) + + +def _snapshot_descriptor( + snapshot: RuntimeSnapshot, + *, + run_dir: Path, + simulator_version: str, +) -> SimulatorSnapshotDescriptor: + import numpy as np + + state_path = run_dir / "snapshots" / "integration_state.npy" + state_path.parent.mkdir(parents=True, exist_ok=True) + with state_path.open("wb") as stream: + np.save(stream, snapshot.integration_state, allow_pickle=False) + + sidecar_path = run_dir / "snapshots" / "runtime_state.json" + _write_json( + sidecar_path, + _runtime_sidecar(snapshot, state_path.relative_to(run_dir).as_posix()), + ) + return SimulatorSnapshotDescriptor( + snapshot_id=CONTEXT_ID, + context_id=CONTEXT_ID, + simulator="LIBERO/robosuite/MuJoCo", + simulator_version=simulator_version, + state_format="numpy-npy-mujoco-mjSTATE_INTEGRATION-float64-v1", + state_uri=state_path.relative_to(run_dir).as_posix(), + state_sha256=_sha256_file(state_path), + state_size_bytes=state_path.stat().st_size, + sidecar_uri=sidecar_path.relative_to(run_dir).as_posix(), + sidecar_sha256=_sha256_file(sidecar_path), + sidecar_size_bytes=sidecar_path.stat().st_size, + timestep=snapshot.timestep, + simulation_time_seconds=snapshot.simulation_time_seconds, + restore_policy=RESTORE_POLICY, + ) + + +def _prepare_environment(run_dir: Path) -> tuple[Any, Any, str]: + import mujoco + import robosuite + from libero.libero import benchmark, get_libero_path + from libero.libero.envs import OffScreenRenderEnv + + revision = _git_revision(LIBERO_ROOT) + if revision != LIBERO_REVISION: + raise RuntimeError(f"LIBERO revision mismatch: expected {LIBERO_REVISION}, got {revision}") + suite = benchmark.get_benchmark_dict()[TASK_SUITE]() + task = suite.get_task(TASK_ID) + if task.language != TASK: + raise RuntimeError(f"pinned LIBERO task changed: expected {TASK!r}, got {task.language!r}") + bddl_path = Path(suite.get_task_bddl_file_path(TASK_ID)) + init_states_path = ( + Path(get_libero_path("init_states")) / task.problem_folder / task.init_states_file + ) + if _sha256_file(bddl_path) != BDDL_SHA256: + raise RuntimeError("LIBERO BDDL hash mismatch") + if _sha256_file(init_states_path) != INIT_STATES_SHA256: + raise RuntimeError("LIBERO init-state hash mismatch") + init_states = _load_init_states(init_states_path) + if init_states.shape != (50, 92): + raise RuntimeError(f"unexpected LIBERO init-state shape: {init_states.shape}") + + environment = OffScreenRenderEnv( + bddl_file_name=str(bddl_path), + camera_heights=256, + camera_widths=256, + ) + environment.seed(SEED) + version = f"libero@{revision}/robosuite@{robosuite.__version__}/mujoco@{mujoco.__version__}" + _write_json( + run_dir / "benchmark_provenance.json", + { + "libero_revision": revision, + "robosuite_version": robosuite.__version__, + "mujoco_version": mujoco.__version__, + "task_suite": TASK_SUITE, + "task_id": TASK_ID, + "task": TASK, + "init_state_index": INIT_STATE_INDEX, + "wait_steps": WAIT_STEPS, + "seed": SEED, + "bddl_uri": str(bddl_path), + "bddl_sha256": BDDL_SHA256, + "init_states_uri": str(init_states_path), + "init_states_sha256": INIT_STATES_SHA256, + "init_states_shape": list(init_states.shape), + "action_space": ACTION_SPACE, + "control_frequency_hz": environment.env.control_freq, + "control_timestep_seconds": environment.env.control_timestep, + "snapshot_observation_semantics": ( + "compare independent force-refreshed observations after restore; robosuite step() " + "camera samples can precede the end-of-control state" + ), + }, + ) + return environment, init_states, version + + +def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: + import numpy as np + + environment, init_states, simulator_version = _prepare_environment(run_dir) + try: + environment.reset() + observation = environment.set_init_state(init_states[INIT_STATE_INDEX]) + for _ in range(WAIT_STEPS): + observation, _, _, _ = environment.step(DUMMY_ACTION) + + snapshot = _capture_runtime_snapshot(environment) + descriptor = _snapshot_descriptor( + snapshot, + run_dir=run_dir, + simulator_version=simulator_version, + ) + original_state = snapshot.integration_state.copy() + + restored_observation = _restore_runtime_snapshot(environment, snapshot) + restored_state = _integration_state(environment).copy() + initial_frame = _typed_frame( + environment, + restored_observation, + run_dir=run_dir, + relative_directory=Path("images") / "initial", + step_index=0, + reward=0.0, + success=bool(environment.check_success()), + done=bool(environment.env.done), + ) + verification_observation = _restore_runtime_snapshot(environment, snapshot) + verification_state = _integration_state(environment).copy() + + branches: list[ActionBranch] = [] + primary_traces: dict[str, BranchTrace] = {} + for branch_id, action_family, action in BRANCH_ACTIONS: + future, trace = _rollout( + environment, + snapshot, + branch_id=branch_id, + action=action, + horizon=horizon, + persist=True, + run_dir=run_dir, + initial_frame=initial_frame, + ) + if future is None: + raise AssertionError("persisted rollout did not return a future") + branch = ActionBranch( + branch_id=branch_id, + action_family=action_family, + initial_snapshot_sha256=descriptor.content_sha256, + action=RobotAction( + action_space=ACTION_SPACE, + values=tuple(action for _ in range(horizon)), + ), + future=future, + ) + branches.append(branch) + primary_traces[branch_id] = trace + _write_json(run_dir / "branches" / f"{branch_id}.json", branch.to_dict()) + + reverse_traces: dict[str, BranchTrace] = {} + for branch_id, _, action in reversed(BRANCH_ACTIONS): + _, trace = _rollout( + environment, + snapshot, + branch_id=branch_id, + action=action, + horizon=horizon, + persist=False, + run_dir=run_dir, + initial_frame=initial_frame, + ) + reverse_traces[branch_id] = trace + + _, repeated_trace = _rollout( + environment, + snapshot, + branch_id="noop", + action=DUMMY_ACTION, + horizon=horizon, + persist=False, + run_dir=run_dir, + initial_frame=initial_frame, + ) + order_comparisons = [ + (primary_traces[branch_id], reverse_traces[branch_id]) + for branch_id, _, _ in BRANCH_ACTIONS + ] + repeat_comparison = (primary_traces["noop"], repeated_trace) + restore_state_error = max( + float(np.max(np.abs(original_state - restored_state))), + float(np.max(np.abs(original_state - verification_state))), + ) + validation = CounterfactualValidation( + restore_trials=2, + restored_initial_state_exact=bool( + np.array_equal(original_state, restored_state) + and np.array_equal(original_state, verification_state) + ), + restored_initial_observation_exact=( + _observation_sha256(restored_observation) + == _observation_sha256(verification_observation) + ), + repeated_branch_exact=_trace_exact(*repeat_comparison), + branch_order_invariant=all(_trace_exact(*pair) for pair in order_comparisons), + max_state_abs_error=max( + restore_state_error, + _max_state_error([*order_comparisons, repeat_comparison]), + ), + ) + group = RobotInterventionGroup( + context_id=CONTEXT_ID, + task=TASK, + snapshot=descriptor, + branches=tuple(branches), + ) + artifact = CounterfactualArtifact(group=group, validation=validation) + artifact.write_json(run_dir / "intervention_group.json") + _write_json(run_dir / "validation.json", validation.to_dict()) + if not validation.passed: + raise RuntimeError( + f"counterfactual restore validation failed; inspect {run_dir / 'validation.json'}" + ) + return artifact + finally: + environment.close() + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gpu-index", type=int, default=0, help="physical EGL GPU index") + parser.add_argument( + "--run-dir", + type=Path, + default=REPOSITORY_ROOT / "runs" / "libero-cf-pilot", + ) + parser.add_argument("--horizon", type=int, default=8, help="control steps per action branch") + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.gpu_index < 0: + raise ValueError("gpu-index must be non-negative") + if args.horizon <= 0: + raise ValueError("horizon must be positive") + run_dir = args.run_dir.expanduser().resolve() + _configure_process(args.gpu_index, run_dir) + _set_seed(SEED) + artifact = _generate(run_dir, args.horizon) + print( + json.dumps( + { + "artifact": str(run_dir / "intervention_group.json"), + "artifact_sha256": artifact.artifact_sha256, + "group_sha256": artifact.group.content_sha256, + "snapshot_sha256": artifact.group.snapshot.content_sha256, + "branches": len(artifact.group.branches), + "horizon": artifact.group.horizon, + "metrics": artifact.metrics.to_dict(), + "validation": artifact.validation.to_dict(), + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/wamprobe/api/__init__.py b/src/wamprobe/api/__init__.py index b303c9f..f9cffc0 100644 --- a/src/wamprobe/api/__init__.py +++ b/src/wamprobe/api/__init__.py @@ -1,6 +1,16 @@ """Public, model-agnostic WAMProbe API.""" from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.counterfactual import ( + ActionBranch, + CounterfactualValidation, + RGBFrameReference, + RobotAction, + RobotFuture, + RobotInterventionGroup, + RobotStateFrame, + SimulatorSnapshotDescriptor, +) from wamprobe.api.model import ActionPredictorAdapter, WAMAdapter from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RGBFrame, RobotObservation from wamprobe.api.types import ( @@ -14,16 +24,24 @@ __all__ = [ "Action2D", + "ActionBranch", "ActionPrediction", "ActionPredictorAdapter", "Context2D", + "CounterfactualValidation", "FutureRepresentation", "InterventionGroup", "InterventionSuite", "InferenceRuntime", "ModelCapabilities", "RGBFrame", + "RGBFrameReference", + "RobotAction", + "RobotFuture", + "RobotInterventionGroup", "RobotObservation", + "RobotStateFrame", + "SimulatorSnapshotDescriptor", "Trajectory2D", "Vec2", "WAMAdapter", diff --git a/src/wamprobe/api/counterfactual.py b/src/wamprobe/api/counterfactual.py new file mode 100644 index 0000000..817a0ed --- /dev/null +++ b/src/wamprobe/api/counterfactual.py @@ -0,0 +1,502 @@ +"""Dependency-free contracts for paired robot counterfactual rollouts.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass + + +def _require_non_empty(value: str, field_name: str) -> None: + if not value.strip(): + raise ValueError(f"{field_name} must not be empty") + + +def _validate_sha256(value: str, field_name: str) -> None: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{field_name} must be a lowercase 64-character SHA256 digest") + + +def _stable_sha256(payload: dict[str, object]) -> str: + encoded = json.dumps( + payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _all_finite(values: tuple[float, ...]) -> bool: + return all(math.isfinite(value) for value in values) + + +@dataclass(frozen=True, slots=True) +class SimulatorSnapshotDescriptor: + """Content-addressed simulator snapshot without embedding its large payloads.""" + + snapshot_id: str + context_id: str + simulator: str + simulator_version: str + state_format: str + state_uri: str + state_sha256: str + state_size_bytes: int + sidecar_uri: str + sidecar_sha256: str + sidecar_size_bytes: int + timestep: int + simulation_time_seconds: float + restore_policy: str + + def __post_init__(self) -> None: + for value, field_name in ( + (self.snapshot_id, "snapshot_id"), + (self.context_id, "snapshot context_id"), + (self.simulator, "simulator"), + (self.simulator_version, "simulator_version"), + (self.state_format, "state_format"), + (self.state_uri, "state_uri"), + (self.sidecar_uri, "sidecar_uri"), + (self.restore_policy, "restore_policy"), + ): + _require_non_empty(value, field_name) + _validate_sha256(self.state_sha256, "state_sha256") + _validate_sha256(self.sidecar_sha256, "sidecar_sha256") + if self.state_size_bytes <= 0 or self.sidecar_size_bytes <= 0: + raise ValueError("snapshot payload sizes must be positive") + if ( + isinstance(self.timestep, bool) + or not isinstance(self.timestep, int) + or self.timestep < 0 + ): + raise ValueError("snapshot timestep must be a non-negative integer") + if not math.isfinite(self.simulation_time_seconds) or self.simulation_time_seconds < 0: + raise ValueError("snapshot simulation time must be finite and non-negative") + + def _payload(self) -> dict[str, object]: + return { + "snapshot_id": self.snapshot_id, + "context_id": self.context_id, + "simulator": self.simulator, + "simulator_version": self.simulator_version, + "state_format": self.state_format, + "state_uri": self.state_uri, + "state_sha256": self.state_sha256, + "state_size_bytes": self.state_size_bytes, + "sidecar_uri": self.sidecar_uri, + "sidecar_sha256": self.sidecar_sha256, + "sidecar_size_bytes": self.sidecar_size_bytes, + "timestep": self.timestep, + "simulation_time_seconds": self.simulation_time_seconds, + "restore_policy": self.restore_policy, + } + + @property + def content_sha256(self) -> str: + """Return a semantic digest over the complete restore contract.""" + + return _stable_sha256(self._payload()) + + def to_dict(self) -> dict[str, object]: + """Return the JSON descriptor and its semantic digest.""" + + return {**self._payload(), "content_sha256": self.content_sha256} + + +@dataclass(frozen=True, slots=True) +class RGBFrameReference: + """Reference to one encoded RGB frame stored outside the JSON contract.""" + + camera_name: str + height: int + width: int + uri: str + sha256: str + size_bytes: int + + def __post_init__(self) -> None: + _require_non_empty(self.camera_name, "camera_name") + _require_non_empty(self.uri, "RGB frame URI") + _validate_sha256(self.sha256, "RGB frame sha256") + if self.height <= 0 or self.width <= 0: + raise ValueError("RGB frame dimensions must be positive") + if self.size_bytes <= 0: + raise ValueError("RGB frame size must be positive") + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible external frame reference.""" + + return { + "camera_name": self.camera_name, + "shape": [self.height, self.width, 3], + "encoding": "png", + "uri": self.uri, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + } + + +@dataclass(frozen=True, slots=True) +class RobotAction: + """A typed robot action sequence in one explicitly named action space.""" + + action_space: str + values: tuple[tuple[float, ...], ...] + + def __post_init__(self) -> None: + _require_non_empty(self.action_space, "action_space") + if not self.values or not self.values[0]: + raise ValueError("robot action must contain a non-empty sequence") + action_dim = len(self.values[0]) + if any(len(action) != action_dim for action in self.values): + raise ValueError("all robot actions must have the same dimension") + if any(not math.isfinite(value) for action in self.values for value in action): + raise ValueError("robot action values must be finite") + + @property + def horizon(self) -> int: + """Return the number of control steps.""" + + return len(self.values) + + @property + def action_dim(self) -> int: + """Return the action-vector width.""" + + return len(self.values[0]) + + @property + def content_sha256(self) -> str: + """Return a stable digest over action semantics and values.""" + + return _stable_sha256(self.to_dict()) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible action sequence.""" + + return { + "action_space": self.action_space, + "values": [list(action) for action in self.values], + "horizon": self.horizon, + "action_dim": self.action_dim, + } + + +@dataclass(frozen=True, slots=True) +class RobotStateFrame: + """One lightweight simulator state and image-reference record.""" + + step_index: int + simulation_time_seconds: float + eef_position: tuple[float, float, float] + eef_axis_angle: tuple[float, float, float] + gripper_qpos: tuple[float, ...] + object_state: tuple[float, ...] + rgb_frames: tuple[RGBFrameReference, ...] + simulator_state_sha256: str + reward: float + success: bool + done: bool + + def __post_init__(self) -> None: + if ( + isinstance(self.step_index, bool) + or not isinstance(self.step_index, int) + or self.step_index < 0 + ): + raise ValueError("step_index must be a non-negative integer") + if not math.isfinite(self.simulation_time_seconds) or self.simulation_time_seconds < 0: + raise ValueError("frame simulation time must be finite and non-negative") + if len(self.eef_position) != 3 or len(self.eef_axis_angle) != 3: + raise ValueError("end-effector position and axis-angle vectors must be 3D") + if not self.gripper_qpos or not self.object_state: + raise ValueError("gripper and object state vectors must not be empty") + state_values = ( + *self.eef_position, + *self.eef_axis_angle, + *self.gripper_qpos, + *self.object_state, + ) + if not _all_finite(state_values): + raise ValueError("robot state values must be finite") + if not self.rgb_frames: + raise ValueError("robot state frame requires at least one RGB reference") + camera_names = [frame.camera_name for frame in self.rgb_frames] + if len(camera_names) != len(set(camera_names)): + raise ValueError("RGB camera names must be unique within a state frame") + _validate_sha256(self.simulator_state_sha256, "simulator_state_sha256") + if not math.isfinite(self.reward): + raise ValueError("frame reward must be finite") + if not isinstance(self.success, bool) or not isinstance(self.done, bool): + raise ValueError("frame success and done values must be booleans") + + @property + def content_sha256(self) -> str: + """Return a stable digest without loading referenced PNG bytes.""" + + return _stable_sha256(self.to_dict()) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible state descriptor.""" + + return { + "step_index": self.step_index, + "simulation_time_seconds": self.simulation_time_seconds, + "eef_position": list(self.eef_position), + "eef_axis_angle": list(self.eef_axis_angle), + "gripper_qpos": list(self.gripper_qpos), + "object_state": list(self.object_state), + "rgb_frames": [frame.to_dict() for frame in self.rgb_frames], + "simulator_state_sha256": self.simulator_state_sha256, + "reward": self.reward, + "success": self.success, + "done": self.done, + } + + +@dataclass(frozen=True, slots=True) +class RobotFuture: + """Ground-truth state future for one branch from a recorded initial frame.""" + + context_id: str + branch_id: str + initial_frame: RobotStateFrame + frames: tuple[RobotStateFrame, ...] + cumulative_return: float + success: bool + termination_reason: str + + def __post_init__(self) -> None: + _require_non_empty(self.context_id, "future context_id") + _require_non_empty(self.branch_id, "future branch_id") + _require_non_empty(self.termination_reason, "termination_reason") + if self.initial_frame.step_index != 0: + raise ValueError("future initial frame must use step_index 0") + if not self.frames: + raise ValueError("robot future must contain at least one post-action frame") + expected_indices = tuple(range(1, len(self.frames) + 1)) + if tuple(frame.step_index for frame in self.frames) != expected_indices: + raise ValueError("future frame step indices must be contiguous and start at 1") + if not math.isfinite(self.cumulative_return): + raise ValueError("cumulative_return must be finite") + observed_return = sum(frame.reward for frame in self.frames) + if not math.isclose(self.cumulative_return, observed_return, abs_tol=1e-12): + raise ValueError("cumulative_return must equal the sum of frame rewards") + if not isinstance(self.success, bool): + raise ValueError("future success must be a boolean") + if self.success != any(frame.success for frame in self.frames): + raise ValueError("future success must match its state frames") + + expected_object_dim = len(self.initial_frame.object_state) + expected_gripper_dim = len(self.initial_frame.gripper_qpos) + expected_cameras = tuple( + (frame.camera_name, frame.height, frame.width) + for frame in self.initial_frame.rgb_frames + ) + for frame in self.frames: + if len(frame.object_state) != expected_object_dim: + raise ValueError("future object-state dimensions must remain constant") + if len(frame.gripper_qpos) != expected_gripper_dim: + raise ValueError("future gripper-state dimensions must remain constant") + cameras = tuple( + (camera.camera_name, camera.height, camera.width) for camera in frame.rgb_frames + ) + if cameras != expected_cameras: + raise ValueError("future camera order and dimensions must remain constant") + + @property + def horizon(self) -> int: + """Return the number of post-intervention state frames.""" + + return len(self.frames) + + @property + def final_frame(self) -> RobotStateFrame: + """Return the last observed state.""" + + return self.frames[-1] + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible future descriptor.""" + + return { + "context_id": self.context_id, + "branch_id": self.branch_id, + "initial_frame": self.initial_frame.to_dict(), + "frames": [frame.to_dict() for frame in self.frames], + "horizon": self.horizon, + "cumulative_return": self.cumulative_return, + "success": self.success, + "termination_reason": self.termination_reason, + } + + +@dataclass(frozen=True, slots=True) +class ActionBranch: + """One named action intervention and its simulator ground-truth future.""" + + branch_id: str + action_family: str + initial_snapshot_sha256: str + action: RobotAction + future: RobotFuture + + def __post_init__(self) -> None: + _require_non_empty(self.branch_id, "branch_id") + _require_non_empty(self.action_family, "action_family") + _validate_sha256(self.initial_snapshot_sha256, "initial_snapshot_sha256") + if self.future.branch_id != self.branch_id: + raise ValueError("action branch and future branch_id values must match") + if self.action.horizon != self.future.horizon: + raise ValueError("action and future horizons must match") + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible paired branch.""" + + return { + "branch_id": self.branch_id, + "action_family": self.action_family, + "initial_snapshot_sha256": self.initial_snapshot_sha256, + "action": self.action.to_dict(), + "future": self.future.to_dict(), + } + + +@dataclass(frozen=True, slots=True) +class RobotInterventionGroup: + """At least two action branches restored from one identical simulator snapshot.""" + + context_id: str + task: str + snapshot: SimulatorSnapshotDescriptor + branches: tuple[ActionBranch, ...] + + def __post_init__(self) -> None: + _require_non_empty(self.context_id, "intervention context_id") + _require_non_empty(self.task, "intervention task") + if len(self.branches) < 2: + raise ValueError("robot intervention group requires at least two branches") + if self.snapshot.context_id != self.context_id: + raise ValueError("snapshot and intervention context_id values must match") + + branch_ids = [branch.branch_id for branch in self.branches] + action_families = [branch.action_family for branch in self.branches] + if len(branch_ids) != len(set(branch_ids)): + raise ValueError("branch_id values must be unique within an intervention group") + if len(action_families) != len(set(action_families)): + raise ValueError("action_family values must be unique within an intervention group") + if any(branch.future.context_id != self.context_id for branch in self.branches): + raise ValueError("branch future context_id values must match the intervention") + if any( + branch.initial_snapshot_sha256 != self.snapshot.content_sha256 + for branch in self.branches + ): + raise ValueError("all branches must reference the same semantic snapshot") + + action_shapes = { + (branch.action.horizon, branch.action.action_dim) for branch in self.branches + } + if len(action_shapes) != 1: + raise ValueError("all branch action shapes must match") + action_spaces = {branch.action.action_space for branch in self.branches} + if len(action_spaces) != 1: + raise ValueError("all branches must use the same action_space") + initial_hashes = {branch.future.initial_frame.content_sha256 for branch in self.branches} + if len(initial_hashes) != 1: + raise ValueError("all branches must share an identical initial frame") + + @property + def horizon(self) -> int: + """Return the common branch horizon.""" + + return self.branches[0].action.horizon + + @property + def action_dim(self) -> int: + """Return the common action-vector width.""" + + return self.branches[0].action.action_dim + + @property + def action_space(self) -> str: + """Return the common semantic action-space identifier.""" + + return self.branches[0].action.action_space + + @property + def content_sha256(self) -> str: + """Return a stable digest over the complete lightweight intervention group.""" + + return _stable_sha256(self.to_dict()) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible intervention group without raw state or RGB bytes.""" + + return { + "context_id": self.context_id, + "task": self.task, + "snapshot": self.snapshot.to_dict(), + "action_space": self.action_space, + "horizon": self.horizon, + "action_dim": self.action_dim, + "branches": [branch.to_dict() for branch in self.branches], + } + + +@dataclass(frozen=True, slots=True) +class CounterfactualValidation: + """Measured restore and branch-order reproducibility checks.""" + + restore_trials: int + restored_initial_state_exact: bool + restored_initial_observation_exact: bool + repeated_branch_exact: bool + branch_order_invariant: bool + max_state_abs_error: float + + def __post_init__(self) -> None: + if ( + isinstance(self.restore_trials, bool) + or not isinstance(self.restore_trials, int) + or self.restore_trials < 2 + ): + raise ValueError("restore_trials must be an integer of at least two") + exactness = ( + self.restored_initial_state_exact, + self.restored_initial_observation_exact, + self.repeated_branch_exact, + self.branch_order_invariant, + ) + if any(not isinstance(value, bool) for value in exactness): + raise ValueError("counterfactual exactness checks must be booleans") + if not math.isfinite(self.max_state_abs_error) or self.max_state_abs_error < 0: + raise ValueError("max_state_abs_error must be finite and non-negative") + + @property + def passed(self) -> bool: + """Return whether all exact restore checks passed.""" + + return ( + self.restored_initial_state_exact + and self.restored_initial_observation_exact + and self.repeated_branch_exact + and self.branch_order_invariant + and self.max_state_abs_error == 0.0 + ) + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible validation record.""" + + return { + "restore_trials": self.restore_trials, + "restored_initial_state_exact": self.restored_initial_state_exact, + "restored_initial_observation_exact": self.restored_initial_observation_exact, + "repeated_branch_exact": self.repeated_branch_exact, + "branch_order_invariant": self.branch_order_invariant, + "max_state_abs_error": self.max_state_abs_error, + "passed": self.passed, + } diff --git a/src/wamprobe/counterfactual.py b/src/wamprobe/counterfactual.py new file mode 100644 index 0000000..454b2f3 --- /dev/null +++ b/src/wamprobe/counterfactual.py @@ -0,0 +1,141 @@ +"""Scoring and deterministic artifacts for paired robot interventions.""" + +from __future__ import annotations + +import hashlib +import json +import math +import uuid +from dataclasses import dataclass +from itertools import combinations +from pathlib import Path + +from wamprobe.api.counterfactual import CounterfactualValidation, RobotInterventionGroup + + +@dataclass(frozen=True, slots=True) +class CounterfactualScores: + """Small ground-truth branch-separation and control-outcome profile.""" + + mean_final_eef_pairwise_distance: float + mean_final_object_pairwise_distance: float + noop_final_eef_drift: float | None + return_spread: float + success_rate: float + + def to_dict(self) -> dict[str, float | None]: + """Return a JSON-compatible metric profile.""" + + return { + "mean_final_eef_pairwise_distance": self.mean_final_eef_pairwise_distance, + "mean_final_object_pairwise_distance": self.mean_final_object_pairwise_distance, + "noop_final_eef_drift": self.noop_final_eef_drift, + "return_spread": self.return_spread, + "success_rate": self.success_rate, + } + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) + + +def score_intervention_group(group: RobotInterventionGroup) -> CounterfactualScores: + """Measure real branch separation, no-op drift, return spread, and success rate.""" + + pairs = list(combinations(group.branches, 2)) + eef_distances = [ + math.dist(left.future.final_frame.eef_position, right.future.final_frame.eef_position) + for left, right in pairs + ] + object_distances = [ + math.dist(left.future.final_frame.object_state, right.future.final_frame.object_state) + for left, right in pairs + ] + noop_branches = [ + branch + for branch in group.branches + if branch.action_family == "no-op" or branch.branch_id == "noop" + ] + noop_drift = ( + math.dist( + noop_branches[0].future.initial_frame.eef_position, + noop_branches[0].future.final_frame.eef_position, + ) + if noop_branches + else None + ) + returns = [branch.future.cumulative_return for branch in group.branches] + return CounterfactualScores( + mean_final_eef_pairwise_distance=_mean(eef_distances), + mean_final_object_pairwise_distance=_mean(object_distances), + noop_final_eef_drift=noop_drift, + return_spread=max(returns) - min(returns), + success_rate=_mean([float(branch.future.success) for branch in group.branches]), + ) + + +@dataclass(frozen=True, slots=True) +class CounterfactualArtifact: + """One cacheable intervention group with real outcomes and restore validation.""" + + group: RobotInterventionGroup + validation: CounterfactualValidation + schema_version: str = "0.1" + + def __post_init__(self) -> None: + if self.schema_version != "0.1": + raise ValueError(f"unsupported counterfactual artifact schema: {self.schema_version}") + + @property + def metrics(self) -> CounterfactualScores: + """Return the deterministic metric profile for the stored group.""" + + return score_intervention_group(self.group) + + def _payload(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "group_sha256": self.group.content_sha256, + "group": self.group.to_dict(), + "metrics": self.metrics.to_dict(), + "validation": self.validation.to_dict(), + } + + @property + def artifact_sha256(self) -> str: + """Return a deterministic digest over group, scores, and validation.""" + + encoded = json.dumps( + self._payload(), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def to_dict(self) -> dict[str, object]: + """Return the complete JSON-compatible counterfactual artifact.""" + + return {**self._payload(), "artifact_sha256": self.artifact_sha256} + + def write_json(self, path: Path) -> None: + """Atomically write the lightweight artifact without raw state or image bytes.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps( + self.to_dict(), + allow_nan=False, + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) diff --git a/tests/test_counterfactual_robotics.py b/tests/test_counterfactual_robotics.py new file mode 100644 index 0000000..0c56e69 --- /dev/null +++ b/tests/test_counterfactual_robotics.py @@ -0,0 +1,265 @@ +import json +import math +from pathlib import Path +from typing import cast + +import pytest + +from wamprobe.api.counterfactual import ( + ActionBranch, + CounterfactualValidation, + RGBFrameReference, + RobotAction, + RobotFuture, + RobotInterventionGroup, + RobotStateFrame, + SimulatorSnapshotDescriptor, +) +from wamprobe.counterfactual import CounterfactualArtifact, score_intervention_group + + +def _snapshot() -> SimulatorSnapshotDescriptor: + return SimulatorSnapshotDescriptor( + snapshot_id="libero-spatial-task0-init0-wait30", + context_id="libero-spatial-task0-init0-wait30", + simulator="LIBERO/robosuite/MuJoCo", + simulator_version="libero@8f1084e/robosuite@1.4.0/mujoco@3.9.0", + state_format="mujoco-mjSTATE_INTEGRATION-float64-v1", + state_uri="snapshots/integration_state.npy", + state_sha256="a" * 64, + state_size_bytes=1024, + sidecar_uri="snapshots/runtime_state.json", + sidecar_sha256="b" * 64, + sidecar_size_bytes=2048, + timestep=30, + simulation_time_seconds=1.5, + restore_policy="libero-integration-and-python-side-state-v1", + ) + + +def _frame( + step_index: int, + *, + eef_x: float, + object_x: float, + reward: float = 0.0, + success: bool = False, +) -> RobotStateFrame: + return RobotStateFrame( + step_index=step_index, + simulation_time_seconds=1.5 + step_index * 0.05, + eef_position=(eef_x, 0.0, 1.0), + eef_axis_angle=(0.0, 0.0, 0.0), + gripper_qpos=(0.02, -0.02), + object_state=(object_x, 0.0), + rgb_frames=( + RGBFrameReference( + camera_name="agentview", + height=2, + width=2, + uri=f"images/branch/step-{step_index:03d}-agentview.png", + sha256=f"{step_index + 1:064x}", + size_bytes=12, + ), + ), + simulator_state_sha256=f"{step_index + 10:064x}", + reward=reward, + success=success, + done=success, + ) + + +def _branch( + branch_id: str, + action_family: str, + action_x: float, + final_eef_x: float, + final_object_x: float, + *, + reward: float = 0.0, + success: bool = False, +) -> ActionBranch: + initial = _frame(0, eef_x=0.0, object_x=0.0) + future = RobotFuture( + context_id=_snapshot().context_id, + branch_id=branch_id, + initial_frame=initial, + frames=( + _frame( + 1, + eef_x=final_eef_x, + object_x=final_object_x, + reward=reward, + success=success, + ), + ), + cumulative_return=reward, + success=success, + termination_reason="success" if success else "horizon", + ) + return ActionBranch( + branch_id=branch_id, + action_family=action_family, + initial_snapshot_sha256=_snapshot().content_sha256, + action=RobotAction( + action_space="libero-eef-delta-pose-gripper-v1", + values=((action_x, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0),), + ), + future=future, + ) + + +def _group() -> RobotInterventionGroup: + return RobotInterventionGroup( + context_id=_snapshot().context_id, + task="pick up the black bowl and place it on the plate", + snapshot=_snapshot(), + branches=( + _branch("noop", "no-op", 0.0, 0.1, 0.0), + _branch( + "move-x-positive", + "directional-positive-x", + 0.5, + 1.0, + 0.5, + reward=2.0, + success=True, + ), + ), + ) + + +def _validation() -> CounterfactualValidation: + return CounterfactualValidation( + restore_trials=2, + restored_initial_state_exact=True, + restored_initial_observation_exact=True, + repeated_branch_exact=True, + branch_order_invariant=True, + max_state_abs_error=0.0, + ) + + +def test_robot_intervention_group_has_stable_digest_without_raw_payloads() -> None: + group = _group() + + assert group.content_sha256 == _group().content_sha256 + assert group.horizon == 1 + assert group.action_dim == 7 + payload = group.to_dict() + snapshot_payload = payload["snapshot"] + assert isinstance(snapshot_payload, dict) + assert snapshot_payload["content_sha256"] == group.snapshot.content_sha256 + assert "data" not in json.dumps(payload) + + +def test_robot_action_rejects_non_finite_or_ragged_values() -> None: + with pytest.raises(ValueError, match="finite"): + RobotAction("space", ((math.nan,),)) + + with pytest.raises(ValueError, match="same dimension"): + RobotAction("space", ((0.0, 1.0), (0.0,))) + + +def test_robot_state_frame_requires_three_dimensional_eef_vectors() -> None: + with pytest.raises(ValueError, match="must be 3D"): + RobotStateFrame( + step_index=0, + simulation_time_seconds=0.0, + eef_position=cast(tuple[float, float, float], (0.0, 0.0)), + eef_axis_angle=(0.0, 0.0, 0.0), + gripper_qpos=(0.0,), + object_state=(0.0,), + rgb_frames=_frame(0, eef_x=0.0, object_x=0.0).rgb_frames, + simulator_state_sha256="a" * 64, + reward=0.0, + success=False, + done=False, + ) + + +def test_group_rejects_branches_from_different_snapshots() -> None: + branch = _branch("noop", "no-op", 0.0, 0.0, 0.0) + mismatched = ActionBranch( + branch_id=branch.branch_id, + action_family=branch.action_family, + initial_snapshot_sha256="f" * 64, + action=branch.action, + future=branch.future, + ) + + with pytest.raises(ValueError, match="snapshot"): + RobotInterventionGroup( + context_id=_snapshot().context_id, + task="task", + snapshot=_snapshot(), + branches=(mismatched, _branch("move", "directional", 0.5, 0.5, 0.0)), + ) + + +def test_group_rejects_mixed_action_shapes_and_initial_frames() -> None: + first = _branch("noop", "no-op", 0.0, 0.0, 0.0) + second = _branch("move", "directional", 0.5, 0.5, 0.0) + mixed_action = ActionBranch( + branch_id=second.branch_id, + action_family=second.action_family, + initial_snapshot_sha256=second.initial_snapshot_sha256, + action=RobotAction(second.action.action_space, ((0.5, 0.0),)), + future=second.future, + ) + + with pytest.raises(ValueError, match="action shape"): + RobotInterventionGroup( + context_id=_snapshot().context_id, + task="task", + snapshot=_snapshot(), + branches=(first, mixed_action), + ) + + changed_initial = RobotFuture( + context_id=second.future.context_id, + branch_id=second.future.branch_id, + initial_frame=_frame(0, eef_x=0.01, object_x=0.0), + frames=second.future.frames, + cumulative_return=second.future.cumulative_return, + success=second.future.success, + termination_reason=second.future.termination_reason, + ) + mixed_initial = ActionBranch( + branch_id=second.branch_id, + action_family=second.action_family, + initial_snapshot_sha256=second.initial_snapshot_sha256, + action=second.action, + future=changed_initial, + ) + with pytest.raises(ValueError, match="initial frame"): + RobotInterventionGroup( + context_id=_snapshot().context_id, + task="task", + snapshot=_snapshot(), + branches=(first, mixed_initial), + ) + + +def test_counterfactual_scores_report_separation_drift_and_return_profile() -> None: + scores = score_intervention_group(_group()) + + assert scores.mean_final_eef_pairwise_distance == pytest.approx(0.9) + assert scores.mean_final_object_pairwise_distance == pytest.approx(0.5) + assert scores.noop_final_eef_drift == pytest.approx(0.1) + assert scores.return_spread == pytest.approx(2.0) + assert scores.success_rate == pytest.approx(0.5) + + +def test_counterfactual_artifact_writes_deterministic_json(tmp_path: Path) -> None: + artifact = CounterfactualArtifact(group=_group(), validation=_validation()) + output = tmp_path / "intervention_group.json" + + artifact.write_json(output) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == "0.1" + assert payload["artifact_sha256"] == artifact.artifact_sha256 + assert payload["metrics"]["success_rate"] == 0.5 + assert payload["validation"]["branch_order_invariant"] is True + assert "data" not in json.dumps(payload) From b5c611fa44bf0349477a35777198bec27a045ba6 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 12:05:34 +0000 Subject: [PATCH 08/20] add context statistics and HTML reports --- README.md | 22 +- docs/QUICKSTART.md | 14 + docs/WAMProbe_PLAN.md | 32 +- examples/pointmass-demo/report.html | 53 + examples/pointmass-demo/report.md | 103 +- examples/pointmass-demo/summary.json | 2094 +++++++++++++++++++++++++- schemas/result-v0.1.schema.json | 4 +- src/wamprobe/adapters/baselines.py | 35 + src/wamprobe/cli.py | 8 +- src/wamprobe/evaluation.py | 36 +- src/wamprobe/metrics.py | 129 ++ src/wamprobe/reporting.py | 238 ++- src/wamprobe/stats.py | 173 +++ tests/test_cli.py | 8 + tests/test_evaluation.py | 39 + tests/test_metrics.py | 57 + tests/test_statistics.py | 71 + 17 files changed, 3018 insertions(+), 98 deletions(-) create mode 100644 examples/pointmass-demo/report.html create mode 100644 src/wamprobe/stats.py create mode 100644 tests/test_metrics.py create mode 100644 tests/test_statistics.py diff --git a/README.md b/README.md index ec821ed..6aa2b27 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ The command creates: ```text runs/pointmass-demo/ ├── summary.json # versioned machine-readable results -└── report.md # human-readable metric comparison +├── report.md # human-readable metric comparison +└── report.html # standalone interactive-friendly report ``` See the committed [example report](examples/pointmass-demo/report.md) for the expected @@ -81,10 +82,11 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - typed, model-agnostic `WAMAdapter` and `ActionPredictorAdapter` protocols; - capability manifest data model and JSON Schema; - paired PointMass-2D counterfactual interventions; -- oracle, copy-last-frame, wrong-direction, and action-agnostic baselines; -- Action Dependence, Counterfactual Direction Accuracy, No-op Stability, state ADE, - and Top-1 Regret; -- JSON and Markdown reports; +- oracle, noisy-linear, copy-last-frame, wrong-direction, and action-agnostic baselines; +- Action Dependence with a within-context permutation null, Counterfactual Direction + Accuracy, No-op Stability, state ADE/FDE, and Top-1 Regret; +- context-block bootstrap intervals and exact-context paired model comparisons; +- versioned JSON plus Markdown and standalone HTML reports; - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - typed robot observations, action predictions, deterministic prediction artifacts, and a pinned StarWAM/LIBERO smoke runner; @@ -96,11 +98,11 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo The next milestones are: -1. context-block bootstrap confidence intervals; -2. a small image-rendered BlockPush benchmark; -3. the versioned intervention dataset loader; -4. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; -5. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference. +1. a small image-rendered BlockPush benchmark; +2. the versioned intervention dataset loader; +3. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; +4. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; +5. add a GPU nightly workflow for the isolated real-model integration. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 75ad2ea..7649d1e 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -15,11 +15,16 @@ The suite creates multiple actions from each shared initial state. The goal is p the final state of one directional action, so the simulator's optimal candidate is known exactly. +The output directory contains `summary.json`, `report.md`, and a standalone +`report.html`. The JSON preserves per-context values, descriptive statistics, +context-block bootstrap intervals, and paired model differences. + ## Built-in baselines | Baseline | Behavior | Diagnostic purpose | |---|---|---| | `oracle-pointmass` | Uses exact dynamics | Expected upper bound | +| `noisy-linear` | Applies the action plus seeded transition noise | Checks smooth accuracy degradation | | `copy-last-frame` | Predicts no movement | Detects action ignorance | | `wrong-direction` | Applies the negative action | Separates dependence from correctness | | `action-agnostic` | Moves to the goal for every action | Mimics a plausible success prior | @@ -28,11 +33,16 @@ exactly. - **Action Dependence** measures endpoint separation across action branches. It does not establish that the direction is correct. +- **Permutation Effect** correlates true and predicted endpoint-distance geometry, then + standardizes it against action-label permutations within the same context. Its p-value + is reported separately; neither value replaces direction or dynamics metrics. - **Counterfactual Direction Accuracy** is mean cosine alignment between predicted and true non-noop displacement. `1` is aligned, `0` is uninformative, and `-1` is reversed. - **No-op Stability** is the fraction of no-op predictions whose final state remains at the shared initial position. - **State ADE** is average Euclidean state error over branches and time; lower is better. +- **State FDE** is final-state Euclidean error averaged over every branch, including the + no-op branch; lower is better. - **Top-1 Regret** measures how much true return is lost by selecting the candidate that looks best under the predicted future; lower is better. @@ -40,6 +50,10 @@ WAMProbe intentionally reports a metric profile, not a composite score. A model action-dependent but physically wrong, or visually accurate but useless for candidate selection. +Uncertainty is computed at the context level. Each bootstrap draw resamples whole shared +initial states, never individual action branches or adjacent frames. Paired comparisons +align exact context IDs and report the left-minus-right difference with a 95% interval. + ## Validate real-model artifacts The first real adapter uses pinned StarWAM and Wan2.2 files stored outside Git. Follow diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 4c13f90..92f69b6 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1036,6 +1036,12 @@ CLI 验收要求： ### Phase 2：Toy Benchmark 与核心指标（第 5–7 周） +当前进度（2026-07-15）：PointMass-2D 已具有 5 个 reference baselines；ADS 的 context +内 action-permutation null、CDA、NOS、state ADE/FDE、Top-1 Regret 已实现。每项指标先在 +context 层计算，再做 context-block bootstrap 和严格 context-ID 对齐的 paired comparison； +示例同时生成 versioned JSON、Markdown 和 standalone HTML。Phase 2 尚缺 #10 的 +BlockPush/Gripper-Catch 与 CRC，不能据此宣称整个 Phase 2 完成。 + 任务： - 实现 PointMass、BlockPush、Gripper-Catch； @@ -1129,6 +1135,10 @@ v0.1 发布门槛： | #19 | Add GPU nightly smoke workflow | P1 | 2 天 | #18 | | #20 | Write benchmark card and v0.1 reproducibility guide | P0 | 4 天 | 全部 | +代码落地状态（2026-07-15）：本轮补齐并验证 #8、#11、#13、#15、#16；此前已完成 +#1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#6 和 #18 已有可运行的 StarWAM +prediction artifact/adapter 切片，但更通用的 cache 与真实 WAM 反事实控制评测仍需继续扩展。 + 建议 labels： ```text @@ -1244,30 +1254,30 @@ priority:p1 ### 功能 - [ ] `pip install wamprobe` 可以安装核心包； -- [ ] `wamprobe doctor` 能检查 capability/benchmark 兼容性； -- [ ] Toy benchmark 可在 CPU 完整运行； -- [ ] 至少 4 个 reference baselines； -- [ ] 至少 1 个真实 WAM adapter； -- [ ] 至少 6 个核心指标； +- [x] `wamprobe doctor` 能检查 capability/benchmark 兼容性； +- [x] Toy benchmark 可在 CPU 完整运行； +- [x] 至少 4 个 reference baselines； +- [x] 至少 1 个真实 WAM adapter； +- [x] 至少 6 个核心指标； - [ ] prediction cache 可恢复中断运行； - [ ] JSONL/JSON/HTML 输出齐全； -- [ ] paired comparison 和 bootstrap CI 可用。 +- [x] paired comparison 和 bootstrap CI 可用。 ### 研究有效性 -- [ ] oracle 与错误 baseline 能被稳定区分； +- [x] oracle 与错误 baseline 能被稳定区分； - [ ] action shuffle/mask 产生预期指标下降； -- [ ] 噪声增加时准确性指标总体退化； +- [x] 噪声增加时准确性指标总体退化； - [ ] 至少一个 causal/ranking 指标与 simulator return 相关； - [ ] 报告传统视频指标与控制指标的差异； - [ ] 公开所有主要失败率和 skipped metrics。 ### 工程质量 -- [ ] 核心 coverage ≥ 85%； -- [ ] schema 有版本和兼容策略； +- [x] 核心 coverage ≥ 85%； +- [x] schema 有版本和兼容策略； - [ ] 所有 release artifact 可追溯； -- [ ] README 15 分钟 quickstart 可执行； +- [x] README 15 分钟 quickstart 可执行； - [ ] benchmark/model/metric cards 完整； - [ ] 至少一个外部用户复现 smoke test。 diff --git a/examples/pointmass-demo/report.html b/examples/pointmass-demo/report.html new file mode 100644 index 0000000..b2f35f3 --- /dev/null +++ b/examples/pointmass-demo/report.html @@ -0,0 +1,53 @@ + + + + + + WAMProbe PointMass-2D Demo + + + +

WAMProbe PointMass-2D Demo

+

Action-aware and deliberately broken baselines are shown as a metric + profile. No composite score is produced.

+
+

Metric profile

+
+ + +
ModelAction DependencePermutation EffectPermutation p-value ↓Counterfactual DirectionNo-op StabilityState ADE ↓State FDE ↓Top-1 Regret ↓
oracle-pointmass1.00002.85420.06671.00001.00000.00000.00000.0000
noisy-linear0.98862.78930.06670.99720.00000.07360.09270.0000
copy-last-frame0.00000.00001.00000.00001.00000.50000.80001.0000
wrong-direction1.00002.85420.0667-1.00001.00001.00001.60002.0000
action-agnostic0.00000.00001.00000.00000.00000.72861.16571.0000
+
+
+

Context-block 95% confidence intervals

+

Whole contexts are resampled; correlated branches and frames are never + treated as independent observations.

+
+ + + +
ModelMetricMean95% CIMedianStdp05 / p95
oracle-pointmassAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
oracle-pointmassPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
oracle-pointmassCounterfactual Direction1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassState ADE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassState FDE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearAction Dependence0.9886[0.9787, 0.9966]0.99670.01490.9625 / 1.0000
noisy-linearPermutation Effect2.7893[2.7724, 2.8072]2.78650.03012.7476 / 2.8319
noisy-linearPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
noisy-linearCounterfactual Direction0.9972[0.9965, 0.9979]0.99730.00130.9951 / 0.9991
noisy-linearNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearState ADE ↓0.0736[0.0667, 0.0814]0.07360.01280.0596 / 0.0929
noisy-linearState FDE ↓0.0927[0.0825, 0.1027]0.08950.01840.0700 / 0.1229
noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameState ADE ↓0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameState FDE ↓0.8000[0.8000, 0.8000]0.80000.00000.8000 / 0.8000
copy-last-frameTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
wrong-directionPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
wrong-directionCounterfactual Direction-1.0000[-1.0000, -1.0000]-1.00000.0000-1.0000 / -1.0000
wrong-directionNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState ADE ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState FDE ↓1.6000[1.6000, 1.6000]1.60000.00001.6000 / 1.6000
wrong-directionTop-1 Regret ↓2.0000[2.0000, 2.0000]2.00000.00002.0000 / 2.0000
action-agnosticAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticState ADE ↓0.7286[0.7286, 0.7286]0.72860.00000.7286 / 0.7286
action-agnosticState FDE ↓1.1657[1.1657, 1.1657]1.16570.00001.1657 / 1.1657
action-agnosticTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
+
+
+

Paired model differences

+

Each value is left minus right, aligned by exact context ID.

+
+ + + +
ModelsMetricMean difference95% CIContexts
oracle-pointmass − noisy-linearAction Dependence0.0114[0.0035, 0.0215]12
oracle-pointmass − noisy-linearPermutation Effect0.0649[0.0487, 0.0818]12
oracle-pointmass − noisy-linearPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − noisy-linearCounterfactual Direction0.0028[0.0020, 0.0035]12
oracle-pointmass − noisy-linearNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − noisy-linearState ADE ↓-0.0736[-0.0810, -0.0667]12
oracle-pointmass − noisy-linearState FDE ↓-0.0927[-0.1032, -0.0831]12
oracle-pointmass − noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-framePermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − copy-last-framePermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − copy-last-frameCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-frameNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameState ADE ↓-0.5000[-0.5000, -0.5000]12
oracle-pointmass − copy-last-frameState FDE ↓-0.8000[-0.8000, -0.8000]12
oracle-pointmass − copy-last-frameTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionAction Dependence0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation Effect0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionCounterfactual Direction2.0000[2.0000, 2.0000]12
oracle-pointmass − wrong-directionNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionState ADE ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionState FDE ↓-1.6000[-1.6000, -1.6000]12
oracle-pointmass − wrong-directionTop-1 Regret ↓-2.0000[-2.0000, -2.0000]12
oracle-pointmass − action-agnosticAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticPermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − action-agnosticPermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − action-agnosticCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticState ADE ↓-0.7286[-0.7286, -0.7286]12
oracle-pointmass − action-agnosticState FDE ↓-1.1657[-1.1657, -1.1657]12
oracle-pointmass − action-agnosticTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
+
+ + diff --git a/examples/pointmass-demo/report.md b/examples/pointmass-demo/report.md index ec3a2ad..4a1d2c0 100644 --- a/examples/pointmass-demo/report.md +++ b/examples/pointmass-demo/report.md @@ -2,13 +2,100 @@ This report compares action-aware and deliberately broken reference baselines. -| Model | Action Dependence | Counterfactual Direction | No-op Stability | State ADE ↓ | Top-1 Regret ↓ | -|---|---|---|---|---|---| -| oracle-pointmass | 1.0000 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | -| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.5000 | 1.0000 | -| wrong-direction | 1.0000 | -1.0000 | 1.0000 | 1.0000 | 2.0000 | -| action-agnostic | 0.0000 | 0.0000 | 0.0000 | 0.7286 | 1.0000 | +| Model | Action Dependence | Permutation Effect | Permutation p-value ↓ | Counterfactual Direction | No-op Stability | State ADE ↓ | State FDE ↓ | Top-1 Regret ↓ | +|---|---|---|---|---|---|---|---|---| +| oracle-pointmass | 1.0000 | 2.8542 | 0.0667 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 0.0000 | +| noisy-linear | 0.9886 | 2.7893 | 0.0667 | 0.9972 | 0.0000 | 0.0736 | 0.0927 | 0.0000 | +| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 1.0000 | 0.5000 | 0.8000 | 1.0000 | +| wrong-direction | 1.0000 | 2.8542 | 0.0667 | -1.0000 | 1.0000 | 1.0000 | 1.6000 | 2.0000 | +| action-agnostic | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 0.0000 | 0.7286 | 1.1657 | 1.0000 | + +## Context-block 95% confidence intervals + +Intervals resample whole contexts, not correlated action branches or frames. + +| Model | Metric | Mean | 95% CI | Median | Std | +|---|---|---:|---:|---:|---:| +| oracle-pointmass | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 2.8542 | 0.0000 | +| oracle-pointmass | Permutation p-value ↓ | 0.0667 | [0.0667, 0.0667] | 0.0667 | 0.0000 | +| oracle-pointmass | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | State ADE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-pointmass | State FDE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-pointmass | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-linear | Action Dependence | 0.9886 | [0.9787, 0.9966] | 0.9967 | 0.0149 | +| noisy-linear | Permutation Effect | 2.7893 | [2.7724, 2.8072] | 2.7865 | 0.0301 | +| noisy-linear | Permutation p-value ↓ | 0.0667 | [0.0667, 0.0667] | 0.0667 | 0.0000 | +| noisy-linear | Counterfactual Direction | 0.9972 | [0.9965, 0.9979] | 0.9973 | 0.0013 | +| noisy-linear | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-linear | State ADE ↓ | 0.0736 | [0.0667, 0.0814] | 0.0736 | 0.0128 | +| noisy-linear | State FDE ↓ | 0.0927 | [0.0825, 0.1027] | 0.0895 | 0.0184 | +| noisy-linear | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| copy-last-frame | Counterfactual Direction | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| copy-last-frame | State ADE ↓ | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| copy-last-frame | State FDE ↓ | 0.8000 | [0.8000, 0.8000] | 0.8000 | 0.0000 | +| copy-last-frame | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 2.8542 | 0.0000 | +| wrong-direction | Permutation p-value ↓ | 0.0667 | [0.0667, 0.0667] | 0.0667 | 0.0000 | +| wrong-direction | Counterfactual Direction | -1.0000 | [-1.0000, -1.0000] | -1.0000 | 0.0000 | +| wrong-direction | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | State ADE ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | State FDE ↓ | 1.6000 | [1.6000, 1.6000] | 1.6000 | 0.0000 | +| wrong-direction | Top-1 Regret ↓ | 2.0000 | [2.0000, 2.0000] | 2.0000 | 0.0000 | +| action-agnostic | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| action-agnostic | Counterfactual Direction | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | State ADE ↓ | 0.7286 | [0.7286, 0.7286] | 0.7286 | 0.0000 | +| action-agnostic | State FDE ↓ | 1.1657 | [1.1657, 1.1657] | 1.1657 | 0.0000 | +| action-agnostic | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | + +## Paired model differences + +Differences are left minus right over exactly aligned context IDs. + +| Models | Metric | Mean difference | 95% CI | Contexts | +|---|---|---:|---:|---:| +| oracle-pointmass − noisy-linear | Action Dependence | 0.0114 | [0.0035, 0.0215] | 12 | +| oracle-pointmass − noisy-linear | Permutation Effect | 0.0649 | [0.0487, 0.0818] | 12 | +| oracle-pointmass − noisy-linear | Permutation p-value ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − noisy-linear | Counterfactual Direction | 0.0028 | [0.0020, 0.0035] | 12 | +| oracle-pointmass − noisy-linear | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − noisy-linear | State ADE ↓ | -0.0736 | [-0.0810, -0.0667] | 12 | +| oracle-pointmass − noisy-linear | State FDE ↓ | -0.0927 | [-0.1032, -0.0831] | 12 | +| oracle-pointmass − noisy-linear | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − copy-last-frame | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − copy-last-frame | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 12 | +| oracle-pointmass − copy-last-frame | Permutation p-value ↓ | -0.9333 | [-0.9333, -0.9333] | 12 | +| oracle-pointmass − copy-last-frame | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − copy-last-frame | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − copy-last-frame | State ADE ↓ | -0.5000 | [-0.5000, -0.5000] | 12 | +| oracle-pointmass − copy-last-frame | State FDE ↓ | -0.8000 | [-0.8000, -0.8000] | 12 | +| oracle-pointmass − copy-last-frame | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | +| oracle-pointmass − wrong-direction | Action Dependence | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − wrong-direction | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − wrong-direction | Permutation p-value ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − wrong-direction | Counterfactual Direction | 2.0000 | [2.0000, 2.0000] | 12 | +| oracle-pointmass − wrong-direction | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − wrong-direction | State ADE ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | +| oracle-pointmass − wrong-direction | State FDE ↓ | -1.6000 | [-1.6000, -1.6000] | 12 | +| oracle-pointmass − wrong-direction | Top-1 Regret ↓ | -2.0000 | [-2.0000, -2.0000] | 12 | +| oracle-pointmass − action-agnostic | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − action-agnostic | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 12 | +| oracle-pointmass − action-agnostic | Permutation p-value ↓ | -0.9333 | [-0.9333, -0.9333] | 12 | +| oracle-pointmass − action-agnostic | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − action-agnostic | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − action-agnostic | State ADE ↓ | -0.7286 | [-0.7286, -0.7286] | 12 | +| oracle-pointmass − action-agnostic | State FDE ↓ | -1.1657 | [-1.1657, -1.1657] | 12 | +| oracle-pointmass − action-agnostic | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | A high Action Dependence score is not sufficient: the wrong-direction baseline -depends on the action but fails Counterfactual Direction Accuracy. This is why -WAMProbe reports a metric profile instead of a single aggregate score. +depends on the action but fails Counterfactual Direction Accuracy. WAMProbe +therefore reports a metric profile rather than a single composite score. diff --git a/examples/pointmass-demo/summary.json b/examples/pointmass-demo/summary.json index 5b79386..d4d1e1c 100644 --- a/examples/pointmass-demo/summary.json +++ b/examples/pointmass-demo/summary.json @@ -9,9 +9,12 @@ "contexts": 12, "metrics": { "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 1.0, "noop_stability": 1.0, "state_ade": 0.0, + "state_fde": 0.0, "top1_regret": 0.0 }, "context_results": [ @@ -19,73 +22,403 @@ "context_id": "context-0000", "selected_action": "right", "optimal_action": "right", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0001", "selected_action": "left", "optimal_action": "left", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0002", "selected_action": "up", "optimal_action": "up", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0003", "selected_action": "down", "optimal_action": "down", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0004", "selected_action": "right", "optimal_action": "right", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0005", "selected_action": "left", "optimal_action": "left", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0006", "selected_action": "up", "optimal_action": "up", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0007", "selected_action": "down", "optimal_action": "down", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0008", "selected_action": "right", "optimal_action": "right", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0009", "selected_action": "left", "optimal_action": "left", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0010", "selected_action": "up", "optimal_action": "up", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } }, { "context_id": "context-0011", "selected_action": "down", "optimal_action": "down", - "top1_regret": 0.0 + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "top1_regret": 0.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "noisy-linear", + "benchmark": "pointmass-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.9885662916166548, + "action_dependence_permutation_effect": 2.789304160469617, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9972383636589002, + "noop_stability": 0.0, + "state_ade": 0.07364777227731902, + "state_fde": 0.0927268500618552, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "context-0000", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.7933632915206936, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9967708560731011, + "noop_stability": 0.0, + "state_ade": 0.10417042880012942, + "state_fde": 0.11595533486335921, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0001", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.7567719197334517, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9952079228988657, + "noop_stability": 0.0, + "state_ade": 0.08371865860289475, + "state_fde": 0.13140609465866257, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0002", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.766645072742772, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9990887394827166, + "noop_stability": 0.0, + "state_ade": 0.07170250725169272, + "state_fde": 0.10261584827837915, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0003", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9779592797063741, + "action_dependence_permutation_effect": 2.803768627798615, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9976979471160832, + "noop_stability": 0.0, + "state_ade": 0.05995406721844264, + "state_fde": 0.07761855384937649, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0004", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9933416880608462, + "action_dependence_permutation_effect": 2.812760554371386, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9979549505094192, + "noop_stability": 0.0, + "state_ade": 0.05918016195121363, + "state_fde": 0.08397848391634824, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0005", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9716774799252708, + "action_dependence_permutation_effect": 2.8271977556861567, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9991626197702563, + "noop_stability": 0.0, + "state_ade": 0.061375348243354264, + "state_fde": 0.07705294381294754, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0006", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9807940247100756, + "action_dependence_permutation_effect": 2.8376646772706624, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9973119928159295, + "noop_stability": 0.0, + "state_ade": 0.06069688510282225, + "state_fde": 0.061287791426493245, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0007", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.7795816644304012, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9950554531415062, + "noop_stability": 0.0, + "state_ade": 0.08021175873604955, + "state_fde": 0.08043799051624254, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0008", + "selected_action": "right", + "optimal_action": "right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.746777303186475, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9961388733218819, + "noop_stability": 0.0, + "state_ade": 0.07540788707466936, + "state_fde": 0.09193913292916703, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0009", + "selected_action": "left", + "optimal_action": "left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9512321250999373, + "action_dependence_permutation_effect": 2.8212670000102467, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9979961255644422, + "noop_stability": 0.0, + "state_ade": 0.08324185274139459, + "state_fde": 0.09664726483255356, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0010", + "selected_action": "up", + "optimal_action": "up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.777564510368594, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9973768635583395, + "noop_stability": 0.0, + "state_ade": 0.066462523353616, + "state_fde": 0.08712314154486435, + "top1_regret": 0.0 + } + }, + { + "context_id": "context-0011", + "selected_action": "down", + "optimal_action": "down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9877909018973522, + "action_dependence_permutation_effect": 2.7482875485159477, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": 0.9970980196542607, + "noop_stability": 0.0, + "state_ade": 0.07765118825154912, + "state_fde": 0.10665962011386854, + "top1_regret": 0.0 + } } ] }, @@ -96,9 +429,12 @@ "contexts": 12, "metrics": { "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, "counterfactual_direction_accuracy": 0.0, "noop_stability": 1.0, "state_ade": 0.5, + "state_fde": 0.8000000000000002, "top1_regret": 1.0 }, "context_results": [ @@ -106,73 +442,193 @@ "context_id": "context-0000", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0001", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0002", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0003", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0004", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0005", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0006", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0007", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0008", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0009", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0010", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } }, { "context_id": "context-0011", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.5, + "state_fde": 0.8, + "top1_regret": 1.0 + } } ] }, @@ -183,9 +639,12 @@ "contexts": 12, "metrics": { "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": -1.0, "noop_stability": 1.0, "state_ade": 1.0, + "state_fde": 1.6000000000000003, "top1_regret": 2.0 }, "context_results": [ @@ -193,73 +652,193 @@ "context_id": "context-0000", "selected_action": "left", "optimal_action": "right", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0001", "selected_action": "right", "optimal_action": "left", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0002", "selected_action": "down", "optimal_action": "up", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0003", "selected_action": "up", "optimal_action": "down", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0004", "selected_action": "left", "optimal_action": "right", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0005", "selected_action": "right", "optimal_action": "left", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0006", "selected_action": "down", "optimal_action": "up", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0007", "selected_action": "up", "optimal_action": "down", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0008", "selected_action": "left", "optimal_action": "right", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0009", "selected_action": "right", "optimal_action": "left", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0010", "selected_action": "down", "optimal_action": "up", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } }, { "context_id": "context-0011", "selected_action": "up", "optimal_action": "down", - "top1_regret": 2.0 + "top1_regret": 2.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.8542329002098215, + "action_dependence_permutation_p_value": 0.06666666666666667, + "counterfactual_direction_accuracy": -1.0, + "noop_stability": 1.0, + "state_ade": 1.0, + "state_fde": 1.6, + "top1_regret": 2.0 + } } ] }, @@ -270,9 +849,12 @@ "contexts": 12, "metrics": { "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, "counterfactual_direction_accuracy": 0.0, "noop_stability": 0.0, - "state_ade": 0.7285533905932737, + "state_ade": 0.7285533905932736, + "state_fde": 1.1656854249492379, "top1_regret": 1.0 }, "context_results": [ @@ -280,75 +862,1489 @@ "context_id": "context-0000", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0001", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0002", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0003", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0004", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0005", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0006", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0007", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0008", "selected_action": "noop", "optimal_action": "right", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0009", "selected_action": "noop", "optimal_action": "left", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0010", "selected_action": "noop", "optimal_action": "up", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } }, { "context_id": "context-0011", "selected_action": "noop", "optimal_action": "down", - "top1_regret": 1.0 + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.7285533905932737, + "state_fde": 1.1656854249492379, + "top1_regret": 1.0 + } } ] } + ], + "statistics": { + "oracle-pointmass": { + "action_dependence": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.8542329002098215, + "median": 2.8542329002098215, + "standard_deviation": 0.0, + "quantiles": { + "p05": 2.8542329002098215, + "p25": 2.8542329002098215, + "p75": 2.8542329002098215, + "p95": 2.8542329002098215 + }, + "confidence_interval": { + "estimate": 2.8542329002098215, + "lower": 2.8542329002098215, + "upper": 2.8542329002098215, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.06666666666666667, + "p25": 0.06666666666666667, + "p75": 0.06666666666666667, + "p95": 0.06666666666666667 + }, + "confidence_interval": { + "estimate": 0.06666666666666667, + "lower": 0.06666666666666667, + "upper": 0.06666666666666667, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "noisy-linear": { + "action_dependence": { + "mean": 0.9885662916166548, + "median": 0.996670844030423, + "standard_deviation": 0.014887837735226563, + "quantiles": { + "p05": 0.9624770702538707, + "p25": 0.9800853384591501, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.9885662916166548, + "lower": 0.9786710395255761, + "upper": 0.9966251791249476, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.789304160469617, + "median": 2.7864724779755474, + "standard_deviation": 0.030093202906788793, + "quantiles": { + "p05": 2.747607938117685, + "p25": 2.7641767844904415, + "p75": 2.814887165781101, + "p95": 2.831907870399184 + }, + "confidence_interval": { + "estimate": 2.789304160469617, + "lower": 2.7723705333948603, + "upper": 2.8072052811909356, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.06666666666666667, + "p25": 0.06666666666666667, + "p75": 0.06666666666666667, + "p95": 0.06666666666666667 + }, + "confidence_interval": { + "estimate": 0.06666666666666667, + "lower": 0.06666666666666667, + "upper": 0.06666666666666667, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.9972383636589002, + "median": 0.9973444281871345, + "standard_deviation": 0.001251169430002784, + "quantiles": { + "p05": 0.995139311508054, + "p25": 0.9966128603852964, + "p75": 0.9979652442731749, + "p95": 0.9991219856121094 + }, + "confidence_interval": { + "estimate": 0.9972383636589002, + "lower": 0.9965093884731476, + "upper": 0.997936868274387, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.07364777227731902, + "median": 0.07355519716318104, + "standard_deviation": 0.012773318511371218, + "quantiles": { + "p05": 0.059605809848189584, + "p25": 0.06120573245822126, + "p75": 0.08096928223738581, + "p95": 0.09292195519165034 + }, + "confidence_interval": { + "estimate": 0.07364777227731902, + "lower": 0.06667487718584154, + "upper": 0.08139988454758483, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.0927268500618552, + "median": 0.0895311372370157, + "standard_deviation": 0.01840482018786067, + "quantiles": { + "p05": 0.06995862523904312, + "p25": 0.07973313134952603, + "p75": 0.10362679123725149, + "p95": 0.12290817677124571 + }, + "confidence_interval": { + "estimate": 0.0927268500618552, + "lower": 0.08250106786937707, + "upper": 0.10267687579941069, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "copy-last-frame": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.8000000000000002, + "median": 0.8, + "standard_deviation": 1.1102230246251565e-16, + "quantiles": { + "p05": 0.8, + "p25": 0.8, + "p75": 0.8, + "p95": 0.8 + }, + "confidence_interval": { + "estimate": 0.8000000000000002, + "lower": 0.8000000000000002, + "upper": 0.8000000000000002, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "wrong-direction": { + "action_dependence": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.8542329002098215, + "median": 2.8542329002098215, + "standard_deviation": 0.0, + "quantiles": { + "p05": 2.8542329002098215, + "p25": 2.8542329002098215, + "p75": 2.8542329002098215, + "p95": 2.8542329002098215 + }, + "confidence_interval": { + "estimate": 2.8542329002098215, + "lower": 2.8542329002098215, + "upper": 2.8542329002098215, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.06666666666666667, + "median": 0.06666666666666667, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.06666666666666667, + "p25": 0.06666666666666667, + "p75": 0.06666666666666667, + "p95": 0.06666666666666667 + }, + "confidence_interval": { + "estimate": 0.06666666666666667, + "lower": 0.06666666666666667, + "upper": 0.06666666666666667, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": -1.0, + "median": -1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": -1.0, + "p25": -1.0, + "p75": -1.0, + "p95": -1.0 + }, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 1.6000000000000003, + "median": 1.6, + "standard_deviation": 2.220446049250313e-16, + "quantiles": { + "p05": 1.6, + "p25": 1.6, + "p75": 1.6, + "p95": 1.6 + }, + "confidence_interval": { + "estimate": 1.6000000000000003, + "lower": 1.6000000000000003, + "upper": 1.6000000000000003, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 2.0, + "median": 2.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 2.0, + "p25": 2.0, + "p75": 2.0, + "p95": 2.0 + }, + "confidence_interval": { + "estimate": 2.0, + "lower": 2.0, + "upper": 2.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "action-agnostic": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.7285533905932736, + "median": 0.7285533905932737, + "standard_deviation": 1.1102230246251565e-16, + "quantiles": { + "p05": 0.7285533905932737, + "p25": 0.7285533905932737, + "p75": 0.7285533905932737, + "p95": 0.7285533905932737 + }, + "confidence_interval": { + "estimate": 0.7285533905932736, + "lower": 0.7285533905932736, + "upper": 0.7285533905932736, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 1.1656854249492379, + "median": 1.1656854249492379, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.1656854249492379, + "p25": 1.1656854249492379, + "p75": 1.1656854249492379, + "p95": 1.1656854249492379 + }, + "confidence_interval": { + "estimate": 1.1656854249492379, + "lower": 1.1656854249492379, + "upper": 1.1656854249492379, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + } + }, + "paired_comparisons": [ + { + "metric": "action_dependence", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.011433708383345323, + "confidence_interval": { + "estimate": 0.011433708383345323, + "lower": 0.0035013046759239386, + "upper": 0.02153456131879754, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.06492873974020463, + "confidence_interval": { + "estimate": 0.06492873974020463, + "lower": 0.04872470548355673, + "upper": 0.08179704843704588, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.0027616363410998177, + "confidence_interval": { + "estimate": 0.0027616363410998177, + "lower": 0.0020221098691746453, + "upper": 0.0034724181651196485, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": -0.07364777227731902, + "confidence_interval": { + "estimate": -0.07364777227731902, + "lower": -0.0809928859181078, + "upper": -0.06674810105199652, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": -0.0927268500618552, + "confidence_interval": { + "estimate": -0.0927268500618552, + "lower": -0.10318039761689358, + "upper": -0.08305408918838206, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 2.8542329002098215, + "confidence_interval": { + "estimate": 2.8542329002098215, + "lower": 2.8542329002098215, + "upper": 2.8542329002098215, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.9333333333333332, + "confidence_interval": { + "estimate": -0.9333333333333332, + "lower": -0.9333333333333332, + "upper": -0.9333333333333332, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.5, + "confidence_interval": { + "estimate": -0.5, + "lower": -0.5, + "upper": -0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.8000000000000002, + "confidence_interval": { + "estimate": -0.8000000000000002, + "lower": -0.8000000000000002, + "upper": -0.8000000000000002, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 2.0, + "confidence_interval": { + "estimate": 2.0, + "lower": 2.0, + "upper": 2.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -1.6000000000000003, + "confidence_interval": { + "estimate": -1.6000000000000003, + "lower": -1.6000000000000003, + "upper": -1.6000000000000003, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -2.0, + "confidence_interval": { + "estimate": -2.0, + "lower": -2.0, + "upper": -2.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 2.8542329002098215, + "confidence_interval": { + "estimate": 2.8542329002098215, + "lower": 2.8542329002098215, + "upper": 2.8542329002098215, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.9333333333333332, + "confidence_interval": { + "estimate": -0.9333333333333332, + "lower": -0.9333333333333332, + "upper": -0.9333333333333332, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.7285533905932736, + "confidence_interval": { + "estimate": -0.7285533905932736, + "lower": -0.7285533905932736, + "upper": -0.7285533905932736, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -1.1656854249492379, + "confidence_interval": { + "estimate": -1.1656854249492379, + "lower": -1.1656854249492379, + "upper": -1.1656854249492379, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + } ] } diff --git a/schemas/result-v0.1.schema.json b/schemas/result-v0.1.schema.json index 15bcae3..d406494 100644 --- a/schemas/result-v0.1.schema.json +++ b/schemas/result-v0.1.schema.json @@ -32,6 +32,8 @@ "context_results": {"type": "array"} } } - } + }, + "statistics": {"type": "object"}, + "paired_comparisons": {"type": "array"} } } diff --git a/src/wamprobe/adapters/baselines.py b/src/wamprobe/adapters/baselines.py index 2420774..ff029e1 100644 --- a/src/wamprobe/adapters/baselines.py +++ b/src/wamprobe/adapters/baselines.py @@ -2,6 +2,9 @@ from __future__ import annotations +import hashlib +from random import Random + from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities from wamprobe.api.types import Action2D, Context2D, Trajectory2D, Vec2 from wamprobe.benchmarks.pointmass import PointMass2D @@ -83,6 +86,38 @@ def predict_future( return self._benchmark.rollout(context, reversed_action) +class NoisyLinearAdapter(_StateBaseline): + """Action-aware linear dynamics with deterministic Gaussian transition noise.""" + + model_id = "noisy-linear" + + def __init__(self, *, noise_std: float = 0.04) -> None: + if noise_std <= 0: + raise ValueError("noise_std must be positive") + self._noise_std = noise_std + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + if horizon <= 0: + raise ValueError("horizon must be positive") + seed_material = f"{seed}\0{context.context_id}\0{action.name}".encode() + local_seed = int.from_bytes(hashlib.sha256(seed_material).digest()[:8], "big") + rng = Random(local_seed) + position = context.position + states: list[Vec2] = [] + for _ in range(horizon): + noise = Vec2(rng.gauss(0.0, self._noise_std), rng.gauss(0.0, self._noise_std)) + position = position + action.delta + noise + states.append(position) + return Trajectory2D(context.context_id, action.name, tuple(states)) + + class ActionAgnosticAdapter(_StateBaseline): """Plausible-looking baseline that moves to the goal for every input action.""" diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index 29e2c29..c21e7bb 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -11,6 +11,7 @@ from wamprobe.adapters.baselines import ( ActionAgnosticAdapter, CopyLastFrameAdapter, + NoisyLinearAdapter, OraclePointMassAdapter, WrongDirectionAdapter, ) @@ -35,7 +36,7 @@ def _build_parser() -> argparse.ArgumentParser: "--output", type=Path, default=Path("runs/pointmass-demo"), - help="directory for summary.json and report.md", + help="directory for summary.json, report.md, and report.html", ) doctor = subparsers.add_parser("doctor", help="validate pinned local model artifacts") doctor.add_argument( @@ -64,19 +65,22 @@ def _run_demo(args: argparse.Namespace) -> int: suite = benchmark.make_suite(contexts=args.contexts, seed=args.seed) adapters: list[WAMAdapter] = [ OraclePointMassAdapter(benchmark), + NoisyLinearAdapter(), CopyLastFrameAdapter(), WrongDirectionAdapter(benchmark), ActionAgnosticAdapter(), ] results = [evaluate(adapter, benchmark, suite, seed=args.seed) for adapter in adapters] - summary_path, report_path = write_reports( + summary_path, report_path, html_path = write_reports( args.output, benchmark=benchmark.benchmark_id, results=results, + seed=args.seed, ) print("WAMProbe demo complete") print(f"JSON: {summary_path}") print(f"Markdown: {report_path}") + print(f"HTML: {html_path}") return 0 diff --git a/src/wamprobe/evaluation.py b/src/wamprobe/evaluation.py index 97e7c87..c8dee1e 100644 --- a/src/wamprobe/evaluation.py +++ b/src/wamprobe/evaluation.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib from dataclasses import asdict, dataclass from typing import Any @@ -10,9 +11,11 @@ from wamprobe.benchmarks.pointmass import PointMass2D from wamprobe.metrics import ( action_dependence, + action_dependence_permutation_test, counterfactual_direction_accuracy, noop_stability, state_ade, + state_fde, ) @@ -24,6 +27,7 @@ class ContextResult: selected_action: str optimal_action: str top1_regret: float + metrics: dict[str, float] @dataclass(frozen=True, slots=True) @@ -61,7 +65,6 @@ def evaluate( if suite.benchmark_id != benchmark.benchmark_id: raise ValueError("suite and benchmark IDs must match") - grouped_samples: list[list[tuple[Context2D, Trajectory2D, Trajectory2D]]] = [] context_results: list[ContextResult] = [] for group in suite.groups: samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]] = [] @@ -86,23 +89,38 @@ def evaluate( selected = max(predicted_scores, key=predicted_scores.__getitem__) optimal = max(true_scores, key=true_scores.__getitem__) regret = true_scores[optimal] - true_scores[selected] + permutation_seed = int.from_bytes( + hashlib.sha256(f"{seed}\0{group.context.context_id}".encode()).digest()[:8], + "big", + ) + permutation_result = action_dependence_permutation_test( + samples, + permutations=128, + seed=permutation_seed, + ) + context_metrics = { + "action_dependence": action_dependence([samples]), + "action_dependence_permutation_effect": permutation_result.effect_size, + "action_dependence_permutation_p_value": permutation_result.p_value, + "counterfactual_direction_accuracy": counterfactual_direction_accuracy(samples), + "noop_stability": noop_stability(samples), + "state_ade": state_ade(samples), + "state_fde": state_fde(samples), + "top1_regret": regret, + } context_results.append( ContextResult( context_id=group.context.context_id, selected_action=selected, optimal_action=optimal, top1_regret=regret, + metrics=context_metrics, ) ) - grouped_samples.append(samples) - - flat_samples = [sample for samples in grouped_samples for sample in samples] + metric_names = context_results[0].metrics.keys() metrics = { - "action_dependence": action_dependence(grouped_samples), - "counterfactual_direction_accuracy": counterfactual_direction_accuracy(flat_samples), - "noop_stability": noop_stability(flat_samples), - "state_ade": state_ade(flat_samples), - "top1_regret": sum(result.top1_regret for result in context_results) / len(context_results), + metric: sum(result.metrics[metric] for result in context_results) / len(context_results) + for metric in metric_names } adapter.close() return EvaluationResult( diff --git a/src/wamprobe/metrics.py b/src/wamprobe/metrics.py index f98649e..97648d8 100644 --- a/src/wamprobe/metrics.py +++ b/src/wamprobe/metrics.py @@ -2,7 +2,11 @@ from __future__ import annotations +from dataclasses import dataclass from itertools import combinations +from itertools import permutations as generate_permutations +from math import factorial, sqrt +from random import Random from wamprobe.api.types import Context2D, Trajectory2D, Vec2 @@ -45,6 +49,131 @@ def state_ade(samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]]) -> fl return sum(errors) / len(errors) if errors else 0.0 +def state_fde(samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]]) -> float: + """Return average final-state displacement error over all action branches.""" + + errors: list[float] = [] + for _, predicted, truth in samples: + if len(predicted.states) != len(truth.states): + raise ValueError("predicted and true trajectories must have equal horizons") + errors.append((predicted.final_state - truth.final_state).norm()) + return sum(errors) / len(errors) if errors else 0.0 + + +@dataclass(frozen=True, slots=True) +class PermutationTestResult: + """Action-dependence score relative to a context-local permutation null.""" + + observed: float + null_mean: float + null_standard_deviation: float + effect_size: float + p_value: float + permutations: int + + +def _pearson(left: list[float], right: list[float]) -> float: + if len(left) != len(right): + raise ValueError("correlation vectors must have equal lengths") + if not left: + return 0.0 + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + left_centered = [value - left_mean for value in left] + right_centered = [value - right_mean for value in right] + denominator = sqrt( + sum(value * value for value in left_centered) + * sum(value * value for value in right_centered) + ) + if denominator <= 1e-15: + return 0.0 + correlation = ( + sum( + left_value * right_value + for left_value, right_value in zip(left_centered, right_centered, strict=True) + ) + / denominator + ) + return max(-1.0, min(1.0, correlation)) + + +def _pairwise_distances(states: list[Vec2]) -> list[float]: + return [ + (states[left] - states[right]).norm() for left, right in combinations(range(len(states)), 2) + ] + + +def _permutation_indices(size: int, *, count: int, seed: int) -> list[tuple[int, ...]]: + identity = tuple(range(size)) + available = factorial(size) - 1 + if available <= count: + return [ + permutation + for permutation in generate_permutations(range(size)) + if permutation != identity + ] + + rng = Random(seed) + selected: set[tuple[int, ...]] = set() + while len(selected) < count: + candidate = list(identity) + rng.shuffle(candidate) + candidate_tuple = tuple(candidate) + if candidate_tuple != identity: + selected.add(candidate_tuple) + return sorted(selected) + + +def action_dependence_permutation_test( + samples: list[tuple[Context2D, Trajectory2D, Trajectory2D]], + *, + permutations: int = 128, + seed: int = 0, +) -> PermutationTestResult: + """Compare action/future geometry with a within-context permutation null. + + Ground-truth endpoint distances define the action-conditioned geometry. Predicted + endpoints are reassigned to action labels to estimate how much alignment would be + expected if the model's futures were unrelated to the supplied actions. + """ + + if len(samples) < 2: + raise ValueError("permutation testing requires at least two action branches") + if permutations <= 0: + raise ValueError("permutations must be positive") + context_ids = {context.context_id for context, _, _ in samples} + if len(context_ids) != 1: + raise ValueError("permutation samples must share one context") + + truth_distances = _pairwise_distances([truth.final_state for _, _, truth in samples]) + predicted_states = [predicted.final_state for _, predicted, _ in samples] + observed = _pearson(truth_distances, _pairwise_distances(predicted_states)) + + indices = _permutation_indices(len(samples), count=permutations, seed=seed) + null_scores = [ + _pearson( + truth_distances, + _pairwise_distances([predicted_states[index] for index in permutation]), + ) + for permutation in indices + ] + null_mean = sum(null_scores) / len(null_scores) + null_variance = sum((score - null_mean) ** 2 for score in null_scores) / len(null_scores) + null_standard_deviation = sqrt(null_variance) + effect_size = ( + (observed - null_mean) / null_standard_deviation if null_standard_deviation > 1e-15 else 0.0 + ) + p_value = (1 + sum(score >= observed - 1e-15 for score in null_scores)) / (1 + len(null_scores)) + return PermutationTestResult( + observed=observed, + null_mean=null_mean, + null_standard_deviation=null_standard_deviation, + effect_size=effect_size, + p_value=p_value, + permutations=len(null_scores), + ) + + def action_dependence( groups: list[list[tuple[Context2D, Trajectory2D, Trajectory2D]]], ) -> float: diff --git a/src/wamprobe/reporting.py b/src/wamprobe/reporting.py index afc96cc..7c26263 100644 --- a/src/wamprobe/reporting.py +++ b/src/wamprobe/reporting.py @@ -2,22 +2,81 @@ from __future__ import annotations +import hashlib import json +from dataclasses import asdict +from html import escape from pathlib import Path from wamprobe.evaluation import EvaluationResult +from wamprobe.stats import MetricSummary, PairedComparison, paired_metric_comparison, summarize _METRIC_COLUMNS = ( ("action_dependence", "Action Dependence"), + ("action_dependence_permutation_effect", "Permutation Effect"), + ("action_dependence_permutation_p_value", "Permutation p-value ↓"), ("counterfactual_direction_accuracy", "Counterfactual Direction"), ("noop_stability", "No-op Stability"), ("state_ade", "State ADE ↓"), + ("state_fde", "State FDE ↓"), ("top1_regret", "Top-1 Regret ↓"), ) +StatisticsByModel = dict[str, dict[str, MetricSummary]] -def render_markdown(results: list[EvaluationResult]) -> str: - """Render a compact comparison table.""" + +def _stable_seed(seed: int, *parts: str) -> int: + material = "\0".join((str(seed), *parts)).encode() + return int.from_bytes(hashlib.sha256(material).digest()[:8], "big") + + +def _build_statistics( + results: list[EvaluationResult], + *, + resamples: int, + seed: int, +) -> tuple[StatisticsByModel, list[PairedComparison]]: + statistics: StatisticsByModel = {} + for result in results: + model_statistics: dict[str, MetricSummary] = {} + for metric, _ in _METRIC_COLUMNS: + values = tuple(context.metrics[metric] for context in result.context_results) + model_statistics[metric] = summarize( + values, + resamples=resamples, + seed=_stable_seed(seed, result.model_id, metric), + ) + statistics[result.model_id] = model_statistics + + comparisons: list[PairedComparison] = [] + if len(results) > 1: + reference = next( + (result for result in results if result.model_id == "oracle-pointmass"), + results[0], + ) + for result in results: + if result is reference: + continue + for metric, _ in _METRIC_COLUMNS: + comparisons.append( + paired_metric_comparison( + reference, + result, + metric=metric, + resamples=resamples, + seed=_stable_seed(seed, reference.model_id, result.model_id, metric), + ) + ) + return statistics, comparisons + + +def render_markdown( + results: list[EvaluationResult], + *, + statistics: StatisticsByModel | None = None, + comparisons: list[PairedComparison] | None = None, +) -> str: + """Render a comparison table and context-block uncertainty estimates.""" headers = ["Model", *(label for _, label in _METRIC_COLUMNS)] lines = [ @@ -32,34 +91,197 @@ def render_markdown(results: list[EvaluationResult]) -> str: values = [result.model_id] values.extend(f"{result.metrics[key]:.4f}" for key, _ in _METRIC_COLUMNS) lines.append("| " + " | ".join(values) + " |") + + if statistics: + lines.extend( + [ + "", + "## Context-block 95% confidence intervals", + "", + "Intervals resample whole contexts, not correlated action branches or frames.", + "", + "| Model | Metric | Mean | 95% CI | Median | Std |", + "|---|---|---:|---:|---:|---:|", + ] + ) + for result in results: + for metric, label in _METRIC_COLUMNS: + summary = statistics[result.model_id][metric] + interval = summary.confidence_interval + lines.append( + f"| {result.model_id} | {label} | {summary.mean:.4f} | " + f"[{interval.lower:.4f}, {interval.upper:.4f}] | " + f"{summary.median:.4f} | {summary.standard_deviation:.4f} |" + ) + + if comparisons: + lines.extend( + [ + "", + "## Paired model differences", + "", + "Differences are left minus right over exactly aligned context IDs.", + "", + "| Models | Metric | Mean difference | 95% CI | Contexts |", + "|---|---|---:|---:|---:|", + ] + ) + labels = dict(_METRIC_COLUMNS) + for comparison in comparisons: + interval = comparison.confidence_interval + lines.append( + f"| {comparison.left_model} − {comparison.right_model} | " + f"{labels[comparison.metric]} | {comparison.mean_difference:.4f} | " + f"[{interval.lower:.4f}, {interval.upper:.4f}] | " + f"{comparison.contexts} |" + ) + lines.extend( [ "", "A high Action Dependence score is not sufficient: the wrong-direction baseline", - "depends on the action but fails Counterfactual Direction Accuracy. This is why", - "WAMProbe reports a metric profile instead of a single aggregate score.", + "depends on the action but fails Counterfactual Direction Accuracy. WAMProbe", + "therefore reports a metric profile rather than a single composite score.", "", ] ) return "\n".join(lines) +def render_html( + results: list[EvaluationResult], + *, + statistics: StatisticsByModel, + comparisons: list[PairedComparison], +) -> str: + """Render a standalone, dependency-free HTML report.""" + + metric_headers = "".join(f"{escape(label)}" for _, label in _METRIC_COLUMNS) + result_rows: list[str] = [] + for result in results: + cells = "".join(f"{result.metrics[metric]:.4f}" for metric, _ in _METRIC_COLUMNS) + result_rows.append(f"{escape(result.model_id)}{cells}") + + statistic_rows: list[str] = [] + for result in results: + for metric, label in _METRIC_COLUMNS: + summary = statistics[result.model_id][metric] + interval = summary.confidence_interval + statistic_rows.append( + "" + f"{escape(result.model_id)}" + f"{escape(label)}" + f"{summary.mean:.4f}" + f"[{interval.lower:.4f}, {interval.upper:.4f}]" + f"{summary.median:.4f}" + f"{summary.standard_deviation:.4f}" + f"{summary.quantiles['p05']:.4f} / {summary.quantiles['p95']:.4f}" + "" + ) + + labels = dict(_METRIC_COLUMNS) + comparison_rows = "".join( + "" + f"{escape(comparison.left_model)} − {escape(comparison.right_model)}" + f"{escape(labels[comparison.metric])}" + f"{comparison.mean_difference:.4f}" + f"[{comparison.confidence_interval.lower:.4f}, " + f"{comparison.confidence_interval.upper:.4f}]" + f"{comparison.contexts}" + "" + for comparison in comparisons + ) + + return f""" + + + + + WAMProbe PointMass-2D Demo + + + +

WAMProbe PointMass-2D Demo

+

Action-aware and deliberately broken baselines are shown as a metric + profile. No composite score is produced.

+
+

Metric profile

+
+ {metric_headers} + {"".join(result_rows)} +
Model
+
+
+

Context-block 95% confidence intervals

+

Whole contexts are resampled; correlated branches and frames are never + treated as independent observations.

+
+ + + {"".join(statistic_rows)} +
ModelMetricMean95% CIMedianStdp05 / p95
+
+
+

Paired model differences

+

Each value is left minus right, aligned by exact context ID.

+
+ + + {comparison_rows} +
ModelsMetricMean difference95% CIContexts
+
+ + +""" + + def write_reports( output_dir: Path, *, benchmark: str, results: list[EvaluationResult], -) -> tuple[Path, Path]: - """Write versioned JSON and Markdown reports.""" + resamples: int = 1000, + seed: int = 0, +) -> tuple[Path, Path, Path]: + """Write versioned JSON, Markdown, and standalone HTML reports.""" + if not results: + raise ValueError("at least one evaluation result is required") + statistics, comparisons = _build_statistics(results, resamples=resamples, seed=seed) output_dir.mkdir(parents=True, exist_ok=True) summary_path = output_dir / "summary.json" report_path = output_dir / "report.md" + html_path = output_dir / "report.html" payload = { "schema_version": "0.1", "benchmark": benchmark, "results": [result.to_dict() for result in results], + "statistics": { + model_id: {metric: asdict(summary) for metric, summary in metrics.items()} + for model_id, metrics in statistics.items() + }, + "paired_comparisons": [asdict(comparison) for comparison in comparisons], } summary_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - report_path.write_text(render_markdown(results), encoding="utf-8") - return summary_path, report_path + report_path.write_text( + render_markdown(results, statistics=statistics, comparisons=comparisons), + encoding="utf-8", + ) + html_path.write_text( + render_html(results, statistics=statistics, comparisons=comparisons), + encoding="utf-8", + ) + return summary_path, report_path, html_path diff --git a/src/wamprobe/stats.py b/src/wamprobe/stats.py new file mode 100644 index 0000000..7b83f8f --- /dev/null +++ b/src/wamprobe/stats.py @@ -0,0 +1,173 @@ +"""Dependency-free context-block statistics for paired WAM evaluation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import sqrt +from random import Random +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + + from wamprobe.evaluation import EvaluationResult + + +@dataclass(frozen=True, slots=True) +class ConfidenceInterval: + """Percentile bootstrap interval whose resampling unit is one context.""" + + estimate: float + lower: float + upper: float + confidence_level: float + resamples: int + unit: str = "context" + + +@dataclass(frozen=True, slots=True) +class MetricSummary: + """Descriptive statistics and a context-block confidence interval.""" + + mean: float + median: float + standard_deviation: float + quantiles: dict[str, float] + confidence_interval: ConfidenceInterval + contexts: int + + +@dataclass(frozen=True, slots=True) +class PairedComparison: + """Paired left-minus-right metric difference over shared contexts.""" + + metric: str + left_model: str + right_model: str + contexts: int + mean_difference: float + confidence_interval: ConfidenceInterval + + +def _mean(values: Sequence[float]) -> float: + return sum(values) / len(values) + + +def _quantile(sorted_values: Sequence[float], probability: float) -> float: + if not 0.0 <= probability <= 1.0: + raise ValueError("quantile probability must be in [0, 1]") + if len(sorted_values) == 1: + return sorted_values[0] + position = probability * (len(sorted_values) - 1) + lower_index = int(position) + upper_index = min(lower_index + 1, len(sorted_values) - 1) + fraction = position - lower_index + return sorted_values[lower_index] * (1.0 - fraction) + sorted_values[upper_index] * fraction + + +def _bootstrap_interval( + values: Sequence[float], + *, + resamples: int, + seed: int, + confidence_level: float, +) -> ConfidenceInterval: + if resamples <= 0: + raise ValueError("resamples must be positive") + if not 0.0 < confidence_level < 1.0: + raise ValueError("confidence_level must be between zero and one") + + rng = Random(seed) + count = len(values) + bootstrap_means = sorted( + _mean([values[rng.randrange(count)] for _ in range(count)]) for _ in range(resamples) + ) + tail = (1.0 - confidence_level) / 2.0 + return ConfidenceInterval( + estimate=_mean(values), + lower=_quantile(bootstrap_means, tail), + upper=_quantile(bootstrap_means, 1.0 - tail), + confidence_level=confidence_level, + resamples=resamples, + ) + + +def summarize( + values: Sequence[float], + *, + resamples: int = 1000, + seed: int = 0, + confidence_level: float = 0.95, +) -> MetricSummary: + """Summarize context scores without treating branches or frames as independent.""" + + if not values: + raise ValueError("at least one context value is required") + if resamples <= 0: + raise ValueError("resamples must be positive") + ordered = sorted(float(value) for value in values) + mean = _mean(ordered) + standard_deviation = sqrt(sum((value - mean) ** 2 for value in ordered) / len(ordered)) + quantiles = { + "p05": _quantile(ordered, 0.05), + "p25": _quantile(ordered, 0.25), + "p75": _quantile(ordered, 0.75), + "p95": _quantile(ordered, 0.95), + } + return MetricSummary( + mean=mean, + median=_quantile(ordered, 0.5), + standard_deviation=standard_deviation, + quantiles=quantiles, + confidence_interval=_bootstrap_interval( + ordered, + resamples=resamples, + seed=seed, + confidence_level=confidence_level, + ), + contexts=len(ordered), + ) + + +def paired_metric_comparison( + left: EvaluationResult, + right: EvaluationResult, + *, + metric: str, + resamples: int = 1000, + seed: int = 0, + confidence_level: float = 0.95, +) -> PairedComparison: + """Bootstrap paired per-context differences after exact context-ID alignment.""" + + if left.benchmark != right.benchmark: + raise ValueError("paired results must use the same benchmark") + + left_values = {result.context_id: result.metrics for result in left.context_results} + right_values = {result.context_id: result.metrics for result in right.context_results} + if len(left_values) != len(left.context_results) or len(right_values) != len( + right.context_results + ): + raise ValueError("paired context IDs must be unique") + if left_values.keys() != right_values.keys(): + raise ValueError("paired context IDs must match exactly") + + differences: list[float] = [] + for context_id in sorted(left_values): + if metric not in left_values[context_id] or metric not in right_values[context_id]: + raise ValueError(f"metric is unavailable for paired context: {metric}") + differences.append(left_values[context_id][metric] - right_values[context_id][metric]) + interval = _bootstrap_interval( + differences, + resamples=resamples, + seed=seed, + confidence_level=confidence_level, + ) + return PairedComparison( + metric=metric, + left_model=left.model_id, + right_model=right.model_id, + contexts=len(differences), + mean_difference=interval.estimate, + confidence_interval=interval, + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index cc9bfe8..a1fe0dc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,16 +25,24 @@ def test_demo_cli_writes_machine_and_human_readable_reports( assert exit_code == 0 summary_path = tmp_path / "summary.json" report_path = tmp_path / "report.md" + html_path = tmp_path / "report.html" assert summary_path.is_file() assert report_path.is_file() + assert html_path.is_file() summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["benchmark"] == "pointmass-2d-v0.1" assert {result["model_id"] for result in summary["results"]} == { "action-agnostic", "copy-last-frame", + "noisy-linear", "oracle-pointmass", "wrong-direction", } + assert summary["statistics"] + assert summary["paired_comparisons"] assert "Counterfactual Direction" in report_path.read_text(encoding="utf-8") + html = html_path.read_text(encoding="utf-8") + assert "" in html + assert "Context-block 95% confidence intervals" in html assert "WAMProbe demo complete" in capsys.readouterr().out diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 322a024..96befc5 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,6 +1,7 @@ from wamprobe.adapters.baselines import ( ActionAgnosticAdapter, CopyLastFrameAdapter, + NoisyLinearAdapter, OraclePointMassAdapter, WrongDirectionAdapter, ) @@ -20,10 +21,14 @@ def test_reference_baselines_have_expected_metric_ordering() -> None: assert oracle.metrics["counterfactual_direction_accuracy"] == 1.0 assert wrong.metrics["counterfactual_direction_accuracy"] == -1.0 assert oracle.metrics["state_ade"] == 0.0 + assert oracle.metrics["state_fde"] == 0.0 assert copy_last.metrics["action_dependence"] == 0.0 assert action_agnostic.metrics["action_dependence"] == 0.0 assert oracle.metrics["top1_regret"] == 0.0 assert action_agnostic.metrics["top1_regret"] > oracle.metrics["top1_regret"] + assert oracle.metrics["action_dependence_permutation_effect"] > 1.0 + assert copy_last.metrics["action_dependence_permutation_effect"] == 0.0 + assert all(result.metrics for result in oracle.context_results) def test_noop_stability_catches_action_agnostic_motion() -> None: @@ -35,3 +40,37 @@ def test_noop_stability_catches_action_agnostic_motion() -> None: assert oracle.metrics["noop_stability"] == 1.0 assert action_agnostic.metrics["noop_stability"] == 0.0 + + +def test_noisy_linear_is_seeded_and_degrades_oracle_accuracy() -> None: + benchmark = PointMass2D(horizon=4) + suite = benchmark.make_suite(contexts=12, seed=11) + + first = evaluate(NoisyLinearAdapter(noise_std=0.04), benchmark, suite, seed=29) + second = evaluate(NoisyLinearAdapter(noise_std=0.04), benchmark, suite, seed=29) + + assert first == second + assert 0.0 < first.metrics["state_ade"] < 0.5 + assert 0.0 < first.metrics["state_fde"] < 1.0 + assert 0.9 < first.metrics["counterfactual_direction_accuracy"] < 1.0 + + +def test_noisy_linear_metrics_degrade_monotonically_with_noise() -> None: + benchmark = PointMass2D(horizon=4) + suite = benchmark.make_suite(contexts=24, seed=11) + + results = [ + evaluate(NoisyLinearAdapter(noise_std=noise), benchmark, suite, seed=29) + for noise in (0.01, 0.04, 0.12) + ] + + assert [result.metrics["state_ade"] for result in results] == sorted( + result.metrics["state_ade"] for result in results + ) + assert [result.metrics["state_fde"] for result in results] == sorted( + result.metrics["state_fde"] for result in results + ) + assert [result.metrics["counterfactual_direction_accuracy"] for result in results] == sorted( + (result.metrics["counterfactual_direction_accuracy"] for result in results), + reverse=True, + ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..3bf4bef --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,57 @@ +from wamprobe.adapters.baselines import CopyLastFrameAdapter, OraclePointMassAdapter +from wamprobe.api.types import Context2D, Trajectory2D, Vec2 +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.metrics import action_dependence_permutation_test, state_fde + + +def test_state_fde_measures_only_final_state_error() -> None: + context = Context2D("context-0", Vec2(0.0, 0.0), Vec2(1.0, 0.0)) + predicted = Trajectory2D( + context_id=context.context_id, + action_name="right", + states=(Vec2(100.0, 100.0), Vec2(0.75, 0.0)), + ) + truth = Trajectory2D( + context_id=context.context_id, + action_name="right", + states=(Vec2(0.5, 0.0), Vec2(1.0, 0.0)), + ) + + assert state_fde([(context, predicted, truth)]) == 0.25 + + +def test_action_dependence_permutation_null_separates_oracle_from_copy_last() -> None: + benchmark = PointMass2D(horizon=4) + group = benchmark.make_suite(contexts=1, seed=7).groups[0] + oracle = OraclePointMassAdapter(benchmark) + copy_last = CopyLastFrameAdapter() + + oracle_samples = [ + ( + group.context, + oracle.predict_future(group.context, action, horizon=4, seed=11), + benchmark.rollout(group.context, action), + ) + for action in group.actions + ] + copy_samples = [ + ( + group.context, + copy_last.predict_future(group.context, action, horizon=4, seed=11), + benchmark.rollout(group.context, action), + ) + for action in group.actions + ] + + oracle_result = action_dependence_permutation_test(oracle_samples, permutations=128, seed=19) + copy_result = action_dependence_permutation_test(copy_samples, permutations=128, seed=19) + + assert oracle_result.observed == 1.0 + assert oracle_result.effect_size > 1.0 + assert oracle_result.p_value < 0.1 + assert oracle_result == action_dependence_permutation_test( + oracle_samples, permutations=128, seed=19 + ) + assert copy_result.observed == 0.0 + assert copy_result.effect_size == 0.0 + assert copy_result.p_value == 1.0 diff --git a/tests/test_statistics.py b/tests/test_statistics.py new file mode 100644 index 0000000..7279f3b --- /dev/null +++ b/tests/test_statistics.py @@ -0,0 +1,71 @@ +import pytest + +from wamprobe.adapters.baselines import CopyLastFrameAdapter, OraclePointMassAdapter +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.evaluation import evaluate +from wamprobe.stats import paired_metric_comparison, summarize + + +def test_context_bootstrap_summary_is_deterministic_and_reports_quantiles() -> None: + first = summarize((1.0, 2.0, 3.0, 4.0), resamples=400, seed=17) + second = summarize((1.0, 2.0, 3.0, 4.0), resamples=400, seed=17) + + assert first == second + assert first.mean == 2.5 + assert first.median == 2.5 + assert first.standard_deviation == pytest.approx(1.11803398875) + assert first.quantiles["p05"] == pytest.approx(1.15) + assert first.quantiles["p95"] == pytest.approx(3.85) + assert first.confidence_interval.lower <= first.mean <= first.confidence_interval.upper + assert first.confidence_interval.unit == "context" + + +def test_constant_context_values_have_a_zero_width_bootstrap_interval() -> None: + summary = summarize((3.0, 3.0, 3.0), resamples=100, seed=1) + + assert summary.confidence_interval.lower == 3.0 + assert summary.confidence_interval.upper == 3.0 + + +def test_paired_metric_comparison_resamples_context_differences() -> None: + benchmark = PointMass2D(horizon=4) + suite = benchmark.make_suite(contexts=8, seed=5) + oracle = evaluate(OraclePointMassAdapter(benchmark), benchmark, suite, seed=3) + copy_last = evaluate(CopyLastFrameAdapter(), benchmark, suite, seed=3) + + comparison = paired_metric_comparison( + oracle, + copy_last, + metric="state_fde", + resamples=300, + seed=23, + ) + + assert comparison.left_model == "oracle-pointmass" + assert comparison.right_model == "copy-last-frame" + assert comparison.contexts == 8 + # Four directional branches have FDE 1.0; the valid no-op branch has FDE 0.0. + assert comparison.mean_difference == -0.8 + assert comparison.confidence_interval.lower == -0.8 + assert comparison.confidence_interval.upper == -0.8 + + +def test_statistics_reject_invalid_inputs_and_unpaired_results() -> None: + with pytest.raises(ValueError, match="at least one"): + summarize((), resamples=100) + with pytest.raises(ValueError, match="resamples"): + summarize((1.0,), resamples=0) + + benchmark = PointMass2D(horizon=4) + left = evaluate( + OraclePointMassAdapter(benchmark), + benchmark, + benchmark.make_suite(contexts=4, seed=1), + ) + right = evaluate( + CopyLastFrameAdapter(), + benchmark, + benchmark.make_suite(contexts=3, seed=1), + ) + with pytest.raises(ValueError, match="paired context"): + paired_metric_comparison(left, right, metric="state_fde", resamples=100) From c917632bc0db2beda64572136209aa1d84a958d1 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 12:09:03 +0000 Subject: [PATCH 09/20] make paired statistics test float tolerant --- tests/test_statistics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_statistics.py b/tests/test_statistics.py index 7279f3b..89d9304 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -45,9 +45,9 @@ def test_paired_metric_comparison_resamples_context_differences() -> None: assert comparison.right_model == "copy-last-frame" assert comparison.contexts == 8 # Four directional branches have FDE 1.0; the valid no-op branch has FDE 0.0. - assert comparison.mean_difference == -0.8 - assert comparison.confidence_interval.lower == -0.8 - assert comparison.confidence_interval.upper == -0.8 + assert comparison.mean_difference == pytest.approx(-0.8) + assert comparison.confidence_interval.lower == pytest.approx(-0.8) + assert comparison.confidence_interval.upper == pytest.approx(-0.8) def test_statistics_reject_invalid_inputs_and_unpaired_results() -> None: From 4319fd8cf609a4e4e89912c48eb6f145d0a42da4 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 12:48:27 +0000 Subject: [PATCH 10/20] add manipulation toy benchmarks and CRC --- README.md | 28 +- docs/QUICKSTART.md | 26 +- docs/WAMProbe_PLAN.md | 19 +- docs/benchmarks/TOY_BENCHMARKS.md | 74 + examples/blockpush-demo/report.html | 53 + examples/blockpush-demo/report.md | 137 + examples/blockpush-demo/summary.json | 3250 ++++++++++++++++++++++ examples/gripper-catch-demo/report.html | 53 + examples/gripper-catch-demo/report.md | 137 + examples/gripper-catch-demo/summary.json | 3250 ++++++++++++++++++++++ examples/pointmass-demo/report.html | 12 +- examples/pointmass-demo/report.md | 58 +- examples/pointmass-demo/summary.json | 924 +++++- src/wamprobe/adapters/__init__.py | 14 + src/wamprobe/adapters/manipulation.py | 152 + src/wamprobe/api/__init__.py | 18 + src/wamprobe/api/manipulation.py | 201 ++ src/wamprobe/benchmarks/__init__.py | 4 +- src/wamprobe/benchmarks/blockpush.py | 156 ++ src/wamprobe/benchmarks/gripper_catch.py | 180 ++ src/wamprobe/benchmarks/rendering.py | 81 + src/wamprobe/cli.py | 52 +- src/wamprobe/evaluation.py | 6 + src/wamprobe/manipulation_evaluation.py | 197 ++ src/wamprobe/metrics.py | 159 +- src/wamprobe/reporting.py | 24 +- tests/test_cli.py | 46 + tests/test_evaluation.py | 5 + tests/test_manipulation_benchmarks.py | 115 + tests/test_ranking_metrics.py | 41 + 30 files changed, 9404 insertions(+), 68 deletions(-) create mode 100644 docs/benchmarks/TOY_BENCHMARKS.md create mode 100644 examples/blockpush-demo/report.html create mode 100644 examples/blockpush-demo/report.md create mode 100644 examples/blockpush-demo/summary.json create mode 100644 examples/gripper-catch-demo/report.html create mode 100644 examples/gripper-catch-demo/report.md create mode 100644 examples/gripper-catch-demo/summary.json create mode 100644 src/wamprobe/adapters/manipulation.py create mode 100644 src/wamprobe/api/manipulation.py create mode 100644 src/wamprobe/benchmarks/blockpush.py create mode 100644 src/wamprobe/benchmarks/gripper_catch.py create mode 100644 src/wamprobe/benchmarks/rendering.py create mode 100644 src/wamprobe/manipulation_evaluation.py create mode 100644 tests/test_manipulation_benchmarks.py create mode 100644 tests/test_ranking_metrics.py diff --git a/README.md b/README.md index 6aa2b27..ed837c6 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ source .venv/bin/activate python -m pip install -e . wamprobe demo --contexts 12 --seed 7 --output runs/pointmass-demo + +# Contact and attachment diagnostics use the same report pipeline. +wamprobe demo --benchmark blockpush --horizon 6 --output runs/blockpush-demo +wamprobe demo --benchmark gripper-catch --horizon 5 --output runs/gripper-catch-demo ``` The command creates: @@ -57,8 +61,10 @@ runs/pointmass-demo/ └── report.html # standalone interactive-friendly report ``` -See the committed [example report](examples/pointmass-demo/report.md) for the expected -baseline ordering. +See the committed [PointMass](examples/pointmass-demo/report.md), +[BlockPush](examples/blockpush-demo/report.md), and +[Gripper-Catch](examples/gripper-catch-demo/report.md) reports for the expected baseline +profiles. Real-model weights are never committed to Git. Before running the StarWAM integration, follow the [model-store layout and download rules](checkpoints/README.md); the first spike requires @@ -81,10 +87,12 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - typed, model-agnostic `WAMAdapter` and `ActionPredictorAdapter` protocols; - capability manifest data model and JSON Schema; -- paired PointMass-2D counterfactual interventions; +- paired PointMass-2D, contact-aware BlockPush-2D, and attachment-aware Gripper-Catch + counterfactual interventions, including dependency-free RGB observations; - oracle, noisy-linear, copy-last-frame, wrong-direction, and action-agnostic baselines; - Action Dependence with a within-context permutation null, Counterfactual Direction - Accuracy, No-op Stability, state ADE/FDE, and Top-1 Regret; + Accuracy, No-op Stability, state ADE/FDE, four-view Candidate Ranking Correlation, + and Top-1 Regret; - context-block bootstrap intervals and exact-context paired model comparisons; - versioned JSON plus Markdown and standalone HTML reports; - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; @@ -98,15 +106,17 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo The next milestones are: -1. a small image-rendered BlockPush benchmark; -2. the versioned intervention dataset loader; -3. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; -4. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; -5. add a GPU nightly workflow for the isolated real-model integration. +1. the versioned intervention dataset loader; +2. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; +3. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; +4. add a GPU nightly workflow for the isolated real-model integration; +5. add the Occluded-Object memory diagnostic to the broader Toy tier. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), [adapter selection record](docs/research/ADAPTER_SELECTION.md), and [design RFCs](docs/rfcs/). +The exact toy dynamics and limitations are documented in the +[toy benchmark card](docs/benchmarks/TOY_BENCHMARKS.md). ## Development diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 7649d1e..35fa4a4 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -6,6 +6,7 @@ From a source checkout: ```bash PYTHONPATH=src python -m wamprobe demo \ + --benchmark pointmass \ --contexts 12 \ --seed 7 \ --output runs/pointmass-demo @@ -19,12 +20,29 @@ The output directory contains `summary.json`, `report.md`, and a standalone `report.html`. The JSON preserves per-context values, descriptive statistics, context-block bootstrap intervals, and paired model differences. +Run the contact and attachment variants through the same evaluator: + +```bash +wamprobe demo --benchmark blockpush --contexts 12 --seed 7 --horizon 6 \ + --output runs/blockpush-demo +wamprobe demo --benchmark gripper-catch --contexts 12 --seed 7 --horizon 5 \ + --output runs/gripper-catch-demo +``` + +BlockPush has an explicit pre-contact approach followed by object motion only under a +directed contact. Gripper-Catch requires both alignment and a close command before the +falling object attaches. Both expose exact state and dependency-free 64×64 RGB +observations. See the [toy benchmark card](benchmarks/TOY_BENCHMARKS.md) for equations, +state fields, and limitations. + ## Built-in baselines | Baseline | Behavior | Diagnostic purpose | |---|---|---| | `oracle-pointmass` | Uses exact dynamics | Expected upper bound | | `noisy-linear` | Applies the action plus seeded transition noise | Checks smooth accuracy degradation | +| `oracle-simulator` | Uses exact BlockPush/Gripper-Catch dynamics | Manipulation upper bound | +| `noisy-dynamics` | Adds content-addressed transition noise | Contact/grasp robustness check | | `copy-last-frame` | Predicts no movement | Detects action ignorance | | `wrong-direction` | Applies the negative action | Separates dependence from correctness | | `action-agnostic` | Moves to the goal for every action | Mimics a plausible success prior | @@ -38,13 +56,17 @@ context-block bootstrap intervals, and paired model differences. is reported separately; neither value replaces direction or dynamics metrics. - **Counterfactual Direction Accuracy** is mean cosine alignment between predicted and true non-noop displacement. `1` is aligned, `0` is uninformative, and `-1` is reversed. -- **No-op Stability** is the fraction of no-op predictions whose final state remains at - the shared initial position. +- **No-op Stability** checks a stationary no-op in PointMass. In a passive dynamic scene + such as a falling object, the same field measures agreement with the true no-op future + rather than incorrectly requiring the world to freeze. - **State ADE** is average Euclidean state error over branches and time; lower is better. - **State FDE** is final-state Euclidean error averaged over every branch, including the no-op branch; lower is better. - **Top-1 Regret** measures how much true return is lost by selecting the candidate that looks best under the predicted future; lower is better. +- **Candidate Ranking Correlation (CRC)** compares all predicted candidate returns with + simulator returns. WAMProbe reports Spearman, Kendall tau-b, NDCG, and pairwise + preference accuracy rather than hiding their different tie behavior in one number. WAMProbe intentionally reports a metric profile, not a composite score. A model may be action-dependent but physically wrong, or visually accurate but useless for candidate diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 92f69b6..d638092 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1036,11 +1036,12 @@ CLI 验收要求： ### Phase 2：Toy Benchmark 与核心指标（第 5–7 周） -当前进度（2026-07-15）：PointMass-2D 已具有 5 个 reference baselines；ADS 的 context -内 action-permutation null、CDA、NOS、state ADE/FDE、Top-1 Regret 已实现。每项指标先在 -context 层计算，再做 context-block bootstrap 和严格 context-ID 对齐的 paired comparison； -示例同时生成 versioned JSON、Markdown 和 standalone HTML。Phase 2 尚缺 #10 的 -BlockPush/Gripper-Catch 与 CRC，不能据此宣称整个 Phase 2 完成。 +完成状态（2026-07-15）：Phase 2 约定范围已经落地。PointMass-2D、带显式接触阶段的 +BlockPush-2D、带闭合/附着语义的 Gripper-Catch 均可在 CPU 运行；后两者提供确定性/带噪声 +状态 rollout 与 dependency-free RGB 观察。ADS permutation null、CDA、NOS、ADE/FDE、 +四视角 CRC、Top-1 Regret 均先在 context 层计算，再做 context-block bootstrap 和严格 +context-ID 对齐的 paired comparison；三个 benchmark 都能生成 JSON、Markdown 和 HTML。 +8.1 节更广义 Toy tier 中的 Occluded-Object 仍是后续扩展，不属于 #10/Phase 2 退出条件。 任务： @@ -1135,8 +1136,8 @@ v0.1 发布门槛： | #19 | Add GPU nightly smoke workflow | P1 | 2 天 | #18 | | #20 | Write benchmark card and v0.1 reproducibility guide | P0 | 4 天 | 全部 | -代码落地状态（2026-07-15）：本轮补齐并验证 #8、#11、#13、#15、#16；此前已完成 -#1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#6 和 #18 已有可运行的 StarWAM +代码落地状态（2026-07-15）：已补齐并验证 #8、#10、#11、#13、#15、#16 以及 CRC； +此前已完成 #1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#6 和 #18 已有可运行的 StarWAM prediction artifact/adapter 切片，但更通用的 cache 与真实 WAM 反事实控制评测仍需继续扩展。 建议 labels： @@ -1266,9 +1267,9 @@ priority:p1 ### 研究有效性 - [x] oracle 与错误 baseline 能被稳定区分； -- [ ] action shuffle/mask 产生预期指标下降； +- [x] action shuffle/permutation 产生预期指标下降； - [x] 噪声增加时准确性指标总体退化； -- [ ] 至少一个 causal/ranking 指标与 simulator return 相关； +- [x] 至少一个 causal/ranking 指标与 simulator return 相关； - [ ] 报告传统视频指标与控制指标的差异； - [ ] 公开所有主要失败率和 skipped metrics。 diff --git a/docs/benchmarks/TOY_BENCHMARKS.md b/docs/benchmarks/TOY_BENCHMARKS.md new file mode 100644 index 0000000..367d9ff --- /dev/null +++ b/docs/benchmarks/TOY_BENCHMARKS.md @@ -0,0 +1,74 @@ +# WAMProbe analytic toy benchmarks + +The toy tier exists to test evaluator behavior against dynamics whose failure modes are +known exactly. It is not a manipulation-success benchmark and should not be used to make +claims about photorealism or real-robot transfer. + +## Shared contract + +`BlockPush2D` and `GripperCatch` expose the same dependency-free contract: + +- one shared initial `ManipulationState` per context; +- two or more typed `ManipulationAction` branches; +- an exact state transition and an optional seeded noisy transition; +- a simulator-return function and an analytic optimal candidate; +- a 64×64 tightly packed RGB observation plus named proprioceptive fields. + +The metric state vector is: + +```text +[effector_x, effector_y, object_x, object_y, + gripper_closed, object_attached, in_contact] +``` + +Boolean contact semantics remain explicit instead of being inferred from action names. +Noise streams are content-addressed by the run seed, context ID, and action name, so +reordering contexts or branches does not change a prediction. + +## BlockPush-2D v0.1 + +The pusher begins three action steps behind the block. The first step is necessarily a +pre-contact approach. Once the pusher is within the contact reach and moving toward the +block, the block follows its planar transition. Perpendicular and away-from-object +commands cannot move the block under deterministic dynamics. + +Candidate actions are no-op and pushes in four cardinal directions. The generated goal +is reached by exactly one direction after the approach phase. Return is negative final +block-to-goal distance. + +This model deliberately omits mass, friction, rotation, and rigid-body collision +resolution. Those belong in the LIBERO tier; here, the two-stage contact causal boundary +is the behavior under test. + +## Gripper-Catch v0.1 + +An object falls by a fixed vertical displacement while the gripper chooses open no-op, +closed-left, closed-stationary, or closed-right motion. Attachment requires both spatial +alignment and an explicit close command. Once attached, the object follows the gripper +and an attached state with an open gripper is rejected by the data model. + +The three generated context families place the falling object at the endpoint of the +left, stationary, or right closed candidate. Return rewards a valid attachment and then +penalizes final object-to-goal distance. + +This is a semantic grasp diagnostic, not a model of finger geometry, impact, bounce, or +force closure. + +## Run + +```bash +wamprobe demo --benchmark pointmass --contexts 12 --seed 7 +wamprobe demo --benchmark blockpush --contexts 12 --seed 7 --horizon 6 \ + --output runs/blockpush-demo +wamprobe demo --benchmark gripper-catch --contexts 12 --seed 7 --horizon 5 \ + --output runs/gripper-catch-demo +``` + +Each command writes versioned JSON, Markdown, and standalone HTML. CRC reports Spearman, +Kendall tau-b, NDCG over shifted non-negative relevance, and pairwise preference +accuracy. Tied model predictions receive half credit in pairwise accuracy; ground-truth +ties are excluded from that denominator. + +The suite generator accepts `contexts=1000` without external assets or simulator +installation. Generated suites are deterministic for a fixed benchmark configuration and +seed; a versioned on-disk intervention dataset loader remains a separate roadmap item. diff --git a/examples/blockpush-demo/report.html b/examples/blockpush-demo/report.html new file mode 100644 index 0000000..9c23925 --- /dev/null +++ b/examples/blockpush-demo/report.html @@ -0,0 +1,53 @@ + + + + + + WAMProbe blockpush-2d-v0.1 Demo + + + +

WAMProbe blockpush-2d-v0.1 Demo

+

Action-aware and deliberately broken baselines are shown as a metric + profile. No composite score is produced.

+
+

Metric profile

+
+ + +
ModelAction DependencePermutation EffectPermutation p-value ↓Counterfactual DirectionNo-op StabilityState ADE ↓State FDE ↓CRC SpearmanCRC Kendall τCRC NDCGCRC Pairwise AccuracyTop-1 Regret ↓
oracle-simulator1.00002.11380.01671.00001.00000.00000.00001.00001.00001.00001.00000.0000
noisy-dynamics0.99072.11190.01670.99780.00000.04270.03551.00001.00001.00001.00000.0000
copy-last-frame0.00000.00001.00000.00001.00000.30310.43690.00000.00000.48710.50000.3000
wrong-direction1.0000-0.15780.4833-0.66301.00000.55480.7953-0.2500-0.25000.43710.37500.3000
action-agnostic0.00000.00001.00000.16850.00000.73610.93910.00000.00000.48710.50000.3000
+
+
+

Context-block 95% confidence intervals

+

Whole contexts are resampled; correlated branches and frames are never + treated as independent observations.

+
+ + + +
ModelMetricMean95% CIMedianStdp05 / p95
oracle-simulatorAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorPermutation Effect2.1138[2.1138, 2.1138]2.11380.00002.1138 / 2.1138
oracle-simulatorPermutation p-value ↓0.0167[0.0167, 0.0167]0.01670.00000.0167 / 0.0167
oracle-simulatorCounterfactual Direction1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorState ADE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-simulatorState FDE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-simulatorCRC Spearman1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC Kendall τ1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-dynamicsAction Dependence0.9907[0.9849, 0.9959]0.99340.01000.9732 / 1.0000
noisy-dynamicsPermutation Effect2.1119[2.1091, 2.1149]2.11360.00522.1032 / 2.1185
noisy-dynamicsPermutation p-value ↓0.0167[0.0167, 0.0167]0.01670.00000.0167 / 0.0167
noisy-dynamicsCounterfactual Direction0.9978[0.9968, 0.9988]0.99820.00180.9943 / 0.9998
noisy-dynamicsNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-dynamicsState ADE ↓0.0427[0.0319, 0.0538]0.04280.01960.0170 / 0.0653
noisy-dynamicsState FDE ↓0.0355[0.0293, 0.0416]0.03530.01140.0190 / 0.0512
noisy-dynamicsCRC Spearman1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC Kendall τ1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameState ADE ↓0.3031[0.3031, 0.3031]0.30310.00000.3031 / 0.3031
copy-last-frameState FDE ↓0.4369[0.4369, 0.4369]0.43690.00000.4369 / 0.4369
copy-last-frameCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC NDCG0.4871[0.4407, 0.5423]0.46530.09230.3869 / 0.6309
copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameTop-1 Regret ↓0.3000[0.3000, 0.3000]0.30000.00000.3000 / 0.3000
wrong-directionAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionPermutation Effect-0.1578[-0.1578, -0.1578]-0.15780.0000-0.1578 / -0.1578
wrong-directionPermutation p-value ↓0.4833[0.4833, 0.4833]0.48330.00000.4833 / 0.4833
wrong-directionCounterfactual Direction-0.6630[-0.6630, -0.6630]-0.66300.0000-0.6630 / -0.6630
wrong-directionNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState ADE ↓0.5548[0.5548, 0.5548]0.55480.00000.5548 / 0.5548
wrong-directionState FDE ↓0.7953[0.7953, 0.7953]0.79530.00000.7953 / 0.7953
wrong-directionCRC Spearman-0.2500[-0.2500, -0.2500]-0.25000.0000-0.2500 / -0.2500
wrong-directionCRC Kendall τ-0.2500[-0.2500, -0.2500]-0.25000.0000-0.2500 / -0.2500
wrong-directionCRC NDCG0.4371[0.4160, 0.4596]0.43070.04050.3869 / 0.5000
wrong-directionCRC Pairwise Accuracy0.3750[0.3750, 0.3750]0.37500.00000.3750 / 0.3750
wrong-directionTop-1 Regret ↓0.3000[0.3000, 0.3000]0.30000.00000.3000 / 0.3000
action-agnosticAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticCounterfactual Direction0.1685[0.1685, 0.1685]0.16850.00000.1685 / 0.1685
action-agnosticNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticState ADE ↓0.7361[0.7361, 0.7361]0.73610.00000.7361 / 0.7361
action-agnosticState FDE ↓0.9391[0.9391, 0.9391]0.93910.00000.9391 / 0.9391
action-agnosticCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC NDCG0.4871[0.4392, 0.5387]0.46530.09230.3869 / 0.6309
action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
action-agnosticTop-1 Regret ↓0.3000[0.3000, 0.3000]0.30000.00000.3000 / 0.3000
+
+
+

Paired model differences

+

Each value is left minus right, aligned by exact context ID.

+
+ + + +
ModelsMetricMean difference95% CIContexts
oracle-simulator − noisy-dynamicsAction Dependence0.0093[0.0041, 0.0152]12
oracle-simulator − noisy-dynamicsPermutation Effect0.0019[-0.0008, 0.0050]12
oracle-simulator − noisy-dynamicsPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCounterfactual Direction0.0022[0.0012, 0.0032]12
oracle-simulator − noisy-dynamicsNo-op Stability1.0000[1.0000, 1.0000]12
oracle-simulator − noisy-dynamicsState ADE ↓-0.0427[-0.0531, -0.0314]12
oracle-simulator − noisy-dynamicsState FDE ↓-0.0355[-0.0419, -0.0285]12
oracle-simulator − noisy-dynamicsCRC Spearman0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC Kendall τ0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC NDCG0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC Pairwise Accuracy0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsTop-1 Regret ↓0.0000[0.0000, 0.0000]12
oracle-simulator − copy-last-frameAction Dependence1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-framePermutation Effect2.1138[2.1138, 2.1138]12
oracle-simulator − copy-last-framePermutation p-value ↓-0.9833[-0.9833, -0.9833]12
oracle-simulator − copy-last-frameCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameNo-op Stability0.0000[0.0000, 0.0000]12
oracle-simulator − copy-last-frameState ADE ↓-0.3031[-0.3031, -0.3031]12
oracle-simulator − copy-last-frameState FDE ↓-0.4369[-0.4369, -0.4369]12
oracle-simulator − copy-last-frameCRC Spearman1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameCRC NDCG0.5129[0.4634, 0.5615]12
oracle-simulator − copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-simulator − copy-last-frameTop-1 Regret ↓-0.3000[-0.3000, -0.3000]12
oracle-simulator − wrong-directionAction Dependence0.0000[0.0000, 0.0000]12
oracle-simulator − wrong-directionPermutation Effect2.2716[2.2716, 2.2716]12
oracle-simulator − wrong-directionPermutation p-value ↓-0.4667[-0.4667, -0.4667]12
oracle-simulator − wrong-directionCounterfactual Direction1.6630[1.6630, 1.6630]12
oracle-simulator − wrong-directionNo-op Stability0.0000[0.0000, 0.0000]12
oracle-simulator − wrong-directionState ADE ↓-0.5548[-0.5548, -0.5548]12
oracle-simulator − wrong-directionState FDE ↓-0.7953[-0.7953, -0.7953]12
oracle-simulator − wrong-directionCRC Spearman1.2500[1.2500, 1.2500]12
oracle-simulator − wrong-directionCRC Kendall τ1.2500[1.2500, 1.2500]12
oracle-simulator − wrong-directionCRC NDCG0.5629[0.5404, 0.5855]12
oracle-simulator − wrong-directionCRC Pairwise Accuracy0.6250[0.6250, 0.6250]12
oracle-simulator − wrong-directionTop-1 Regret ↓-0.3000[-0.3000, -0.3000]12
oracle-simulator − action-agnosticAction Dependence1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticPermutation Effect2.1138[2.1138, 2.1138]12
oracle-simulator − action-agnosticPermutation p-value ↓-0.9833[-0.9833, -0.9833]12
oracle-simulator − action-agnosticCounterfactual Direction0.8315[0.8315, 0.8315]12
oracle-simulator − action-agnosticNo-op Stability1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticState ADE ↓-0.7361[-0.7361, -0.7361]12
oracle-simulator − action-agnosticState FDE ↓-0.9391[-0.9391, -0.9391]12
oracle-simulator − action-agnosticCRC Spearman1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticCRC NDCG0.5129[0.4570, 0.5645]12
oracle-simulator − action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-simulator − action-agnosticTop-1 Regret ↓-0.3000[-0.3000, -0.3000]12
+
+ + diff --git a/examples/blockpush-demo/report.md b/examples/blockpush-demo/report.md new file mode 100644 index 0000000..5e12084 --- /dev/null +++ b/examples/blockpush-demo/report.md @@ -0,0 +1,137 @@ +# WAMProbe blockpush-2d-v0.1 Demo + +This report compares action-aware and deliberately broken reference baselines. + +| Model | Action Dependence | Permutation Effect | Permutation p-value ↓ | Counterfactual Direction | No-op Stability | State ADE ↓ | State FDE ↓ | CRC Spearman | CRC Kendall τ | CRC NDCG | CRC Pairwise Accuracy | Top-1 Regret ↓ | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| oracle-simulator | 1.0000 | 2.1138 | 0.0167 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | 0.0000 | +| noisy-dynamics | 0.9907 | 2.1119 | 0.0167 | 0.9978 | 0.0000 | 0.0427 | 0.0355 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | 0.0000 | +| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 1.0000 | 0.3031 | 0.4369 | 0.0000 | 0.0000 | 0.4871 | 0.5000 | 0.3000 | +| wrong-direction | 1.0000 | -0.1578 | 0.4833 | -0.6630 | 1.0000 | 0.5548 | 0.7953 | -0.2500 | -0.2500 | 0.4371 | 0.3750 | 0.3000 | +| action-agnostic | 0.0000 | 0.0000 | 1.0000 | 0.1685 | 0.0000 | 0.7361 | 0.9391 | 0.0000 | 0.0000 | 0.4871 | 0.5000 | 0.3000 | + +## Context-block 95% confidence intervals + +Intervals resample whole contexts, not correlated action branches or frames. + +| Model | Metric | Mean | 95% CI | Median | Std | +|---|---|---:|---:|---:|---:| +| oracle-simulator | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | Permutation Effect | 2.1138 | [2.1138, 2.1138] | 2.1138 | 0.0000 | +| oracle-simulator | Permutation p-value ↓ | 0.0167 | [0.0167, 0.0167] | 0.0167 | 0.0000 | +| oracle-simulator | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | State ADE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-simulator | State FDE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-simulator | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-dynamics | Action Dependence | 0.9907 | [0.9849, 0.9959] | 0.9934 | 0.0100 | +| noisy-dynamics | Permutation Effect | 2.1119 | [2.1091, 2.1149] | 2.1136 | 0.0052 | +| noisy-dynamics | Permutation p-value ↓ | 0.0167 | [0.0167, 0.0167] | 0.0167 | 0.0000 | +| noisy-dynamics | Counterfactual Direction | 0.9978 | [0.9968, 0.9988] | 0.9982 | 0.0018 | +| noisy-dynamics | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-dynamics | State ADE ↓ | 0.0427 | [0.0319, 0.0538] | 0.0428 | 0.0196 | +| noisy-dynamics | State FDE ↓ | 0.0355 | [0.0293, 0.0416] | 0.0353 | 0.0114 | +| noisy-dynamics | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| copy-last-frame | Counterfactual Direction | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| copy-last-frame | State ADE ↓ | 0.3031 | [0.3031, 0.3031] | 0.3031 | 0.0000 | +| copy-last-frame | State FDE ↓ | 0.4369 | [0.4369, 0.4369] | 0.4369 | 0.0000 | +| copy-last-frame | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC NDCG | 0.4871 | [0.4407, 0.5423] | 0.4653 | 0.0923 | +| copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| copy-last-frame | Top-1 Regret ↓ | 0.3000 | [0.3000, 0.3000] | 0.3000 | 0.0000 | +| wrong-direction | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | Permutation Effect | -0.1578 | [-0.1578, -0.1578] | -0.1578 | 0.0000 | +| wrong-direction | Permutation p-value ↓ | 0.4833 | [0.4833, 0.4833] | 0.4833 | 0.0000 | +| wrong-direction | Counterfactual Direction | -0.6630 | [-0.6630, -0.6630] | -0.6630 | 0.0000 | +| wrong-direction | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | State ADE ↓ | 0.5548 | [0.5548, 0.5548] | 0.5548 | 0.0000 | +| wrong-direction | State FDE ↓ | 0.7953 | [0.7953, 0.7953] | 0.7953 | 0.0000 | +| wrong-direction | CRC Spearman | -0.2500 | [-0.2500, -0.2500] | -0.2500 | 0.0000 | +| wrong-direction | CRC Kendall τ | -0.2500 | [-0.2500, -0.2500] | -0.2500 | 0.0000 | +| wrong-direction | CRC NDCG | 0.4371 | [0.4160, 0.4596] | 0.4307 | 0.0405 | +| wrong-direction | CRC Pairwise Accuracy | 0.3750 | [0.3750, 0.3750] | 0.3750 | 0.0000 | +| wrong-direction | Top-1 Regret ↓ | 0.3000 | [0.3000, 0.3000] | 0.3000 | 0.0000 | +| action-agnostic | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| action-agnostic | Counterfactual Direction | 0.1685 | [0.1685, 0.1685] | 0.1685 | 0.0000 | +| action-agnostic | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | State ADE ↓ | 0.7361 | [0.7361, 0.7361] | 0.7361 | 0.0000 | +| action-agnostic | State FDE ↓ | 0.9391 | [0.9391, 0.9391] | 0.9391 | 0.0000 | +| action-agnostic | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC NDCG | 0.4871 | [0.4392, 0.5387] | 0.4653 | 0.0923 | +| action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| action-agnostic | Top-1 Regret ↓ | 0.3000 | [0.3000, 0.3000] | 0.3000 | 0.0000 | + +## Paired model differences + +Differences are left minus right over exactly aligned context IDs. + +| Models | Metric | Mean difference | 95% CI | Contexts | +|---|---|---:|---:|---:| +| oracle-simulator − noisy-dynamics | Action Dependence | 0.0093 | [0.0041, 0.0152] | 12 | +| oracle-simulator − noisy-dynamics | Permutation Effect | 0.0019 | [-0.0008, 0.0050] | 12 | +| oracle-simulator − noisy-dynamics | Permutation p-value ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | Counterfactual Direction | 0.0022 | [0.0012, 0.0032] | 12 | +| oracle-simulator − noisy-dynamics | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − noisy-dynamics | State ADE ↓ | -0.0427 | [-0.0531, -0.0314] | 12 | +| oracle-simulator − noisy-dynamics | State FDE ↓ | -0.0355 | [-0.0419, -0.0285] | 12 | +| oracle-simulator − noisy-dynamics | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC NDCG | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC Pairwise Accuracy | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − copy-last-frame | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | Permutation Effect | 2.1138 | [2.1138, 2.1138] | 12 | +| oracle-simulator − copy-last-frame | Permutation p-value ↓ | -0.9833 | [-0.9833, -0.9833] | 12 | +| oracle-simulator − copy-last-frame | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − copy-last-frame | State ADE ↓ | -0.3031 | [-0.3031, -0.3031] | 12 | +| oracle-simulator − copy-last-frame | State FDE ↓ | -0.4369 | [-0.4369, -0.4369] | 12 | +| oracle-simulator − copy-last-frame | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | CRC NDCG | 0.5129 | [0.4634, 0.5615] | 12 | +| oracle-simulator − copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | +| oracle-simulator − copy-last-frame | Top-1 Regret ↓ | -0.3000 | [-0.3000, -0.3000] | 12 | +| oracle-simulator − wrong-direction | Action Dependence | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − wrong-direction | Permutation Effect | 2.2716 | [2.2716, 2.2716] | 12 | +| oracle-simulator − wrong-direction | Permutation p-value ↓ | -0.4667 | [-0.4667, -0.4667] | 12 | +| oracle-simulator − wrong-direction | Counterfactual Direction | 1.6630 | [1.6630, 1.6630] | 12 | +| oracle-simulator − wrong-direction | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − wrong-direction | State ADE ↓ | -0.5548 | [-0.5548, -0.5548] | 12 | +| oracle-simulator − wrong-direction | State FDE ↓ | -0.7953 | [-0.7953, -0.7953] | 12 | +| oracle-simulator − wrong-direction | CRC Spearman | 1.2500 | [1.2500, 1.2500] | 12 | +| oracle-simulator − wrong-direction | CRC Kendall τ | 1.2500 | [1.2500, 1.2500] | 12 | +| oracle-simulator − wrong-direction | CRC NDCG | 0.5629 | [0.5404, 0.5855] | 12 | +| oracle-simulator − wrong-direction | CRC Pairwise Accuracy | 0.6250 | [0.6250, 0.6250] | 12 | +| oracle-simulator − wrong-direction | Top-1 Regret ↓ | -0.3000 | [-0.3000, -0.3000] | 12 | +| oracle-simulator − action-agnostic | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | Permutation Effect | 2.1138 | [2.1138, 2.1138] | 12 | +| oracle-simulator − action-agnostic | Permutation p-value ↓ | -0.9833 | [-0.9833, -0.9833] | 12 | +| oracle-simulator − action-agnostic | Counterfactual Direction | 0.8315 | [0.8315, 0.8315] | 12 | +| oracle-simulator − action-agnostic | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | State ADE ↓ | -0.7361 | [-0.7361, -0.7361] | 12 | +| oracle-simulator − action-agnostic | State FDE ↓ | -0.9391 | [-0.9391, -0.9391] | 12 | +| oracle-simulator − action-agnostic | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | CRC NDCG | 0.5129 | [0.4570, 0.5645] | 12 | +| oracle-simulator − action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | +| oracle-simulator − action-agnostic | Top-1 Regret ↓ | -0.3000 | [-0.3000, -0.3000] | 12 | + +Action separation alone does not establish correct direction, state dynamics, +or control ranking. WAMProbe therefore reports a metric profile rather than a +single composite score. diff --git a/examples/blockpush-demo/summary.json b/examples/blockpush-demo/summary.json new file mode 100644 index 0000000..91e917c --- /dev/null +++ b/examples/blockpush-demo/summary.json @@ -0,0 +1,3250 @@ +{ + "schema_version": "0.1", + "benchmark": "blockpush-2d-v0.1", + "results": [ + { + "schema_version": "0.1", + "model_id": "oracle-simulator", + "benchmark": "blockpush-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "blockpush-context-0000", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0001", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0002", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0003", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0004", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0005", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0006", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0007", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0008", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0009", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0010", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0011", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1138324205383303, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "noisy-dynamics", + "benchmark": "blockpush-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.9906688707175885, + "action_dependence_permutation_effect": 2.1119401408417615, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9978195270405896, + "noop_stability": 0.0, + "state_ade": 0.04274783427553582, + "state_fde": 0.03553171008466385, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "blockpush-context-0000", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9817815890396523, + "action_dependence_permutation_effect": 2.1091809909428987, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9969551873915743, + "noop_stability": 0.0, + "state_ade": 0.02035434408671741, + "state_fde": 0.022607915115848927, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0001", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9945494304184874, + "action_dependence_permutation_effect": 2.1088165685096842, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9941826242605436, + "noop_stability": 0.0, + "state_ade": 0.03004732783228947, + "state_fde": 0.04112968142306165, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0002", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9922510372452186, + "action_dependence_permutation_effect": 2.1153655401103295, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9980594331827886, + "noop_stability": 0.0, + "state_ade": 0.06525933270504829, + "state_fde": 0.045883637258629296, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0003", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9858251571494614, + "action_dependence_permutation_effect": 2.108731212091623, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9990203438654953, + "noop_stability": 0.0, + "state_ade": 0.05561090617900592, + "state_fde": 0.032455029559460155, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0004", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.116495718861126, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9995581784531452, + "noop_stability": 0.0, + "state_ade": 0.018852062452394115, + "state_fde": 0.02080931977007242, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0005", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9751748937516358, + "action_dependence_permutation_effect": 2.1010329725463004, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9997309387186614, + "noop_stability": 0.0, + "state_ade": 0.05623762917115013, + "state_fde": 0.02857786019425229, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0006", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1200160858550188, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9982340456764643, + "noop_stability": 0.0, + "state_ade": 0.029881047965936765, + "state_fde": 0.03788294733341109, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0007", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1050145977908206, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9981677540573998, + "noop_stability": 0.0, + "state_ade": 0.02927357567895715, + "state_fde": 0.054780155976736064, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0008", + "selected_action": "push-right", + "optimal_action": "push-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9877364113800982, + "action_dependence_permutation_effect": 2.1141967877459513, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9986969257504722, + "noop_stability": 0.0, + "state_ade": 0.06312572604549081, + "state_fde": 0.03267939684317012, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0009", + "selected_action": "push-left", + "optimal_action": "push-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1171857916168753, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9997969209655514, + "noop_stability": 0.0, + "state_ade": 0.014652079949477514, + "state_fde": 0.016746171367139153, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0010", + "selected_action": "push-up", + "optimal_action": "push-up", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.1136685138714264, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9943985204065345, + "noop_stability": 0.0, + "state_ade": 0.06425953795292311, + "state_fde": 0.04456550064353025, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "blockpush-context-0011", + "selected_action": "push-down", + "optimal_action": "push-down", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9707079296265068, + "action_dependence_permutation_effect": 2.1135769101590864, + "action_dependence_permutation_p_value": 0.016666666666666666, + "counterfactual_direction_accuracy": 0.9970334517584456, + "noop_stability": 0.0, + "state_ade": 0.06542044128703912, + "state_fde": 0.0482629055306548, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "copy-last-frame", + "benchmark": "blockpush-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764564, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.48711477971984807, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + }, + "context_results": [ + { + "context_id": "blockpush-context-0000", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0001", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0002", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0003", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0004", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.30305796369951177, + "state_fde": 0.43687100307645643, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0005", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0006", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0007", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0008", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0009", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.30000000000000004, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.30000000000000004 + } + }, + { + "context_id": "blockpush-context-0010", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.43687100307645643, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0011", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 1.0, + "state_ade": 0.3030579636995117, + "state_fde": 0.4368710030764563, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "wrong-direction", + "benchmark": "blockpush-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214387, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.7952908435996061, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4370514808453319, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + }, + "context_results": [ + { + "context_id": "blockpush-context-0000", + "selected_action": "push-left", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0001", + "selected_action": "push-right", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214392, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0002", + "selected_action": "push-down", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214392, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0003", + "selected_action": "push-up", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0004", + "selected_action": "push-left", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214326, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.6629910649137509, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.7952908435996061, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0005", + "selected_action": "push-right", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0006", + "selected_action": "push-down", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0007", + "selected_action": "push-up", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0008", + "selected_action": "push-left", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214395, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0009", + "selected_action": "push-right", + "optimal_action": "push-left", + "top1_regret": 0.30000000000000004, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.157813983772144, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.30000000000000004 + } + }, + { + "context_id": "blockpush-context-0010", + "selected_action": "push-down", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214353, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0011", + "selected_action": "push-up", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": -0.15781398377214395, + "action_dependence_permutation_p_value": 0.48333333333333334, + "counterfactual_direction_accuracy": -0.662991064913751, + "noop_stability": 1.0, + "state_ade": 0.5548217276442767, + "state_fde": 0.795290843599606, + "candidate_ranking_spearman": -0.25, + "candidate_ranking_kendall_tau": -0.25, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.375, + "top1_regret": 0.3 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "action-agnostic", + "benchmark": "blockpush-2d-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.1685044675431245, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.48711477971984807, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + }, + "context_results": [ + { + "context_id": "blockpush-context-0000", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0001", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0002", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0003", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0004", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.1685044675431245, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0005", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0006", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0007", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0008", + "selected_action": "noop", + "optimal_action": "push-right", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.4306765580733931, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0009", + "selected_action": "noop", + "optimal_action": "push-left", + "top1_regret": 0.30000000000000004, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.30000000000000004 + } + }, + { + "context_id": "blockpush-context-0010", + "selected_action": "noop", + "optimal_action": "push-up", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.3868528072345416, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + }, + { + "context_id": "blockpush-context-0011", + "selected_action": "noop", + "optimal_action": "push-down", + "top1_regret": 0.3, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.16850446754312448, + "noop_stability": 0.0, + "state_ade": 0.7361391966023326, + "state_fde": 0.9391366999396707, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 0.3 + } + } + ] + } + ], + "statistics": { + "oracle-simulator": { + "action_dependence": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.1138324205383303, + "median": 2.1138324205383303, + "standard_deviation": 0.0, + "quantiles": { + "p05": 2.1138324205383303, + "p25": 2.1138324205383303, + "p75": 2.1138324205383303, + "p95": 2.1138324205383303 + }, + "confidence_interval": { + "estimate": 2.1138324205383303, + "lower": 2.1138324205383303, + "upper": 2.1138324205383303, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.016666666666666666, + "median": 0.016666666666666666, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.016666666666666666, + "p25": 0.016666666666666666, + "p75": 0.016666666666666666, + "p95": 0.016666666666666666 + }, + "confidence_interval": { + "estimate": 0.016666666666666666, + "lower": 0.016666666666666666, + "upper": 0.016666666666666666, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "noisy-dynamics": { + "action_dependence": { + "mean": 0.9906688707175885, + "median": 0.993400233831853, + "standard_deviation": 0.010010240626981961, + "quantiles": { + "p05": 0.9731647598953277, + "p25": 0.9848142651220091, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.9906688707175885, + "lower": 0.984943113044481, + "upper": 0.9959408525622048, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.1119401408417615, + "median": 2.1136227120152564, + "standard_deviation": 0.005248646145330853, + "quantiles": { + "p05": 2.1032228664307864, + "p25": 2.108795229405169, + "p75": 2.115648084798029, + "p95": 2.1184594240240395 + }, + "confidence_interval": { + "estimate": 2.1119401408417615, + "lower": 2.1090758879267675, + "upper": 2.1148769274822197, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.016666666666666666, + "median": 0.016666666666666666, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.016666666666666666, + "p25": 0.016666666666666666, + "p75": 0.016666666666666666, + "p95": 0.016666666666666666 + }, + "confidence_interval": { + "estimate": 0.016666666666666666, + "lower": 0.016666666666666666, + "upper": 0.016666666666666666, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.9978195270405896, + "median": 0.9982008998669321, + "standard_deviation": 0.0018130147068397878, + "quantiles": { + "p05": 0.9943013671408386, + "p25": 0.9970138856667277, + "p75": 0.9991548025124077, + "p95": 0.999760630729762 + }, + "confidence_interval": { + "estimate": 0.9978195270405896, + "lower": 0.9968021097256851, + "upper": 0.9988292797040705, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.04274783427553582, + "median": 0.0428291170056477, + "standard_deviation": 0.01961402590631085, + "quantiles": { + "p05": 0.016962070326081644, + "p25": 0.027043767780897215, + "p75": 0.06340917902234888, + "p95": 0.06533183156694417 + }, + "confidence_interval": { + "estimate": 0.04274783427553582, + "lower": 0.03193523461282041, + "upper": 0.05379214274230163, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.03553171008466385, + "median": 0.03528117208829061, + "standard_deviation": 0.011404862905010178, + "quantiles": { + "p05": 0.01898090298875245, + "p25": 0.02708537392465145, + "p75": 0.04489503479730501, + "p95": 0.051195668231391364 + }, + "confidence_interval": { + "estimate": 0.03553171008466385, + "lower": 0.02931087256733419, + "upper": 0.041613250773388726, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "copy-last-frame": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.3030579636995117, + "median": 0.3030579636995117, + "standard_deviation": 1.6024689053196365e-17, + "quantiles": { + "p05": 0.3030579636995117, + "p25": 0.3030579636995117, + "p75": 0.3030579636995117, + "p95": 0.30305796369951177 + }, + "confidence_interval": { + "estimate": 0.3030579636995117, + "lower": 0.3030579636995117, + "upper": 0.3030579636995117, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.4368710030764564, + "median": 0.4368710030764563, + "standard_deviation": 5.551115123125783e-17, + "quantiles": { + "p05": 0.4368710030764563, + "p25": 0.4368710030764563, + "p75": 0.4368710030764563, + "p95": 0.43687100307645643 + }, + "confidence_interval": { + "estimate": 0.4368710030764564, + "lower": 0.43687100307645627, + "upper": 0.4368710030764564, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.48711477971984807, + "median": 0.46533827903669656, + "standard_deviation": 0.09231266380468323, + "quantiles": { + "p05": 0.3868528072345416, + "p25": 0.41972062036368024, + "p75": 0.5327324383928644, + "p95": 0.6309297535714575 + }, + "confidence_interval": { + "estimate": 0.48711477971984807, + "lower": 0.44065720704700917, + "upper": 0.542313066020904, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.3, + "median": 0.3, + "standard_deviation": 1.6024689053196365e-17, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "confidence_interval": { + "estimate": 0.3, + "lower": 0.3, + "upper": 0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "wrong-direction": { + "action_dependence": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": -0.15781398377214387, + "median": -0.15781398377214398, + "standard_deviation": 2.271891778003035e-16, + "quantiles": { + "p05": -0.157813983772144, + "p25": -0.157813983772144, + "p75": -0.15781398377214392, + "p95": -0.1578139837721434 + }, + "confidence_interval": { + "estimate": -0.15781398377214387, + "lower": -0.15781398377214398, + "upper": -0.15781398377214376, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.48333333333333334, + "median": 0.48333333333333334, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.48333333333333334, + "p25": 0.48333333333333334, + "p75": 0.48333333333333334, + "p95": 0.4833333333333334 + }, + "confidence_interval": { + "estimate": 0.48333333333333334, + "lower": 0.48333333333333334, + "upper": 0.48333333333333334, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": -0.662991064913751, + "median": -0.662991064913751, + "standard_deviation": 3.204937810639273e-17, + "quantiles": { + "p05": -0.662991064913751, + "p25": -0.662991064913751, + "p75": -0.662991064913751, + "p95": -0.6629910649137509 + }, + "confidence_interval": { + "estimate": -0.662991064913751, + "lower": -0.662991064913751, + "upper": -0.662991064913751, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.5548217276442767, + "median": 0.5548217276442767, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5548217276442767, + "p25": 0.5548217276442767, + "p75": 0.5548217276442767, + "p95": 0.5548217276442767 + }, + "confidence_interval": { + "estimate": 0.5548217276442767, + "lower": 0.5548217276442767, + "upper": 0.5548217276442767, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.7952908435996061, + "median": 0.795290843599606, + "standard_deviation": 1.0629576194313574e-16, + "quantiles": { + "p05": 0.795290843599606, + "p25": 0.795290843599606, + "p75": 0.795290843599606, + "p95": 0.795290843599606 + }, + "confidence_interval": { + "estimate": 0.7952908435996061, + "lower": 0.7952908435996061, + "upper": 0.7952908435996061, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": -0.25, + "median": -0.25, + "standard_deviation": 0.0, + "quantiles": { + "p05": -0.25, + "p25": -0.25, + "p75": -0.25, + "p95": -0.25 + }, + "confidence_interval": { + "estimate": -0.25, + "lower": -0.25, + "upper": -0.25, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": -0.25, + "median": -0.25, + "standard_deviation": 0.0, + "quantiles": { + "p05": -0.25, + "p25": -0.25, + "p75": -0.25, + "p95": -0.25 + }, + "confidence_interval": { + "estimate": -0.25, + "lower": -0.25, + "upper": -0.25, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.4370514808453319, + "median": 0.4306765580733931, + "standard_deviation": 0.040508339193136025, + "quantiles": { + "p05": 0.3868528072345416, + "p25": 0.41972062036368024, + "p75": 0.44800741855504483, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.4370514808453319, + "lower": 0.4160304660026279, + "upper": 0.45956132554281265, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.375, + "median": 0.375, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.375, + "p25": 0.375, + "p75": 0.375, + "p95": 0.375 + }, + "confidence_interval": { + "estimate": 0.375, + "lower": 0.375, + "upper": 0.375, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.3, + "median": 0.3, + "standard_deviation": 1.6024689053196365e-17, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "confidence_interval": { + "estimate": 0.3, + "lower": 0.3, + "upper": 0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "action-agnostic": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.1685044675431245, + "median": 0.16850446754312448, + "standard_deviation": 2.6573940485783934e-17, + "quantiles": { + "p05": 0.16850446754312448, + "p25": 0.16850446754312448, + "p75": 0.16850446754312448, + "p95": 0.16850446754312448 + }, + "confidence_interval": { + "estimate": 0.1685044675431245, + "lower": 0.1685044675431245, + "upper": 0.1685044675431245, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.7361391966023326, + "median": 0.7361391966023326, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.7361391966023326, + "p25": 0.7361391966023326, + "p75": 0.7361391966023326, + "p95": 0.7361391966023326 + }, + "confidence_interval": { + "estimate": 0.7361391966023326, + "lower": 0.7361391966023326, + "upper": 0.7361391966023326, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.9391366999396707, + "median": 0.9391366999396707, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.9391366999396709, + "p25": 0.9391366999396707, + "p75": 0.9391366999396707, + "p95": 0.9391366999396706 + }, + "confidence_interval": { + "estimate": 0.9391366999396707, + "lower": 0.9391366999396707, + "upper": 0.9391366999396707, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.48711477971984807, + "median": 0.46533827903669656, + "standard_deviation": 0.09231266380468323, + "quantiles": { + "p05": 0.3868528072345416, + "p25": 0.41972062036368024, + "p75": 0.5327324383928644, + "p95": 0.6309297535714575 + }, + "confidence_interval": { + "estimate": 0.48711477971984807, + "lower": 0.43916037774789124, + "upper": 0.5387211609283759, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.3, + "median": 0.3, + "standard_deviation": 1.6024689053196365e-17, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "confidence_interval": { + "estimate": 0.3, + "lower": 0.3, + "upper": 0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + } + }, + "paired_comparisons": [ + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.009331129282411635, + "confidence_interval": { + "estimate": 0.009331129282411635, + "lower": 0.004075300138524986, + "upper": 0.015187504142470621, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0018922796965685773, + "confidence_interval": { + "estimate": 0.0018922796965685773, + "lower": -0.0008359966668372833, + "upper": 0.004988668245693784, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.002180472959410318, + "confidence_interval": { + "estimate": 0.002180472959410318, + "lower": 0.0012253024585918491, + "upper": 0.003238825384234412, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": -0.04274783427553582, + "confidence_interval": { + "estimate": -0.04274783427553582, + "lower": -0.05308207616478455, + "upper": -0.031399256035883505, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": -0.03553171008466385, + "confidence_interval": { + "estimate": -0.03553171008466385, + "lower": -0.04192893710319782, + "upper": -0.028507391886291974, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 2.1138324205383303, + "confidence_interval": { + "estimate": 2.1138324205383303, + "lower": 2.1138324205383303, + "upper": 2.1138324205383303, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.9833333333333333, + "confidence_interval": { + "estimate": -0.9833333333333333, + "lower": -0.9833333333333333, + "upper": -0.9833333333333333, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.3030579636995117, + "confidence_interval": { + "estimate": -0.3030579636995117, + "lower": -0.3030579636995117, + "upper": -0.3030579636995117, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.4368710030764564, + "confidence_interval": { + "estimate": -0.4368710030764564, + "lower": -0.4368710030764564, + "upper": -0.43687100307645627, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.5128852202801519, + "confidence_interval": { + "estimate": 0.5128852202801519, + "lower": 0.463366766327258, + "upper": 0.5615036860891806, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.3, + "confidence_interval": { + "estimate": -0.3, + "lower": -0.3, + "upper": -0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 2.271646404310474, + "confidence_interval": { + "estimate": 2.271646404310474, + "lower": 2.271646404310474, + "upper": 2.2716464043104745, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.4666666666666666, + "confidence_interval": { + "estimate": -0.4666666666666666, + "lower": -0.4666666666666666, + "upper": -0.4666666666666666, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.662991064913751, + "confidence_interval": { + "estimate": 1.662991064913751, + "lower": 1.662991064913751, + "upper": 1.6629910649137514, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.5548217276442767, + "confidence_interval": { + "estimate": -0.5548217276442767, + "lower": -0.5548217276442767, + "upper": -0.5548217276442767, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.7952908435996061, + "confidence_interval": { + "estimate": -0.7952908435996061, + "lower": -0.7952908435996061, + "upper": -0.7952908435996061, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.25, + "confidence_interval": { + "estimate": 1.25, + "lower": 1.25, + "upper": 1.25, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.25, + "confidence_interval": { + "estimate": 1.25, + "lower": 1.25, + "upper": 1.25, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.562948519154668, + "confidence_interval": { + "estimate": 0.562948519154668, + "lower": 0.5404386744571873, + "upper": 0.5854583638521488, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.625, + "confidence_interval": { + "estimate": 0.625, + "lower": 0.625, + "upper": 0.625, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.3, + "confidence_interval": { + "estimate": -0.3, + "lower": -0.3, + "upper": -0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 2.1138324205383303, + "confidence_interval": { + "estimate": 2.1138324205383303, + "lower": 2.1138324205383303, + "upper": 2.1138324205383303, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.9833333333333333, + "confidence_interval": { + "estimate": -0.9833333333333333, + "lower": -0.9833333333333333, + "upper": -0.9833333333333333, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.8314955324568755, + "confidence_interval": { + "estimate": 0.8314955324568755, + "lower": 0.8314955324568755, + "upper": 0.8314955324568757, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.7361391966023326, + "confidence_interval": { + "estimate": -0.7361391966023326, + "lower": -0.7361391966023326, + "upper": -0.7361391966023326, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.9391366999396707, + "confidence_interval": { + "estimate": -0.9391366999396707, + "lower": -0.9391366999396707, + "upper": -0.9391366999396707, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.5128852202801519, + "confidence_interval": { + "estimate": 0.5128852202801519, + "lower": 0.4569627959979814, + "upper": 0.564475524133926, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.3, + "confidence_interval": { + "estimate": -0.3, + "lower": -0.3, + "upper": -0.3, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + } + ] +} diff --git a/examples/gripper-catch-demo/report.html b/examples/gripper-catch-demo/report.html new file mode 100644 index 0000000..83bcc8b --- /dev/null +++ b/examples/gripper-catch-demo/report.html @@ -0,0 +1,53 @@ + + + + + + WAMProbe gripper-catch-v0.1 Demo + + + +

WAMProbe gripper-catch-v0.1 Demo

+

Action-aware and deliberately broken baselines are shown as a metric + profile. No composite score is produced.

+
+

Metric profile

+
+ + +
ModelAction DependencePermutation EffectPermutation p-value ↓Counterfactual DirectionNo-op StabilityState ADE ↓State FDE ↓CRC SpearmanCRC Kendall τCRC NDCGCRC Pairwise AccuracyTop-1 Regret ↓
oracle-simulator1.00002.00960.05561.00001.00000.00000.00001.00001.00001.00001.00000.0000
noisy-dynamics0.99852.00990.05560.99860.00000.05630.03051.00001.00001.00001.00000.0000
copy-last-frame0.00000.00001.00000.00000.00000.88561.04470.00000.00000.71030.50001.0000
wrong-direction0.1735-0.23890.55560.36871.00000.83870.96630.00000.00000.71030.50001.0000
action-agnostic0.00000.00001.00000.58110.00000.64391.15600.00000.00000.71030.50001.0000
+
+
+

Context-block 95% confidence intervals

+

Whole contexts are resampled; correlated branches and frames are never + treated as independent observations.

+
+ + + +
ModelMetricMean95% CIMedianStdp05 / p95
oracle-simulatorAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorPermutation Effect2.0096[1.9929, 2.0262]2.03180.03141.9651 / 2.0318
oracle-simulatorPermutation p-value ↓0.0556[0.0451, 0.0660]0.04170.01960.0417 / 0.0833
oracle-simulatorCounterfactual Direction1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorState ADE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-simulatorState FDE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-simulatorCRC Spearman1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC Kendall τ1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-simulatorTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-dynamicsAction Dependence0.9985[0.9964, 0.9999]1.00000.00320.9926 / 1.0000
noisy-dynamicsPermutation Effect2.0099[1.9928, 2.0265]2.02760.03141.9649 / 2.0377
noisy-dynamicsPermutation p-value ↓0.0556[0.0451, 0.0660]0.04170.01960.0417 / 0.0833
noisy-dynamicsCounterfactual Direction0.9986[0.9982, 0.9990]0.99870.00080.9975 / 0.9996
noisy-dynamicsNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-dynamicsState ADE ↓0.0563[0.0372, 0.0765]0.05500.03660.0169 / 0.0975
noisy-dynamicsState FDE ↓0.0305[0.0272, 0.0336]0.03040.00590.0220 / 0.0387
noisy-dynamicsCRC Spearman1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC Kendall τ1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-dynamicsTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameState ADE ↓0.8856[0.8855, 0.8858]0.88540.00030.8854 / 0.8860
copy-last-frameState FDE ↓1.0447[1.0442, 1.0453]1.04410.00091.0441 / 1.0460
copy-last-frameCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC NDCG0.7103[0.5981, 0.8353]0.63090.21170.5000 / 1.0000
copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionAction Dependence0.1735[0.1725, 0.1745]0.17490.00200.1707 / 0.1749
wrong-directionPermutation Effect-0.2389[-0.7944, 0.3167]0.50191.0476-1.7204 / 0.5019
wrong-directionPermutation p-value ↓0.5556[0.3889, 0.7222]0.33330.31430.3333 / 1.0000
wrong-directionCounterfactual Direction0.3687[0.3646, 0.3727]0.37400.00760.3579 / 0.3740
wrong-directionNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState ADE ↓0.8387[0.8380, 0.8393]0.83780.00120.8378 / 0.8404
wrong-directionState FDE ↓0.9663[0.9643, 0.9683]0.96370.00370.9637 / 0.9715
wrong-directionCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
wrong-directionCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
wrong-directionCRC NDCG0.7103[0.5962, 0.8244]0.63090.21170.5000 / 1.0000
wrong-directionCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
wrong-directionTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticCounterfactual Direction0.5811[0.5799, 0.5823]0.57950.00230.5795 / 0.5843
action-agnosticNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticState ADE ↓0.6439[0.6396, 0.6481]0.64950.00800.6325 / 0.6495
action-agnosticState FDE ↓1.1560[1.1527, 1.1593]1.16040.00621.1472 / 1.1604
action-agnosticCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC NDCG0.7103[0.5962, 0.8353]0.63090.21170.5000 / 1.0000
action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
action-agnosticTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
+
+
+

Paired model differences

+

Each value is left minus right, aligned by exact context ID.

+
+ + + +
ModelsMetricMean difference95% CIContexts
oracle-simulator − noisy-dynamicsAction Dependence0.0015[0.0001, 0.0034]12
oracle-simulator − noisy-dynamicsPermutation Effect-0.0004[-0.0028, 0.0019]12
oracle-simulator − noisy-dynamicsPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCounterfactual Direction0.0014[0.0010, 0.0018]12
oracle-simulator − noisy-dynamicsNo-op Stability1.0000[1.0000, 1.0000]12
oracle-simulator − noisy-dynamicsState ADE ↓-0.0563[-0.0760, -0.0367]12
oracle-simulator − noisy-dynamicsState FDE ↓-0.0305[-0.0340, -0.0270]12
oracle-simulator − noisy-dynamicsCRC Spearman0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC Kendall τ0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC NDCG0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsCRC Pairwise Accuracy0.0000[0.0000, 0.0000]12
oracle-simulator − noisy-dynamicsTop-1 Regret ↓0.0000[0.0000, 0.0000]12
oracle-simulator − copy-last-frameAction Dependence1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-framePermutation Effect2.0096[1.9929, 2.0262]12
oracle-simulator − copy-last-framePermutation p-value ↓-0.9444[-0.9549, -0.9340]12
oracle-simulator − copy-last-frameCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameNo-op Stability1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameState ADE ↓-0.8856[-0.8858, -0.8855]12
oracle-simulator − copy-last-frameState FDE ↓-1.0447[-1.0452, -1.0442]12
oracle-simulator − copy-last-frameCRC Spearman1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-simulator − copy-last-frameCRC NDCG0.2897[0.1667, 0.4038]12
oracle-simulator − copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-simulator − copy-last-frameTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-simulator − wrong-directionAction Dependence0.8265[0.8255, 0.8275]12
oracle-simulator − wrong-directionPermutation Effect2.2484[1.7095, 2.7873]12
oracle-simulator − wrong-directionPermutation p-value ↓-0.5000[-0.6562, -0.3438]12
oracle-simulator − wrong-directionCounterfactual Direction0.6313[0.6273, 0.6354]12
oracle-simulator − wrong-directionNo-op Stability0.0000[0.0000, 0.0000]12
oracle-simulator − wrong-directionState ADE ↓-0.8387[-0.8393, -0.8380]12
oracle-simulator − wrong-directionState FDE ↓-0.9663[-0.9683, -0.9643]12
oracle-simulator − wrong-directionCRC Spearman1.0000[1.0000, 1.0000]12
oracle-simulator − wrong-directionCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-simulator − wrong-directionCRC NDCG0.2897[0.1754, 0.4038]12
oracle-simulator − wrong-directionCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-simulator − wrong-directionTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-simulator − action-agnosticAction Dependence1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticPermutation Effect2.0096[1.9929, 2.0262]12
oracle-simulator − action-agnosticPermutation p-value ↓-0.9444[-0.9549, -0.9340]12
oracle-simulator − action-agnosticCounterfactual Direction0.4189[0.4177, 0.4201]12
oracle-simulator − action-agnosticNo-op Stability1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticState ADE ↓-0.6439[-0.6481, -0.6396]12
oracle-simulator − action-agnosticState FDE ↓-1.1560[-1.1593, -1.1527]12
oracle-simulator − action-agnosticCRC Spearman1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-simulator − action-agnosticCRC NDCG0.2897[0.1647, 0.4147]12
oracle-simulator − action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-simulator − action-agnosticTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
+
+ + diff --git a/examples/gripper-catch-demo/report.md b/examples/gripper-catch-demo/report.md new file mode 100644 index 0000000..d5a0da1 --- /dev/null +++ b/examples/gripper-catch-demo/report.md @@ -0,0 +1,137 @@ +# WAMProbe gripper-catch-v0.1 Demo + +This report compares action-aware and deliberately broken reference baselines. + +| Model | Action Dependence | Permutation Effect | Permutation p-value ↓ | Counterfactual Direction | No-op Stability | State ADE ↓ | State FDE ↓ | CRC Spearman | CRC Kendall τ | CRC NDCG | CRC Pairwise Accuracy | Top-1 Regret ↓ | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| oracle-simulator | 1.0000 | 2.0096 | 0.0556 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | 0.0000 | +| noisy-dynamics | 0.9985 | 2.0099 | 0.0556 | 0.9986 | 0.0000 | 0.0563 | 0.0305 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | 0.0000 | +| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 0.0000 | 0.8856 | 1.0447 | 0.0000 | 0.0000 | 0.7103 | 0.5000 | 1.0000 | +| wrong-direction | 0.1735 | -0.2389 | 0.5556 | 0.3687 | 1.0000 | 0.8387 | 0.9663 | 0.0000 | 0.0000 | 0.7103 | 0.5000 | 1.0000 | +| action-agnostic | 0.0000 | 0.0000 | 1.0000 | 0.5811 | 0.0000 | 0.6439 | 1.1560 | 0.0000 | 0.0000 | 0.7103 | 0.5000 | 1.0000 | + +## Context-block 95% confidence intervals + +Intervals resample whole contexts, not correlated action branches or frames. + +| Model | Metric | Mean | 95% CI | Median | Std | +|---|---|---:|---:|---:|---:| +| oracle-simulator | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | Permutation Effect | 2.0096 | [1.9929, 2.0262] | 2.0318 | 0.0314 | +| oracle-simulator | Permutation p-value ↓ | 0.0556 | [0.0451, 0.0660] | 0.0417 | 0.0196 | +| oracle-simulator | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | State ADE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-simulator | State FDE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-simulator | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-simulator | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-dynamics | Action Dependence | 0.9985 | [0.9964, 0.9999] | 1.0000 | 0.0032 | +| noisy-dynamics | Permutation Effect | 2.0099 | [1.9928, 2.0265] | 2.0276 | 0.0314 | +| noisy-dynamics | Permutation p-value ↓ | 0.0556 | [0.0451, 0.0660] | 0.0417 | 0.0196 | +| noisy-dynamics | Counterfactual Direction | 0.9986 | [0.9982, 0.9990] | 0.9987 | 0.0008 | +| noisy-dynamics | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| noisy-dynamics | State ADE ↓ | 0.0563 | [0.0372, 0.0765] | 0.0550 | 0.0366 | +| noisy-dynamics | State FDE ↓ | 0.0305 | [0.0272, 0.0336] | 0.0304 | 0.0059 | +| noisy-dynamics | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| copy-last-frame | Counterfactual Direction | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | State ADE ↓ | 0.8856 | [0.8855, 0.8858] | 0.8854 | 0.0003 | +| copy-last-frame | State FDE ↓ | 1.0447 | [1.0442, 1.0453] | 1.0441 | 0.0009 | +| copy-last-frame | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC NDCG | 0.7103 | [0.5981, 0.8353] | 0.6309 | 0.2117 | +| copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| copy-last-frame | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | Action Dependence | 0.1735 | [0.1725, 0.1745] | 0.1749 | 0.0020 | +| wrong-direction | Permutation Effect | -0.2389 | [-0.7944, 0.3167] | 0.5019 | 1.0476 | +| wrong-direction | Permutation p-value ↓ | 0.5556 | [0.3889, 0.7222] | 0.3333 | 0.3143 | +| wrong-direction | Counterfactual Direction | 0.3687 | [0.3646, 0.3727] | 0.3740 | 0.0076 | +| wrong-direction | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| wrong-direction | State ADE ↓ | 0.8387 | [0.8380, 0.8393] | 0.8378 | 0.0012 | +| wrong-direction | State FDE ↓ | 0.9663 | [0.9643, 0.9683] | 0.9637 | 0.0037 | +| wrong-direction | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| wrong-direction | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| wrong-direction | CRC NDCG | 0.7103 | [0.5962, 0.8244] | 0.6309 | 0.2117 | +| wrong-direction | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| wrong-direction | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| action-agnostic | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | Permutation p-value ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| action-agnostic | Counterfactual Direction | 0.5811 | [0.5799, 0.5823] | 0.5795 | 0.0023 | +| action-agnostic | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | State ADE ↓ | 0.6439 | [0.6396, 0.6481] | 0.6495 | 0.0080 | +| action-agnostic | State FDE ↓ | 1.1560 | [1.1527, 1.1593] | 1.1604 | 0.0062 | +| action-agnostic | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC NDCG | 0.7103 | [0.5962, 0.8353] | 0.6309 | 0.2117 | +| action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | +| action-agnostic | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | + +## Paired model differences + +Differences are left minus right over exactly aligned context IDs. + +| Models | Metric | Mean difference | 95% CI | Contexts | +|---|---|---:|---:|---:| +| oracle-simulator − noisy-dynamics | Action Dependence | 0.0015 | [0.0001, 0.0034] | 12 | +| oracle-simulator − noisy-dynamics | Permutation Effect | -0.0004 | [-0.0028, 0.0019] | 12 | +| oracle-simulator − noisy-dynamics | Permutation p-value ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | Counterfactual Direction | 0.0014 | [0.0010, 0.0018] | 12 | +| oracle-simulator − noisy-dynamics | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − noisy-dynamics | State ADE ↓ | -0.0563 | [-0.0760, -0.0367] | 12 | +| oracle-simulator − noisy-dynamics | State FDE ↓ | -0.0305 | [-0.0340, -0.0270] | 12 | +| oracle-simulator − noisy-dynamics | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC NDCG | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | CRC Pairwise Accuracy | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − noisy-dynamics | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − copy-last-frame | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | Permutation Effect | 2.0096 | [1.9929, 2.0262] | 12 | +| oracle-simulator − copy-last-frame | Permutation p-value ↓ | -0.9444 | [-0.9549, -0.9340] | 12 | +| oracle-simulator − copy-last-frame | Counterfactual Direction | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | State ADE ↓ | -0.8856 | [-0.8858, -0.8855] | 12 | +| oracle-simulator − copy-last-frame | State FDE ↓ | -1.0447 | [-1.0452, -1.0442] | 12 | +| oracle-simulator − copy-last-frame | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − copy-last-frame | CRC NDCG | 0.2897 | [0.1667, 0.4038] | 12 | +| oracle-simulator − copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | +| oracle-simulator − copy-last-frame | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | +| oracle-simulator − wrong-direction | Action Dependence | 0.8265 | [0.8255, 0.8275] | 12 | +| oracle-simulator − wrong-direction | Permutation Effect | 2.2484 | [1.7095, 2.7873] | 12 | +| oracle-simulator − wrong-direction | Permutation p-value ↓ | -0.5000 | [-0.6562, -0.3438] | 12 | +| oracle-simulator − wrong-direction | Counterfactual Direction | 0.6313 | [0.6273, 0.6354] | 12 | +| oracle-simulator − wrong-direction | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-simulator − wrong-direction | State ADE ↓ | -0.8387 | [-0.8393, -0.8380] | 12 | +| oracle-simulator − wrong-direction | State FDE ↓ | -0.9663 | [-0.9683, -0.9643] | 12 | +| oracle-simulator − wrong-direction | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − wrong-direction | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − wrong-direction | CRC NDCG | 0.2897 | [0.1754, 0.4038] | 12 | +| oracle-simulator − wrong-direction | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | +| oracle-simulator − wrong-direction | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | +| oracle-simulator − action-agnostic | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | Permutation Effect | 2.0096 | [1.9929, 2.0262] | 12 | +| oracle-simulator − action-agnostic | Permutation p-value ↓ | -0.9444 | [-0.9549, -0.9340] | 12 | +| oracle-simulator − action-agnostic | Counterfactual Direction | 0.4189 | [0.4177, 0.4201] | 12 | +| oracle-simulator − action-agnostic | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | State ADE ↓ | -0.6439 | [-0.6481, -0.6396] | 12 | +| oracle-simulator − action-agnostic | State FDE ↓ | -1.1560 | [-1.1593, -1.1527] | 12 | +| oracle-simulator − action-agnostic | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-simulator − action-agnostic | CRC NDCG | 0.2897 | [0.1647, 0.4147] | 12 | +| oracle-simulator − action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | +| oracle-simulator − action-agnostic | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | + +Action separation alone does not establish correct direction, state dynamics, +or control ranking. WAMProbe therefore reports a metric profile rather than a +single composite score. diff --git a/examples/gripper-catch-demo/summary.json b/examples/gripper-catch-demo/summary.json new file mode 100644 index 0000000..daef4cd --- /dev/null +++ b/examples/gripper-catch-demo/summary.json @@ -0,0 +1,3250 @@ +{ + "schema_version": "0.1", + "benchmark": "gripper-catch-v0.1", + "results": [ + { + "schema_version": "0.1", + "model_id": "oracle-simulator", + "benchmark": "gripper-catch-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0095517494564104, + "action_dependence_permutation_p_value": 0.05555555555555555, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "gripper-catch-context-0000", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0001", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9650800460424103, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0002", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0003", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0004", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9650800460424103, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 1.0000000000000002, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0005", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0006", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0007", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9650800460424103, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 1.0000000000000002, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0008", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0009", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0010", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9650800460424103, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0011", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0317876011634106, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 1.0, + "noop_stability": 1.0, + "state_ade": 0.0, + "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "noisy-dynamics", + "benchmark": "gripper-catch-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.9984849062033448, + "action_dependence_permutation_effect": 2.0099300435884997, + "action_dependence_permutation_p_value": 0.05555555555555555, + "counterfactual_direction_accuracy": 0.9985736550394932, + "noop_stability": 0.0, + "state_ade": 0.0563249614434043, + "state_fde": 0.03046040283421649, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + }, + "context_results": [ + { + "context_id": "gripper-catch-context-0000", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.03328627964326, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.998751222150501, + "noop_stability": 0.0, + "state_ade": 0.01783554477574189, + "state_fde": 0.026143430717877662, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0001", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9959234095134148, + "action_dependence_permutation_effect": 1.9683076955377539, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 0.9995639264516412, + "noop_stability": 0.0, + "state_ade": 0.09173022758531302, + "state_fde": 0.034067814957425814, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0002", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0280144534404685, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9990347568707167, + "noop_stability": 0.0, + "state_ade": 0.09347966057905202, + "state_fde": 0.030746605551571854, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0003", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.028866065063458, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9977935046168772, + "noop_stability": 0.0, + "state_ade": 0.09897130733313156, + "state_fde": 0.03504915675765663, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0004", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9649762037923153, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 0.9980722775867971, + "noop_stability": 0.0, + "state_ade": 0.08687304588393416, + "state_fde": 0.02834071234381706, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0005", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0271076012916223, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9994667512281166, + "noop_stability": 0.0, + "state_ade": 0.018318001389661775, + "state_fde": 0.020428044596131067, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0006", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9977227317152318, + "action_dependence_permutation_effect": 2.035343362360904, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9985705304120333, + "noop_stability": 0.0, + "state_ade": 0.022804654309996276, + "state_fde": 0.02334611649230101, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0007", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9995430737260003, + "action_dependence_permutation_effect": 1.9655369130929543, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 0.997844337576858, + "noop_stability": 0.0, + "state_ade": 0.02303561309428863, + "state_fde": 0.03361532668044107, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0008", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 0.9886296594854924, + "action_dependence_permutation_effect": 2.0406675383673174, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9991041271849446, + "noop_stability": 0.0, + "state_ade": 0.08950559915447696, + "state_fde": 0.03550807952748611, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0009", + "selected_action": "catch-left", + "optimal_action": "catch-left", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0352356743135878, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9978654032378947, + "noop_stability": 0.0, + "state_ade": 0.021171616306822315, + "state_fde": 0.029971376354966165, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0010", + "selected_action": "catch-stay", + "optimal_action": "catch-stay", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 1.9649004841406483, + "action_dependence_permutation_p_value": 0.08333333333333333, + "counterfactual_direction_accuracy": 0.9995858403895441, + "noop_stability": 0.0, + "state_ade": 0.01585892153034818, + "state_fde": 0.02564925575065324, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + }, + { + "context_id": "gripper-catch-context-0011", + "selected_action": "catch-right", + "optimal_action": "catch-right", + "top1_regret": 0.0, + "metrics": { + "action_dependence": 1.0, + "action_dependence_permutation_effect": 2.0269182520177056, + "action_dependence_permutation_p_value": 0.041666666666666664, + "counterfactual_direction_accuracy": 0.9972311827679945, + "noop_stability": 0.0, + "state_ade": 0.09631534537808481, + "state_fde": 0.042658914280270216, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, + "top1_regret": 0.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "copy-last-frame", + "benchmark": "gripper-catch-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8856122005706287, + "state_fde": 1.0446933677159576, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7103099178571526, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + }, + "context_results": [ + { + "context_id": "gripper-catch-context-0000", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0001", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8860220415648333, + "state_fde": 1.045967186417906, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0002", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0003", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0004", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8860220415648333, + "state_fde": 1.045967186417906, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0005", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0006", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0007", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8860220415648333, + "state_fde": 1.045967186417906, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0008", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649834, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0009", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649832, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0010", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8860220415648333, + "state_fde": 1.045967186417906, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0011", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.0, + "noop_stability": 0.0, + "state_ade": 0.8854072800735265, + "state_fde": 1.0440564583649834, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "wrong-direction", + "benchmark": "gripper-catch-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.17350858337580802, + "action_dependence_permutation_effect": -0.23885297302335362, + "action_dependence_permutation_p_value": 0.5555555555555555, + "counterfactual_direction_accuracy": 0.3686621169012934, + "noop_stability": 1.0, + "state_ade": 0.8386650303332343, + "state_fde": 0.9662883683502264, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7103099178571526, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + }, + "context_results": [ + { + "context_id": "gripper-catch-context-0000", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0001", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.1707400017727927, + "action_dependence_permutation_effect": -1.7204145585509087, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.35789326558483675, + "noop_stability": 1.0, + "state_ade": 0.8403866427306597, + "state_fde": 0.9715291826056698, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0002", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0003", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.5019278197404238, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0004", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.1707400017727927, + "action_dependence_permutation_effect": -1.7204145585509087, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.35789326558483675, + "noop_stability": 1.0, + "state_ade": 0.8403866427306597, + "state_fde": 0.9715291826056698, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0005", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0006", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.5019278197404238, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0007", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.1707400017727927, + "action_dependence_permutation_effect": -1.7204145585509087, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.35789326558483675, + "noop_stability": 1.0, + "state_ade": 0.8403866427306597, + "state_fde": 0.9715291826056698, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0008", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0009", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0010", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.1707400017727927, + "action_dependence_permutation_effect": -1.7204145585509087, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.35789326558483675, + "noop_stability": 1.0, + "state_ade": 0.8403866427306597, + "state_fde": 0.9715291826056698, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0011", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.17489287417731567, + "action_dependence_permutation_effect": 0.501927819740424, + "action_dependence_permutation_p_value": 0.3333333333333333, + "counterfactual_direction_accuracy": 0.37404654255952174, + "noop_stability": 1.0, + "state_ade": 0.8378042241345215, + "state_fde": 0.9636679612225048, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + } + ] + }, + { + "schema_version": "0.1", + "model_id": "action-agnostic", + "benchmark": "gripper-catch-v0.1", + "contexts": 12, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5811096409230095, + "noop_stability": 0.0, + "state_ade": 0.6438600690131464, + "state_fde": 1.155975033047959, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7103099178571526, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + }, + "context_results": [ + { + "context_id": "gripper-catch-context-0000", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0001", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5843267946125047, + "noop_stability": 0.0, + "state_ade": 0.6325355864978516, + "state_fde": 1.1471555447465043, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0002", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0003", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0004", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5843267946125048, + "noop_stability": 0.0, + "state_ade": 0.6325355864978516, + "state_fde": 1.1471555447465043, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0005", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0006", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0007", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5843267946125048, + "noop_stability": 0.0, + "state_ade": 0.6325355864978516, + "state_fde": 1.1471555447465043, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0008", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.579501064078262, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0009", + "selected_action": "noop", + "optimal_action": "catch-left", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5795010640782621, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0010", + "selected_action": "noop", + "optimal_action": "catch-stay", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.5843267946125047, + "noop_stability": 0.0, + "state_ade": 0.6325355864978516, + "state_fde": 1.1471555447465043, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + }, + { + "context_id": "gripper-catch-context-0011", + "selected_action": "noop", + "optimal_action": "catch-right", + "top1_regret": 1.0, + "metrics": { + "action_dependence": 0.0, + "action_dependence_permutation_effect": 0.0, + "action_dependence_permutation_p_value": 1.0, + "counterfactual_direction_accuracy": 0.579501064078262, + "noop_stability": 0.0, + "state_ade": 0.6495223102707939, + "state_fde": 1.1603847771986866, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6309297535714575, + "candidate_ranking_pairwise_accuracy": 0.5, + "top1_regret": 1.0 + } + } + ] + } + ], + "statistics": { + "oracle-simulator": { + "action_dependence": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.0095517494564104, + "median": 2.0317876011634106, + "standard_deviation": 0.031446243054956496, + "quantiles": { + "p05": 1.9650800460424103, + "p25": 1.9650800460424103, + "p75": 2.0317876011634106, + "p95": 2.0317876011634106 + }, + "confidence_interval": { + "estimate": 2.0095517494564104, + "lower": 1.9928748606761604, + "upper": 2.0262286382366606, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.05555555555555555, + "median": 0.041666666666666664, + "standard_deviation": 0.019641855032959652, + "quantiles": { + "p05": 0.041666666666666664, + "p25": 0.041666666666666664, + "p75": 0.08333333333333333, + "p95": 0.08333333333333333 + }, + "confidence_interval": { + "estimate": 0.05555555555555555, + "lower": 0.04513888888888889, + "upper": 0.06597222222222222, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 9.064933036736789e-17, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0000000000000002 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0000000000000002, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "noisy-dynamics": { + "action_dependence": { + "mean": 0.9984849062033448, + "median": 1.0, + "standard_deviation": 0.003211332217319107, + "quantiles": { + "p05": 0.9926412220008497, + "p25": 0.9990879882233081, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.9984849062033448, + "lower": 0.996439905934109, + "upper": 0.9998857684315001, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 2.0099300435884997, + "median": 2.0275610273660454, + "standard_deviation": 0.03135480345651305, + "quantiles": { + "p05": 1.964942129949065, + "p25": 1.967614999926554, + "p75": 2.033773628310842, + "p95": 2.03773924156379 + }, + "confidence_interval": { + "estimate": 2.0099300435884997, + "lower": 1.9928348841574257, + "upper": 2.0265215453962866, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.05555555555555555, + "median": 0.041666666666666664, + "standard_deviation": 0.019641855032959652, + "quantiles": { + "p05": 0.041666666666666664, + "p25": 0.041666666666666664, + "p75": 0.08333333333333333, + "p95": 0.08333333333333333 + }, + "confidence_interval": { + "estimate": 0.05555555555555555, + "lower": 0.04513888888888889, + "upper": 0.06597222222222222, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.9985736550394932, + "median": 0.9986608762812672, + "standard_deviation": 0.0007650764286586958, + "quantiles": { + "p05": 0.99754045978488, + "p25": 0.9978601368226356, + "p75": 0.9991947831957376, + "p95": 0.9995737877236974 + }, + "confidence_interval": { + "estimate": 0.9985736550394932, + "lower": 0.9981601566556876, + "upper": 0.9990146259172817, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.0563249614434043, + "median": 0.05495432948911139, + "standard_deviation": 0.03664821541543745, + "quantiles": { + "p05": 0.016946064315314723, + "p25": 0.020458212577532178, + "p75": 0.09216758583374776, + "p95": 0.09751052825785583 + }, + "confidence_interval": { + "estimate": 0.0563249614434043, + "lower": 0.03715961420223628, + "upper": 0.07654673654357069, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.03046040283421649, + "median": 0.030358990953269008, + "standard_deviation": 0.00588900928230019, + "quantiles": { + "p05": 0.022032984139024533, + "p25": 0.026019886976071556, + "p75": 0.03431315040748352, + "p95": 0.03872595516623895 + }, + "confidence_interval": { + "estimate": 0.03046040283421649, + "lower": 0.02723484772669842, + "upper": 0.03360871025156912, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "copy-last-frame": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.8856122005706287, + "median": 0.8854072800735265, + "standard_deviation": 0.00028980134621025834, + "quantiles": { + "p05": 0.8854072800735265, + "p25": 0.8854072800735265, + "p75": 0.8860220415648333, + "p95": 0.8860220415648333 + }, + "confidence_interval": { + "estimate": 0.8856122005706287, + "lower": 0.8854585101978021, + "upper": 0.8857658909434555, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 1.0446933677159576, + "median": 1.0440564583649832, + "standard_deviation": 0.0009007258421500823, + "quantiles": { + "p05": 1.0440564583649832, + "p25": 1.0440564583649832, + "p75": 1.045967186417906, + "p95": 1.045967186417906 + }, + "confidence_interval": { + "estimate": 1.0446933677159576, + "lower": 1.0442156857027267, + "upper": 1.0453302770669317, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.7103099178571526, + "median": 0.6309297535714575, + "standard_deviation": 0.21170090200015357, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.7103099178571526, + "lower": 0.5981479005654977, + "upper": 0.8353099178571526, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "wrong-direction": { + "action_dependence": { + "mean": 0.17350858337580802, + "median": 0.17489287417731567, + "standard_deviation": 0.0019576828257604514, + "quantiles": { + "p05": 0.1707400017727927, + "p25": 0.1707400017727927, + "p75": 0.17489287417731567, + "p95": 0.17489287417731567 + }, + "confidence_interval": { + "estimate": 0.17350858337580802, + "lower": 0.17247036527467727, + "upper": 0.17454680147693877, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": -0.23885297302335362, + "median": 0.5019278197404239, + "standard_deviation": 1.0476222438720273, + "quantiles": { + "p05": -1.7204145585509087, + "p25": -1.7204145585509087, + "p75": 0.501927819740424, + "p95": 0.501927819740424 + }, + "confidence_interval": { + "estimate": -0.23885297302335362, + "lower": -0.7944385675961868, + "upper": 0.31673262154947956, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 0.5555555555555555, + "median": 0.3333333333333333, + "standard_deviation": 0.31426968052735443, + "quantiles": { + "p05": 0.3333333333333333, + "p25": 0.3333333333333333, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.5555555555555555, + "lower": 0.38888888888888884, + "upper": 0.7222222222222222, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.3686621169012934, + "median": 0.37404654255952174, + "standard_deviation": 0.007614727791456185, + "quantiles": { + "p05": 0.35789326558483675, + "p25": 0.35789326558483675, + "p75": 0.37404654255952174, + "p95": 0.37404654255952174 + }, + "confidence_interval": { + "estimate": 0.3686621169012934, + "lower": 0.36462379765762215, + "upper": 0.3727004361449647, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.8386650303332343, + "median": 0.8378042241345215, + "standard_deviation": 0.0012173638007943626, + "quantiles": { + "p05": 0.8378042241345215, + "p25": 0.8378042241345215, + "p75": 0.8403866427306597, + "p95": 0.8403866427306597 + }, + "confidence_interval": { + "estimate": 0.8386650303332343, + "lower": 0.8380194256841996, + "upper": 0.8393106349822688, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 0.9662883683502264, + "median": 0.9636679612225048, + "standard_deviation": 0.0037058152989631215, + "quantiles": { + "p05": 0.9636679612225048, + "p25": 0.9636679612225048, + "p75": 0.9715291826056698, + "p95": 0.9715291826056698 + }, + "confidence_interval": { + "estimate": 0.9662883683502264, + "lower": 0.9643230630044352, + "upper": 0.9682536736960178, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.7103099178571526, + "median": 0.6309297535714575, + "standard_deviation": 0.21170090200015357, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.7103099178571526, + "lower": 0.596220730654774, + "upper": 0.8243991050595311, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + }, + "action-agnostic": { + "action_dependence": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_effect": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "action_dependence_permutation_p_value": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "counterfactual_direction_accuracy": { + "mean": 0.5811096409230095, + "median": 0.5795010640782621, + "standard_deviation": 0.002274871189961332, + "quantiles": { + "p05": 0.579501064078262, + "p25": 0.5795010640782621, + "p75": 0.5843267946125047, + "p95": 0.5843267946125048 + }, + "confidence_interval": { + "estimate": 0.5811096409230095, + "lower": 0.579903208289449, + "upper": 0.5823160735565703, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "noop_stability": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_ade": { + "mean": 0.6438600690131464, + "median": 0.6495223102707939, + "standard_deviation": 0.008007618379993493, + "quantiles": { + "p05": 0.6325355864978516, + "p25": 0.6325355864978516, + "p75": 0.6495223102707939, + "p95": 0.6495223102707939 + }, + "confidence_interval": { + "estimate": 0.6438600690131464, + "lower": 0.6396133880699109, + "upper": 0.6481067499563821, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "state_fde": { + "mean": 1.155975033047959, + "median": 1.1603847771986866, + "standard_deviation": 0.006236319984554175, + "quantiles": { + "p05": 1.1471555447465043, + "p25": 1.1471555447465043, + "p75": 1.1603847771986866, + "p95": 1.1603847771986866 + }, + "confidence_interval": { + "estimate": 1.155975033047959, + "lower": 1.1526677249349135, + "upper": 1.1592823411610047, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.7103099178571526, + "median": 0.6309297535714575, + "standard_deviation": 0.21170090200015357, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 0.7103099178571526, + "lower": 0.596220730654774, + "upper": 0.8353099178571526, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "top1_regret": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + } + } + }, + "paired_comparisons": [ + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.001515093796655052, + "confidence_interval": { + "estimate": 0.001515093796655052, + "lower": 7.615437899995599e-05, + "upper": 0.0034128541295564323, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": -0.00037829413208911866, + "confidence_interval": { + "estimate": -0.00037829413208911866, + "lower": -0.0028239938699153298, + "upper": 0.0018500427037220954, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0014263449605067906, + "confidence_interval": { + "estimate": 0.0014263449605067906, + "lower": 0.0010300486869430296, + "upper": 0.0018451662707829543, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": -0.0563249614434043, + "confidence_interval": { + "estimate": -0.0563249614434043, + "lower": -0.07596181393834078, + "upper": -0.036706954525698554, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": -0.03046040283421649, + "confidence_interval": { + "estimate": -0.03046040283421649, + "lower": -0.03402953235186622, + "upper": -0.027013881455695568, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "noisy-dynamics", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 2.0095517494564104, + "confidence_interval": { + "estimate": 2.0095517494564104, + "lower": 1.9928748606761604, + "upper": 2.0262286382366606, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.9444444444444445, + "confidence_interval": { + "estimate": -0.9444444444444445, + "lower": -0.9548611111111112, + "upper": -0.9340277777777778, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0000000000000002, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -0.8856122005706287, + "confidence_interval": { + "estimate": -0.8856122005706287, + "lower": -0.8857658909434555, + "upper": -0.8854585101978021, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -1.0446933677159576, + "confidence_interval": { + "estimate": -1.0446933677159576, + "lower": -1.0451710497291884, + "upper": -1.0442156857027267, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.28969008214284747, + "confidence_interval": { + "estimate": 0.28969008214284747, + "lower": 0.16666666666666666, + "upper": 0.403779269345226, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.8264914166241919, + "confidence_interval": { + "estimate": 0.8264914166241919, + "lower": 0.8254531985230612, + "upper": 0.8275296347253227, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 2.2484047224797643, + "confidence_interval": { + "estimate": 2.2484047224797643, + "lower": 1.709496016687181, + "upper": 2.787313428272347, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.5, + "confidence_interval": { + "estimate": -0.5, + "lower": -0.65625, + "upper": -0.34375, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.6313378830987065, + "confidence_interval": { + "estimate": 0.6313378830987065, + "lower": 0.6272995638550353, + "upper": 0.6353762023423779, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.8386650303332343, + "confidence_interval": { + "estimate": -0.8386650303332343, + "lower": -0.8393106349822688, + "upper": -0.8380194256841996, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -0.9662883683502264, + "confidence_interval": { + "estimate": -0.9662883683502264, + "lower": -0.9682536736960178, + "upper": -0.9643230630044352, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.28969008214284747, + "confidence_interval": { + "estimate": 0.28969008214284747, + "lower": 0.17537753923362412, + "upper": 0.403779269345226, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_effect", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 2.0095517494564104, + "confidence_interval": { + "estimate": 2.0095517494564104, + "lower": 1.9928748606761604, + "upper": 2.0262286382366606, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "action_dependence_permutation_p_value", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.9444444444444445, + "confidence_interval": { + "estimate": -0.9444444444444445, + "lower": -0.9548611111111112, + "upper": -0.9340277777777778, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "counterfactual_direction_accuracy", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.4188903590769904, + "confidence_interval": { + "estimate": 0.4188903590769904, + "lower": 0.4176839264434298, + "upper": 0.4200967917105511, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "noop_stability", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_ade", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -0.6438600690131464, + "confidence_interval": { + "estimate": -0.6438600690131464, + "lower": -0.6481067499563821, + "upper": -0.6396133880699109, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "state_fde", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -1.155975033047959, + "confidence_interval": { + "estimate": -1.155975033047959, + "lower": -1.1592823411610047, + "upper": -1.1526677249349135, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.28969008214284747, + "confidence_interval": { + "estimate": 0.28969008214284747, + "lower": 0.1646900821428475, + "upper": 0.41469008214284747, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "top1_regret", + "left_model": "oracle-simulator", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": -1.0, + "confidence_interval": { + "estimate": -1.0, + "lower": -1.0, + "upper": -1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + } + ] +} diff --git a/examples/pointmass-demo/report.html b/examples/pointmass-demo/report.html index b2f35f3..453e7ab 100644 --- a/examples/pointmass-demo/report.html +++ b/examples/pointmass-demo/report.html @@ -3,7 +3,7 @@ - WAMProbe PointMass-2D Demo + WAMProbe pointmass-2d-v0.1 Demo -

WAMProbe PointMass-2D Demo

+

WAMProbe pointmass-2d-v0.1 Demo

Action-aware and deliberately broken baselines are shown as a metric profile. No composite score is produced.

Metric profile

- - + +
ModelAction DependencePermutation EffectPermutation p-value ↓Counterfactual DirectionNo-op StabilityState ADE ↓State FDE ↓Top-1 Regret ↓
oracle-pointmass1.00002.85420.06671.00001.00000.00000.00000.0000
noisy-linear0.98862.78930.06670.99720.00000.07360.09270.0000
copy-last-frame0.00000.00001.00000.00001.00000.50000.80001.0000
wrong-direction1.00002.85420.0667-1.00001.00001.00001.60002.0000
action-agnostic0.00000.00001.00000.00000.00000.72861.16571.0000
ModelAction DependencePermutation EffectPermutation p-value ↓Counterfactual DirectionNo-op StabilityState ADE ↓State FDE ↓CRC SpearmanCRC Kendall τCRC NDCGCRC Pairwise AccuracyTop-1 Regret ↓
oracle-pointmass1.00002.85420.06671.00001.00000.00000.00001.00001.00001.00001.00000.0000
noisy-linear0.98862.78930.06670.99720.00000.07360.09270.97680.95301.00001.00000.0000
copy-last-frame0.00000.00001.00000.00001.00000.50000.80000.00000.00000.71990.50001.0000
wrong-direction1.00002.85420.0667-1.00001.00001.00001.6000-0.6855-0.55930.55100.22042.0000
action-agnostic0.00000.00001.00000.00000.00000.72861.16570.00000.00000.71990.50001.0000
@@ -37,7 +37,7 @@

Context-block 95% confidence intervals

- +
ModelMetricMean95% CI MedianStdp05 / p95
oracle-pointmassAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
oracle-pointmassPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
oracle-pointmassCounterfactual Direction1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassState ADE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassState FDE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearAction Dependence0.9886[0.9787, 0.9966]0.99670.01490.9625 / 1.0000
noisy-linearPermutation Effect2.7893[2.7724, 2.8072]2.78650.03012.7476 / 2.8319
noisy-linearPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
noisy-linearCounterfactual Direction0.9972[0.9965, 0.9979]0.99730.00130.9951 / 0.9991
noisy-linearNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearState ADE ↓0.0736[0.0667, 0.0814]0.07360.01280.0596 / 0.0929
noisy-linearState FDE ↓0.0927[0.0825, 0.1027]0.08950.01840.0700 / 0.1229
noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameState ADE ↓0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameState FDE ↓0.8000[0.8000, 0.8000]0.80000.00000.8000 / 0.8000
copy-last-frameTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
wrong-directionPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
wrong-directionCounterfactual Direction-1.0000[-1.0000, -1.0000]-1.00000.0000-1.0000 / -1.0000
wrong-directionNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState ADE ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState FDE ↓1.6000[1.6000, 1.6000]1.60000.00001.6000 / 1.6000
wrong-directionTop-1 Regret ↓2.0000[2.0000, 2.0000]2.00000.00002.0000 / 2.0000
action-agnosticAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticState ADE ↓0.7286[0.7286, 0.7286]0.72860.00000.7286 / 0.7286
action-agnosticState FDE ↓1.1657[1.1657, 1.1657]1.16570.00001.1657 / 1.1657
action-agnosticTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
oracle-pointmassPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
oracle-pointmassCounterfactual Direction1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassState ADE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassState FDE ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
oracle-pointmassCRC Spearman1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassCRC Kendall τ1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
oracle-pointmassTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearAction Dependence0.9886[0.9787, 0.9966]0.99670.01490.9625 / 1.0000
noisy-linearPermutation Effect2.7893[2.7724, 2.8072]2.78650.03012.7476 / 2.8319
noisy-linearPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
noisy-linearCounterfactual Direction0.9972[0.9965, 0.9979]0.99730.00130.9951 / 0.9991
noisy-linearNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
noisy-linearState ADE ↓0.0736[0.0667, 0.0814]0.07360.01280.0596 / 0.0929
noisy-linearState FDE ↓0.0927[0.0825, 0.1027]0.08950.01840.0700 / 0.1229
noisy-linearCRC Spearman0.9768[0.9747, 0.9810]0.97470.00700.9747 / 0.9861
noisy-linearCRC Kendall τ0.9530[0.9487, 0.9615]0.94870.01420.9487 / 0.9718
noisy-linearCRC NDCG1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-linearCRC Pairwise Accuracy1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-framePermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
copy-last-frameState ADE ↓0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameState FDE ↓0.8000[0.8000, 0.8000]0.80000.00000.8000 / 0.8000
copy-last-frameCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
copy-last-frameCRC NDCG0.7199[0.6329, 0.8212]0.68020.17160.5351 / 0.9840
copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
copy-last-frameTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionAction Dependence1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionPermutation Effect2.8542[2.8542, 2.8542]2.85420.00002.8542 / 2.8542
wrong-directionPermutation p-value ↓0.0667[0.0667, 0.0667]0.06670.00000.0667 / 0.0667
wrong-directionCounterfactual Direction-1.0000[-1.0000, -1.0000]-1.00000.0000-1.0000 / -1.0000
wrong-directionNo-op Stability1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState ADE ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
wrong-directionState FDE ↓1.6000[1.6000, 1.6000]1.60000.00001.6000 / 1.6000
wrong-directionCRC Spearman-0.6855[-0.6882, -0.6842]-0.68420.0044-0.6913 / -0.6842
wrong-directionCRC Kendall τ-0.5593[-0.5667, -0.5556]-0.55560.0123-0.5756 / -0.5556
wrong-directionCRC NDCG0.5510[0.5510, 0.5510]0.55100.00000.5510 / 0.5510
wrong-directionCRC Pairwise Accuracy0.2204[0.2167, 0.2222]0.22220.00610.2122 / 0.2222
wrong-directionTop-1 Regret ↓2.0000[2.0000, 2.0000]2.00000.00002.0000 / 2.0000
action-agnosticAction Dependence0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation Effect0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticPermutation p-value ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
action-agnosticCounterfactual Direction0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticNo-op Stability0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticState ADE ↓0.7286[0.7286, 0.7286]0.72860.00000.7286 / 0.7286
action-agnosticState FDE ↓1.1657[1.1657, 1.1657]1.16570.00001.1657 / 1.1657
action-agnosticCRC Spearman0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC Kendall τ0.0000[0.0000, 0.0000]0.00000.00000.0000 / 0.0000
action-agnosticCRC NDCG0.7199[0.6315, 0.8143]0.68020.17160.5351 / 0.9840
action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]0.50000.00000.5000 / 0.5000
action-agnosticTop-1 Regret ↓1.0000[1.0000, 1.0000]1.00000.00001.0000 / 1.0000
@@ -46,7 +46,7 @@

Paired model differences

- +
ModelsMetricMean difference 95% CIContexts
oracle-pointmass − noisy-linearAction Dependence0.0114[0.0035, 0.0215]12
oracle-pointmass − noisy-linearPermutation Effect0.0649[0.0487, 0.0818]12
oracle-pointmass − noisy-linearPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − noisy-linearCounterfactual Direction0.0028[0.0020, 0.0035]12
oracle-pointmass − noisy-linearNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − noisy-linearState ADE ↓-0.0736[-0.0810, -0.0667]12
oracle-pointmass − noisy-linearState FDE ↓-0.0927[-0.1032, -0.0831]12
oracle-pointmass − noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-framePermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − copy-last-framePermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − copy-last-frameCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-frameNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameState ADE ↓-0.5000[-0.5000, -0.5000]12
oracle-pointmass − copy-last-frameState FDE ↓-0.8000[-0.8000, -0.8000]12
oracle-pointmass − copy-last-frameTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionAction Dependence0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation Effect0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionCounterfactual Direction2.0000[2.0000, 2.0000]12
oracle-pointmass − wrong-directionNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionState ADE ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionState FDE ↓-1.6000[-1.6000, -1.6000]12
oracle-pointmass − wrong-directionTop-1 Regret ↓-2.0000[-2.0000, -2.0000]12
oracle-pointmass − action-agnosticAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticPermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − action-agnosticPermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − action-agnosticCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticState ADE ↓-0.7286[-0.7286, -0.7286]12
oracle-pointmass − action-agnosticState FDE ↓-1.1657[-1.1657, -1.1657]12
oracle-pointmass − action-agnosticTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − noisy-linearAction Dependence0.0114[0.0035, 0.0215]12
oracle-pointmass − noisy-linearPermutation Effect0.0649[0.0487, 0.0818]12
oracle-pointmass − noisy-linearPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − noisy-linearCounterfactual Direction0.0028[0.0020, 0.0035]12
oracle-pointmass − noisy-linearNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − noisy-linearState ADE ↓-0.0736[-0.0810, -0.0667]12
oracle-pointmass − noisy-linearState FDE ↓-0.0927[-0.1032, -0.0831]12
oracle-pointmass − noisy-linearCRC Spearman0.0232[0.0190, 0.0253]12
oracle-pointmass − noisy-linearCRC Kendall τ0.0470[0.0385, 0.0513]12
oracle-pointmass − noisy-linearCRC NDCG0.0000[0.0000, 0.0000]12
oracle-pointmass − noisy-linearCRC Pairwise Accuracy0.0000[0.0000, 0.0000]12
oracle-pointmass − noisy-linearTop-1 Regret ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-framePermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − copy-last-framePermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − copy-last-frameCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-frameNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − copy-last-frameState ADE ↓-0.5000[-0.5000, -0.5000]12
oracle-pointmass − copy-last-frameState FDE ↓-0.8000[-0.8000, -0.8000]12
oracle-pointmass − copy-last-frameCRC Spearman1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-frameCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-pointmass − copy-last-frameCRC NDCG0.2801[0.1801, 0.3671]12
oracle-pointmass − copy-last-frameCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-pointmass − copy-last-frameTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionAction Dependence0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation Effect0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionPermutation p-value ↓0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionCounterfactual Direction2.0000[2.0000, 2.0000]12
oracle-pointmass − wrong-directionNo-op Stability0.0000[0.0000, 0.0000]12
oracle-pointmass − wrong-directionState ADE ↓-1.0000[-1.0000, -1.0000]12
oracle-pointmass − wrong-directionState FDE ↓-1.6000[-1.6000, -1.6000]12
oracle-pointmass − wrong-directionCRC Spearman1.6855[1.6842, 1.6882]12
oracle-pointmass − wrong-directionCRC Kendall τ1.5593[1.5556, 1.5667]12
oracle-pointmass − wrong-directionCRC NDCG0.4490[0.4490, 0.4490]12
oracle-pointmass − wrong-directionCRC Pairwise Accuracy0.7796[0.7778, 0.7833]12
oracle-pointmass − wrong-directionTop-1 Regret ↓-2.0000[-2.0000, -2.0000]12
oracle-pointmass − action-agnosticAction Dependence1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticPermutation Effect2.8542[2.8542, 2.8542]12
oracle-pointmass − action-agnosticPermutation p-value ↓-0.9333[-0.9333, -0.9333]12
oracle-pointmass − action-agnosticCounterfactual Direction1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticNo-op Stability1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticState ADE ↓-0.7286[-0.7286, -0.7286]12
oracle-pointmass − action-agnosticState FDE ↓-1.1657[-1.1657, -1.1657]12
oracle-pointmass − action-agnosticCRC Spearman1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticCRC Kendall τ1.0000[1.0000, 1.0000]12
oracle-pointmass − action-agnosticCRC NDCG0.2801[0.1738, 0.3681]12
oracle-pointmass − action-agnosticCRC Pairwise Accuracy0.5000[0.5000, 0.5000]12
oracle-pointmass − action-agnosticTop-1 Regret ↓-1.0000[-1.0000, -1.0000]12
diff --git a/examples/pointmass-demo/report.md b/examples/pointmass-demo/report.md index 4a1d2c0..9f53aba 100644 --- a/examples/pointmass-demo/report.md +++ b/examples/pointmass-demo/report.md @@ -1,14 +1,14 @@ -# WAMProbe PointMass-2D Demo +# WAMProbe pointmass-2d-v0.1 Demo This report compares action-aware and deliberately broken reference baselines. -| Model | Action Dependence | Permutation Effect | Permutation p-value ↓ | Counterfactual Direction | No-op Stability | State ADE ↓ | State FDE ↓ | Top-1 Regret ↓ | -|---|---|---|---|---|---|---|---|---| -| oracle-pointmass | 1.0000 | 2.8542 | 0.0667 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 0.0000 | -| noisy-linear | 0.9886 | 2.7893 | 0.0667 | 0.9972 | 0.0000 | 0.0736 | 0.0927 | 0.0000 | -| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 1.0000 | 0.5000 | 0.8000 | 1.0000 | -| wrong-direction | 1.0000 | 2.8542 | 0.0667 | -1.0000 | 1.0000 | 1.0000 | 1.6000 | 2.0000 | -| action-agnostic | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 0.0000 | 0.7286 | 1.1657 | 1.0000 | +| Model | Action Dependence | Permutation Effect | Permutation p-value ↓ | Counterfactual Direction | No-op Stability | State ADE ↓ | State FDE ↓ | CRC Spearman | CRC Kendall τ | CRC NDCG | CRC Pairwise Accuracy | Top-1 Regret ↓ | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| oracle-pointmass | 1.0000 | 2.8542 | 0.0667 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | 0.0000 | +| noisy-linear | 0.9886 | 2.7893 | 0.0667 | 0.9972 | 0.0000 | 0.0736 | 0.0927 | 0.9768 | 0.9530 | 1.0000 | 1.0000 | 0.0000 | +| copy-last-frame | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 1.0000 | 0.5000 | 0.8000 | 0.0000 | 0.0000 | 0.7199 | 0.5000 | 1.0000 | +| wrong-direction | 1.0000 | 2.8542 | 0.0667 | -1.0000 | 1.0000 | 1.0000 | 1.6000 | -0.6855 | -0.5593 | 0.5510 | 0.2204 | 2.0000 | +| action-agnostic | 0.0000 | 0.0000 | 1.0000 | 0.0000 | 0.0000 | 0.7286 | 1.1657 | 0.0000 | 0.0000 | 0.7199 | 0.5000 | 1.0000 | ## Context-block 95% confidence intervals @@ -23,6 +23,10 @@ Intervals resample whole contexts, not correlated action branches or frames. | oracle-pointmass | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | oracle-pointmass | State ADE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | oracle-pointmass | State FDE ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| oracle-pointmass | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| oracle-pointmass | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | oracle-pointmass | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | noisy-linear | Action Dependence | 0.9886 | [0.9787, 0.9966] | 0.9967 | 0.0149 | | noisy-linear | Permutation Effect | 2.7893 | [2.7724, 2.8072] | 2.7865 | 0.0301 | @@ -31,6 +35,10 @@ Intervals resample whole contexts, not correlated action branches or frames. | noisy-linear | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | noisy-linear | State ADE ↓ | 0.0736 | [0.0667, 0.0814] | 0.0736 | 0.0128 | | noisy-linear | State FDE ↓ | 0.0927 | [0.0825, 0.1027] | 0.0895 | 0.0184 | +| noisy-linear | CRC Spearman | 0.9768 | [0.9747, 0.9810] | 0.9747 | 0.0070 | +| noisy-linear | CRC Kendall τ | 0.9530 | [0.9487, 0.9615] | 0.9487 | 0.0142 | +| noisy-linear | CRC NDCG | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-linear | CRC Pairwise Accuracy | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | noisy-linear | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | copy-last-frame | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | copy-last-frame | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | @@ -39,6 +47,10 @@ Intervals resample whole contexts, not correlated action branches or frames. | copy-last-frame | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | copy-last-frame | State ADE ↓ | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | | copy-last-frame | State FDE ↓ | 0.8000 | [0.8000, 0.8000] | 0.8000 | 0.0000 | +| copy-last-frame | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| copy-last-frame | CRC NDCG | 0.7199 | [0.6329, 0.8212] | 0.6802 | 0.1716 | +| copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | | copy-last-frame | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | wrong-direction | Action Dependence | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | wrong-direction | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 2.8542 | 0.0000 | @@ -47,6 +59,10 @@ Intervals resample whole contexts, not correlated action branches or frames. | wrong-direction | No-op Stability | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | wrong-direction | State ADE ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | | wrong-direction | State FDE ↓ | 1.6000 | [1.6000, 1.6000] | 1.6000 | 0.0000 | +| wrong-direction | CRC Spearman | -0.6855 | [-0.6882, -0.6842] | -0.6842 | 0.0044 | +| wrong-direction | CRC Kendall τ | -0.5593 | [-0.5667, -0.5556] | -0.5556 | 0.0123 | +| wrong-direction | CRC NDCG | 0.5510 | [0.5510, 0.5510] | 0.5510 | 0.0000 | +| wrong-direction | CRC Pairwise Accuracy | 0.2204 | [0.2167, 0.2222] | 0.2222 | 0.0061 | | wrong-direction | Top-1 Regret ↓ | 2.0000 | [2.0000, 2.0000] | 2.0000 | 0.0000 | | action-agnostic | Action Dependence | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | action-agnostic | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | @@ -55,6 +71,10 @@ Intervals resample whole contexts, not correlated action branches or frames. | action-agnostic | No-op Stability | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | | action-agnostic | State ADE ↓ | 0.7286 | [0.7286, 0.7286] | 0.7286 | 0.0000 | | action-agnostic | State FDE ↓ | 1.1657 | [1.1657, 1.1657] | 1.1657 | 0.0000 | +| action-agnostic | CRC Spearman | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC Kendall τ | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 | +| action-agnostic | CRC NDCG | 0.7199 | [0.6315, 0.8143] | 0.6802 | 0.1716 | +| action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 0.5000 | 0.0000 | | action-agnostic | Top-1 Regret ↓ | 1.0000 | [1.0000, 1.0000] | 1.0000 | 0.0000 | ## Paired model differences @@ -70,6 +90,10 @@ Differences are left minus right over exactly aligned context IDs. | oracle-pointmass − noisy-linear | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | | oracle-pointmass − noisy-linear | State ADE ↓ | -0.0736 | [-0.0810, -0.0667] | 12 | | oracle-pointmass − noisy-linear | State FDE ↓ | -0.0927 | [-0.1032, -0.0831] | 12 | +| oracle-pointmass − noisy-linear | CRC Spearman | 0.0232 | [0.0190, 0.0253] | 12 | +| oracle-pointmass − noisy-linear | CRC Kendall τ | 0.0470 | [0.0385, 0.0513] | 12 | +| oracle-pointmass − noisy-linear | CRC NDCG | 0.0000 | [0.0000, 0.0000] | 12 | +| oracle-pointmass − noisy-linear | CRC Pairwise Accuracy | 0.0000 | [0.0000, 0.0000] | 12 | | oracle-pointmass − noisy-linear | Top-1 Regret ↓ | 0.0000 | [0.0000, 0.0000] | 12 | | oracle-pointmass − copy-last-frame | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | | oracle-pointmass − copy-last-frame | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 12 | @@ -78,6 +102,10 @@ Differences are left minus right over exactly aligned context IDs. | oracle-pointmass − copy-last-frame | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | | oracle-pointmass − copy-last-frame | State ADE ↓ | -0.5000 | [-0.5000, -0.5000] | 12 | | oracle-pointmass − copy-last-frame | State FDE ↓ | -0.8000 | [-0.8000, -0.8000] | 12 | +| oracle-pointmass − copy-last-frame | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − copy-last-frame | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − copy-last-frame | CRC NDCG | 0.2801 | [0.1801, 0.3671] | 12 | +| oracle-pointmass − copy-last-frame | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | | oracle-pointmass − copy-last-frame | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | | oracle-pointmass − wrong-direction | Action Dependence | 0.0000 | [0.0000, 0.0000] | 12 | | oracle-pointmass − wrong-direction | Permutation Effect | 0.0000 | [0.0000, 0.0000] | 12 | @@ -86,6 +114,10 @@ Differences are left minus right over exactly aligned context IDs. | oracle-pointmass − wrong-direction | No-op Stability | 0.0000 | [0.0000, 0.0000] | 12 | | oracle-pointmass − wrong-direction | State ADE ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | | oracle-pointmass − wrong-direction | State FDE ↓ | -1.6000 | [-1.6000, -1.6000] | 12 | +| oracle-pointmass − wrong-direction | CRC Spearman | 1.6855 | [1.6842, 1.6882] | 12 | +| oracle-pointmass − wrong-direction | CRC Kendall τ | 1.5593 | [1.5556, 1.5667] | 12 | +| oracle-pointmass − wrong-direction | CRC NDCG | 0.4490 | [0.4490, 0.4490] | 12 | +| oracle-pointmass − wrong-direction | CRC Pairwise Accuracy | 0.7796 | [0.7778, 0.7833] | 12 | | oracle-pointmass − wrong-direction | Top-1 Regret ↓ | -2.0000 | [-2.0000, -2.0000] | 12 | | oracle-pointmass − action-agnostic | Action Dependence | 1.0000 | [1.0000, 1.0000] | 12 | | oracle-pointmass − action-agnostic | Permutation Effect | 2.8542 | [2.8542, 2.8542] | 12 | @@ -94,8 +126,12 @@ Differences are left minus right over exactly aligned context IDs. | oracle-pointmass − action-agnostic | No-op Stability | 1.0000 | [1.0000, 1.0000] | 12 | | oracle-pointmass − action-agnostic | State ADE ↓ | -0.7286 | [-0.7286, -0.7286] | 12 | | oracle-pointmass − action-agnostic | State FDE ↓ | -1.1657 | [-1.1657, -1.1657] | 12 | +| oracle-pointmass − action-agnostic | CRC Spearman | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − action-agnostic | CRC Kendall τ | 1.0000 | [1.0000, 1.0000] | 12 | +| oracle-pointmass − action-agnostic | CRC NDCG | 0.2801 | [0.1738, 0.3681] | 12 | +| oracle-pointmass − action-agnostic | CRC Pairwise Accuracy | 0.5000 | [0.5000, 0.5000] | 12 | | oracle-pointmass − action-agnostic | Top-1 Regret ↓ | -1.0000 | [-1.0000, -1.0000] | 12 | -A high Action Dependence score is not sufficient: the wrong-direction baseline -depends on the action but fails Counterfactual Direction Accuracy. WAMProbe -therefore reports a metric profile rather than a single composite score. +Action separation alone does not establish correct direction, state dynamics, +or control ranking. WAMProbe therefore reports a metric profile rather than a +single composite score. diff --git a/examples/pointmass-demo/summary.json b/examples/pointmass-demo/summary.json index d4d1e1c..a68440f 100644 --- a/examples/pointmass-demo/summary.json +++ b/examples/pointmass-demo/summary.json @@ -15,6 +15,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 }, "context_results": [ @@ -31,6 +35,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -47,6 +55,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -63,6 +75,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -79,6 +95,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -95,6 +115,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -111,6 +135,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -127,6 +155,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -143,6 +175,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -159,6 +195,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -175,6 +215,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -191,6 +235,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -207,6 +255,10 @@ "noop_stability": 1.0, "state_ade": 0.0, "state_fde": 0.0, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } } @@ -225,6 +277,10 @@ "noop_stability": 0.0, "state_ade": 0.07364777227731902, "state_fde": 0.0927268500618552, + "candidate_ranking_spearman": 0.9767894816074884, + "candidate_ranking_kendall_tau": 0.9529596898796376, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 }, "context_results": [ @@ -241,6 +297,10 @@ "noop_stability": 0.0, "state_ade": 0.10417042880012942, "state_fde": 0.11595533486335921, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -257,6 +317,10 @@ "noop_stability": 0.0, "state_ade": 0.08371865860289475, "state_fde": 0.13140609465866257, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -267,12 +331,16 @@ "top1_regret": 0.0, "metrics": { "action_dependence": 1.0, - "action_dependence_permutation_effect": 2.766645072742772, + "action_dependence_permutation_effect": 2.7666450727427723, "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 0.9990887394827166, "noop_stability": 0.0, "state_ade": 0.07170250725169272, "state_fde": 0.10261584827837915, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -289,6 +357,10 @@ "noop_stability": 0.0, "state_ade": 0.05995406721844264, "state_fde": 0.07761855384937649, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -305,6 +377,10 @@ "noop_stability": 0.0, "state_ade": 0.05918016195121363, "state_fde": 0.08397848391634824, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -315,12 +391,16 @@ "top1_regret": 0.0, "metrics": { "action_dependence": 0.9716774799252708, - "action_dependence_permutation_effect": 2.8271977556861567, + "action_dependence_permutation_effect": 2.8271977556861563, "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 0.9991626197702563, "noop_stability": 0.0, "state_ade": 0.061375348243354264, "state_fde": 0.07705294381294754, + "candidate_ranking_spearman": 1.0, + "candidate_ranking_kendall_tau": 1.0, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -337,6 +417,10 @@ "noop_stability": 0.0, "state_ade": 0.06069688510282225, "state_fde": 0.061287791426493245, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -353,6 +437,10 @@ "noop_stability": 0.0, "state_ade": 0.08021175873604955, "state_fde": 0.08043799051624254, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -369,6 +457,10 @@ "noop_stability": 0.0, "state_ade": 0.07540788707466936, "state_fde": 0.09193913292916703, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -379,12 +471,16 @@ "top1_regret": 0.0, "metrics": { "action_dependence": 0.9512321250999373, - "action_dependence_permutation_effect": 2.8212670000102467, + "action_dependence_permutation_effect": 2.821267000010246, "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 0.9979961255644422, "noop_stability": 0.0, "state_ade": 0.08324185274139459, "state_fde": 0.09664726483255356, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -395,12 +491,16 @@ "top1_regret": 0.0, "metrics": { "action_dependence": 1.0, - "action_dependence_permutation_effect": 2.777564510368594, + "action_dependence_permutation_effect": 2.7775645103685944, "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 0.9973768635583395, "noop_stability": 0.0, "state_ade": 0.066462523353616, "state_fde": 0.08712314154486435, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } }, @@ -411,12 +511,16 @@ "top1_regret": 0.0, "metrics": { "action_dependence": 0.9877909018973522, - "action_dependence_permutation_effect": 2.7482875485159477, + "action_dependence_permutation_effect": 2.748287548515948, "action_dependence_permutation_p_value": 0.06666666666666667, "counterfactual_direction_accuracy": 0.9970980196542607, "noop_stability": 0.0, "state_ade": 0.07765118825154912, "state_fde": 0.10665962011386854, + "candidate_ranking_spearman": 0.9746794344808964, + "candidate_ranking_kendall_tau": 0.9486832980505138, + "candidate_ranking_ndcg": 1.0, + "candidate_ranking_pairwise_accuracy": 1.0, "top1_regret": 0.0 } } @@ -435,6 +539,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8000000000000002, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7199074911151153, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 }, "context_results": [ @@ -451,6 +559,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -467,6 +579,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -483,6 +599,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -499,6 +619,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -515,6 +639,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -531,6 +659,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -547,6 +679,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -563,6 +699,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -579,6 +719,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -595,6 +739,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -611,6 +759,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -627,6 +779,10 @@ "noop_stability": 1.0, "state_ade": 0.5, "state_fde": 0.8, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } } @@ -645,6 +801,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6000000000000003, + "candidate_ranking_spearman": -0.6855263157894737, + "candidate_ranking_kendall_tau": -0.5592592592592592, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.22037037037037036, "top1_regret": 2.0 }, "context_results": [ @@ -661,6 +821,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -677,6 +841,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -693,6 +861,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -709,6 +881,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -725,6 +901,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -741,6 +921,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.7, + "candidate_ranking_kendall_tau": -0.6, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2, "top1_regret": 2.0 } }, @@ -757,6 +941,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -773,6 +961,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -789,6 +981,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -805,6 +1001,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -821,6 +1021,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } }, @@ -837,6 +1041,10 @@ "noop_stability": 1.0, "state_ade": 1.0, "state_fde": 1.6, + "candidate_ranking_spearman": -0.6842105263157895, + "candidate_ranking_kendall_tau": -0.5555555555555556, + "candidate_ranking_ndcg": 0.551034377809143, + "candidate_ranking_pairwise_accuracy": 0.2222222222222222, "top1_regret": 2.0 } } @@ -855,6 +1063,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932736, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7199074911151153, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 }, "context_results": [ @@ -871,6 +1083,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -887,6 +1103,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -903,6 +1123,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -919,6 +1143,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -935,6 +1163,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -951,6 +1183,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -967,6 +1203,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -983,6 +1223,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -999,6 +1243,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.6069330503943371, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -1015,6 +1263,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.7535647263985802, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -1031,6 +1283,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.5350832827383436, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } }, @@ -1047,6 +1303,10 @@ "noop_stability": 0.0, "state_ade": 0.7285533905932737, "state_fde": 1.1656854249492379, + "candidate_ranking_spearman": 0.0, + "candidate_ranking_kendall_tau": 0.0, + "candidate_ranking_ndcg": 0.9840489049292005, + "candidate_ranking_pairwise_accuracy": 0.5, "top1_regret": 1.0 } } @@ -1195,6 +1455,86 @@ }, "contexts": 12 }, + "candidate_ranking_spearman": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, "top1_regret": { "mean": 0.0, "median": 0.0, @@ -1240,10 +1580,10 @@ "action_dependence_permutation_effect": { "mean": 2.789304160469617, "median": 2.7864724779755474, - "standard_deviation": 0.030093202906788793, + "standard_deviation": 0.030093202906788616, "quantiles": { - "p05": 2.747607938117685, - "p25": 2.7641767844904415, + "p05": 2.7476079381176852, + "p25": 2.7641767844904424, "p75": 2.814887165781101, "p95": 2.831907870399184 }, @@ -1357,6 +1697,86 @@ }, "contexts": 12 }, + "candidate_ranking_spearman": { + "mean": 0.9767894816074884, + "median": 0.9746794344808964, + "standard_deviation": 0.006998234608873083, + "quantiles": { + "p05": 0.9746794344808964, + "p25": 0.9746794344808964, + "p75": 0.9746794344808964, + "p95": 0.986073688964493 + }, + "confidence_interval": { + "estimate": 0.9767894816074884, + "lower": 0.9746794344808963, + "upper": 0.9810095758606723, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.9529596898796376, + "median": 0.9486832980505138, + "standard_deviation": 0.014183187153745442, + "quantiles": { + "p05": 0.9486832980505138, + "p25": 0.9486832980505138, + "p75": 0.9486832980505138, + "p95": 0.9717758139277826 + }, + "confidence_interval": { + "estimate": 0.9529596898796376, + "lower": 0.9486832980505137, + "upper": 0.9615124735378853, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 1.0, + "median": 1.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, "top1_regret": { "mean": 0.0, "median": 0.0, @@ -1519,6 +1939,86 @@ }, "contexts": 12 }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.7199074911151153, + "median": 0.6802488883964586, + "standard_deviation": 0.17162947607921494, + "quantiles": { + "p05": 0.5350832827383436, + "p25": 0.5889706084803388, + "p75": 0.8111857710312352, + "p95": 0.9840489049292005 + }, + "confidence_interval": { + "estimate": 0.7199074911151153, + "lower": 0.6328605810829523, + "upper": 0.821174163292696, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, "top1_regret": { "mean": 1.0, "median": 1.0, @@ -1681,6 +2181,86 @@ }, "contexts": 12 }, + "candidate_ranking_spearman": { + "mean": -0.6855263157894737, + "median": -0.6842105263157895, + "standard_deviation": 0.00436397998730972, + "quantiles": { + "p05": -0.6913157894736842, + "p25": -0.6842105263157895, + "p75": -0.6842105263157895, + "p95": -0.6842105263157895 + }, + "confidence_interval": { + "estimate": -0.6855263157894737, + "lower": -0.6881578947368422, + "upper": -0.6842105263157895, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": -0.5592592592592592, + "median": -0.5555555555555556, + "standard_deviation": 0.012283795519834802, + "quantiles": { + "p05": -0.5755555555555556, + "p25": -0.5555555555555556, + "p75": -0.5555555555555556, + "p95": -0.5555555555555556 + }, + "confidence_interval": { + "estimate": -0.5592592592592592, + "lower": -0.5666666666666667, + "upper": -0.5555555555555556, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.551034377809143, + "median": 0.551034377809143, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.551034377809143, + "p25": 0.551034377809143, + "p75": 0.551034377809143, + "p95": 0.551034377809143 + }, + "confidence_interval": { + "estimate": 0.551034377809143, + "lower": 0.551034377809143, + "upper": 0.551034377809143, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.22037037037037036, + "median": 0.2222222222222222, + "standard_deviation": 0.006141897759917401, + "quantiles": { + "p05": 0.21222222222222223, + "p25": 0.2222222222222222, + "p75": 0.2222222222222222, + "p95": 0.2222222222222222 + }, + "confidence_interval": { + "estimate": 0.22037037037037036, + "lower": 0.21666666666666667, + "upper": 0.2222222222222222, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, "top1_regret": { "mean": 2.0, "median": 2.0, @@ -1843,6 +2423,86 @@ }, "contexts": 12 }, + "candidate_ranking_spearman": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_kendall_tau": { + "mean": 0.0, + "median": 0.0, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_ndcg": { + "mean": 0.7199074911151153, + "median": 0.6802488883964586, + "standard_deviation": 0.17162947607921494, + "quantiles": { + "p05": 0.5350832827383436, + "p25": 0.5889706084803388, + "p75": 0.8111857710312352, + "p95": 0.9840489049292005 + }, + "confidence_interval": { + "estimate": 0.7199074911151153, + "lower": 0.6314913268010958, + "upper": 0.8143111360671345, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, + "candidate_ranking_pairwise_accuracy": { + "mean": 0.5, + "median": 0.5, + "standard_deviation": 0.0, + "quantiles": { + "p05": 0.5, + "p25": 0.5, + "p75": 0.5, + "p95": 0.5 + }, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + }, + "contexts": 12 + }, "top1_regret": { "mean": 1.0, "median": 1.0, @@ -1886,11 +2546,11 @@ "left_model": "oracle-pointmass", "right_model": "noisy-linear", "contexts": 12, - "mean_difference": 0.06492873974020463, + "mean_difference": 0.06492873974020459, "confidence_interval": { - "estimate": 0.06492873974020463, - "lower": 0.04872470548355673, - "upper": 0.08179704843704588, + "estimate": 0.06492873974020459, + "lower": 0.04872470548355685, + "upper": 0.08179704843704583, "confidence_level": 0.95, "resamples": 1000, "unit": "context" @@ -1971,6 +2631,66 @@ "unit": "context" } }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.023210518392511592, + "confidence_interval": { + "estimate": 0.023210518392511592, + "lower": 0.018990424139327666, + "upper": 0.025320565519103555, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.04704031012036238, + "confidence_interval": { + "estimate": 0.04704031012036238, + "lower": 0.038487526462114674, + "upper": 0.05131670194948623, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-pointmass", + "right_model": "noisy-linear", + "contexts": 12, + "mean_difference": 0.0, + "confidence_interval": { + "estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, { "metric": "top1_regret", "left_model": "oracle-pointmass", @@ -2091,6 +2811,66 @@ "unit": "context" } }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.28009250888488463, + "confidence_interval": { + "estimate": 0.28009250888488463, + "lower": 0.18006430104441512, + "upper": 0.36714552754349, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-pointmass", + "right_model": "copy-last-frame", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, { "metric": "top1_regret", "left_model": "oracle-pointmass", @@ -2211,6 +2991,66 @@ "unit": "context" } }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.6855263157894738, + "confidence_interval": { + "estimate": 1.6855263157894738, + "lower": 1.6842105263157894, + "upper": 1.688157894736842, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 1.5592592592592593, + "confidence_interval": { + "estimate": 1.5592592592592593, + "lower": 1.5555555555555556, + "upper": 1.5666666666666667, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.44896562219085706, + "confidence_interval": { + "estimate": 0.44896562219085706, + "lower": 0.44896562219085706, + "upper": 0.44896562219085706, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-pointmass", + "right_model": "wrong-direction", + "contexts": 12, + "mean_difference": 0.7796296296296297, + "confidence_interval": { + "estimate": 0.7796296296296297, + "lower": 0.7777777777777778, + "upper": 0.7833333333333333, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, { "metric": "top1_regret", "left_model": "oracle-pointmass", @@ -2331,6 +3171,66 @@ "unit": "context" } }, + { + "metric": "candidate_ranking_spearman", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_kendall_tau", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 1.0, + "confidence_interval": { + "estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_ndcg", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.28009250888488463, + "confidence_interval": { + "estimate": 0.28009250888488463, + "lower": 0.1738385839751699, + "upper": 0.36813964682291306, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, + { + "metric": "candidate_ranking_pairwise_accuracy", + "left_model": "oracle-pointmass", + "right_model": "action-agnostic", + "contexts": 12, + "mean_difference": 0.5, + "confidence_interval": { + "estimate": 0.5, + "lower": 0.5, + "upper": 0.5, + "confidence_level": 0.95, + "resamples": 1000, + "unit": "context" + } + }, { "metric": "top1_regret", "left_model": "oracle-pointmass", diff --git a/src/wamprobe/adapters/__init__.py b/src/wamprobe/adapters/__init__.py index 042a8e2..fde3982 100644 --- a/src/wamprobe/adapters/__init__.py +++ b/src/wamprobe/adapters/__init__.py @@ -3,17 +3,31 @@ from wamprobe.adapters.baselines import ( ActionAgnosticAdapter, CopyLastFrameAdapter, + NoisyLinearAdapter, OraclePointMassAdapter, WrongDirectionAdapter, ) +from wamprobe.adapters.manipulation import ( + ActionAgnosticManipulationAdapter, + CopyLastManipulationAdapter, + NoisyManipulationAdapter, + OracleManipulationAdapter, + WrongDirectionManipulationAdapter, +) from wamprobe.adapters.starwam import StarWAMAdapter, StarWAMBackendResult, StarWAMRelease __all__ = [ "ActionAgnosticAdapter", "CopyLastFrameAdapter", + "CopyLastManipulationAdapter", + "ActionAgnosticManipulationAdapter", + "NoisyManipulationAdapter", + "NoisyLinearAdapter", "OraclePointMassAdapter", + "OracleManipulationAdapter", "StarWAMAdapter", "StarWAMBackendResult", "StarWAMRelease", "WrongDirectionAdapter", + "WrongDirectionManipulationAdapter", ] diff --git a/src/wamprobe/adapters/manipulation.py b/src/wamprobe/adapters/manipulation.py new file mode 100644 index 0000000..6ce8f71 --- /dev/null +++ b/src/wamprobe/adapters/manipulation.py @@ -0,0 +1,152 @@ +"""Reference baselines for dependency-free analytic manipulation benchmarks.""" + +from __future__ import annotations + +from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationBenchmark, + ManipulationContext, + ManipulationTrajectory, +) + + +class _ManipulationBaseline: + model_id = "manipulation-baseline" + + @property + def capabilities(self) -> ModelCapabilities: + return ModelCapabilities( + model_id=self.model_id, + future_representation=FutureRepresentation.STATES, + ) + + def close(self) -> None: + """Analytic baselines own no external resources.""" + + +class OracleManipulationAdapter(_ManipulationBaseline): + """Upper bound backed by the benchmark's exact transition function.""" + + model_id = "oracle-simulator" + + def __init__(self, benchmark: ManipulationBenchmark) -> None: + self._benchmark = benchmark + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + return self._benchmark.rollout(context, action, seed=seed) + + +class NoisyManipulationAdapter(_ManipulationBaseline): + """Action-aware benchmark dynamics with seeded transition noise.""" + + model_id = "noisy-dynamics" + + def __init__(self, benchmark: ManipulationBenchmark, *, noise_std: float = 0.01) -> None: + if noise_std <= 0: + raise ValueError("noise_std must be positive") + self._benchmark = benchmark + self._noise_std = noise_std + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + return self._benchmark.rollout( + context, + action, + noise_std=self._noise_std, + seed=seed, + ) + + +class CopyLastManipulationAdapter(_ManipulationBaseline): + """Action-ignoring baseline that freezes the complete current state.""" + + model_id = "copy-last-frame" + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + del seed + return ManipulationTrajectory( + context.context_id, + action.name, + (context.initial_state,) * horizon, + ) + + +class WrongDirectionManipulationAdapter(_ManipulationBaseline): + """Deliberately reverse motion and, for catches, gripper semantics.""" + + model_id = "wrong-direction" + + def __init__(self, benchmark: ManipulationBenchmark) -> None: + self._benchmark = benchmark + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + reversed_trajectory = self._benchmark.rollout( + context, + self._benchmark.reverse_action(action), + seed=seed, + ) + return ManipulationTrajectory( + context.context_id, + action.name, + reversed_trajectory.states, + ) + + +class ActionAgnosticManipulationAdapter(_ManipulationBaseline): + """Predict the context-optimal future regardless of the candidate action.""" + + model_id = "action-agnostic" + + def __init__(self, benchmark: ManipulationBenchmark) -> None: + self._benchmark = benchmark + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + if horizon != self._benchmark.horizon: + raise ValueError("adapter and benchmark horizons must match") + optimal = self._benchmark.rollout( + context, + self._benchmark.optimal_action(context), + seed=seed, + ) + return ManipulationTrajectory(context.context_id, action.name, optimal.states) diff --git a/src/wamprobe/api/__init__.py b/src/wamprobe/api/__init__.py index f9cffc0..564bc49 100644 --- a/src/wamprobe/api/__init__.py +++ b/src/wamprobe/api/__init__.py @@ -11,6 +11,16 @@ RobotStateFrame, SimulatorSnapshotDescriptor, ) +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationAdapter, + ManipulationBenchmark, + ManipulationContext, + ManipulationInterventionGroup, + ManipulationInterventionSuite, + ManipulationState, + ManipulationTrajectory, +) from wamprobe.api.model import ActionPredictorAdapter, WAMAdapter from wamprobe.api.robotics import ActionPrediction, InferenceRuntime, RGBFrame, RobotObservation from wamprobe.api.types import ( @@ -34,6 +44,14 @@ "InterventionSuite", "InferenceRuntime", "ModelCapabilities", + "ManipulationAction", + "ManipulationAdapter", + "ManipulationBenchmark", + "ManipulationContext", + "ManipulationInterventionGroup", + "ManipulationInterventionSuite", + "ManipulationState", + "ManipulationTrajectory", "RGBFrame", "RGBFrameReference", "RobotAction", diff --git a/src/wamprobe/api/manipulation.py b/src/wamprobe/api/manipulation.py new file mode 100644 index 0000000..c28a19f --- /dev/null +++ b/src/wamprobe/api/manipulation.py @@ -0,0 +1,201 @@ +"""Typed dependency-free contracts for analytic manipulation benchmarks.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + +from wamprobe.api.capabilities import ModelCapabilities +from wamprobe.api.robotics import RobotObservation +from wamprobe.api.types import Vec2 + + +@dataclass(frozen=True, slots=True) +class ManipulationState: + """Minimal planar robot/object state with explicit contact and grasp semantics.""" + + effector_position: Vec2 + object_position: Vec2 + gripper_closed: bool = False + object_attached: bool = False + in_contact: bool = False + + def __post_init__(self) -> None: + values = ( + self.effector_position.x, + self.effector_position.y, + self.object_position.x, + self.object_position.y, + ) + if any(not math.isfinite(value) for value in values): + raise ValueError("manipulation state coordinates must be finite") + if self.object_attached and not self.gripper_closed: + raise ValueError("an attached object requires a closed gripper") + + def vector(self) -> tuple[float, ...]: + """Return a metric-ready state vector with explicit semantic flags.""" + + return ( + self.effector_position.x, + self.effector_position.y, + self.object_position.x, + self.object_position.y, + float(self.gripper_closed), + float(self.object_attached), + float(self.in_contact), + ) + + +@dataclass(frozen=True, slots=True) +class ManipulationContext: + """Shared initial state and object-space goal for one intervention group.""" + + context_id: str + initial_state: ManipulationState + goal: Vec2 + + def __post_init__(self) -> None: + if not self.context_id.strip(): + raise ValueError("context_id must not be empty") + if not math.isfinite(self.goal.x) or not math.isfinite(self.goal.y): + raise ValueError("manipulation goal coordinates must be finite") + + +@dataclass(frozen=True, slots=True) +class ManipulationAction: + """Constant planar effector command plus explicit gripper-close semantics.""" + + name: str + effector_delta: Vec2 + close_gripper: bool = False + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("manipulation action name must not be empty") + + +@dataclass(frozen=True, slots=True) +class ManipulationTrajectory: + """State rollout for one context/action branch.""" + + context_id: str + action_name: str + states: tuple[ManipulationState, ...] + + def __post_init__(self) -> None: + if not self.states: + raise ValueError("manipulation trajectory must contain at least one state") + + @property + def final_state(self) -> ManipulationState: + """Return the final manipulation state.""" + + return self.states[-1] + + +@dataclass(frozen=True, slots=True) +class ManipulationInterventionGroup: + """Candidate actions branched from one manipulation context.""" + + context: ManipulationContext + actions: tuple[ManipulationAction, ...] + + def __post_init__(self) -> None: + if len(self.actions) < 2: + raise ValueError("a manipulation group requires at least two actions") + names = [action.name for action in self.actions] + if len(names) != len(set(names)): + raise ValueError("manipulation action names must be unique") + + +@dataclass(frozen=True, slots=True) +class ManipulationInterventionSuite: + """Versioned collection of analytic manipulation intervention groups.""" + + benchmark_id: str + groups: tuple[ManipulationInterventionGroup, ...] + + def __post_init__(self) -> None: + if not self.benchmark_id.strip(): + raise ValueError("benchmark_id must not be empty") + if not self.groups: + raise ValueError("a manipulation suite requires at least one group") + + +@runtime_checkable +class ManipulationBenchmark(Protocol): + """Known-dynamics benchmark used by manipulation adapters and evaluators.""" + + benchmark_id: str + horizon: int + + def actions(self) -> tuple[ManipulationAction, ...]: + """Return the benchmark's candidate action set.""" + + ... + + def rollout( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + noise_std: float = 0.0, + seed: int = 0, + ) -> ManipulationTrajectory: + """Roll out known dynamics, optionally with seeded transition noise.""" + + ... + + def score(self, context: ManipulationContext, trajectory: ManipulationTrajectory) -> float: + """Score one candidate future using ground-truth task semantics.""" + + ... + + def make_suite(self, *, contexts: int = 8, seed: int = 0) -> ManipulationInterventionSuite: + """Generate deterministic paired interventions.""" + + ... + + def optimal_action(self, context: ManipulationContext) -> ManipulationAction: + """Return the analytically optimal candidate for a generated context.""" + + ... + + def reverse_action(self, action: ManipulationAction) -> ManipulationAction: + """Return a deliberately wrong semantic/directional command.""" + + ... + + def observe(self, context: ManipulationContext, state: ManipulationState) -> RobotObservation: + """Render a state as RGB plus named proprioception.""" + + ... + + +@runtime_checkable +class ManipulationAdapter(Protocol): + """Adapter protocol for analytic planar manipulation futures.""" + + @property + def capabilities(self) -> ModelCapabilities: + """Describe the adapter output and runtime behavior.""" + + ... + + def predict_future( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + horizon: int, + seed: int, + ) -> ManipulationTrajectory: + """Predict a manipulation state trajectory.""" + + ... + + def close(self) -> None: + """Release adapter resources.""" + + ... diff --git a/src/wamprobe/benchmarks/__init__.py b/src/wamprobe/benchmarks/__init__.py index 48f1402..baa4a01 100644 --- a/src/wamprobe/benchmarks/__init__.py +++ b/src/wamprobe/benchmarks/__init__.py @@ -1,5 +1,7 @@ """Built-in WAMProbe benchmarks.""" +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch from wamprobe.benchmarks.pointmass import PointMass2D -__all__ = ["PointMass2D"] +__all__ = ["BlockPush2D", "GripperCatch", "PointMass2D"] diff --git a/src/wamprobe/benchmarks/blockpush.py b/src/wamprobe/benchmarks/blockpush.py new file mode 100644 index 0000000..8d511df --- /dev/null +++ b/src/wamprobe/benchmarks/blockpush.py @@ -0,0 +1,156 @@ +"""Analytic planar block-pushing benchmark with explicit contact phases.""" + +from __future__ import annotations + +import hashlib +from random import Random + +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationContext, + ManipulationInterventionGroup, + ManipulationInterventionSuite, + ManipulationState, + ManipulationTrajectory, +) +from wamprobe.api.robotics import RobotObservation +from wamprobe.api.types import Vec2 +from wamprobe.benchmarks.rendering import render_scene + + +class BlockPush2D: + """Known two-stage dynamics: approach first, then contact-driven object motion.""" + + benchmark_id = "blockpush-2d-v0.1" + + def __init__(self, *, horizon: int = 6, action_scale: float = 0.06) -> None: + if horizon < 3: + raise ValueError("BlockPush horizon must be at least three") + if action_scale <= 0: + raise ValueError("action_scale must be positive") + self.horizon = horizon + self.action_scale = action_scale + self._contact_reach = action_scale * 2.2 + + def actions(self) -> tuple[ManipulationAction, ...]: + """Return no-op and four constant planar push candidates.""" + + scale = self.action_scale + return ( + ManipulationAction("noop", Vec2(0.0, 0.0)), + ManipulationAction("push-right", Vec2(scale, 0.0)), + ManipulationAction("push-left", Vec2(-scale, 0.0)), + ManipulationAction("push-up", Vec2(0.0, scale)), + ManipulationAction("push-down", Vec2(0.0, -scale)), + ) + + @staticmethod + def _rng(context: ManipulationContext, action: ManipulationAction, seed: int) -> Random: + material = f"{seed}\0{context.context_id}\0{action.name}".encode() + local_seed = int.from_bytes(hashlib.sha256(material).digest()[:8], "big") + return Random(local_seed) + + def rollout( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + noise_std: float = 0.0, + seed: int = 0, + ) -> ManipulationTrajectory: + """Advance the pusher and move the block only after directed contact.""" + + if noise_std < 0: + raise ValueError("noise_std must be non-negative") + rng = self._rng(context, action, seed) + state = context.initial_state + states: list[ManipulationState] = [] + for _ in range(self.horizon): + noise = ( + Vec2(rng.gauss(0.0, noise_std), rng.gauss(0.0, noise_std)) + if noise_std + else Vec2(0.0, 0.0) + ) + commanded_delta = action.effector_delta + noise + object_direction = state.object_position - state.effector_position + directed_toward_object = commanded_delta.dot(object_direction) > 1e-15 + can_reach = object_direction.norm() <= self._contact_reach + in_contact = directed_toward_object and can_reach + object_position = ( + state.object_position + commanded_delta if in_contact else state.object_position + ) + state = ManipulationState( + effector_position=state.effector_position + commanded_delta, + object_position=object_position, + in_contact=in_contact, + ) + states.append(state) + return ManipulationTrajectory(context.context_id, action.name, tuple(states)) + + def score(self, context: ManipulationContext, trajectory: ManipulationTrajectory) -> float: + """Return negative final block distance to the object-space goal.""" + + return -(trajectory.final_state.object_position - context.goal).norm() + + def make_suite(self, *, contexts: int = 8, seed: int = 0) -> ManipulationInterventionSuite: + """Generate paired pushes with one analytically optimal contact direction.""" + + if contexts <= 0: + raise ValueError("contexts must be positive") + rng = Random(seed) + candidates = self.actions() + directions = candidates[1:] + groups: list[ManipulationInterventionGroup] = [] + for index in range(contexts): + optimal = directions[index % len(directions)] + block = Vec2(rng.uniform(-0.2, 0.2), rng.uniform(-0.2, 0.2)) + pusher = block - optimal.effector_delta * 3.0 + goal = block + optimal.effector_delta * (self.horizon - 1) + context = ManipulationContext( + context_id=f"blockpush-context-{index:04d}", + initial_state=ManipulationState(pusher, block), + goal=goal, + ) + groups.append(ManipulationInterventionGroup(context, candidates)) + return ManipulationInterventionSuite(self.benchmark_id, tuple(groups)) + + def optimal_action(self, context: ManipulationContext) -> ManipulationAction: + """Select the candidate aligned with block-to-goal displacement.""" + + desired = context.goal - context.initial_state.object_position + return max(self.actions()[1:], key=lambda action: action.effector_delta.dot(desired)) + + def reverse_action(self, action: ManipulationAction) -> ManipulationAction: + """Reverse planar motion while retaining branch identity.""" + + return ManipulationAction(action.name, action.effector_delta * -1.0) + + def observe(self, context: ManipulationContext, state: ManipulationState) -> RobotObservation: + """Return a 64×64 RGB view and explicit planar state fields.""" + + frame = render_scene( + effector_position=state.effector_position, + object_position=state.object_position, + goal=context.goal, + gripper_closed=False, + object_attached=False, + ) + return RobotObservation( + context_id=context.context_id, + task="push the block to the green target", + frames=(frame,), + proprio=( + state.effector_position.x, + state.effector_position.y, + state.object_position.x, + state.object_position.y, + float(state.in_contact), + ), + proprio_fields=( + "pusher_x", + "pusher_y", + "block_x", + "block_y", + "in_contact", + ), + ) diff --git a/src/wamprobe/benchmarks/gripper_catch.py b/src/wamprobe/benchmarks/gripper_catch.py new file mode 100644 index 0000000..cd72d3d --- /dev/null +++ b/src/wamprobe/benchmarks/gripper_catch.py @@ -0,0 +1,180 @@ +"""Analytic falling-object catch benchmark with explicit gripper attachment.""" + +from __future__ import annotations + +import hashlib +from random import Random + +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationContext, + ManipulationInterventionGroup, + ManipulationInterventionSuite, + ManipulationState, + ManipulationTrajectory, +) +from wamprobe.api.robotics import RobotObservation +from wamprobe.api.types import Vec2 +from wamprobe.benchmarks.rendering import render_scene + + +class GripperCatch: + """Falling-object dynamics where attachment requires alignment and closure.""" + + benchmark_id = "gripper-catch-v0.1" + + def __init__(self, *, horizon: int = 5, action_scale: float = 0.04) -> None: + if horizon < 3: + raise ValueError("GripperCatch horizon must be at least three") + if action_scale <= 0: + raise ValueError("action_scale must be positive") + self.horizon = horizon + self.action_scale = action_scale + self._gravity_step = 0.06 + self._catch_radius = 0.075 + + def actions(self) -> tuple[ManipulationAction, ...]: + """Return open no-op plus closed stationary/left/right catch candidates.""" + + scale = self.action_scale + return ( + ManipulationAction("noop", Vec2(0.0, 0.0), close_gripper=False), + ManipulationAction("catch-left", Vec2(-scale, 0.0), close_gripper=True), + ManipulationAction("catch-stay", Vec2(0.0, 0.0), close_gripper=True), + ManipulationAction("catch-right", Vec2(scale, 0.0), close_gripper=True), + ) + + @staticmethod + def _rng(context: ManipulationContext, action: ManipulationAction, seed: int) -> Random: + material = f"{seed}\0{context.context_id}\0{action.name}".encode() + local_seed = int.from_bytes(hashlib.sha256(material).digest()[:8], "big") + return Random(local_seed) + + def rollout( + self, + context: ManipulationContext, + action: ManipulationAction, + *, + noise_std: float = 0.0, + seed: int = 0, + ) -> ManipulationTrajectory: + """Apply gravity and attach only when the closed gripper reaches the object.""" + + if noise_std < 0: + raise ValueError("noise_std must be non-negative") + rng = self._rng(context, action, seed) + state = context.initial_state + states: list[ManipulationState] = [] + for _ in range(self.horizon): + noise = ( + Vec2(rng.gauss(0.0, noise_std), rng.gauss(0.0, noise_std)) + if noise_std + else Vec2(0.0, 0.0) + ) + effector = state.effector_position + action.effector_delta + noise + attached = state.object_attached and action.close_gripper + if attached: + object_position = effector + else: + object_position = Vec2( + state.object_position.x, + max(0.0, state.object_position.y - self._gravity_step), + ) + if ( + action.close_gripper + and (object_position - effector).norm() <= self._catch_radius + ): + attached = True + object_position = effector + state = ManipulationState( + effector_position=effector, + object_position=object_position, + gripper_closed=action.close_gripper, + object_attached=attached, + in_contact=attached, + ) + states.append(state) + return ManipulationTrajectory(context.context_id, action.name, tuple(states)) + + def score(self, context: ManipulationContext, trajectory: ManipulationTrajectory) -> float: + """Reward a valid attachment, then prefer object proximity to the catch goal.""" + + final = trajectory.final_state + return float(final.object_attached) - (final.object_position - context.goal).norm() + + def make_suite(self, *, contexts: int = 8, seed: int = 0) -> ManipulationInterventionSuite: + """Generate left/stationary/right catches from shared falling-object states.""" + + if contexts <= 0: + raise ValueError("contexts must be positive") + rng = Random(seed) + candidates = self.actions() + catches = candidates[1:] + groups: list[ManipulationInterventionGroup] = [] + for index in range(contexts): + optimal = catches[index % len(catches)] + initial_effector = Vec2(rng.uniform(-0.05, 0.05), 0.0) + target_x = initial_effector.x + optimal.effector_delta.x * self.horizon + goal = Vec2(target_x, 0.0) + falling_object = Vec2(target_x, self._gravity_step * self.horizon) + context = ManipulationContext( + context_id=f"gripper-catch-context-{index:04d}", + initial_state=ManipulationState(initial_effector, falling_object), + goal=goal, + ) + groups.append(ManipulationInterventionGroup(context, candidates)) + return ManipulationInterventionSuite(self.benchmark_id, tuple(groups)) + + def optimal_action(self, context: ManipulationContext) -> ManipulationAction: + """Select the closed command whose endpoint aligns with the falling object.""" + + initial_x = context.initial_state.effector_position.x + return min( + self.actions()[1:], + key=lambda action: abs( + initial_x + action.effector_delta.x * self.horizon - context.goal.x + ), + ) + + def reverse_action(self, action: ManipulationAction) -> ManipulationAction: + """Invert motion and open the gripper for deliberately wrong dynamics.""" + + if action.name == "noop": + return action + return ManipulationAction( + action.name, + action.effector_delta * -1.0, + close_gripper=False, + ) + + def observe(self, context: ManipulationContext, state: ManipulationState) -> RobotObservation: + """Return a 64×64 RGB view and explicit catch-state fields.""" + + frame = render_scene( + effector_position=state.effector_position, + object_position=state.object_position, + goal=context.goal, + gripper_closed=state.gripper_closed, + object_attached=state.object_attached, + ) + return RobotObservation( + context_id=context.context_id, + task="catch the falling object with the gripper", + frames=(frame,), + proprio=( + state.effector_position.x, + state.effector_position.y, + state.object_position.x, + state.object_position.y, + float(state.gripper_closed), + float(state.object_attached), + ), + proprio_fields=( + "gripper_x", + "gripper_y", + "object_x", + "object_y", + "gripper_closed", + "object_attached", + ), + ) diff --git a/src/wamprobe/benchmarks/rendering.py b/src/wamprobe/benchmarks/rendering.py new file mode 100644 index 0000000..b014c58 --- /dev/null +++ b/src/wamprobe/benchmarks/rendering.py @@ -0,0 +1,81 @@ +"""Small dependency-free rasterizer for analytic manipulation observations.""" + +from __future__ import annotations + +from wamprobe.api.robotics import RGBFrame +from wamprobe.api.types import Vec2 + +Color = tuple[int, int, int] + + +def _pixel_coordinate(position: Vec2, *, size: int) -> tuple[int, int]: + column = round((position.x + 0.6) / 1.2 * (size - 1)) + row = round((0.6 - position.y) / 1.2 * (size - 1)) + return max(0, min(size - 1, column)), max(0, min(size - 1, row)) + + +def _draw_disc( + pixels: bytearray, + *, + size: int, + center: tuple[int, int], + radius: int, + color: Color, +) -> None: + center_x, center_y = center + for row in range(max(0, center_y - radius), min(size, center_y + radius + 1)): + for column in range(max(0, center_x - radius), min(size, center_x + radius + 1)): + if (column - center_x) ** 2 + (row - center_y) ** 2 > radius**2: + continue + offset = (row * size + column) * 3 + pixels[offset : offset + 3] = bytes(color) + + +def _draw_goal(pixels: bytearray, *, size: int, goal: Vec2) -> None: + center_x, center_y = _pixel_coordinate(goal, size=size) + color = (52, 168, 83) + radius = 5 + for delta in range(-radius, radius + 1): + for column, row in ( + (center_x + delta, center_y - radius), + (center_x + delta, center_y + radius), + (center_x - radius, center_y + delta), + (center_x + radius, center_y + delta), + ): + if 0 <= column < size and 0 <= row < size: + offset = (row * size + column) * 3 + pixels[offset : offset + 3] = bytes(color) + + +def render_scene( + *, + effector_position: Vec2, + object_position: Vec2, + goal: Vec2, + gripper_closed: bool, + object_attached: bool, + size: int = 64, +) -> RGBFrame: + """Render goal, object, and effector into one tightly packed RGB frame.""" + + if size < 16: + raise ValueError("render size must be at least 16 pixels") + pixels = bytearray((244, 247, 252) * (size * size)) + _draw_goal(pixels, size=size, goal=goal) + object_color = (241, 143, 45) if not object_attached else (170, 94, 220) + _draw_disc( + pixels, + size=size, + center=_pixel_coordinate(object_position, size=size), + radius=4, + color=object_color, + ) + effector_color = (31, 111, 235) if not gripper_closed else (10, 70, 170) + _draw_disc( + pixels, + size=size, + center=_pixel_coordinate(effector_position, size=size), + radius=3, + color=effector_color, + ) + return RGBFrame("toy-overhead", height=size, width=size, data=bytes(pixels)) diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index c21e7bb..fe8efc9 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -15,10 +15,21 @@ OraclePointMassAdapter, WrongDirectionAdapter, ) +from wamprobe.adapters.manipulation import ( + ActionAgnosticManipulationAdapter, + CopyLastManipulationAdapter, + NoisyManipulationAdapter, + OracleManipulationAdapter, + WrongDirectionManipulationAdapter, +) +from wamprobe.api.manipulation import ManipulationAdapter, ManipulationBenchmark from wamprobe.api.model import WAMAdapter +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch from wamprobe.benchmarks.pointmass import PointMass2D from wamprobe.doctor import ModelManifestError, check_model_store, load_manifest -from wamprobe.evaluation import evaluate +from wamprobe.evaluation import EvaluationResult, evaluate +from wamprobe.manipulation_evaluation import evaluate_manipulation from wamprobe.reporting import write_reports @@ -28,7 +39,13 @@ def _build_parser() -> argparse.ArgumentParser: description="Counterfactual evaluation for World Action Models", ) subparsers = parser.add_subparsers(dest="command", required=True) - demo = subparsers.add_parser("demo", help="run the PointMass-2D baseline comparison") + demo = subparsers.add_parser("demo", help="run an analytic baseline comparison") + demo.add_argument( + "--benchmark", + choices=("pointmass", "blockpush", "gripper-catch"), + default="pointmass", + help="analytic benchmark to evaluate", + ) demo.add_argument("--contexts", type=int, default=12, help="number of shared contexts") demo.add_argument("--seed", type=int, default=7, help="deterministic suite seed") demo.add_argument("--horizon", type=int, default=4, help="prediction horizon") @@ -60,7 +77,7 @@ def _build_parser() -> argparse.ArgumentParser: return parser -def _run_demo(args: argparse.Namespace) -> int: +def _pointmass_results(args: argparse.Namespace) -> tuple[str, list[EvaluationResult]]: benchmark = PointMass2D(horizon=args.horizon) suite = benchmark.make_suite(contexts=args.contexts, seed=args.seed) adapters: list[WAMAdapter] = [ @@ -71,9 +88,36 @@ def _run_demo(args: argparse.Namespace) -> int: ActionAgnosticAdapter(), ] results = [evaluate(adapter, benchmark, suite, seed=args.seed) for adapter in adapters] + return benchmark.benchmark_id, results + + +def _manipulation_results(args: argparse.Namespace) -> tuple[str, list[EvaluationResult]]: + benchmark: ManipulationBenchmark + if args.benchmark == "blockpush": + benchmark = BlockPush2D(horizon=args.horizon) + else: + benchmark = GripperCatch(horizon=args.horizon) + suite = benchmark.make_suite(contexts=args.contexts, seed=args.seed) + adapters: list[ManipulationAdapter] = [ + OracleManipulationAdapter(benchmark), + NoisyManipulationAdapter(benchmark), + CopyLastManipulationAdapter(), + WrongDirectionManipulationAdapter(benchmark), + ActionAgnosticManipulationAdapter(benchmark), + ] + results = [ + evaluate_manipulation(adapter, benchmark, suite, seed=args.seed) for adapter in adapters + ] + return benchmark.benchmark_id, results + + +def _run_demo(args: argparse.Namespace) -> int: + benchmark_id, results = ( + _pointmass_results(args) if args.benchmark == "pointmass" else _manipulation_results(args) + ) summary_path, report_path, html_path = write_reports( args.output, - benchmark=benchmark.benchmark_id, + benchmark=benchmark_id, results=results, seed=args.seed, ) diff --git a/src/wamprobe/evaluation.py b/src/wamprobe/evaluation.py index c8dee1e..1397bc9 100644 --- a/src/wamprobe/evaluation.py +++ b/src/wamprobe/evaluation.py @@ -12,6 +12,7 @@ from wamprobe.metrics import ( action_dependence, action_dependence_permutation_test, + candidate_ranking_correlation, counterfactual_direction_accuracy, noop_stability, state_ade, @@ -98,6 +99,7 @@ def evaluate( permutations=128, seed=permutation_seed, ) + ranking_result = candidate_ranking_correlation(predicted_scores, true_scores) context_metrics = { "action_dependence": action_dependence([samples]), "action_dependence_permutation_effect": permutation_result.effect_size, @@ -106,6 +108,10 @@ def evaluate( "noop_stability": noop_stability(samples), "state_ade": state_ade(samples), "state_fde": state_fde(samples), + "candidate_ranking_spearman": ranking_result.spearman, + "candidate_ranking_kendall_tau": ranking_result.kendall_tau, + "candidate_ranking_ndcg": ranking_result.ndcg, + "candidate_ranking_pairwise_accuracy": (ranking_result.pairwise_preference_accuracy), "top1_regret": regret, } context_results.append( diff --git a/src/wamprobe/manipulation_evaluation.py b/src/wamprobe/manipulation_evaluation.py new file mode 100644 index 0000000..22af1ae --- /dev/null +++ b/src/wamprobe/manipulation_evaluation.py @@ -0,0 +1,197 @@ +"""Counterfactual evaluation for analytic manipulation state trajectories.""" + +from __future__ import annotations + +import hashlib +from itertools import combinations +from math import sqrt + +from wamprobe.api.manipulation import ( + ManipulationAdapter, + ManipulationBenchmark, + ManipulationContext, + ManipulationInterventionSuite, + ManipulationTrajectory, +) +from wamprobe.evaluation import ContextResult, EvaluationResult +from wamprobe.metrics import ( + action_dependence_permutation_vectors, + candidate_ranking_correlation, +) + +ManipulationSample = tuple[ + ManipulationContext, + ManipulationTrajectory, + ManipulationTrajectory, +] + + +def _distance(left: tuple[float, ...], right: tuple[float, ...]) -> float: + if len(left) != len(right): + raise ValueError("manipulation state vectors must have equal dimensions") + return sqrt( + sum( + (left_value - right_value) ** 2 + for left_value, right_value in zip(left, right, strict=True) + ) + ) + + +def _displacement(start: tuple[float, ...], end: tuple[float, ...]) -> tuple[float, ...]: + return tuple(end_value - start_value for start_value, end_value in zip(start, end, strict=True)) + + +def _dot(left: tuple[float, ...], right: tuple[float, ...]) -> float: + return sum( + left_value * right_value for left_value, right_value in zip(left, right, strict=True) + ) + + +def _state_errors(samples: list[ManipulationSample]) -> tuple[float, float]: + step_errors: list[float] = [] + final_errors: list[float] = [] + for _, predicted, truth in samples: + if len(predicted.states) != len(truth.states): + raise ValueError("predicted and true trajectories must have equal horizons") + errors = [ + _distance(predicted_state.vector(), true_state.vector()) + for predicted_state, true_state in zip(predicted.states, truth.states, strict=True) + ] + step_errors.extend(errors) + final_errors.append(errors[-1]) + return sum(step_errors) / len(step_errors), sum(final_errors) / len(final_errors) + + +def _counterfactual_direction(samples: list[ManipulationSample]) -> float: + scores: list[float] = [] + for context, predicted, truth in samples: + start = context.initial_state.vector() + predicted_delta = _displacement(start, predicted.final_state.vector()) + truth_delta = _displacement(start, truth.final_state.vector()) + truth_norm = sqrt(_dot(truth_delta, truth_delta)) + if truth_norm <= 1e-12: + continue + predicted_norm = sqrt(_dot(predicted_delta, predicted_delta)) + if predicted_norm <= 1e-12: + scores.append(0.0) + continue + scores.append(_dot(predicted_delta, truth_delta) / (predicted_norm * truth_norm)) + return sum(scores) / len(scores) if scores else 0.0 + + +def _action_dependence(samples: list[ManipulationSample]) -> float: + predicted_endpoints = [predicted.final_state.vector() for _, predicted, _ in samples] + truth_endpoints = [truth.final_state.vector() for _, _, truth in samples] + predicted_distances = [ + _distance(predicted_endpoints[left], predicted_endpoints[right]) + for left, right in combinations(range(len(samples)), 2) + ] + truth_distances = [ + _distance(truth_endpoints[left], truth_endpoints[right]) + for left, right in combinations(range(len(samples)), 2) + ] + predicted_mean = sum(predicted_distances) / len(predicted_distances) + truth_mean = sum(truth_distances) / len(truth_distances) + return min(1.0, predicted_mean / truth_mean) if truth_mean > 1e-15 else 0.0 + + +def _noop_fidelity(samples: list[ManipulationSample]) -> float: + noop_scores = [ + 1.0 + if _distance(predicted.final_state.vector(), truth.final_state.vector()) <= 1e-12 + else 0.0 + for _, predicted, truth in samples + if truth.action_name == "noop" + ] + return sum(noop_scores) / len(noop_scores) if noop_scores else 0.0 + + +def evaluate_manipulation( + adapter: ManipulationAdapter, + benchmark: ManipulationBenchmark, + suite: ManipulationInterventionSuite, + *, + seed: int = 0, +) -> EvaluationResult: + """Evaluate one adapter over paired contact/grasp intervention groups.""" + + if suite.benchmark_id != benchmark.benchmark_id: + raise ValueError("suite and benchmark IDs must match") + + context_results: list[ContextResult] = [] + try: + for group in suite.groups: + samples: list[ManipulationSample] = [] + predicted_scores: dict[str, float] = {} + true_scores: dict[str, float] = {} + for action in group.actions: + truth = benchmark.rollout(group.context, action) + predicted = adapter.predict_future( + group.context, + action, + horizon=benchmark.horizon, + seed=seed, + ) + if predicted.context_id != group.context.context_id: + raise ValueError("adapter returned a mismatched context_id") + if predicted.action_name != action.name: + raise ValueError("adapter returned a mismatched action_name") + samples.append((group.context, predicted, truth)) + predicted_scores[action.name] = benchmark.score(group.context, predicted) + true_scores[action.name] = benchmark.score(group.context, truth) + + selected = max(predicted_scores, key=predicted_scores.__getitem__) + optimal = max(true_scores, key=true_scores.__getitem__) + regret = true_scores[optimal] - true_scores[selected] + permutation_seed = int.from_bytes( + hashlib.sha256(f"{seed}\0{group.context.context_id}".encode()).digest()[:8], + "big", + ) + permutation_result = action_dependence_permutation_vectors( + predicted_endpoints=[predicted.final_state.vector() for _, predicted, _ in samples], + truth_endpoints=[truth.final_state.vector() for _, _, truth in samples], + permutations=128, + seed=permutation_seed, + ) + ranking_result = candidate_ranking_correlation(predicted_scores, true_scores) + state_ade, state_fde = _state_errors(samples) + context_metrics = { + "action_dependence": _action_dependence(samples), + "action_dependence_permutation_effect": permutation_result.effect_size, + "action_dependence_permutation_p_value": permutation_result.p_value, + "counterfactual_direction_accuracy": _counterfactual_direction(samples), + "noop_stability": _noop_fidelity(samples), + "state_ade": state_ade, + "state_fde": state_fde, + "candidate_ranking_spearman": ranking_result.spearman, + "candidate_ranking_kendall_tau": ranking_result.kendall_tau, + "candidate_ranking_ndcg": ranking_result.ndcg, + "candidate_ranking_pairwise_accuracy": ( + ranking_result.pairwise_preference_accuracy + ), + "top1_regret": regret, + } + context_results.append( + ContextResult( + context_id=group.context.context_id, + selected_action=selected, + optimal_action=optimal, + top1_regret=regret, + metrics=context_metrics, + ) + ) + finally: + adapter.close() + + metric_names = context_results[0].metrics.keys() + metrics = { + metric: sum(result.metrics[metric] for result in context_results) / len(context_results) + for metric in metric_names + } + return EvaluationResult( + model_id=adapter.capabilities.model_id, + benchmark=benchmark.benchmark_id, + contexts=len(suite.groups), + metrics=metrics, + context_results=tuple(context_results), + ) diff --git a/src/wamprobe/metrics.py b/src/wamprobe/metrics.py index 97648d8..3ccd42e 100644 --- a/src/wamprobe/metrics.py +++ b/src/wamprobe/metrics.py @@ -5,11 +5,15 @@ from dataclasses import dataclass from itertools import combinations from itertools import permutations as generate_permutations -from math import factorial, sqrt +from math import factorial, isfinite, log2, sqrt from random import Random +from typing import TYPE_CHECKING from wamprobe.api.types import Context2D, Trajectory2D, Vec2 +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + def _displacement(context: Context2D, trajectory: Trajectory2D) -> Vec2: return trajectory.final_state - context.position @@ -72,6 +76,16 @@ class PermutationTestResult: permutations: int +@dataclass(frozen=True, slots=True) +class CandidateRankingResult: + """Per-context candidate ranking agreement with simulator returns.""" + + spearman: float + kendall_tau: float + ndcg: float + pairwise_preference_accuracy: float + + def _pearson(left: list[float], right: list[float]) -> float: if len(left) != len(right): raise ValueError("correlation vectors must have equal lengths") @@ -97,9 +111,21 @@ def _pearson(left: list[float], right: list[float]) -> float: return max(-1.0, min(1.0, correlation)) -def _pairwise_distances(states: list[Vec2]) -> list[float]: +def _vector_distance(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right): + raise ValueError("state vectors must have equal dimensions") + return sqrt( + sum( + (left_value - right_value) ** 2 + for left_value, right_value in zip(left, right, strict=True) + ) + ) + + +def _pairwise_vector_distances(states: Sequence[Sequence[float]]) -> list[float]: return [ - (states[left] - states[right]).norm() for left, right in combinations(range(len(states)), 2) + _vector_distance(states[left], states[right]) + for left, right in combinations(range(len(states)), 2) ] @@ -145,15 +171,40 @@ def action_dependence_permutation_test( if len(context_ids) != 1: raise ValueError("permutation samples must share one context") - truth_distances = _pairwise_distances([truth.final_state for _, _, truth in samples]) - predicted_states = [predicted.final_state for _, predicted, _ in samples] - observed = _pearson(truth_distances, _pairwise_distances(predicted_states)) + return action_dependence_permutation_vectors( + predicted_endpoints=[ + (predicted.final_state.x, predicted.final_state.y) for _, predicted, _ in samples + ], + truth_endpoints=[(truth.final_state.x, truth.final_state.y) for _, _, truth in samples], + permutations=permutations, + seed=seed, + ) + + +def action_dependence_permutation_vectors( + *, + predicted_endpoints: Sequence[Sequence[float]], + truth_endpoints: Sequence[Sequence[float]], + permutations: int = 128, + seed: int = 0, +) -> PermutationTestResult: + """Evaluate action/future geometry for arbitrary fixed-width state vectors.""" + + if len(predicted_endpoints) != len(truth_endpoints): + raise ValueError("predicted and true endpoints must have equal lengths") + if len(predicted_endpoints) < 2: + raise ValueError("permutation testing requires at least two action branches") + if permutations <= 0: + raise ValueError("permutations must be positive") + + truth_distances = _pairwise_vector_distances(truth_endpoints) + observed = _pearson(truth_distances, _pairwise_vector_distances(predicted_endpoints)) - indices = _permutation_indices(len(samples), count=permutations, seed=seed) + indices = _permutation_indices(len(predicted_endpoints), count=permutations, seed=seed) null_scores = [ _pearson( truth_distances, - _pairwise_distances([predicted_states[index] for index in permutation]), + _pairwise_vector_distances([predicted_endpoints[index] for index in permutation]), ) for permutation in indices ] @@ -174,6 +225,98 @@ def action_dependence_permutation_test( ) +def _average_ranks(scores: Mapping[str, float], names: Sequence[str]) -> list[float]: + ordered = sorted(names, key=lambda name: (scores[name], name)) + rank_by_name: dict[str, float] = {} + start = 0 + while start < len(ordered): + end = start + 1 + while end < len(ordered) and scores[ordered[end]] == scores[ordered[start]]: + end += 1 + average_rank = ((start + 1) + end) / 2.0 + for index in range(start, end): + rank_by_name[ordered[index]] = average_rank + start = end + return [rank_by_name[name] for name in names] + + +def candidate_ranking_correlation( + predicted_scores: Mapping[str, float], + true_scores: Mapping[str, float], +) -> CandidateRankingResult: + """Compare candidate ordering with simulator returns using four CRC views.""" + + if predicted_scores.keys() != true_scores.keys(): + raise ValueError("predicted and true candidate names must match exactly") + if len(predicted_scores) < 2: + raise ValueError("candidate ranking requires at least two candidates") + if any( + not isfinite(value) + for scores in (predicted_scores, true_scores) + for value in scores.values() + ): + raise ValueError("candidate scores must be finite") + + names = sorted(predicted_scores) + predicted_ranks = _average_ranks(predicted_scores, names) + true_ranks = _average_ranks(true_scores, names) + spearman = _pearson(predicted_ranks, true_ranks) + + concordant = 0 + discordant = 0 + predicted_only_ties = 0 + truth_only_ties = 0 + preference_total = 0 + preference_score = 0.0 + for left, right in combinations(names, 2): + predicted_delta = predicted_scores[left] - predicted_scores[right] + true_delta = true_scores[left] - true_scores[right] + if predicted_delta == 0.0 and true_delta == 0.0: + continue + if predicted_delta == 0.0: + predicted_only_ties += 1 + elif true_delta == 0.0: + truth_only_ties += 1 + elif predicted_delta * true_delta > 0.0: + concordant += 1 + else: + discordant += 1 + + if true_delta != 0.0: + preference_total += 1 + if predicted_delta == 0.0: + preference_score += 0.5 + elif predicted_delta * true_delta > 0.0: + preference_score += 1.0 + + denominator = sqrt( + (concordant + discordant + predicted_only_ties) + * (concordant + discordant + truth_only_ties) + ) + kendall_tau = (concordant - discordant) / denominator if denominator > 1e-15 else 0.0 + pairwise_accuracy = preference_score / preference_total if preference_total else 0.0 + + minimum_relevance = min(true_scores.values()) + + def discounted_gain(order: Sequence[str]) -> float: + gain = 0.0 + for index, name in enumerate(order): + relevance = true_scores[name] - minimum_relevance + gain += (pow(2.0, relevance) - 1.0) / log2(index + 2) + return gain + + predicted_order = sorted(names, key=lambda name: (-predicted_scores[name], name)) + ideal_order = sorted(names, key=lambda name: (-true_scores[name], name)) + ideal_gain = discounted_gain(ideal_order) + ndcg = discounted_gain(predicted_order) / ideal_gain if ideal_gain > 1e-15 else 1.0 + return CandidateRankingResult( + spearman=spearman, + kendall_tau=kendall_tau, + ndcg=max(0.0, min(1.0, ndcg)), + pairwise_preference_accuracy=pairwise_accuracy, + ) + + def action_dependence( groups: list[list[tuple[Context2D, Trajectory2D, Trajectory2D]]], ) -> float: diff --git a/src/wamprobe/reporting.py b/src/wamprobe/reporting.py index 7c26263..1a70c4c 100644 --- a/src/wamprobe/reporting.py +++ b/src/wamprobe/reporting.py @@ -19,12 +19,21 @@ ("noop_stability", "No-op Stability"), ("state_ade", "State ADE ↓"), ("state_fde", "State FDE ↓"), + ("candidate_ranking_spearman", "CRC Spearman"), + ("candidate_ranking_kendall_tau", "CRC Kendall τ"), + ("candidate_ranking_ndcg", "CRC NDCG"), + ("candidate_ranking_pairwise_accuracy", "CRC Pairwise Accuracy"), ("top1_regret", "Top-1 Regret ↓"), ) StatisticsByModel = dict[str, dict[str, MetricSummary]] +def _report_title(results: list[EvaluationResult]) -> str: + benchmark = results[0].benchmark if results else "evaluation" + return f"WAMProbe {benchmark} Demo" + + def _stable_seed(seed: int, *parts: str) -> int: material = "\0".join((str(seed), *parts)).encode() return int.from_bytes(hashlib.sha256(material).digest()[:8], "big") @@ -80,7 +89,7 @@ def render_markdown( headers = ["Model", *(label for _, label in _METRIC_COLUMNS)] lines = [ - "# WAMProbe PointMass-2D Demo", + f"# {_report_title(results)}", "", "This report compares action-aware and deliberately broken reference baselines.", "", @@ -139,9 +148,9 @@ def render_markdown( lines.extend( [ "", - "A high Action Dependence score is not sufficient: the wrong-direction baseline", - "depends on the action but fails Counterfactual Direction Accuracy. WAMProbe", - "therefore reports a metric profile rather than a single composite score.", + "Action separation alone does not establish correct direction, state dynamics,", + "or control ranking. WAMProbe therefore reports a metric profile rather than a", + "single composite score.", "", ] ) @@ -156,6 +165,7 @@ def render_html( ) -> str: """Render a standalone, dependency-free HTML report.""" + title = _report_title(results) metric_headers = "".join(f"{escape(label)}" for _, label in _METRIC_COLUMNS) result_rows: list[str] = [] for result in results: @@ -197,7 +207,7 @@ def render_html( - WAMProbe PointMass-2D Demo + {escape(title)} -

WAMProbe PointMass-2D Demo

+

{escape(title)}

Action-aware and deliberately broken baselines are shown as a metric profile. No composite score is produced.

@@ -260,6 +270,8 @@ def write_reports( if not results: raise ValueError("at least one evaluation result is required") + if any(result.benchmark != benchmark for result in results): + raise ValueError("all evaluation results must match the report benchmark") statistics, comparisons = _build_statistics(results, resamples=resamples, seed=seed) output_dir.mkdir(parents=True, exist_ok=True) summary_path = output_dir / "summary.json" diff --git a/tests/test_cli.py b/tests/test_cli.py index a1fe0dc..fe4ba02 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -46,3 +46,49 @@ def test_demo_cli_writes_machine_and_human_readable_reports( assert "" in html assert "Context-block 95% confidence intervals" in html assert "WAMProbe demo complete" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("benchmark_name", "benchmark_id"), + [ + ("blockpush", "blockpush-2d-v0.1"), + ("gripper-catch", "gripper-catch-v0.1"), + ], +) +def test_demo_cli_runs_manipulation_benchmarks( + tmp_path: Path, + benchmark_name: str, + benchmark_id: str, +) -> None: + output = tmp_path / benchmark_name + + assert ( + main( + [ + "demo", + "--benchmark", + benchmark_name, + "--contexts", + "4", + "--seed", + "5", + "--output", + str(output), + ] + ) + == 0 + ) + + summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) + assert summary["benchmark"] == benchmark_id + assert benchmark_id in (output / "report.html").read_text(encoding="utf-8") + assert {result["model_id"] for result in summary["results"]} == { + "action-agnostic", + "copy-last-frame", + "noisy-dynamics", + "oracle-simulator", + "wrong-direction", + } + assert all( + "candidate_ranking_kendall_tau" in result["metrics"] for result in summary["results"] + ) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 96befc5..25424c8 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -28,6 +28,11 @@ def test_reference_baselines_have_expected_metric_ordering() -> None: assert action_agnostic.metrics["top1_regret"] > oracle.metrics["top1_regret"] assert oracle.metrics["action_dependence_permutation_effect"] > 1.0 assert copy_last.metrics["action_dependence_permutation_effect"] == 0.0 + assert oracle.metrics["candidate_ranking_spearman"] == 1.0 + assert oracle.metrics["candidate_ranking_kendall_tau"] == 1.0 + assert oracle.metrics["candidate_ranking_ndcg"] == 1.0 + assert oracle.metrics["candidate_ranking_pairwise_accuracy"] == 1.0 + assert action_agnostic.metrics["candidate_ranking_spearman"] == 0.0 assert all(result.metrics for result in oracle.context_results) diff --git a/tests/test_manipulation_benchmarks.py b/tests/test_manipulation_benchmarks.py new file mode 100644 index 0000000..4ec32e3 --- /dev/null +++ b/tests/test_manipulation_benchmarks.py @@ -0,0 +1,115 @@ +from wamprobe.adapters.manipulation import ( + ActionAgnosticManipulationAdapter, + CopyLastManipulationAdapter, + NoisyManipulationAdapter, + OracleManipulationAdapter, + WrongDirectionManipulationAdapter, +) +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch +from wamprobe.manipulation_evaluation import evaluate_manipulation + + +def test_blockpush_has_precontact_and_contact_dynamics() -> None: + benchmark = BlockPush2D(horizon=6) + group = benchmark.make_suite(contexts=1, seed=7).groups[0] + optimal = benchmark.optimal_action(group.context) + + trajectory = benchmark.rollout(group.context, optimal) + noop = benchmark.rollout(group.context, group.actions[0]) + reversed_trajectory = benchmark.rollout( + group.context, + benchmark.reverse_action(optimal), + ) + + assert trajectory.states[0].object_position == group.context.initial_state.object_position + assert any(state.in_contact for state in trajectory.states[1:]) + assert trajectory.final_state.object_position == group.context.goal + assert noop.final_state.object_position == group.context.initial_state.object_position + assert ( + reversed_trajectory.final_state.object_position + == group.context.initial_state.object_position + ) + + +def test_manipulation_suite_generator_scales_to_public_pilot_size() -> None: + for benchmark in (BlockPush2D(), GripperCatch()): + suite = benchmark.make_suite(contexts=1000, seed=17) + + assert len(suite.groups) == 1000 + assert len({group.context.context_id for group in suite.groups}) == 1000 + assert suite == benchmark.make_suite(contexts=1000, seed=17) + + +def test_gripper_catch_requires_alignment_and_close_semantics() -> None: + benchmark = GripperCatch(horizon=5) + suite = benchmark.make_suite(contexts=3, seed=3) + + for group in suite.groups: + optimal = benchmark.optimal_action(group.context) + caught = benchmark.rollout(group.context, optimal) + noop = benchmark.rollout(group.context, group.actions[0]) + + assert caught.final_state.object_attached + assert caught.final_state.gripper_closed + assert caught.final_state.object_position == caught.final_state.effector_position + assert not noop.final_state.object_attached + assert benchmark.score(group.context, caught) > benchmark.score(group.context, noop) + + +def test_manipulation_rgb_observation_is_dependency_free_and_state_sensitive() -> None: + for benchmark in (BlockPush2D(horizon=5), GripperCatch(horizon=5)): + group = benchmark.make_suite(contexts=1, seed=2).groups[0] + initial = benchmark.observe(group.context, group.context.initial_state) + final_state = benchmark.rollout( + group.context, + benchmark.optimal_action(group.context), + ).final_state + final = benchmark.observe(group.context, final_state) + + assert initial.frames[0].height == 64 + assert initial.frames[0].width == 64 + assert len(initial.frames[0].data) == 64 * 64 * 3 + assert initial.content_sha256 != final.content_sha256 + + +def test_noisy_manipulation_rollouts_are_content_addressed() -> None: + benchmark = BlockPush2D(horizon=6) + group = benchmark.make_suite(contexts=1, seed=4).groups[0] + action = benchmark.optimal_action(group.context) + + first = benchmark.rollout(group.context, action, noise_std=0.01, seed=19) + second = benchmark.rollout(group.context, action, noise_std=0.01, seed=19) + changed_seed = benchmark.rollout(group.context, action, noise_std=0.01, seed=20) + + assert first == second + assert first != changed_seed + + +def test_manipulation_baselines_produce_expected_metric_profiles() -> None: + for benchmark in (BlockPush2D(horizon=6), GripperCatch(horizon=5)): + suite = benchmark.make_suite(contexts=12, seed=11) + oracle = evaluate_manipulation( + OracleManipulationAdapter(benchmark), benchmark, suite, seed=29 + ) + noisy = evaluate_manipulation( + NoisyManipulationAdapter(benchmark, noise_std=0.01), + benchmark, + suite, + seed=29, + ) + copy_last = evaluate_manipulation(CopyLastManipulationAdapter(), benchmark, suite, seed=29) + wrong = evaluate_manipulation( + WrongDirectionManipulationAdapter(benchmark), benchmark, suite, seed=29 + ) + action_agnostic = evaluate_manipulation( + ActionAgnosticManipulationAdapter(benchmark), benchmark, suite, seed=29 + ) + + assert oracle.metrics["state_fde"] == 0.0 + assert oracle.metrics["top1_regret"] == 0.0 + assert oracle.metrics["candidate_ranking_spearman"] == 1.0 + assert oracle.metrics["candidate_ranking_ndcg"] == 1.0 + assert 0.0 < noisy.metrics["state_fde"] < copy_last.metrics["state_fde"] + assert wrong.metrics["state_fde"] > oracle.metrics["state_fde"] + assert action_agnostic.metrics["candidate_ranking_spearman"] == 0.0 diff --git a/tests/test_ranking_metrics.py b/tests/test_ranking_metrics.py new file mode 100644 index 0000000..7875944 --- /dev/null +++ b/tests/test_ranking_metrics.py @@ -0,0 +1,41 @@ +import pytest + +from wamprobe.metrics import candidate_ranking_correlation + + +def test_candidate_ranking_correlation_has_exact_oracle_and_reversed_bounds() -> None: + truth = {"best": 3.0, "middle": 2.0, "worst": 1.0} + + oracle = candidate_ranking_correlation(truth, truth) + reversed_result = candidate_ranking_correlation( + {"best": 1.0, "middle": 2.0, "worst": 3.0}, + truth, + ) + + assert oracle.spearman == 1.0 + assert oracle.kendall_tau == 1.0 + assert oracle.ndcg == 1.0 + assert oracle.pairwise_preference_accuracy == 1.0 + assert reversed_result.spearman == -1.0 + assert reversed_result.kendall_tau == -1.0 + assert 0.0 <= reversed_result.ndcg < 1.0 + assert reversed_result.pairwise_preference_accuracy == 0.0 + + +def test_candidate_ranking_correlation_handles_tied_predictions() -> None: + result = candidate_ranking_correlation( + {"best": 0.0, "middle": 0.0, "worst": 0.0}, + {"best": 3.0, "middle": 2.0, "worst": 1.0}, + ) + + assert result.spearman == 0.0 + assert result.kendall_tau == 0.0 + assert result.pairwise_preference_accuracy == 0.5 + assert 0.0 <= result.ndcg <= 1.0 + + +def test_candidate_ranking_correlation_rejects_unpaired_candidates() -> None: + with pytest.raises(ValueError, match="candidate names"): + candidate_ranking_correlation({"left": 1.0}, {"right": 1.0}) + with pytest.raises(ValueError, match="at least two"): + candidate_ranking_correlation({"only": 1.0}, {"only": 1.0}) From c8eea56cc91ef56b2eaa2758881b1dce1d9fe09e Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 13:17:25 +0000 Subject: [PATCH 11/20] add resumable evaluation data pipeline --- README.md | 26 ++- docs/WAMProbe_PLAN.md | 16 +- src/wamprobe/cache.py | 263 +++++++++++++++++++++++++++ src/wamprobe/cli.py | 298 ++++++++++++++++++++++++++++-- src/wamprobe/datasets.py | 354 ++++++++++++++++++++++++++++++++++++ src/wamprobe/evaluation.py | 103 ++++++++++- src/wamprobe/reporting.py | 27 ++- tests/test_cli.py | 81 +++++++++ tests/test_dataset_cache.py | 113 ++++++++++++ 9 files changed, 1257 insertions(+), 24 deletions(-) create mode 100644 src/wamprobe/cache.py create mode 100644 src/wamprobe/datasets.py create mode 100644 tests/test_dataset_cache.py diff --git a/README.md b/README.md index ed837c6..2acfebd 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,17 @@ python -m pip install -e . wamprobe demo --contexts 12 --seed 7 --output runs/pointmass-demo +# Resume identical evaluations from verified content-addressed results. +wamprobe demo --contexts 12 --seed 7 --cache-dir runs/cache --output runs/resumed + +# Export, verify, compare, and rebuild reports without rerunning a model. +wamprobe dataset-export --benchmark pointmass --contexts 12 --output data/pointmass.jsonl +wamprobe dataset-validate data/pointmass.jsonl +wamprobe compare runs/pointmass-demo runs/resumed \ + --left-model oracle-pointmass --right-model copy-last-frame \ + --metric state_fde --output runs/comparison.json +wamprobe report runs/pointmass-demo --output runs/rebuilt-report + # Contact and attachment diagnostics use the same report pipeline. wamprobe demo --benchmark blockpush --horizon 6 --output runs/blockpush-demo wamprobe demo --benchmark gripper-catch --horizon 5 --output runs/gripper-catch-demo @@ -57,6 +68,7 @@ The command creates: ```text runs/pointmass-demo/ ├── summary.json # versioned machine-readable results +├── results.jsonl # one stable record per model and shared context ├── report.md # human-readable metric comparison └── report.html # standalone interactive-friendly report ``` @@ -94,7 +106,10 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo Accuracy, No-op Stability, state ADE/FDE, four-view Candidate Ranking Correlation, and Top-1 Regret; - context-block bootstrap intervals and exact-context paired model comparisons; -- versioned JSON plus Markdown and standalone HTML reports; +- deterministic checksummed intervention JSONL with strict typed round trips; +- corruption-detecting, content-addressed prediction cache for resumable runs; +- versioned JSON/JSONL plus Markdown and standalone HTML reports, with standalone + `report` and exact-context `compare` commands; - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - typed robot observations, action predictions, deterministic prediction artifacts, and a pinned StarWAM/LIBERO smoke runner; @@ -106,11 +121,10 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo The next milestones are: -1. the versioned intervention dataset loader; -2. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; -3. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; -4. add a GPU nightly workflow for the isolated real-model integration; -5. add the Occluded-Object memory diagnostic to the broader Toy tier. +1. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; +2. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; +3. add a GPU nightly workflow for the isolated real-model integration; +4. add the Occluded-Object memory diagnostic to the broader Toy tier. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index d638092..baa21aa 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1014,6 +1014,12 @@ CLI 验收要求： ### Phase 1：核心 API 与数据格式（第 3–4 周） +完成状态（2026-07-15）：typed API、capability/result/intervention schema、doctor、dummy +adapters 以及通用 intervention JSONL loader 已落地。Prediction cache 以完整输入和配置生成 +内容地址，使用原子写入与 payload SHA256 校验；相同 demo 可从 5/5 缓存结果恢复，损坏条目 +会明确失败而不会静默重算。`dataset-export`、`dataset-validate`、`report` 和 `compare` CLI +均已通过端到端测试。 + 任务： - 完成 typed data classes 和 `WAMAdapter`； @@ -1136,9 +1142,9 @@ v0.1 发布门槛： | #19 | Add GPU nightly smoke workflow | P1 | 2 天 | #18 | | #20 | Write benchmark card and v0.1 reproducibility guide | P0 | 4 天 | 全部 | -代码落地状态（2026-07-15）：已补齐并验证 #8、#10、#11、#13、#15、#16 以及 CRC； -此前已完成 #1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#6 和 #18 已有可运行的 StarWAM -prediction artifact/adapter 切片，但更通用的 cache 与真实 WAM 反事实控制评测仍需继续扩展。 +代码落地状态（2026-07-15）：已补齐并验证 #6、#8、#10、#11、#13、#15、#16 以及 CRC； +此前已完成 #1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#18 已有可运行的 StarWAM +prediction artifact/adapter 切片，但真实 WAM 反事实控制评测仍需继续扩展。 建议 labels： @@ -1260,8 +1266,8 @@ priority:p1 - [x] 至少 4 个 reference baselines； - [x] 至少 1 个真实 WAM adapter； - [x] 至少 6 个核心指标； -- [ ] prediction cache 可恢复中断运行； -- [ ] JSONL/JSON/HTML 输出齐全； +- [x] prediction cache 可恢复中断运行； +- [x] JSONL/JSON/HTML 输出齐全； - [x] paired comparison 和 bootstrap CI 可用。 ### 研究有效性 diff --git a/src/wamprobe/cache.py b/src/wamprobe/cache.py new file mode 100644 index 0000000..8ae34bc --- /dev/null +++ b/src/wamprobe/cache.py @@ -0,0 +1,263 @@ +"""Content-addressed, corruption-detecting prediction cache.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Callable +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import cast + + +class CacheCorruptionError(RuntimeError): + """Raised when a cache entry no longer matches its content identity.""" + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def _sha256(value: object) -> str: + return hashlib.sha256(_canonical_json(value)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class PredictionCacheRequest: + """All inputs that can change a cached prediction payload.""" + + namespace: str + model_id: str + benchmark_id: str + context_id: str + action_name: str + horizon: int + seed: int + input_sha256: str + configuration_sha256: str + + def __post_init__(self) -> None: + text_fields = ( + self.namespace, + self.model_id, + self.benchmark_id, + self.context_id, + self.action_name, + ) + if any(not value.strip() for value in text_fields): + raise ValueError("cache request text fields must not be empty") + if self.horizon <= 0: + raise ValueError("cache request horizon must be positive") + for field_name, checksum in ( + ("input_sha256", self.input_sha256), + ("configuration_sha256", self.configuration_sha256), + ): + if len(checksum) != 64 or any( + character not in "0123456789abcdef" for character in checksum + ): + raise ValueError(f"{field_name} must be a lowercase SHA256 digest") + + @property + def cache_key(self) -> str: + """Return a stable key over every request field.""" + + return _sha256(asdict(self)) + + @classmethod + def from_object(cls, value: object) -> PredictionCacheRequest: + """Restore and validate a request stored inside a cache entry.""" + + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise CacheCorruptionError("request must be an object") + payload = cast(dict[str, object], value) + expected = { + "namespace", + "model_id", + "benchmark_id", + "context_id", + "action_name", + "horizon", + "seed", + "input_sha256", + "configuration_sha256", + } + if payload.keys() != expected: + raise CacheCorruptionError("request fields do not match the cache schema") + string_fields = ( + "namespace", + "model_id", + "benchmark_id", + "context_id", + "action_name", + "input_sha256", + "configuration_sha256", + ) + if any(not isinstance(payload[field], str) for field in string_fields): + raise CacheCorruptionError("request string field has an invalid type") + horizon = payload["horizon"] + seed = payload["seed"] + if isinstance(horizon, bool) or not isinstance(horizon, int): + raise CacheCorruptionError("request horizon has an invalid type") + if isinstance(seed, bool) or not isinstance(seed, int): + raise CacheCorruptionError("request seed has an invalid type") + try: + return cls( + namespace=cast(str, payload["namespace"]), + model_id=cast(str, payload["model_id"]), + benchmark_id=cast(str, payload["benchmark_id"]), + context_id=cast(str, payload["context_id"]), + action_name=cast(str, payload["action_name"]), + horizon=horizon, + seed=seed, + input_sha256=cast(str, payload["input_sha256"]), + configuration_sha256=cast(str, payload["configuration_sha256"]), + ) + except ValueError as error: + raise CacheCorruptionError(str(error)) from error + + +@dataclass(frozen=True, slots=True) +class CacheAudit: + """Integrity summary over all entries beneath a cache root.""" + + entries: int + corrupt_entries: int + + @property + def passed(self) -> bool: + """Whether every discovered entry passed integrity checks.""" + + return self.corrupt_entries == 0 + + +class PredictionCache: + """Atomic JSON cache that never silently replaces corrupt entries.""" + + schema_version = "0.1" + + def __init__(self, root: Path) -> None: + self.root = root + + def path_for(self, request: PredictionCacheRequest) -> Path: + """Return the sharded path for a request.""" + + key = request.cache_key + return self.root / key[:2] / f"{key}.json" + + @staticmethod + def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(path) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + @staticmethod + def _mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise CacheCorruptionError(f"{field} must be an object") + return cast(dict[str, object], value) + + def _load_path( + self, + path: Path, + expected_request: PredictionCacheRequest | None = None, + ) -> dict[str, object]: + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise CacheCorruptionError(f"cache entry is unreadable: {path}") from error + entry = self._mapping(raw, "cache entry") + if entry.get("schema_version") != self.schema_version: + raise CacheCorruptionError("cache schema_version mismatch") + request = PredictionCacheRequest.from_object(entry.get("request")) + cache_key = entry.get("cache_key") + if not isinstance(cache_key, str) or cache_key != request.cache_key: + raise CacheCorruptionError("cache_key mismatch") + if path.stem != cache_key: + raise CacheCorruptionError("cache filename does not match cache_key") + if expected_request is not None and request != expected_request: + raise CacheCorruptionError("cache request mismatch") + payload = self._mapping(entry.get("payload"), "payload") + payload_sha256 = entry.get("payload_sha256") + if not isinstance(payload_sha256, str) or payload_sha256 != _sha256(payload): + raise CacheCorruptionError("payload_sha256 mismatch") + return payload + + def load(self, request: PredictionCacheRequest) -> dict[str, object] | None: + """Return a verified payload, or ``None`` when no entry exists.""" + + path = self.path_for(request) + if not path.exists(): + return None + return self._load_path(path, request) + + def store( + self, + request: PredictionCacheRequest, + payload: dict[str, object], + ) -> Path: + """Atomically persist a JSON-compatible payload under its request key.""" + + path = self.path_for(request) + if path.exists(): + existing = self._load_path(path, request) + if existing != payload: + raise CacheCorruptionError("valid cache entry already contains another payload") + return path + entry: dict[str, object] = { + "schema_version": self.schema_version, + "cache_key": request.cache_key, + "request": asdict(request), + "payload": payload, + "payload_sha256": _sha256(payload), + } + self._atomic_write(path, _canonical_json(entry) + b"\n") + return path + + def resolve( + self, + request: PredictionCacheRequest, + producer: Callable[[], dict[str, object]], + ) -> tuple[dict[str, object], bool]: + """Load an entry or produce it once and store it for later resumption.""" + + cached = self.load(request) + if cached is not None: + return cached, True + payload = producer() + self.store(request, payload) + return payload, False + + def audit(self) -> CacheAudit: + """Validate every JSON entry beneath the cache root.""" + + paths = sorted(self.root.glob("*/*.json")) if self.root.exists() else [] + corrupt_entries = 0 + for path in paths: + try: + self._load_path(path) + except CacheCorruptionError: + corrupt_entries += 1 + return CacheAudit(entries=len(paths), corrupt_entries=corrupt_entries) diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index fe8efc9..0ca0d87 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -3,10 +3,13 @@ from __future__ import annotations import argparse +import hashlib import json import sys from collections.abc import Sequence +from dataclasses import asdict from pathlib import Path +from typing import cast from wamprobe.adapters.baselines import ( ActionAgnosticAdapter, @@ -22,15 +25,28 @@ OracleManipulationAdapter, WrongDirectionManipulationAdapter, ) -from wamprobe.api.manipulation import ManipulationAdapter, ManipulationBenchmark +from wamprobe.api.manipulation import ( + ManipulationAdapter, + ManipulationBenchmark, + ManipulationInterventionSuite, +) from wamprobe.api.model import WAMAdapter +from wamprobe.api.types import InterventionSuite from wamprobe.benchmarks.blockpush import BlockPush2D from wamprobe.benchmarks.gripper_catch import GripperCatch from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.cache import PredictionCache, PredictionCacheRequest +from wamprobe.datasets import ( + DatasetValidationError, + intervention_dataset_sha256, + load_intervention_dataset, + write_intervention_dataset, +) from wamprobe.doctor import ModelManifestError, check_model_store, load_manifest from wamprobe.evaluation import EvaluationResult, evaluate from wamprobe.manipulation_evaluation import evaluate_manipulation from wamprobe.reporting import write_reports +from wamprobe.stats import paired_metric_comparison def _build_parser() -> argparse.ArgumentParser: @@ -53,7 +69,13 @@ def _build_parser() -> argparse.ArgumentParser: "--output", type=Path, default=Path("runs/pointmass-demo"), - help="directory for summary.json, report.md, and report.html", + help="directory for JSON, JSONL, Markdown, and HTML reports", + ) + demo.add_argument( + "--cache-dir", + type=Path, + default=None, + help="optional content-addressed cache for resumable model results", ) doctor = subparsers.add_parser("doctor", help="validate pinned local model artifacts") doctor.add_argument( @@ -74,10 +96,79 @@ def _build_parser() -> argparse.ArgumentParser: help="stream SHA256 verification for files that declare a checksum", ) doctor.add_argument("--json", action="store_true", help="emit a machine-readable report") + + report = subparsers.add_parser("report", help="rebuild reports from a summary.json") + report.add_argument("summary", type=Path, help="summary.json file or containing run directory") + report.add_argument("--output", type=Path, required=True, help="report output directory") + report.add_argument("--resamples", type=int, default=1000, help="bootstrap resamples") + report.add_argument("--seed", type=int, default=0, help="bootstrap seed") + + compare = subparsers.add_parser("compare", help="compare paired cached evaluation results") + compare.add_argument("left", type=Path, help="left summary file or run directory") + compare.add_argument("right", type=Path, help="right summary file or run directory") + compare.add_argument("--left-model", required=True, help="model ID in the left summary") + compare.add_argument("--right-model", required=True, help="model ID in the right summary") + compare.add_argument("--metric", required=True, help="per-context metric to compare") + compare.add_argument("--resamples", type=int, default=1000, help="bootstrap resamples") + compare.add_argument("--seed", type=int, default=0, help="bootstrap seed") + compare.add_argument("--output", type=Path, required=True, help="comparison JSON path") + + dataset_export = subparsers.add_parser( + "dataset-export", + help="export a deterministic built-in intervention suite", + ) + dataset_export.add_argument( + "--benchmark", + choices=("pointmass", "blockpush", "gripper-catch"), + required=True, + ) + dataset_export.add_argument("--contexts", type=int, default=12) + dataset_export.add_argument("--seed", type=int, default=7) + dataset_export.add_argument("--output", type=Path, required=True) + + dataset_validate = subparsers.add_parser( + "dataset-validate", + help="validate checksums and schema for an intervention JSONL file", + ) + dataset_validate.add_argument("dataset", type=Path) return parser -def _pointmass_results(args: argparse.Namespace) -> tuple[str, list[EvaluationResult]]: +def _stable_sha256(value: object) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _request_for( + *, + model_id: str, + benchmark_id: str, + suite_sha256: str, + horizon: int, + seed: int, + configuration: object, +) -> PredictionCacheRequest: + return PredictionCacheRequest( + namespace="wamprobe-evaluation-v0.1", + model_id=model_id, + benchmark_id=benchmark_id, + context_id=f"suite-{suite_sha256[:16]}", + action_name="all", + horizon=horizon, + seed=seed, + input_sha256=suite_sha256, + configuration_sha256=_stable_sha256(configuration), + ) + + +def _pointmass_results( + args: argparse.Namespace, +) -> tuple[str, list[EvaluationResult], int]: benchmark = PointMass2D(horizon=args.horizon) suite = benchmark.make_suite(contexts=args.contexts, seed=args.seed) adapters: list[WAMAdapter] = [ @@ -87,11 +178,46 @@ def _pointmass_results(args: argparse.Namespace) -> tuple[str, list[EvaluationRe WrongDirectionAdapter(benchmark), ActionAgnosticAdapter(), ] - results = [evaluate(adapter, benchmark, suite, seed=args.seed) for adapter in adapters] - return benchmark.benchmark_id, results + cache = PredictionCache(args.cache_dir) if args.cache_dir is not None else None + suite_sha256 = intervention_dataset_sha256(suite) + results: list[EvaluationResult] = [] + cache_hits = 0 + for adapter in adapters: + if cache is None: + results.append(evaluate(adapter, benchmark, suite, seed=args.seed)) + continue + request = _request_for( + model_id=adapter.capabilities.model_id, + benchmark_id=benchmark.benchmark_id, + suite_sha256=suite_sha256, + horizon=benchmark.horizon, + seed=args.seed, + configuration={ + "capabilities": adapter.capabilities.to_dict(), + "benchmark": { + "horizon": benchmark.horizon, + "action_scale": benchmark.action_scale, + }, + }, + ) + + def produce(current_adapter: WAMAdapter = adapter) -> dict[str, object]: + return cast( + dict[str, object], + evaluate(current_adapter, benchmark, suite, seed=args.seed).to_dict(), + ) + payload, hit = cache.resolve(request, produce) + if hit: + cache_hits += 1 + adapter.close() + results.append(EvaluationResult.from_dict(payload)) + return benchmark.benchmark_id, results, cache_hits -def _manipulation_results(args: argparse.Namespace) -> tuple[str, list[EvaluationResult]]: + +def _manipulation_results( + args: argparse.Namespace, +) -> tuple[str, list[EvaluationResult], int]: benchmark: ManipulationBenchmark if args.benchmark == "blockpush": benchmark = BlockPush2D(horizon=args.horizon) @@ -105,14 +231,50 @@ def _manipulation_results(args: argparse.Namespace) -> tuple[str, list[Evaluatio WrongDirectionManipulationAdapter(benchmark), ActionAgnosticManipulationAdapter(benchmark), ] - results = [ - evaluate_manipulation(adapter, benchmark, suite, seed=args.seed) for adapter in adapters - ] - return benchmark.benchmark_id, results + cache = PredictionCache(args.cache_dir) if args.cache_dir is not None else None + suite_sha256 = intervention_dataset_sha256(suite) + results: list[EvaluationResult] = [] + cache_hits = 0 + for adapter in adapters: + if cache is None: + results.append(evaluate_manipulation(adapter, benchmark, suite, seed=args.seed)) + continue + request = _request_for( + model_id=adapter.capabilities.model_id, + benchmark_id=benchmark.benchmark_id, + suite_sha256=suite_sha256, + horizon=benchmark.horizon, + seed=args.seed, + configuration={ + "capabilities": adapter.capabilities.to_dict(), + "benchmark_type": type(benchmark).__name__, + "horizon": benchmark.horizon, + }, + ) + + def produce( + current_adapter: ManipulationAdapter = adapter, + ) -> dict[str, object]: + return cast( + dict[str, object], + evaluate_manipulation( + current_adapter, + benchmark, + suite, + seed=args.seed, + ).to_dict(), + ) + + payload, hit = cache.resolve(request, produce) + if hit: + cache_hits += 1 + adapter.close() + results.append(EvaluationResult.from_dict(payload)) + return benchmark.benchmark_id, results, cache_hits def _run_demo(args: argparse.Namespace) -> int: - benchmark_id, results = ( + benchmark_id, results, cache_hits = ( _pointmass_results(args) if args.benchmark == "pointmass" else _manipulation_results(args) ) summary_path, report_path, html_path = write_reports( @@ -125,6 +287,112 @@ def _run_demo(args: argparse.Namespace) -> int: print(f"JSON: {summary_path}") print(f"Markdown: {report_path}") print(f"HTML: {html_path}") + if args.cache_dir is not None: + print(f"Cache hits: {cache_hits}/{len(results)}") + return 0 + + +def _load_summary(source: Path) -> tuple[str, list[EvaluationResult]]: + path = source / "summary.json" if source.is_dir() else source + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"could not read summary: {error}") from error + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise ValueError("summary must be a JSON object") + payload = cast(dict[str, object], raw) + if payload.get("schema_version") != "0.1": + raise ValueError("summary uses an unsupported schema_version") + benchmark = payload.get("benchmark") + raw_results = payload.get("results") + if not isinstance(benchmark, str) or not benchmark.strip(): + raise ValueError("summary benchmark must be a non-empty string") + if not isinstance(raw_results, list) or not raw_results: + raise ValueError("summary results must be a non-empty array") + results = [EvaluationResult.from_dict(result) for result in raw_results] + if any(result.benchmark != benchmark for result in results): + raise ValueError("summary contains results for a different benchmark") + return benchmark, results + + +def _run_report(args: argparse.Namespace) -> int: + try: + benchmark, results = _load_summary(args.summary) + summary, markdown, html = write_reports( + args.output, + benchmark=benchmark, + results=results, + resamples=args.resamples, + seed=args.seed, + ) + except (OSError, ValueError) as error: + print(f"WAMProbe report error: {error}", file=sys.stderr) + return 2 + print(f"JSON: {summary}") + print(f"Markdown: {markdown}") + print(f"HTML: {html}") + return 0 + + +def _select_model(results: list[EvaluationResult], model_id: str) -> EvaluationResult: + matches = [result for result in results if result.model_id == model_id] + if len(matches) != 1: + raise ValueError(f"expected exactly one result for model: {model_id}") + return matches[0] + + +def _run_compare(args: argparse.Namespace) -> int: + try: + _, left_results = _load_summary(args.left) + _, right_results = _load_summary(args.right) + comparison = paired_metric_comparison( + _select_model(left_results, args.left_model), + _select_model(right_results, args.right_model), + metric=args.metric, + resamples=args.resamples, + seed=args.seed, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(asdict(comparison), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + except (OSError, ValueError) as error: + print(f"WAMProbe compare error: {error}", file=sys.stderr) + return 2 + print(f"Comparison: {args.output}") + return 0 + + +def _run_dataset_export(args: argparse.Namespace) -> int: + suite: InterventionSuite | ManipulationInterventionSuite + if args.benchmark == "pointmass": + suite = PointMass2D().make_suite(contexts=args.contexts, seed=args.seed) + elif args.benchmark == "blockpush": + suite = BlockPush2D().make_suite(contexts=args.contexts, seed=args.seed) + else: + suite = GripperCatch().make_suite(contexts=args.contexts, seed=args.seed) + try: + summary = write_intervention_dataset(suite, args.output) + except (OSError, TypeError, ValueError) as error: + print(f"WAMProbe dataset export error: {error}", file=sys.stderr) + return 2 + print( + f"Exported {summary.records} intervention groups for " + f"{summary.benchmark_id} ({summary.sha256})" + ) + return 0 + + +def _run_dataset_validate(args: argparse.Namespace) -> int: + try: + suite = load_intervention_dataset(args.dataset) + except DatasetValidationError as error: + print(f"WAMProbe dataset validation error: {error}", file=sys.stderr) + return 2 + print( + f"Validated {len(suite.groups)} intervention groups for {suite.benchmark_id} (schema 0.1)" + ) return 0 @@ -167,5 +435,13 @@ def main(argv: Sequence[str] | None = None) -> int: return _run_demo(args) if args.command == "doctor": return _run_doctor(args) + if args.command == "report": + return _run_report(args) + if args.command == "compare": + return _run_compare(args) + if args.command == "dataset-export": + return _run_dataset_export(args) + if args.command == "dataset-validate": + return _run_dataset_validate(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/datasets.py b/src/wamprobe/datasets.py new file mode 100644 index 0000000..4cea5cf --- /dev/null +++ b/src/wamprobe/datasets.py @@ -0,0 +1,354 @@ +"""Versioned JSONL interchange for paired intervention suites.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationContext, + ManipulationInterventionGroup, + ManipulationInterventionSuite, + ManipulationState, +) +from wamprobe.api.types import ( + Action2D, + Context2D, + InterventionGroup, + InterventionSuite, + Vec2, +) + +SCHEMA_VERSION = "0.1" +InterventionDataset = InterventionSuite | ManipulationInterventionSuite + + +class DatasetValidationError(ValueError): + """Raised when an intervention dataset violates its schema or checksum.""" + + +@dataclass(frozen=True, slots=True) +class DatasetSummary: + """Identity and size of a serialized intervention dataset.""" + + schema_version: str + benchmark_id: str + records: int + sha256: str + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def _sha256(value: object) -> str: + return hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _vec_payload(vector: Vec2) -> list[float]: + return [vector.x, vector.y] + + +def _pointmass_record(group: InterventionGroup, benchmark_id: str) -> dict[str, object]: + return { + "schema_version": SCHEMA_VERSION, + "benchmark_id": benchmark_id, + "state_kind": "pointmass-2d", + "context": { + "context_id": group.context.context_id, + "position": _vec_payload(group.context.position), + "goal": _vec_payload(group.context.goal), + }, + "actions": [ + {"name": action.name, "delta": _vec_payload(action.delta)} for action in group.actions + ], + } + + +def _manipulation_record( + group: ManipulationInterventionGroup, + benchmark_id: str, +) -> dict[str, object]: + state = group.context.initial_state + return { + "schema_version": SCHEMA_VERSION, + "benchmark_id": benchmark_id, + "state_kind": "manipulation-2d", + "context": { + "context_id": group.context.context_id, + "initial_state": { + "effector_position": _vec_payload(state.effector_position), + "object_position": _vec_payload(state.object_position), + "gripper_closed": state.gripper_closed, + "object_attached": state.object_attached, + "in_contact": state.in_contact, + }, + "goal": _vec_payload(group.context.goal), + }, + "actions": [ + { + "name": action.name, + "effector_delta": _vec_payload(action.effector_delta), + "close_gripper": action.close_gripper, + } + for action in group.actions + ], + } + + +def _records(suite: InterventionDataset) -> list[dict[str, object]]: + records: list[dict[str, object]] = [] + if isinstance(suite, InterventionSuite): + base_records = [_pointmass_record(group, suite.benchmark_id) for group in suite.groups] + elif isinstance(suite, ManipulationInterventionSuite): + base_records = [_manipulation_record(group, suite.benchmark_id) for group in suite.groups] + else: + raise TypeError("unsupported intervention suite type") + for base in base_records: + record = dict(base) + record["record_sha256"] = _sha256(base) + records.append(record) + return records + + +def _dataset_bytes(suite: InterventionDataset) -> bytes: + return b"".join(_canonical_json(record) + b"\n" for record in _records(suite)) + + +def intervention_dataset_sha256(suite: InterventionDataset) -> str: + """Return the content identity of a suite's canonical JSONL representation.""" + + return hashlib.sha256(_dataset_bytes(suite)).hexdigest() + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(path) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def write_intervention_dataset( + suite: InterventionDataset, + path: Path, +) -> DatasetSummary: + """Atomically write a deterministic, checksummed JSONL intervention suite.""" + + payload = _dataset_bytes(suite) + _atomic_write(path, payload) + return DatasetSummary( + schema_version=SCHEMA_VERSION, + benchmark_id=suite.benchmark_id, + records=len(suite.groups), + sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def _mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise DatasetValidationError(f"{field} must be an object") + return cast(dict[str, object], value) + + +def _list(value: object, field: str) -> list[object]: + if not isinstance(value, list): + raise DatasetValidationError(f"{field} must be an array") + return cast(list[object], value) + + +def _string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise DatasetValidationError(f"{field} must be a non-empty string") + return value + + +def _boolean(value: object, field: str) -> bool: + if not isinstance(value, bool): + raise DatasetValidationError(f"{field} must be a boolean") + return value + + +def _number(value: object, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise DatasetValidationError(f"{field} must be a finite number") + result = float(value) + if not math.isfinite(result): + raise DatasetValidationError(f"{field} must be a finite number") + return result + + +def _vec(value: object, field: str) -> Vec2: + coordinates = _list(value, field) + if len(coordinates) != 2: + raise DatasetValidationError(f"{field} must contain exactly two coordinates") + return Vec2( + _number(coordinates[0], f"{field}[0]"), + _number(coordinates[1], f"{field}[1]"), + ) + + +def _verify_record(record: dict[str, object], line_number: int) -> dict[str, object]: + checksum = _string(record.get("record_sha256"), f"line {line_number} record_sha256") + base = {key: value for key, value in record.items() if key != "record_sha256"} + if checksum != _sha256(base): + raise DatasetValidationError(f"line {line_number} record_sha256 mismatch") + if record.get("schema_version") != SCHEMA_VERSION: + raise DatasetValidationError(f"line {line_number} uses unsupported schema_version") + return base + + +def _parse_pointmass( + record: dict[str, object], + line_number: int, +) -> InterventionGroup: + context_payload = _mapping(record.get("context"), f"line {line_number} context") + context = Context2D( + _string(context_payload.get("context_id"), "context.context_id"), + _vec(context_payload.get("position"), "context.position"), + _vec(context_payload.get("goal"), "context.goal"), + ) + actions: list[Action2D] = [] + for index, raw_action in enumerate(_list(record.get("actions"), "actions")): + action = _mapping(raw_action, f"actions[{index}]") + actions.append( + Action2D( + _string(action.get("name"), f"actions[{index}].name"), + _vec(action.get("delta"), f"actions[{index}].delta"), + ) + ) + try: + return InterventionGroup(context, tuple(actions)) + except ValueError as error: + raise DatasetValidationError(str(error)) from error + + +def _parse_manipulation( + record: dict[str, object], + line_number: int, +) -> ManipulationInterventionGroup: + context_payload = _mapping(record.get("context"), f"line {line_number} context") + state_payload = _mapping(context_payload.get("initial_state"), "context.initial_state") + try: + state = ManipulationState( + effector_position=_vec( + state_payload.get("effector_position"), + "context.initial_state.effector_position", + ), + object_position=_vec( + state_payload.get("object_position"), + "context.initial_state.object_position", + ), + gripper_closed=_boolean( + state_payload.get("gripper_closed"), + "context.initial_state.gripper_closed", + ), + object_attached=_boolean( + state_payload.get("object_attached"), + "context.initial_state.object_attached", + ), + in_contact=_boolean( + state_payload.get("in_contact"), + "context.initial_state.in_contact", + ), + ) + context = ManipulationContext( + _string(context_payload.get("context_id"), "context.context_id"), + state, + _vec(context_payload.get("goal"), "context.goal"), + ) + actions: list[ManipulationAction] = [] + for index, raw_action in enumerate(_list(record.get("actions"), "actions")): + action = _mapping(raw_action, f"actions[{index}]") + actions.append( + ManipulationAction( + _string(action.get("name"), f"actions[{index}].name"), + _vec( + action.get("effector_delta"), + f"actions[{index}].effector_delta", + ), + _boolean( + action.get("close_gripper"), + f"actions[{index}].close_gripper", + ), + ) + ) + return ManipulationInterventionGroup(context, tuple(actions)) + except ValueError as error: + raise DatasetValidationError(str(error)) from error + + +def load_intervention_dataset(path: Path) -> InterventionDataset: + """Load a checksummed JSONL suite and restore its typed representation.""" + + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as error: + raise DatasetValidationError(f"could not read dataset: {error}") from error + if not lines: + raise DatasetValidationError("intervention dataset must not be empty") + + benchmark_id: str | None = None + state_kind: str | None = None + pointmass_groups: list[InterventionGroup] = [] + manipulation_groups: list[ManipulationInterventionGroup] = [] + context_ids: set[str] = set() + for line_number, line in enumerate(lines, start=1): + try: + raw: object = json.loads(line) + except json.JSONDecodeError as error: + raise DatasetValidationError(f"line {line_number} is not valid JSON") from error + record = _verify_record(_mapping(raw, f"line {line_number}"), line_number) + record_benchmark = _string(record.get("benchmark_id"), "benchmark_id") + record_kind = _string(record.get("state_kind"), "state_kind") + if benchmark_id is None: + benchmark_id = record_benchmark + state_kind = record_kind + elif record_benchmark != benchmark_id or record_kind != state_kind: + raise DatasetValidationError("all records must share benchmark_id and state_kind") + + if record_kind == "pointmass-2d": + pointmass_group = _parse_pointmass(record, line_number) + pointmass_groups.append(pointmass_group) + context_id = pointmass_group.context.context_id + elif record_kind == "manipulation-2d": + manipulation_group = _parse_manipulation(record, line_number) + manipulation_groups.append(manipulation_group) + context_id = manipulation_group.context.context_id + else: + raise DatasetValidationError(f"unsupported state_kind: {record_kind}") + if context_id in context_ids: + raise DatasetValidationError(f"duplicate context_id: {context_id}") + context_ids.add(context_id) + + if benchmark_id is None or state_kind is None: + raise DatasetValidationError("intervention dataset must not be empty") + if state_kind == "pointmass-2d": + return InterventionSuite(benchmark_id, tuple(pointmass_groups)) + return ManipulationInterventionSuite(benchmark_id, tuple(manipulation_groups)) diff --git a/src/wamprobe/evaluation.py b/src/wamprobe/evaluation.py index 1397bc9..052d0a3 100644 --- a/src/wamprobe/evaluation.py +++ b/src/wamprobe/evaluation.py @@ -3,8 +3,9 @@ from __future__ import annotations import hashlib +import math from dataclasses import asdict, dataclass -from typing import Any +from typing import Any, cast from wamprobe.api.model import WAMAdapter from wamprobe.api.types import Context2D, InterventionSuite, Trajectory2D @@ -53,6 +54,106 @@ def to_dict(self) -> dict[str, Any]: "context_results": [asdict(result) for result in self.context_results], } + @classmethod + def from_dict(cls, value: object) -> EvaluationResult: + """Strictly restore a versioned result from JSON-compatible data.""" + + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError("evaluation result must be an object") + payload = cast(dict[str, object], value) + expected_fields = { + "schema_version", + "model_id", + "benchmark", + "contexts", + "metrics", + "context_results", + } + if payload.keys() != expected_fields: + raise ValueError("evaluation result fields do not match schema 0.1") + if payload["schema_version"] != "0.1": + raise ValueError("unsupported evaluation result schema_version") + model_id = _non_empty_string(payload["model_id"], "model_id") + benchmark = _non_empty_string(payload["benchmark"], "benchmark") + contexts = payload["contexts"] + if isinstance(contexts, bool) or not isinstance(contexts, int) or contexts <= 0: + raise ValueError("contexts must be a positive integer") + metrics = _metric_mapping(payload["metrics"], "metrics") + raw_contexts = payload["context_results"] + if not isinstance(raw_contexts, list): + raise ValueError("context_results must be an array") + context_results = tuple( + _context_result(raw, index) for index, raw in enumerate(raw_contexts) + ) + if len(context_results) != contexts: + raise ValueError("contexts does not match context_results length") + context_ids = [result.context_id for result in context_results] + if len(context_ids) != len(set(context_ids)): + raise ValueError("context_result IDs must be unique") + if any(result.metrics.keys() != metrics.keys() for result in context_results): + raise ValueError("aggregated and context metric names must match") + return cls( + model_id=model_id, + benchmark=benchmark, + contexts=contexts, + metrics=metrics, + context_results=context_results, + ) + + +def _non_empty_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _finite_float(value: object, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field} must be a finite number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field} must be a finite number") + return result + + +def _metric_mapping(value: object, field: str) -> dict[str, float]: + if not isinstance(value, dict) or not value: + raise ValueError(f"{field} must be a non-empty object") + result: dict[str, float] = {} + for key, metric_value in value.items(): + if not isinstance(key, str) or not key.strip(): + raise ValueError(f"{field} keys must be non-empty strings") + result[key] = _finite_float(metric_value, f"{field}.{key}") + return result + + +def _context_result(value: object, index: int) -> ContextResult: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError(f"context_results[{index}] must be an object") + payload = cast(dict[str, object], value) + expected = { + "context_id", + "selected_action", + "optimal_action", + "top1_regret", + "metrics", + } + if payload.keys() != expected: + raise ValueError(f"context_results[{index}] fields do not match schema") + top1_regret = _finite_float(payload["top1_regret"], "top1_regret") + metrics = _metric_mapping(payload["metrics"], "context metrics") + if "top1_regret" in metrics and not math.isclose( + metrics["top1_regret"], top1_regret, rel_tol=0.0, abs_tol=1e-12 + ): + raise ValueError("context top1_regret does not match metrics") + return ContextResult( + context_id=_non_empty_string(payload["context_id"], "context_id"), + selected_action=_non_empty_string(payload["selected_action"], "selected_action"), + optimal_action=_non_empty_string(payload["optimal_action"], "optimal_action"), + top1_regret=top1_regret, + metrics=metrics, + ) + def evaluate( adapter: WAMAdapter, diff --git a/src/wamprobe/reporting.py b/src/wamprobe/reporting.py index 1a70c4c..878bb87 100644 --- a/src/wamprobe/reporting.py +++ b/src/wamprobe/reporting.py @@ -277,6 +277,7 @@ def write_reports( summary_path = output_dir / "summary.json" report_path = output_dir / "report.md" html_path = output_dir / "report.html" + jsonl_path = output_dir / "results.jsonl" payload = { "schema_version": "0.1", "benchmark": benchmark, @@ -287,7 +288,10 @@ def write_reports( }, "paired_comparisons": [asdict(comparison) for comparison in comparisons], } - summary_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + summary_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) report_path.write_text( render_markdown(results, statistics=statistics, comparisons=comparisons), encoding="utf-8", @@ -296,4 +300,25 @@ def write_reports( render_html(results, statistics=statistics, comparisons=comparisons), encoding="utf-8", ) + jsonl_records = ( + { + "schema_version": "0.1", + "benchmark": result.benchmark, + "model_id": result.model_id, + "context_id": context.context_id, + "selected_action": context.selected_action, + "optimal_action": context.optimal_action, + "top1_regret": context.top1_regret, + "metrics": context.metrics, + } + for result in results + for context in result.context_results + ) + jsonl_path.write_text( + "".join( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" + for record in jsonl_records + ), + encoding="utf-8", + ) return summary_path, report_path, html_path diff --git a/tests/test_cli.py b/tests/test_cli.py index fe4ba02..46aa4fc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -26,9 +26,11 @@ def test_demo_cli_writes_machine_and_human_readable_reports( summary_path = tmp_path / "summary.json" report_path = tmp_path / "report.md" html_path = tmp_path / "report.html" + jsonl_path = tmp_path / "results.jsonl" assert summary_path.is_file() assert report_path.is_file() assert html_path.is_file() + assert jsonl_path.is_file() summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["benchmark"] == "pointmass-2d-v0.1" @@ -48,6 +50,85 @@ def test_demo_cli_writes_machine_and_human_readable_reports( assert "WAMProbe demo complete" in capsys.readouterr().out +def test_demo_cache_report_compare_and_dataset_commands( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + first_run = tmp_path / "first" + second_run = tmp_path / "second" + cache = tmp_path / "cache" + common = [ + "demo", + "--contexts", + "4", + "--seed", + "5", + "--cache-dir", + str(cache), + ] + assert main([*common, "--output", str(first_run)]) == 0 + capsys.readouterr() + assert main([*common, "--output", str(second_run)]) == 0 + assert "Cache hits: 5/5" in capsys.readouterr().out + assert (first_run / "summary.json").read_bytes() == (second_run / "summary.json").read_bytes() + + rebuilt = tmp_path / "rebuilt" + assert ( + main( + [ + "report", + str(first_run / "summary.json"), + "--output", + str(rebuilt), + ] + ) + == 0 + ) + assert (rebuilt / "report.html").is_file() + + comparison = tmp_path / "comparison.json" + assert ( + main( + [ + "compare", + str(first_run), + str(second_run), + "--left-model", + "oracle-pointmass", + "--right-model", + "copy-last-frame", + "--metric", + "state_fde", + "--output", + str(comparison), + ] + ) + == 0 + ) + comparison_payload = json.loads(comparison.read_text(encoding="utf-8")) + assert comparison_payload["mean_difference"] == pytest.approx(-0.8) + + dataset = tmp_path / "pointmass.jsonl" + assert ( + main( + [ + "dataset-export", + "--benchmark", + "pointmass", + "--contexts", + "7", + "--seed", + "11", + "--output", + str(dataset), + ] + ) + == 0 + ) + assert main(["dataset-validate", str(dataset)]) == 0 + assert "7 intervention groups" in capsys.readouterr().out + + @pytest.mark.parametrize( ("benchmark_name", "benchmark_id"), [ diff --git a/tests/test_dataset_cache.py b/tests/test_dataset_cache.py new file mode 100644 index 0000000..17a0f47 --- /dev/null +++ b/tests/test_dataset_cache.py @@ -0,0 +1,113 @@ +import json +from pathlib import Path + +import pytest + +from wamprobe.api.manipulation import ManipulationInterventionSuite +from wamprobe.api.types import InterventionSuite +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.cache import ( + CacheCorruptionError, + PredictionCache, + PredictionCacheRequest, +) +from wamprobe.datasets import ( + DatasetValidationError, + load_intervention_dataset, + write_intervention_dataset, +) + + +@pytest.mark.parametrize( + "suite", + [ + PointMass2D().make_suite(contexts=5, seed=7), + BlockPush2D().make_suite(contexts=5, seed=7), + GripperCatch().make_suite(contexts=5, seed=7), + ], +) +def test_intervention_jsonl_round_trip_is_versioned_and_deterministic( + tmp_path: Path, + suite: InterventionSuite | ManipulationInterventionSuite, +) -> None: + path = tmp_path / "interventions.jsonl" + + first = write_intervention_dataset(suite, path) + first_bytes = path.read_bytes() + second = write_intervention_dataset(suite, path) + + assert first == second + assert first.records == 5 + assert first.schema_version == "0.1" + assert len(first.sha256) == 64 + assert path.read_bytes() == first_bytes + assert len(path.read_text(encoding="utf-8").splitlines()) == 5 + assert load_intervention_dataset(path) == suite + + +def test_intervention_jsonl_rejects_record_tampering(tmp_path: Path) -> None: + path = tmp_path / "interventions.jsonl" + write_intervention_dataset(PointMass2D().make_suite(contexts=2, seed=3), path) + records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + records[0]["context"]["position"][0] += 0.5 + path.write_text( + "\n".join(json.dumps(record, sort_keys=True) for record in records) + "\n", + encoding="utf-8", + ) + + with pytest.raises(DatasetValidationError, match="record_sha256"): + load_intervention_dataset(path) + + +def _cache_request(*, model_id: str = "oracle") -> PredictionCacheRequest: + return PredictionCacheRequest( + namespace="unit-test-v0.1", + model_id=model_id, + benchmark_id="pointmass-2d-v0.1", + context_id="suite-0001", + action_name="all", + horizon=4, + seed=7, + input_sha256="a" * 64, + configuration_sha256="b" * 64, + ) + + +def test_prediction_cache_resumes_without_recomputing_valid_entries(tmp_path: Path) -> None: + cache = PredictionCache(tmp_path / "cache") + calls = 0 + + def produce() -> dict[str, object]: + nonlocal calls + calls += 1 + return {"states": [[0.0, 1.0]], "complete": True} + + first, first_hit = cache.resolve(_cache_request(), produce) + second, second_hit = cache.resolve(_cache_request(), produce) + + assert first == second + assert not first_hit + assert second_hit + assert calls == 1 + assert cache.audit().passed + assert cache.audit().entries == 1 + + +def test_prediction_cache_detects_corruption_and_keys_configuration(tmp_path: Path) -> None: + cache = PredictionCache(tmp_path / "cache") + request = _cache_request() + cache.store(request, {"value": 1.0}) + cache.store(_cache_request(model_id="copy-last"), {"value": 2.0}) + entry_path = cache.path_for(request) + payload = json.loads(entry_path.read_text(encoding="utf-8")) + payload["payload"]["value"] = 99.0 + entry_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(CacheCorruptionError, match="payload_sha256"): + cache.load(request) + audit = cache.audit() + assert not audit.passed + assert audit.entries == 2 + assert audit.corrupt_entries == 1 From afb77f05cad661864302fb5820fa27201cc2f11e Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 13:30:08 +0000 Subject: [PATCH 12/20] expand LIBERO counterfactual mini suite --- README.md | 10 +- configs/benchmarks/libero_cf_mini_v0.1.json | 76 ++++ docs/QUICKSTART.md | 18 +- docs/WAMProbe_PLAN.md | 8 + docs/benchmarks/LIBERO_CF_MINI.md | 98 +++++ .../libero_cf_mini_verified_v0.1.json | 65 ++++ environments/libero/README.md | 24 +- environments/libero/generate_cf_pilot.py | 359 +++++++++++++++--- src/wamprobe/libero_cf.py | 223 +++++++++++ tests/test_libero_cf.py | 44 +++ 10 files changed, 844 insertions(+), 81 deletions(-) create mode 100644 configs/benchmarks/libero_cf_mini_v0.1.json create mode 100644 docs/benchmarks/LIBERO_CF_MINI.md create mode 100644 docs/benchmarks/libero_cf_mini_verified_v0.1.json create mode 100644 src/wamprobe/libero_cf.py create mode 100644 tests/test_libero_cf.py diff --git a/README.md b/README.md index 2acfebd..7525cc2 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ the same initial state The project is in an early alpha. Its dependency-free core includes an analytic PointMass-2D benchmark, while opt-in isolated runners now cover one released StarWAM -observation-to-action path and one real paired LIBERO simulator pilot. +observation-to-action path and a four-family paired LIBERO-CF-Mini simulator pilot. ## Why another evaluation project? @@ -113,16 +113,16 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - typed robot observations, action predictions, deterministic prediction artifacts, and a pinned StarWAM/LIBERO smoke runner; -- dependency-free robot action-branch/future contracts plus a four-branch, eight-step - LIBERO snapshot/restore pilot with exact branch-order validation; +- dependency-free robot action-branch/future contracts plus four task families × four + branches × eight steps in LIBERO-CF-Mini, with exact restore and branch-order validation; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap The next milestones are: -1. expand the LIBERO-CF-Mini pilot from one context to 3–5 task families; -2. execute cached StarWAM candidate actions, then add LingBot-VA as a published reference; +1. execute cached StarWAM candidate actions across LIBERO-CF-Mini; +2. run action-mask/shuffle, seed, horizon, and NFE ablations; 3. add a GPU nightly workflow for the isolated real-model integration; 4. add the Occluded-Object memory diagnostic to the broader Toy tier. diff --git a/configs/benchmarks/libero_cf_mini_v0.1.json b/configs/benchmarks/libero_cf_mini_v0.1.json new file mode 100644 index 0000000..855ad90 --- /dev/null +++ b/configs/benchmarks/libero_cf_mini_v0.1.json @@ -0,0 +1,76 @@ +{ + "schema_version": "0.1", + "benchmark_id": "libero-cf-mini-v0.1", + "libero_revision": "8f1084e3132a39270c3a13ebe37270a43ece2a01", + "upstream_license": "MIT", + "tasks": [ + { + "key": "spatial-task0", + "task_family": "spatial-relation", + "task_suite": "libero_spatial", + "task_id": 0, + "task": "pick up the black bowl between the plate and the ramekin and place it on the plate", + "init_state_index": 0, + "wait_steps": 30, + "seed": 42, + "bddl_sha256": "9b59eb1287802868ad9bc78d58e6d36d4ba31134e679cfdbdf4b0feb660c959b", + "init_states_sha256": "cbbc73792ce546c9bec181fd328a411d3183074840b282671dee481511381d0a", + "init_states_shape": [50, 92], + "init_states_dtype": "float64", + "problem_folder": "libero_spatial", + "bddl_file": "pick_up_the_black_bowl_between_the_plate_and_the_ramekin_and_place_it_on_the_plate.bddl", + "init_states_file": "pick_up_the_black_bowl_between_the_plate_and_the_ramekin_and_place_it_on_the_plate.pruned_init" + }, + { + "key": "object-task0", + "task_family": "object-identity", + "task_suite": "libero_object", + "task_id": 0, + "task": "pick up the alphabet soup and place it in the basket", + "init_state_index": 0, + "wait_steps": 30, + "seed": 43, + "bddl_sha256": "df088984da13131f8332ee0f13a7896c6a97afd02ee5007a42e8fc5e0893571e", + "init_states_sha256": "b9023e5546b239cacb2aaa3c76df3b73c3de6f97500fea5e27a0db9827002a19", + "init_states_shape": [50, 110], + "init_states_dtype": "float64", + "problem_folder": "libero_object", + "bddl_file": "pick_up_the_alphabet_soup_and_place_it_in_the_basket.bddl", + "init_states_file": "pick_up_the_alphabet_soup_and_place_it_in_the_basket.pruned_init" + }, + { + "key": "goal-task0", + "task_family": "goal-conditioned-articulation", + "task_suite": "libero_goal", + "task_id": 0, + "task": "open the middle drawer of the cabinet", + "init_state_index": 0, + "wait_steps": 30, + "seed": 44, + "bddl_sha256": "832ab48ec476807c767e8985666e295700f92d1f8daeae1c34eee1e75293769b", + "init_states_sha256": "977efe1216bee72bb4e9fa9ec4be1bcfe37fb00ece815eac117feba019b34b9f", + "init_states_shape": [50, 79], + "init_states_dtype": "float64", + "problem_folder": "libero_goal", + "bddl_file": "open_the_middle_drawer_of_the_cabinet.bddl", + "init_states_file": "open_the_middle_drawer_of_the_cabinet.pruned_init" + }, + { + "key": "long-horizon-task0", + "task_family": "long-horizon-composition", + "task_suite": "libero_10", + "task_id": 0, + "task": "put both the alphabet soup and the tomato sauce in the basket", + "init_state_index": 0, + "wait_steps": 30, + "seed": 45, + "bddl_sha256": "2e661058917f683a25bce480015197f0a2c1911bcbc2b9dd9d947199069c9618", + "init_states_sha256": "1899fe54c51e24a41d5f9e32254730d6777f4894ab68e0e873f1fa7ae2062030", + "init_states_shape": [50, 123], + "init_states_dtype": "float64", + "problem_folder": "libero_10", + "bddl_file": "LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket.bddl", + "init_states_file": "LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket.pruned_init" + } + ] +} diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 35fa4a4..8219030 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -117,21 +117,23 @@ This step uses the same isolated simulator environment but loads no model weight ```bash environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ - --gpu-index 0 --horizon 8 + --gpu-index 0 --horizon 8 --run-dir runs/libero-cf-mini-v0.1 ``` -The runner restores one fixed snapshot before `no-op`, positive/negative end-effector X, -and gripper-close branches. It stores two camera views and state descriptors for all eight +The runner restores one fixed snapshot in each of four task families before `no-op`, +positive/negative end-effector X, and gripper-close branches. It stores two camera views +and state descriptors for all eight future steps, repeats no-op, and executes all branches in reverse order. A successful run requires byte-identical MuJoCo integration states, force-refreshed initial observations, repeated rollouts, and order-independent branch results. See [`environments/libero/README.md`](../environments/libero/README.md) for the output layout -and the required Python-side gripper-state restoration. +and the required Python-side gripper-state restoration. Repeat `--task-key` to select a +subset; verified outputs resume without reopening the simulator. ## What this demo does not prove PointMass-2D validates the evaluator and guards against metric bugs. The current LIBERO -pilot validates paired data generation for one context, not policy quality: its diagnostic -actions do not solve the task. Transfer claims still require multiple manipulation tasks, -WAM future predictions, stochastic generation analysis, and correlation with closed-loop -return. +pilot validates paired data generation across four task families, not policy quality: its +diagnostic actions do not solve the tasks. Transfer claims still require more initial +states, WAM future predictions, stochastic generation analysis, and correlation with +closed-loop return. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index baa21aa..ba7bc36 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1065,6 +1065,14 @@ context-ID 对齐的 paired comparison；三个 benchmark 都能生成 JSON、Ma ### Phase 3：LIBERO-CF-Mini（第 8–10 周） +当前进度（2026-07-15）：已固定 spatial/object/goal/long-horizon 四类任务的 BDDL、 +init-state 与上游 commit，批量生成 4 task × 4 branch × 8 step 的真实模拟器数据。每个任务 +均通过两次独立 restore、重复 no-op 与正反 branch order 检查，最大 integration-state 误差 +为 `0.0`；第二次整套运行在校验 JSON、snapshot、sidecar 和全部 PNG 后得到 4/4 cache hit。 +任务选择、MIT 许可证、完整 hash、零稀疏回报和适用边界见 +[`docs/benchmarks/LIBERO_CF_MINI.md`](benchmarks/LIBERO_CF_MINI.md)。扩大 init-state 数量、 +接入真实 WAM 预测与外部全新环境复现仍属于后续工作。 + 任务： - simulator snapshot/restore； diff --git a/docs/benchmarks/LIBERO_CF_MINI.md b/docs/benchmarks/LIBERO_CF_MINI.md new file mode 100644 index 0000000..5f4bcfc --- /dev/null +++ b/docs/benchmarks/LIBERO_CF_MINI.md @@ -0,0 +1,98 @@ +# LIBERO-CF-Mini benchmark card + +## Summary + +LIBERO-CF-Mini v0.1 is WAMProbe's Tier-1 paired simulator benchmark. It starts several +candidate action sequences from an exactly restored simulator state and records their +state and RGB futures. The benchmark measures whether a model distinguishes action +consequences; it is not a LIBERO policy leaderboard and its diagnostic branches are not +expert actions. + +The fixed task manifest covers four intentionally different families: + +| Key | LIBERO suite | Family | Fixed task | +|---|---|---|---| +| `spatial-task0` | `libero_spatial` | spatial relation | black bowl between two objects → plate | +| `object-task0` | `libero_object` | object identity | alphabet soup → basket | +| `goal-task0` | `libero_goal` | articulated goal | open the middle drawer | +| `long-horizon-task0` | `libero_10` | long-horizon composition | two objects → basket | + +The source selection, task text, BDDL/init-state filenames and hashes are pinned in +[`configs/benchmarks/libero_cf_mini_v0.1.json`](../../configs/benchmarks/libero_cf_mini_v0.1.json). +The manifest's SHA256 is +`1660ab49ec0e5aac58f4801f0ec9a12326053f21f5f1974dd3b6ea63260f78c3`. + +## Upstream data and license + +- Source: [LIBERO](https://github.com/Lifelong-Robot-Learning/LIBERO) +- Revision: `8f1084e3132a39270c3a13ebe37270a43ece2a01` +- Upstream code, BDDL assets and fixed init states: MIT license +- WAMProbe-generated metadata: Apache-2.0 +- No LIBERO demonstrations are required or redistributed by this benchmark slice. + +Users redistributing generated image/state bundles should retain both projects' license +and provenance notices. The runner rejects an upstream commit, task description, BDDL, +init-state file, shape or dtype that differs from the manifest. + +## Intervention protocol + +Each task uses init state 0, 30 no-op settling steps, two 256×256 RGB cameras and an +eight-step branch horizon. Four constant commands are applied from one shared snapshot: + +1. open-gripper no-op; +2. positive end-effector X; +3. negative end-effector X; +4. stationary gripper close. + +The snapshot contains MuJoCo `mjSTATE_INTEGRATION` plus robosuite clock, controller, +observable, gripper and Python/NumPy RNG side state. Every task is checked with two +independent restores, a repeated no-op branch, and forward/reverse branch execution. +All state and image references carry size and SHA256 metadata. + +## Verified v0.1 pilot + +The four-task, 16-branch, 128-post-action-frame run completed on 2026-07-15. Every exact +restore check passed and the maximum integration-state error was `0.0`. + +| Task | Snapshot SHA256 | Artifact SHA256 | EEF separation | Object-state separation | No-op drift | +|---|---|---|---:|---:|---:| +| spatial | `22f220ce…e36` | `73014e3d…ae97` | 0.0395949 | 0.0979067 | 0.0000247 | +| object | `d73e2f59…d8d3` | `2bfda04f…db1` | 0.0395781 | 0.1135034 | 0.0000762 | +| goal | `aadc9f6f…bef9` | `9e34e5df…bac3` | 0.0395915 | 0.0873014 | 0.0000340 | +| long-horizon | `ae5737d2…a082` | `014aea0c…df3a` | 0.0396259 | 0.1234343 | 0.0000605 | + +Full hashes and values are available in +[`libero_cf_mini_verified_v0.1.json`](libero_cf_mini_verified_v0.1.json). A second run +verified the artifact JSON, snapshot sidecars and every PNG by size and SHA256, then +reported four cache hits without opening a simulator. + +## Reproduction + +Follow the isolated environment instructions in +[`environments/libero/README.md`](../../environments/libero/README.md), then run: + +```bash +environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ + --gpu-index 0 --horizon 8 --run-dir runs/libero-cf-mini-v0.1 +``` + +The command processes all four tasks by default and writes a structured failure record +to `index.json` after every task. Repeat `--task-key` to reproduce a smaller subset. +Valid existing artifacts resume as cache hits; use `--force` only to regenerate them. + +## Suitable and unsuitable uses + +Suitable uses include snapshot/restore validation, action-condition diagnostics, short +horizon state/video comparisons and model-integration smoke tests. Unsuitable uses +include reporting LIBERO task success, comparing policies, claiming task completion, or +aggregating raw object-state distances across tasks whose state vector dimensions differ. + +## Limitations + +- The four fixed commands are causal probes, not constraint-aware expert trajectories. +- Eight steps are too short for the selected sparse-reward tasks; all verified return + spreads and success rates are zero. This is reported as a limitation, not hidden. +- The current branch set probes X translation and gripper closure only. +- One init state per family is insufficient for paper-level statistical claims. +- RGB similarity, keypoint metrics, real-WAM predictions and closed-loop control utility + are separate evaluation stages and are not established by this data-generation run. diff --git a/docs/benchmarks/libero_cf_mini_verified_v0.1.json b/docs/benchmarks/libero_cf_mini_verified_v0.1.json new file mode 100644 index 0000000..5cc835a --- /dev/null +++ b/docs/benchmarks/libero_cf_mini_verified_v0.1.json @@ -0,0 +1,65 @@ +{ + "schema_version": "0.1", + "benchmark_id": "libero-cf-mini-v0.1", + "verified_on": "2026-07-15", + "manifest_sha256": "1660ab49ec0e5aac58f4801f0ec9a12326053f21f5f1974dd3b6ea63260f78c3", + "artifact_bundle_published": false, + "tasks": [ + { + "task_key": "spatial-task0", + "context_id": "libero-spatial-task0-init0-wait30-seed42", + "snapshot_sha256": "22f220ce36eb42d3770d5a657452218beb690744ceed4ec516537826ece82e36", + "group_sha256": "5dfc75878b3807f7e612f2eb37f411ddf96cd83a820320442c92e4a53a172bff", + "artifact_sha256": "73014e3d079fcef7c8c3b4a8a5524d9c2e26e7e584c6ba95c3176b02eaacae97", + "mean_final_eef_pairwise_distance": 0.03959486715327739, + "mean_final_object_pairwise_distance": 0.09790671954798946, + "noop_final_eef_drift": 0.0000246710317476801, + "return_spread": 0.0, + "success_rate": 0.0, + "max_state_abs_error": 0.0, + "validation_passed": true + }, + { + "task_key": "object-task0", + "context_id": "libero-object-task0-init0-wait30-seed43", + "snapshot_sha256": "d73e2f594d207095f8d0ade12006646d8cd003905c748f9ef70d73256b3548d3", + "group_sha256": "13e315747794370985511a4a228b04e01a54bcb619796398c8b961235326a5e8", + "artifact_sha256": "2bfda04f514e353977158c46306aa058517a56f890c5066e2aa69e57558ccdb1", + "mean_final_eef_pairwise_distance": 0.03957809886870484, + "mean_final_object_pairwise_distance": 0.11350344931133359, + "noop_final_eef_drift": 0.0000762307974349324, + "return_spread": 0.0, + "success_rate": 0.0, + "max_state_abs_error": 0.0, + "validation_passed": true + }, + { + "task_key": "goal-task0", + "context_id": "libero-goal-task0-init0-wait30-seed44", + "snapshot_sha256": "aadc9f6f6a21e190c5793675d22866f341016c4fd98574e6f45ffae44d55bef9", + "group_sha256": "a07a54e94302371b44ddef955b43ecbb72e7c2f8e38bd7a8bbe37cbc38e4162e", + "artifact_sha256": "9e34e5df4cb50bba38ecde068096ccbadc8852a6b9c1c3f98bcc61d0a524bac3", + "mean_final_eef_pairwise_distance": 0.039591519347864125, + "mean_final_object_pairwise_distance": 0.08730143622876134, + "noop_final_eef_drift": 0.00003402985862670911, + "return_spread": 0.0, + "success_rate": 0.0, + "max_state_abs_error": 0.0, + "validation_passed": true + }, + { + "task_key": "long-horizon-task0", + "context_id": "libero-10-task0-init0-wait30-seed45", + "snapshot_sha256": "ae5737d2947014272cce8236efb0d68389fb30c59a3165a9a94e9dd355e9a082", + "group_sha256": "36b78b842ada76c36b5c528df8f12add408dfc9ce7ab9fa739ed1d03c22acf45", + "artifact_sha256": "014aea0cf738521b91b60e14ace0b61b8561fe73de295a3702c878713a7fdf3a", + "mean_final_eef_pairwise_distance": 0.03962585963458628, + "mean_final_object_pairwise_distance": 0.12343427103714627, + "noop_final_eef_drift": 0.00006053874090315244, + "return_spread": 0.0, + "success_rate": 0.0, + "max_state_abs_error": 0.0, + "validation_passed": true + } + ] +} diff --git a/environments/libero/README.md b/environments/libero/README.md index 53c2083..1f00900 100644 --- a/environments/libero/README.md +++ b/environments/libero/README.md @@ -1,15 +1,16 @@ -# LIBERO counterfactual pilot +# LIBERO-CF-Mini counterfactual pilot -This isolated runner generates four real action branches from one pinned LIBERO state. It +This isolated runner generates four real action branches from four pinned LIBERO states. It does not load StarWAM or any other model weights; it reuses the existing Python 3.11 environment because that environment already pins LIBERO, robosuite, MuJoCo, and Pillow. ```bash environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ - --gpu-index 0 --horizon 8 + --gpu-index 0 --horizon 8 --run-dir runs/libero-cf-mini-v0.1 ``` -The default output is ignored by Git and written below `runs/libero-cf-pilot/`: +The default output is ignored by Git and written below `runs/libero-cf-mini/`. Each fixed +task key gets the following directory, and the root receives a resumable `index.json`: ```text benchmark_provenance.json @@ -28,16 +29,25 @@ clock state, controller state, Python and NumPy RNG state, and each gripper's st `current_action`. Omitting `current_action` makes branch results depend on execution order even when MuJoCo state restoration is exact. -The runner executes the branch set in forward and reverse order and repeats the no-op +The pinned manifest covers spatial-relation, object-identity, articulated-goal, and +long-horizon-composition families. Use repeatable `--task-key` arguments to select a +subset. The runner executes each branch set in forward and reverse order and repeats the no-op branch. It fails after writing `validation.json` if restored state/observations, repeated -rollouts, or branch-order invariance are not exact. +rollouts, or branch-order invariance are not exact. Failures remain structured in the root +index; verified existing JSON, snapshots, sidecars, and PNGs resume as cache hits. The canonical initial observation is force-refreshed from the restored end-of-control state. It is intentionally compared against a second independent restore, not against the camera value returned by the preceding `step()`: robosuite can sample camera observables inside the control interval before the final simulator state is reached. -## Verified fixed pilot +## Verified multi-family pilot + +The four-task v0.1 evidence and limitations are recorded in the +[LIBERO-CF-Mini benchmark card](../../docs/benchmarks/LIBERO_CF_MINI.md). The earlier +single-task pilot below remains as a historical fixed-point check. + +### Earlier single-task fixed point The pinned 4-branch × 8-step run was repeated twice on 2026-07-15 with identical hashes: diff --git a/environments/libero/generate_cf_pilot.py b/environments/libero/generate_cf_pilot.py index 3f2e033..77daf5e 100644 --- a/environments/libero/generate_cf_pilot.py +++ b/environments/libero/generate_cf_pilot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Generate one pinned LIBERO counterfactual intervention group. +"""Generate the pinned multi-family LIBERO-CF-Mini intervention suite. This runner needs the isolated StarWAM virtual environment only for its already-pinned LIBERO, robosuite, MuJoCo, PyTorch, and image dependencies. It does not load a WAM or any @@ -25,17 +25,8 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] SOURCE_ROOT = REPOSITORY_ROOT / "src" LIBERO_ROOT = REPOSITORY_ROOT / "vendor" / "upstream" / "libero" +MANIFEST_PATH = REPOSITORY_ROOT / "configs" / "benchmarks" / "libero_cf_mini_v0.1.json" -LIBERO_REVISION = "8f1084e3132a39270c3a13ebe37270a43ece2a01" -TASK_SUITE = "libero_spatial" -TASK_ID = 0 -INIT_STATE_INDEX = 0 -WAIT_STEPS = 30 -SEED = 42 -TASK = "pick up the black bowl between the plate and the ramekin and place it on the plate" -CONTEXT_ID = "libero-spatial-task0-init0-wait30" -BDDL_SHA256 = "9b59eb1287802868ad9bc78d58e6d36d4ba31134e679cfdbdf4b0feb660c959b" -INIT_STATES_SHA256 = "cbbc73792ce546c9bec181fd328a411d3183074840b282671dee481511381d0a" ACTION_SPACE = "libero-eef-delta-pose-gripper-v1" RESTORE_POLICY = "libero-integration-and-python-side-state-v1" CAMERAS = ( @@ -64,6 +55,7 @@ SimulatorSnapshotDescriptor, ) from wamprobe.counterfactual import CounterfactualArtifact # noqa: E402 +from wamprobe.libero_cf import LiberoTaskSpec, load_libero_cf_manifest # noqa: E402 @dataclass(slots=True) @@ -277,10 +269,14 @@ def _numpy_rng_payload(state: tuple[Any, ...]) -> dict[str, object]: } -def _runtime_sidecar(snapshot: RuntimeSnapshot, state_uri: str) -> dict[str, object]: +def _runtime_sidecar( + snapshot: RuntimeSnapshot, + state_uri: str, + spec: LiberoTaskSpec, +) -> dict[str, object]: return { "schema_version": "0.1", - "context_id": CONTEXT_ID, + "context_id": spec.context_id, "state_uri": state_uri, "libero_wrapper_state": [float(value) for value in snapshot.libero_state], "timestep": snapshot.timestep, @@ -484,6 +480,7 @@ def _rollout( persist: bool, run_dir: Path, initial_frame: RobotStateFrame, + context_id: str, ) -> tuple[RobotFuture | None, BranchTrace]: import numpy as np @@ -526,7 +523,7 @@ def _rollout( future = ( RobotFuture( - context_id=CONTEXT_ID, + context_id=context_id, branch_id=branch_id, initial_frame=initial_frame, frames=tuple(typed_frames), @@ -581,6 +578,7 @@ def _snapshot_descriptor( *, run_dir: Path, simulator_version: str, + spec: LiberoTaskSpec, ) -> SimulatorSnapshotDescriptor: import numpy as np @@ -592,11 +590,15 @@ def _snapshot_descriptor( sidecar_path = run_dir / "snapshots" / "runtime_state.json" _write_json( sidecar_path, - _runtime_sidecar(snapshot, state_path.relative_to(run_dir).as_posix()), + _runtime_sidecar( + snapshot, + state_path.relative_to(run_dir).as_posix(), + spec, + ), ) return SimulatorSnapshotDescriptor( - snapshot_id=CONTEXT_ID, - context_id=CONTEXT_ID, + snapshot_id=spec.context_id, + context_id=spec.context_id, simulator="LIBERO/robosuite/MuJoCo", simulator_version=simulator_version, state_format="numpy-npy-mujoco-mjSTATE_INTEGRATION-float64-v1", @@ -612,54 +614,74 @@ def _snapshot_descriptor( ) -def _prepare_environment(run_dir: Path) -> tuple[Any, Any, str]: +def _prepare_environment( + run_dir: Path, + spec: LiberoTaskSpec, + *, + expected_libero_revision: str, + benchmark_id: str, +) -> tuple[Any, Any, str]: import mujoco import robosuite from libero.libero import benchmark, get_libero_path from libero.libero.envs import OffScreenRenderEnv revision = _git_revision(LIBERO_ROOT) - if revision != LIBERO_REVISION: - raise RuntimeError(f"LIBERO revision mismatch: expected {LIBERO_REVISION}, got {revision}") - suite = benchmark.get_benchmark_dict()[TASK_SUITE]() - task = suite.get_task(TASK_ID) - if task.language != TASK: - raise RuntimeError(f"pinned LIBERO task changed: expected {TASK!r}, got {task.language!r}") - bddl_path = Path(suite.get_task_bddl_file_path(TASK_ID)) + if revision != expected_libero_revision: + raise RuntimeError( + f"LIBERO revision mismatch: expected {expected_libero_revision}, got {revision}" + ) + suite = benchmark.get_benchmark_dict()[spec.task_suite]() + task = suite.get_task(spec.task_id) + if task.language != spec.task: + raise RuntimeError( + f"pinned LIBERO task changed: expected {spec.task!r}, got {task.language!r}" + ) + if task.problem_folder != spec.problem_folder: + raise RuntimeError("pinned LIBERO problem folder changed") + if task.bddl_file != spec.bddl_file or task.init_states_file != spec.init_states_file: + raise RuntimeError("pinned LIBERO task filenames changed") + bddl_path = Path(suite.get_task_bddl_file_path(spec.task_id)) init_states_path = ( Path(get_libero_path("init_states")) / task.problem_folder / task.init_states_file ) - if _sha256_file(bddl_path) != BDDL_SHA256: + if _sha256_file(bddl_path) != spec.bddl_sha256: raise RuntimeError("LIBERO BDDL hash mismatch") - if _sha256_file(init_states_path) != INIT_STATES_SHA256: + if _sha256_file(init_states_path) != spec.init_states_sha256: raise RuntimeError("LIBERO init-state hash mismatch") init_states = _load_init_states(init_states_path) - if init_states.shape != (50, 92): + if init_states.shape != spec.init_states_shape: raise RuntimeError(f"unexpected LIBERO init-state shape: {init_states.shape}") + if str(init_states.dtype) != spec.init_states_dtype: + raise RuntimeError(f"unexpected LIBERO init-state dtype: {init_states.dtype}") environment = OffScreenRenderEnv( bddl_file_name=str(bddl_path), camera_heights=256, camera_widths=256, ) - environment.seed(SEED) + environment.seed(spec.seed) version = f"libero@{revision}/robosuite@{robosuite.__version__}/mujoco@{mujoco.__version__}" _write_json( run_dir / "benchmark_provenance.json", { + "benchmark_id": benchmark_id, + "task_key": spec.key, + "task_family": spec.task_family, "libero_revision": revision, "robosuite_version": robosuite.__version__, "mujoco_version": mujoco.__version__, - "task_suite": TASK_SUITE, - "task_id": TASK_ID, - "task": TASK, - "init_state_index": INIT_STATE_INDEX, - "wait_steps": WAIT_STEPS, - "seed": SEED, + "task_suite": spec.task_suite, + "task_id": spec.task_id, + "task": spec.task, + "context_id": spec.context_id, + "init_state_index": spec.init_state_index, + "wait_steps": spec.wait_steps, + "seed": spec.seed, "bddl_uri": str(bddl_path), - "bddl_sha256": BDDL_SHA256, + "bddl_sha256": spec.bddl_sha256, "init_states_uri": str(init_states_path), - "init_states_sha256": INIT_STATES_SHA256, + "init_states_sha256": spec.init_states_sha256, "init_states_shape": list(init_states.shape), "action_space": ACTION_SPACE, "control_frequency_hz": environment.env.control_freq, @@ -673,14 +695,26 @@ def _prepare_environment(run_dir: Path) -> tuple[Any, Any, str]: return environment, init_states, version -def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: +def _generate( + run_dir: Path, + horizon: int, + spec: LiberoTaskSpec, + *, + expected_libero_revision: str, + benchmark_id: str, +) -> CounterfactualArtifact: import numpy as np - environment, init_states, simulator_version = _prepare_environment(run_dir) + environment, init_states, simulator_version = _prepare_environment( + run_dir, + spec, + expected_libero_revision=expected_libero_revision, + benchmark_id=benchmark_id, + ) try: environment.reset() - observation = environment.set_init_state(init_states[INIT_STATE_INDEX]) - for _ in range(WAIT_STEPS): + observation = environment.set_init_state(init_states[spec.init_state_index]) + for _ in range(spec.wait_steps): observation, _, _, _ = environment.step(DUMMY_ACTION) snapshot = _capture_runtime_snapshot(environment) @@ -688,6 +722,7 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: snapshot, run_dir=run_dir, simulator_version=simulator_version, + spec=spec, ) original_state = snapshot.integration_state.copy() @@ -718,6 +753,7 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: persist=True, run_dir=run_dir, initial_frame=initial_frame, + context_id=spec.context_id, ) if future is None: raise AssertionError("persisted rollout did not return a future") @@ -746,6 +782,7 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: persist=False, run_dir=run_dir, initial_frame=initial_frame, + context_id=spec.context_id, ) reverse_traces[branch_id] = trace @@ -758,6 +795,7 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: persist=False, run_dir=run_dir, initial_frame=initial_frame, + context_id=spec.context_id, ) order_comparisons = [ (primary_traces[branch_id], reverse_traces[branch_id]) @@ -786,8 +824,8 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: ), ) group = RobotInterventionGroup( - context_id=CONTEXT_ID, - task=TASK, + context_id=spec.context_id, + task=spec.task, snapshot=descriptor, branches=tuple(branches), ) @@ -803,15 +841,184 @@ def _generate(run_dir: Path, horizon: int) -> CounterfactualArtifact: environment.close() +def _artifact_cache_summary( + run_dir: Path, + spec: LiberoTaskSpec, + horizon: int, +) -> dict[str, object] | None: + """Return verified cached metadata, or ``None`` when regeneration is required.""" + + artifact_path = run_dir / "intervention_group.json" + if not artifact_path.is_file(): + return None + try: + payload = json.loads(artifact_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + return None + artifact_sha256 = payload.get("artifact_sha256") + unsigned = {key: value for key, value in payload.items() if key != "artifact_sha256"} + computed = hashlib.sha256( + json.dumps( + unsigned, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + if artifact_sha256 != computed: + return None + group = payload["group"] + validation = payload["validation"] + if not isinstance(group, dict) or not isinstance(validation, dict): + return None + if group.get("context_id") != spec.context_id or group.get("horizon") != horizon: + return None + if validation.get("passed") is not True: + return None + + snapshot = group["snapshot"] + if not isinstance(snapshot, dict): + return None + _verify_cached_file( + run_dir, + snapshot["state_uri"], + snapshot["state_sha256"], + snapshot["state_size_bytes"], + ) + _verify_cached_file( + run_dir, + snapshot["sidecar_uri"], + snapshot["sidecar_sha256"], + snapshot["sidecar_size_bytes"], + ) + branches = group["branches"] + if not isinstance(branches, list) or len(branches) != len(BRANCH_ACTIONS): + return None + for branch in branches: + if not isinstance(branch, dict): + return None + future = branch["future"] + if not isinstance(future, dict): + return None + frames = [future["initial_frame"], *future["frames"]] + for frame in frames: + if not isinstance(frame, dict): + return None + rgb_frames = frame["rgb_frames"] + if not isinstance(rgb_frames, list): + return None + for reference in rgb_frames: + if not isinstance(reference, dict): + return None + _verify_cached_file( + run_dir, + reference["uri"], + reference["sha256"], + reference["size_bytes"], + ) + return { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "artifact": str(artifact_path), + "artifact_sha256": artifact_sha256, + "group_sha256": payload.get("group_sha256"), + "snapshot_sha256": snapshot.get("content_sha256"), + "branches": len(branches), + "horizon": horizon, + "metrics": payload.get("metrics"), + "validation": validation, + } + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def _verify_cached_file( + run_dir: Path, + raw_uri: object, + raw_sha256: object, + raw_size: object, +) -> None: + if not isinstance(raw_uri, str) or not isinstance(raw_sha256, str): + raise ValueError("cached external reference fields have invalid types") + if isinstance(raw_size, bool) or not isinstance(raw_size, int): + raise ValueError("cached external reference size has an invalid type") + path = (run_dir / raw_uri).resolve() + path.relative_to(run_dir.resolve()) + if path.stat().st_size != raw_size or _sha256_file(path) != raw_sha256: + raise ValueError(f"cached external reference failed verification: {raw_uri}") + + +def _generated_summary( + run_dir: Path, + spec: LiberoTaskSpec, + artifact: CounterfactualArtifact, +) -> dict[str, object]: + return { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "artifact": str(run_dir / "intervention_group.json"), + "artifact_sha256": artifact.artifact_sha256, + "group_sha256": artifact.group.content_sha256, + "snapshot_sha256": artifact.group.snapshot.content_sha256, + "branches": len(artifact.group.branches), + "horizon": artifact.group.horizon, + "metrics": artifact.metrics.to_dict(), + "validation": artifact.validation.to_dict(), + } + + +def _write_index( + run_dir: Path, + *, + benchmark_id: str, + manifest_path: Path, + selected_tasks: tuple[LiberoTaskSpec, ...], + records: list[dict[str, object]], +) -> None: + _write_json( + run_dir / "index.json", + { + "schema_version": "0.1", + "benchmark_id": benchmark_id, + "manifest_uri": str(manifest_path), + "manifest_sha256": _sha256_file(manifest_path), + "selected_task_keys": [task.key for task in selected_tasks], + "tasks": records, + "completed": sum(record["status"] in {"generated", "cache-hit"} for record in records), + "failed": sum(record["status"] == "failed" for record in records), + }, + ) + + def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--gpu-index", type=int, default=0, help="physical EGL GPU index") parser.add_argument( "--run-dir", type=Path, - default=REPOSITORY_ROOT / "runs" / "libero-cf-pilot", + default=REPOSITORY_ROOT / "runs" / "libero-cf-mini", + ) + parser.add_argument( + "--manifest", + type=Path, + default=MANIFEST_PATH, + help="pinned LIBERO-CF-Mini task manifest", + ) + parser.add_argument( + "--task-key", + action="append", + default=[], + help="generate one manifest task key; repeat to select several (default: all)", ) parser.add_argument("--horizon", type=int, default=8, help="control steps per action branch") + parser.add_argument( + "--force", + action="store_true", + help="regenerate even when all content-addressed files verify", + ) return parser.parse_args() @@ -822,25 +1029,55 @@ def main() -> None: if args.horizon <= 0: raise ValueError("horizon must be positive") run_dir = args.run_dir.expanduser().resolve() + manifest_path = args.manifest.expanduser().resolve() + manifest = load_libero_cf_manifest(manifest_path) + selected_tasks = manifest.select(args.task_key) _configure_process(args.gpu_index, run_dir) - _set_seed(SEED) - artifact = _generate(run_dir, args.horizon) - print( - json.dumps( - { - "artifact": str(run_dir / "intervention_group.json"), - "artifact_sha256": artifact.artifact_sha256, - "group_sha256": artifact.group.content_sha256, - "snapshot_sha256": artifact.group.snapshot.content_sha256, - "branches": len(artifact.group.branches), - "horizon": artifact.group.horizon, - "metrics": artifact.metrics.to_dict(), - "validation": artifact.validation.to_dict(), - }, - indent=2, - sort_keys=True, + records: list[dict[str, object]] = [] + failures = 0 + for spec in selected_tasks: + task_run_dir = run_dir / spec.key + cached = None if args.force else _artifact_cache_summary(task_run_dir, spec, args.horizon) + if cached is not None: + record = {**cached, "status": "cache-hit"} + else: + try: + _set_seed(spec.seed) + artifact = _generate( + task_run_dir, + args.horizon, + spec, + expected_libero_revision=manifest.libero_revision, + benchmark_id=manifest.benchmark_id, + ) + record = { + **_generated_summary(task_run_dir, spec, artifact), + "status": "generated", + } + except Exception as error: + failures += 1 + record = { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "status": "failed", + "error_type": type(error).__name__, + "error": str(error), + } + records.append(record) + _write_index( + run_dir, + benchmark_id=manifest.benchmark_id, + manifest_path=manifest_path, + selected_tasks=selected_tasks, + records=records, + ) + print(json.dumps(record, indent=2, sort_keys=True)) + if failures: + raise RuntimeError( + f"{failures}/{len(selected_tasks)} LIBERO-CF-Mini tasks failed; " + f"inspect {run_dir / 'index.json'}" ) - ) if __name__ == "__main__": diff --git a/src/wamprobe/libero_cf.py b/src/wamprobe/libero_cf.py new file mode 100644 index 0000000..bb4a694 --- /dev/null +++ b/src/wamprobe/libero_cf.py @@ -0,0 +1,223 @@ +"""Pinned, dependency-free manifest contract for LIBERO-CF-Mini.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import cast + + +def _sha256_digest(value: object, field: str) -> str: + if not isinstance(value, str) or len(value) != 64: + raise ValueError(f"{field} must be a lowercase SHA256 digest") + if any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{field} must be a lowercase SHA256 digest") + return value + + +def _git_revision(value: object, field: str) -> str: + if not isinstance(value, str) or len(value) != 40: + raise ValueError(f"{field} must be a lowercase Git commit") + if any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{field} must be a lowercase Git commit") + return value + + +def _string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _integer(value: object, field: str, *, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{field} must be an integer >= {minimum}") + return value + + +def _mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{field} must be an object") + return cast(dict[str, object], value) + + +@dataclass(frozen=True, slots=True) +class LiberoTaskSpec: + """One immutable simulator origin selected for the mini benchmark.""" + + key: str + task_family: str + task_suite: str + task_id: int + task: str + init_state_index: int + wait_steps: int + seed: int + bddl_sha256: str + init_states_sha256: str + init_states_shape: tuple[int, int] + init_states_dtype: str + problem_folder: str + bddl_file: str + init_states_file: str + + def __post_init__(self) -> None: + for field_name in ( + "key", + "task_family", + "task_suite", + "task", + "init_states_dtype", + "problem_folder", + "bddl_file", + "init_states_file", + ): + _string(getattr(self, field_name), field_name) + _integer(self.task_id, "task_id") + _integer(self.init_state_index, "init_state_index") + _integer(self.wait_steps, "wait_steps") + _integer(self.seed, "seed") + _sha256_digest(self.bddl_sha256, "bddl_sha256") + _sha256_digest(self.init_states_sha256, "init_states_sha256") + if len(self.init_states_shape) != 2 or any( + dimension <= 0 for dimension in self.init_states_shape + ): + raise ValueError("init_states_shape must contain two positive dimensions") + if self.init_state_index >= self.init_states_shape[0]: + raise ValueError("init_state_index is outside init_states_shape") + + @property + def context_id(self) -> str: + """Return the stable shared-context identifier used by artifacts and caches.""" + + suite = self.task_suite.replace("_", "-") + return ( + f"{suite}-task{self.task_id}-init{self.init_state_index}" + f"-wait{self.wait_steps}-seed{self.seed}" + ) + + @classmethod + def from_object(cls, value: object, index: int) -> LiberoTaskSpec: + """Strictly restore one task from a manifest record.""" + + payload = _mapping(value, f"tasks[{index}]") + expected = { + "key", + "task_family", + "task_suite", + "task_id", + "task", + "init_state_index", + "wait_steps", + "seed", + "bddl_sha256", + "init_states_sha256", + "init_states_shape", + "init_states_dtype", + "problem_folder", + "bddl_file", + "init_states_file", + } + if payload.keys() != expected: + raise ValueError(f"tasks[{index}] fields do not match schema 0.1") + shape = payload["init_states_shape"] + if not isinstance(shape, list) or len(shape) != 2: + raise ValueError("init_states_shape must be a two-element array") + return cls( + key=_string(payload["key"], "key"), + task_family=_string(payload["task_family"], "task_family"), + task_suite=_string(payload["task_suite"], "task_suite"), + task_id=_integer(payload["task_id"], "task_id"), + task=_string(payload["task"], "task"), + init_state_index=_integer(payload["init_state_index"], "init_state_index"), + wait_steps=_integer(payload["wait_steps"], "wait_steps"), + seed=_integer(payload["seed"], "seed"), + bddl_sha256=_sha256_digest(payload["bddl_sha256"], "bddl_sha256"), + init_states_sha256=_sha256_digest( + payload["init_states_sha256"], + "init_states_sha256", + ), + init_states_shape=( + _integer(shape[0], "init_states_shape[0]", minimum=1), + _integer(shape[1], "init_states_shape[1]", minimum=1), + ), + init_states_dtype=_string(payload["init_states_dtype"], "init_states_dtype"), + problem_folder=_string(payload["problem_folder"], "problem_folder"), + bddl_file=_string(payload["bddl_file"], "bddl_file"), + init_states_file=_string(payload["init_states_file"], "init_states_file"), + ) + + +@dataclass(frozen=True, slots=True) +class LiberoCFManifest: + """Versioned benchmark selection and upstream provenance.""" + + benchmark_id: str + libero_revision: str + upstream_license: str + tasks: tuple[LiberoTaskSpec, ...] + schema_version: str = "0.1" + + def __post_init__(self) -> None: + if self.schema_version != "0.1": + raise ValueError("unsupported LIBERO-CF manifest schema_version") + _string(self.benchmark_id, "benchmark_id") + _git_revision(self.libero_revision, "libero_revision") + _string(self.upstream_license, "upstream_license") + if not 3 <= len(self.tasks) <= 5: + raise ValueError("LIBERO-CF-Mini requires three to five task families") + keys = [task.key for task in self.tasks] + families = [task.task_family for task in self.tasks] + context_ids = [task.context_id for task in self.tasks] + if len(keys) != len(set(keys)): + raise ValueError("LIBERO-CF task keys must be unique") + if len(families) != len(set(families)): + raise ValueError("LIBERO-CF task families must be unique") + if len(context_ids) != len(set(context_ids)): + raise ValueError("LIBERO-CF context IDs must be unique") + + def select(self, keys: list[str] | None = None) -> tuple[LiberoTaskSpec, ...]: + """Select manifest-order tasks, rejecting unknown or duplicate keys.""" + + if not keys: + return self.tasks + if len(keys) != len(set(keys)): + raise ValueError("selected LIBERO-CF task keys must be unique") + requested = set(keys) + selected = tuple(task for task in self.tasks if task.key in requested) + missing = requested - {task.key for task in selected} + if missing: + raise ValueError(f"unknown LIBERO-CF task keys: {sorted(missing)}") + return selected + + +def load_libero_cf_manifest(path: Path) -> LiberoCFManifest: + """Load and strictly validate the pinned LIBERO-CF-Mini manifest.""" + + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"could not read LIBERO-CF manifest: {error}") from error + payload = _mapping(raw, "manifest") + expected = { + "schema_version", + "benchmark_id", + "libero_revision", + "upstream_license", + "tasks", + } + if payload.keys() != expected: + raise ValueError("LIBERO-CF manifest fields do not match schema 0.1") + raw_tasks = payload["tasks"] + if not isinstance(raw_tasks, list): + raise ValueError("tasks must be an array") + return LiberoCFManifest( + schema_version=_string(payload["schema_version"], "schema_version"), + benchmark_id=_string(payload["benchmark_id"], "benchmark_id"), + libero_revision=_git_revision(payload["libero_revision"], "libero_revision"), + upstream_license=_string(payload["upstream_license"], "upstream_license"), + tasks=tuple( + LiberoTaskSpec.from_object(task, index) for index, task in enumerate(raw_tasks) + ), + ) diff --git a/tests/test_libero_cf.py b/tests/test_libero_cf.py new file mode 100644 index 0000000..98f679f --- /dev/null +++ b/tests/test_libero_cf.py @@ -0,0 +1,44 @@ +import json +from pathlib import Path + +import pytest + +from wamprobe.libero_cf import load_libero_cf_manifest + +MANIFEST = Path("configs/benchmarks/libero_cf_mini_v0.1.json") + + +def test_libero_cf_manifest_pins_four_distinct_task_families() -> None: + manifest = load_libero_cf_manifest(MANIFEST) + + assert manifest.schema_version == "0.1" + assert manifest.benchmark_id == "libero-cf-mini-v0.1" + assert manifest.upstream_license == "MIT" + assert len(manifest.tasks) == 4 + assert len({task.task_family for task in manifest.tasks}) == 4 + assert len({task.task_suite for task in manifest.tasks}) == 4 + assert all(len(task.bddl_sha256) == 64 for task in manifest.tasks) + assert all(len(task.init_states_sha256) == 64 for task in manifest.tasks) + assert all(task.context_id.endswith(f"seed{task.seed}") for task in manifest.tasks) + + +def test_libero_cf_manifest_selection_preserves_manifest_order() -> None: + manifest = load_libero_cf_manifest(MANIFEST) + + selected = manifest.select(["goal-task0", "spatial-task0"]) + + assert [task.key for task in selected] == ["spatial-task0", "goal-task0"] + with pytest.raises(ValueError, match="unknown"): + manifest.select(["not-a-task"]) + with pytest.raises(ValueError, match="unique"): + manifest.select(["goal-task0", "goal-task0"]) + + +def test_libero_cf_manifest_rejects_tampered_checksum(tmp_path: Path) -> None: + payload = json.loads(MANIFEST.read_text(encoding="utf-8")) + payload["tasks"][0]["bddl_sha256"] = "tampered" + path = tmp_path / "manifest.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="bddl_sha256"): + load_libero_cf_manifest(path) From d23938c8d1cb4d2c4fb9c50c963008b3f82a7516 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 13:55:51 +0000 Subject: [PATCH 13/20] run StarWAM action experiment matrix --- .github/workflows/gpu-nightly.yml | 80 +++ README.md | 15 +- docs/WAMProbe_PLAN.md | 15 +- .../STARWAM_LIBERO_CF_MINI_V0.1.md | 82 +++ docs/metrics/CORE_METRICS.md | 47 ++ docs/models/STARWAM.md | 72 +++ docs/reproducibility/GPU_NIGHTLY.md | 24 + environments/starwam/README.md | 36 +- .../starwam/execute_prediction_matrix.py | 351 +++++++++++ environments/starwam/run_libero_smoke.py | 588 ++++++++++++++++-- src/wamprobe/artifacts.py | 15 +- src/wamprobe/cli.py | 23 + src/wamprobe/experiments.py | 585 +++++++++++++++++ tests/test_artifacts.py | 1 + tests/test_experiments.py | 127 ++++ 15 files changed, 2002 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/gpu-nightly.yml create mode 100644 docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md create mode 100644 docs/metrics/CORE_METRICS.md create mode 100644 docs/models/STARWAM.md create mode 100644 docs/reproducibility/GPU_NIGHTLY.md create mode 100644 environments/starwam/execute_prediction_matrix.py create mode 100644 src/wamprobe/experiments.py create mode 100644 tests/test_experiments.py diff --git a/.github/workflows/gpu-nightly.yml b/.github/workflows/gpu-nightly.yml new file mode 100644 index 0000000..89470a0 --- /dev/null +++ b/.github/workflows/gpu-nightly.yml @@ -0,0 +1,80 @@ +name: GPU nightly + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + full_matrix: + description: Run all four tasks, seeds, and NFE settings + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: wamprobe-gpu-nightly + cancel-in-progress: false + +jobs: + starwam-libero-smoke: + if: ${{ vars.WAMPROBE_GPU_NIGHTLY_ENABLED == 'true' }} + runs-on: [self-hosted, linux, x64, gpu] + timeout-minutes: 90 + + steps: + - uses: actions/checkout@v4 + with: + clean: false + - name: Validate provisioned runner + run: | + test -x environments/starwam/.venv/bin/python + test -d vendor/upstream/starwam + test -d vendor/upstream/libero + nvidia-smi + environments/starwam/.venv/bin/python -m wamprobe doctor --json + - name: Run cached StarWAM matrix + env: + FULL_MATRIX: ${{ inputs.full_matrix || 'false' }} + run: | + args=( + --mode matrix + --gpu-index 0 + --minimum-free-gib 18 + --vae-device cpu + --run-dir runs/gpu-nightly + ) + if [[ "$FULL_MATRIX" != "true" ]]; then + args+=( + --task-key spatial-task0 + --matrix-seed 0 + --matrix-inference-steps 1 + ) + fi + environments/starwam/.venv/bin/python \ + environments/starwam/run_libero_smoke.py "${args[@]}" + - name: Execute action prefixes + run: | + environments/starwam/.venv/bin/python \ + environments/starwam/execute_prediction_matrix.py \ + --gpu-index 0 \ + --matrix-dir runs/gpu-nightly + - name: Build experiment report + run: | + environments/starwam/.venv/bin/python -m wamprobe experiment-report \ + runs/gpu-nightly/matrix-index.json \ + runs/gpu-nightly/execution-index.json \ + --output runs/gpu-nightly/report + - name: Upload structured evidence + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: wamprobe-gpu-nightly-${{ github.run_id }} + if-no-files-found: warn + retention-days: 14 + path: | + runs/gpu-nightly/matrix-index.json + runs/gpu-nightly/execution-index.json + runs/gpu-nightly/report/ diff --git a/README.md b/README.md index 7525cc2..78576b0 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo `report` and exact-context `compare` commands; - `wamprobe doctor` model layout, revision, size, Git LFS pointer, and SHA256 checks; - typed robot observations, action predictions, deterministic prediction artifacts, and a - pinned StarWAM/LIBERO smoke runner; + pinned StarWAM/LIBERO runner with multi-seed/NFE action execution reports; - dependency-free robot action-branch/future contracts plus four task families × four branches × eight steps in LIBERO-CF-Mini, with exact restore and branch-order validation; - Python 3.11–3.13 CI with linting, strict typing, and coverage. @@ -121,16 +121,19 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo The next milestones are: -1. execute cached StarWAM candidate actions across LIBERO-CF-Mini; -2. run action-mask/shuffle, seed, horizon, and NFE ablations; -3. add a GPU nightly workflow for the isolated real-model integration; -4. add the Occluded-Object memory diagnostic to the broader Toy tier. +1. add a traditional video-quality versus control-value baseline study; +2. expand fixed states and add a minimal replanning/closed-loop experiment; +3. add the Occluded-Object memory diagnostic to the broader Toy tier; +4. prepare a signed v0.1 release candidate and external reproduction smoke. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), [adapter selection record](docs/research/ADAPTER_SELECTION.md), and [design RFCs](docs/rfcs/). The exact toy dynamics and limitations are documented in the -[toy benchmark card](docs/benchmarks/TOY_BENCHMARKS.md). +[toy benchmark card](docs/benchmarks/TOY_BENCHMARKS.md), +[LIBERO-CF-Mini benchmark card](docs/benchmarks/LIBERO_CF_MINI.md), +[StarWAM model card](docs/models/STARWAM.md), and +[core metric cards](docs/metrics/CORE_METRICS.md). ## Development diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index ba7bc36..ad99962 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1090,6 +1090,14 @@ init-state 与上游 commit，批量生成 4 task × 4 branch × 8 step 的真 ### Phase 4：真实 WAM Adapter（第 11–13 周） +当前进度（2026-07-15）：StarWAM matrix runner 已在一次模型加载中完成 4 task × 3 seed × +3 NFE 共 36 次真实推理，第二次运行得到 36/36 output-SHA-verified cache hit；随后从配对 +snapshot 执行全部动作块，并在 horizon 8/16/32 记录状态、回报和成功。预测与执行失败率 +均为 0，NFE 1/4/8 平均延迟为 0.780/0.972/1.216 秒，峰值约 11.39 GiB。所有短 horizon +稀疏成功均为 0，EEF 平均位移随 horizon 为 0.1716/0.1121/0.0000，保留为“更长 rollout +不单调改善控制”的负结果。候选 action mask/shuffle 因已验证 adapter 不接受候选 action +输入而结构化跳过。模型卡、实验报告和 opt-in self-hosted GPU nightly 已补齐。 + 任务： - 完成 StarWAM 或其他首选 adapter； @@ -1150,7 +1158,8 @@ v0.1 发布门槛： | #19 | Add GPU nightly smoke workflow | P1 | 2 天 | #18 | | #20 | Write benchmark card and v0.1 reproducibility guide | P0 | 4 天 | 全部 | -代码落地状态（2026-07-15）：已补齐并验证 #6、#8、#10、#11、#13、#15、#16 以及 CRC； +代码落地状态（2026-07-15）：已补齐并验证 #6、#8、#10、#11、#13、#15、#16、#19 +以及 CRC； 此前已完成 #1–#5、#7、#9、#12、#14、#17 的当前 v0.1 切片。#18 已有可运行的 StarWAM prediction artifact/adapter 切片，但真实 WAM 反事实控制评测仍需继续扩展。 @@ -1285,7 +1294,7 @@ priority:p1 - [x] 噪声增加时准确性指标总体退化； - [x] 至少一个 causal/ranking 指标与 simulator return 相关； - [ ] 报告传统视频指标与控制指标的差异； -- [ ] 公开所有主要失败率和 skipped metrics。 +- [x] 公开所有主要失败率和 skipped metrics。 ### 工程质量 @@ -1293,7 +1302,7 @@ priority:p1 - [x] schema 有版本和兼容策略； - [ ] 所有 release artifact 可追溯； - [x] README 15 分钟 quickstart 可执行； -- [ ] benchmark/model/metric cards 完整； +- [x] benchmark/model/metric cards 完整； - [ ] 至少一个外部用户复现 smoke test。 --- diff --git a/docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md b/docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md new file mode 100644 index 0000000..ee8818d --- /dev/null +++ b/docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md @@ -0,0 +1,82 @@ +# StarWAM × LIBERO-CF-Mini v0.1 experiment + +## Result + +The fixed 2026-07-15 run completed 36/36 predictions and 36/36 simulator executions with +no failed samples or failed tasks. Four prediction executions were repeated from their +snapshots and matched exactly. Re-running inference produced 36/36 verified cache hits. + +Configuration: + +- model: `starwam-wan22-5b-mot-libero`; +- tasks: spatial, object, articulated-goal and long-horizon composition; +- diffusion seeds: 0, 1, 2; +- action NFE: 1, 4, 8; +- output action shape: 32 × 7; +- executed prefix horizons: 8, 16, 32. + +## Efficiency + +| NFE | Samples | Mean latency | Peak allocated | Action RMS | Mean step change | +|---:|---:|---:|---:|---:|---:| +| 1 | 12 | 0.7798 s | 11.388 GiB | 0.4594 | 0.0638 | +| 4 | 12 | 0.9722 s | 11.388 GiB | 0.4604 | 0.0519 | +| 8 | 12 | 1.2160 s | 11.388 GiB | 0.4615 | 0.0514 | + +NFE and latency had Pearson correlation 0.8668 in this small repeated matrix. Increasing +NFE cost more latency but did not yield any sparse success. + +## Simulator outcomes + +| Prefix | Executions | Success | Mean return | Mean EEF displacement | +|---:|---:|---:|---:|---:| +| 8 | 36 | 0.0000 | 0.0000 | 0.1716 m | +| 16 | 36 | 0.0000 | 0.0000 | 0.1121 m | +| 32 | 36 | 0.0000 | 0.0000 | 0.0000 m | + +The 32-step chunks returned the EEF to its task-specific initial position even though +simulator and object state changed. Longer open-loop execution therefore did not +monotonically increase control displacement or utility. This supports reporting horizon +profiles rather than assuming the largest action chunk is best. + +Mean action RMS differences were 0.009067 across seeds and 0.011779 across NFE settings. +Mean executed EEF differences across NFE were 0.001790/0.003121/0.004164 m at horizons +8/16/32; across seeds they were 0.000798/0.001142/0.001882 m. + +## Explicit skips and negative results + +- Candidate-action mask/shuffle: skipped because the verified adapter has no candidate + action input or action-conditioned future endpoint. +- Predicted-video quality versus control: skipped because the adapter returns no future + video/state artifact; inventing PSNR/SSIM/FVD would be invalid. +- Success correlation: undefined because sparse success and return are constant zero. +- The zero-success outcome is retained. It is a fixed-context, short open-loop diagnostic, + not a claim that the upstream policy has zero LIBERO benchmark success. + +## Reproduction and hashes + +```bash +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode matrix --gpu-index 0 --minimum-free-gib 18 --vae-device cpu \ + --run-dir runs/starwam-libero-cf-mini-v0.1 + +environments/starwam/.venv/bin/python \ + environments/starwam/execute_prediction_matrix.py \ + --gpu-index 0 --matrix-dir runs/starwam-libero-cf-mini-v0.1 + +wamprobe experiment-report \ + runs/starwam-libero-cf-mini-v0.1/matrix-index.json \ + runs/starwam-libero-cf-mini-v0.1/execution-index.json \ + --output runs/starwam-libero-cf-mini-v0.1/report +``` + +Verified evidence identities: + +- matrix index SHA256: `61fd988ea648922652ef2ab23f08b14a9849d37353d2e4894eb839fe3dfdb12d`; +- execution index SHA256: `3bb9d5bdab9437cf07b0ed6567a8ddb146191447931a59a098b853e2fd3e9846`; +- LIBERO-CF manifest SHA256: `1660ab49ec0e5aac58f4801f0ec9a12326053f21f5f1974dd3b6ea63260f78c3`. + +The indexes and raw predictions remain ignored locally until a user-approved release +bundle is published. Each prediction artifact carries its own input cache key and output +SHA256; the committed report records the aggregate evidence without committing weights, +RGB frames or simulator states. diff --git a/docs/metrics/CORE_METRICS.md b/docs/metrics/CORE_METRICS.md new file mode 100644 index 0000000..4a8963e --- /dev/null +++ b/docs/metrics/CORE_METRICS.md @@ -0,0 +1,47 @@ +# Core metric cards + +WAMProbe reports a profile rather than one composite score. All aggregate confidence +intervals resample complete shared contexts; action branches and frames from one context +are never treated as independent samples. + +| Metric | Requires | Definition | Direction / range | Principal limitation | +|---|---|---|---|---| +| Action Dependence | paired candidate futures | mean predicted pair separation normalized by true separation | higher; analytic implementation clipped to [0,1] | separation can be confidently wrong | +| ADS permutation effect | paired action/future alignment | observed action-future association minus within-context permutation null | higher is stronger alignment | small branch sets give coarse p-values | +| ADS permutation p-value | same | add-one Monte Carlo randomization probability | lower is stronger evidence | not an effect size and not independent across frames | +| Counterfactual Direction | vector states | cosine of predicted and true displacement | -1 opposite, 0 absent/orthogonal, 1 aligned | ignores calibrated magnitude | +| No-op Stability/Fidelity | explicit no-op truth | point-mass drift score or exact manipulation no-op fidelity | higher is better | a static model can pass no-op alone | +| State ADE | typed state trajectory | mean Euclidean error over steps and branches | lower, ≥0 | state dimensions and units must be semantically aligned | +| State FDE | typed state trajectory | mean final-state Euclidean error | lower, ≥0 | can hide transient physical violations | +| CRC Spearman | candidate returns | rank correlation with average tie ranks | higher, [-1,1] | unstable for very small/tied candidate sets | +| CRC Kendall tau-b | candidate returns | concordant-minus-discordant pairs with tie correction | higher, [-1,1] | ignores return magnitude | +| CRC NDCG | candidate returns | discounted gain after shifting relevance non-negative | higher, [0,1] | shift convention must stay fixed | +| CRC pairwise accuracy | candidate returns | correct preference fraction; model ties receive 0.5 | higher, [0,1] | true-return ties are excluded | +| Top-1 Regret | candidate returns | oracle return minus return of model-selected candidate | lower, ≥0 | depends on candidate-set coverage | +| Latency | timed model call | synchronized wall time per prediction | lower, seconds | hardware and warm-up dependent | +| Peak allocated/reserved memory | CUDA runtime | maximum allocator bytes during one prediction | lower, GiB | not total board memory or energy | +| NFE sensitivity | matched inputs/seeds | paired action RMS and executed endpoint distance across NFE | lower means more stable | stability does not establish correctness | +| Seed sensitivity | matched inputs/NFE | paired action RMS and executed endpoint distance across seeds | lower means more stable | deterministic collapse can also score low | +| Branch EEF/object separation | paired simulator futures | mean pairwise final-state distance | diagnostic, ≥0 | raw object vectors are comparable only within a task | +| No-op EEF drift | explicit simulator no-op | distance from initial to final EEF | lower, ≥0 | controller settling can produce small nonzero drift | + +## Applicability rules + +Metrics are gated by the adapter capability manifest. Missing inputs produce an explicit +skip, never a surrogate value. In particular, the released StarWAM adapter emits actions +but no action-conditioned future artifact. WAMProbe therefore skips candidate-action +mask/shuffle, predicted-video PSNR/SSIM/FVD and future-state ADE/FDE for that adapter. + +An undefined correlation caused by a constant outcome is serialized as JSON `null` and +explained in the report. A zero sparse success rate remains a reported negative result; +it is not dropped from aggregation. + +## Reference-baseline expectations + +- Oracle dynamics should minimize ADE/FDE and regret while maximizing direction and CRC. +- Copy-last and action-agnostic baselines should have weak action dependence and ranking. +- Wrong-direction should separate futures but have negative direction correctness. +- Seeded noisy dynamics should degrade ADE/FDE and ranking as noise rises. + +Any new metric must add a failure-mode statement, capability requirement, reference +ordering test, aggregation unit and range/direction before it enters a public report. diff --git a/docs/models/STARWAM.md b/docs/models/STARWAM.md new file mode 100644 index 0000000..27f0cc9 --- /dev/null +++ b/docs/models/STARWAM.md @@ -0,0 +1,72 @@ +# StarWAM adapter model card + +## Model and adapter identity + +| Field | Value | +|---|---| +| WAMProbe model ID | `starwam-wan22-5b-mot-libero` | +| Upstream code | `shaohua-pan/StarWAM@f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7` | +| Released model | `panshaohua/starwam@7d4bfe3ec76172ca17169fa959d21da099d386fe` | +| Checkpoint SHA256 | `d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378` | +| Backbone | Wan2.2 TI2V-5B revision `921dbaf3f1674a56f47e83fb80a34bac8a8f203e` | +| Adapter version | `0.1.0a0` | +| Upstream code/model license | Apache-2.0 as declared by upstream repositories | + +The WAMProbe adapter is an isolated integration target, not a reimplementation or an +endorsement by the StarWAM authors. Weights remain outside Git. + +## Verified interface + +Input: + +- two ordered 256×256 RGB cameras (`agentview`, `wrist`), rotated 180° and resized to + 224×224 before horizontal concatenation; +- eight proprioceptive values: EEF position, axis-angle orientation and two gripper + positions; +- the LIBERO natural-language task, encoded with the pinned UMT5-XXL text encoder; +- an explicit non-negative diffusion seed. + +Output: + +- 32 × 7 denormalized LIBERO delta-pose/gripper action chunk; +- measured latency, device, dtype and peak allocated/reserved GPU memory; +- full code/model/preprocessing/inference provenance and a prediction payload SHA256. + +## Declared capabilities + +The released integration is verified for observation/task → action prediction. It does +not expose an action-conditioned `predict_future(candidate_action)` endpoint, future RGB +video, future state, uncertainty or likelihood through the WAMProbe adapter. Therefore: + +- StarWAM action chunks can be executed and scored in LIBERO; +- NFE, seed and executed-prefix horizon can be evaluated; +- candidate-action mask/shuffle, PSNR/SSIM/FVD and future-state ADE/FDE are skipped with + structured reasons for this adapter; +- internal video/world features are not relabeled as predicted videos. + +## Verified evaluation + +The 2026-07-15 matrix used four LIBERO-CF-Mini task families, seeds 0/1/2 and NFE +1/4/8: 36 predictions and 36 action executions completed with zero runtime failures. A +second inference pass returned 36 SHA-verified cache hits. Mean latency rose from 0.780 s +at NFE 1 to 1.216 s at NFE 8 while peak allocation remained about 11.39 GiB. + +Actions were executed from the same content-addressed snapshots with checkpoints after +8/16/32 controls. Sparse success and return were zero for every fixed context. Mean EEF +displacement was 0.1716/0.1121/0.0000 m, showing that a longer open-loop action prefix did +not monotonically improve control. This is a negative short-horizon diagnostic result, +not a LIBERO policy success-rate estimate. + +See the [experiment report](../experiments/STARWAM_LIBERO_CF_MINI_V0.1.md) and +[LIBERO-CF-Mini benchmark card](../benchmarks/LIBERO_CF_MINI.md). + +## Limitations and risks + +- Four fixed init states cannot estimate general LIBERO success. +- The checkpoint was evaluated without retraining or calibration. +- Open-loop prefixes can leave the training distribution; WAMProbe does not assert robot + safety or constraint satisfaction. +- Seed/NFE sensitivity is descriptive with only three values per axis. +- The current adapter cannot isolate whether internal future features caused an action. +- Results depend on pinned CUDA/PyTorch/MuJoCo hardware and software; GPU nightly evidence + should be treated as a reproducibility check, not numerical bit-equivalence across GPUs. diff --git a/docs/reproducibility/GPU_NIGHTLY.md b/docs/reproducibility/GPU_NIGHTLY.md new file mode 100644 index 0000000..4fad7d1 --- /dev/null +++ b/docs/reproducibility/GPU_NIGHTLY.md @@ -0,0 +1,24 @@ +# GPU nightly runner guide + +The GPU workflow is intentionally disabled until a trusted self-hosted runner is +provisioned. Set repository variable `WAMPROBE_GPU_NIGHTLY_ENABLED=true` only after the +runner has labels `self-hosted`, `linux`, `x64`, `gpu` and the following ignored assets: + +- `environments/starwam/.venv/` with the pinned Python 3.11 environment; +- `vendor/upstream/starwam` and `vendor/upstream/libero` at documented commits; +- `checkpoints/upstream/` matching `wamprobe doctor`. + +Checkout uses `clean: false` so Git does not delete these provisioned ignored assets. Do +not attach the runner to untrusted pull-request workflows. The scheduled job reads only +the default branch, has `contents: read`, times out after 90 minutes and uploads structured +indexes/reports for 14 days. The default nightly is one task/seed/NFE; manual dispatch can +select the complete 36-prediction matrix. + +Before enabling the schedule, run locally: + +```bash +environments/starwam/.venv/bin/python -m wamprobe doctor --verify-hashes +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode matrix --task-key spatial-task0 --matrix-seed 0 \ + --matrix-inference-steps 1 --run-dir runs/gpu-nightly +``` diff --git a/environments/starwam/README.md b/environments/starwam/README.md index d933f19..10ec2d5 100644 --- a/environments/starwam/README.md +++ b/environments/starwam/README.md @@ -70,9 +70,9 @@ The pinned artifacts and environment passed the following checks on 2026-07-15: with the same measured GPU peak and cache key `c6c46faab2a33940fe40d20aba3fab311f8e17885ed753f963c8b51161d939e1`. -These runs prove the typed observation-to-action integration path, not policy quality or -rollout success. No predicted action was executed in the simulator, so neither artifact -may be reported as a LIBERO benchmark result. +These fixed smoke runs prove the typed observation-to-action integration path, not policy +quality or rollout success. The later matrix below executes predicted actions but remains +a fixed-context diagnostic rather than a LIBERO benchmark score. ## Reproduce the fixed smoke run @@ -95,6 +95,31 @@ Add `--verify-large-hashes` to rehash the checkpoint, VAE, and T5 before a run. PNGs, text caches, observation metadata, and prediction JSON remain below `runs/starwam-libero-smoke/` and are ignored by Git. +## Run the LIBERO-CF-Mini matrix + +Matrix mode prepares all four text caches before loading the action model once. It runs +three seeds × NFE 1/4/8 for every fixed task and resumes from output-SHA-verified +prediction artifacts: + +```bash +environments/starwam/.venv/bin/python environments/starwam/run_libero_smoke.py \ + --mode matrix --gpu-index 0 --minimum-free-gib 18 --vae-device cpu \ + --run-dir runs/starwam-libero-cf-mini-v0.1 + +environments/starwam/.venv/bin/python \ + environments/starwam/execute_prediction_matrix.py \ + --gpu-index 0 --matrix-dir runs/starwam-libero-cf-mini-v0.1 + +wamprobe experiment-report \ + runs/starwam-libero-cf-mini-v0.1/matrix-index.json \ + runs/starwam-libero-cf-mini-v0.1/execution-index.json \ + --output runs/starwam-libero-cf-mini-v0.1/report +``` + +The verified run completed 36 predictions and executions with no runtime failure. See the +[experiment report](../../docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md) for NFE/seed +sensitivity, 8/16/32-step outcomes, negative results and capability-gated skips. + ## Local paths ```text @@ -104,5 +129,6 @@ checkpoints/upstream/modelscope/panshaohua/starwam/ environments/starwam/.venv/ ``` -Only this environment documentation is tracked. The virtual environment, upstream source, -weights, caches, LIBERO assets, and predictions stay local. +Only code, documentation and aggregate evidence are tracked. The virtual environment, +upstream source, weights, caches, LIBERO assets, simulator states and predictions stay +local until a user-approved artifact release. diff --git a/environments/starwam/execute_prediction_matrix.py b/environments/starwam/execute_prediction_matrix.py new file mode 100644 index 0000000..229775c --- /dev/null +++ b/environments/starwam/execute_prediction_matrix.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python +"""Execute cached StarWAM action chunks from pinned LIBERO-CF-Mini snapshots.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +import uuid +from pathlib import Path +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +SOURCE_ROOT = REPOSITORY_ROOT / "src" +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from environments.libero import generate_cf_pilot as cf # noqa: E402 +from wamprobe.libero_cf import LiberoTaskSpec, load_libero_cf_manifest # noqa: E402 + +DEFAULT_MANIFEST = REPOSITORY_ROOT / "configs" / "benchmarks" / "libero_cf_mini_v0.1.json" +DEFAULT_MATRIX_DIR = REPOSITORY_ROOT / "runs" / "starwam-libero-cf-mini-v0.1" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(8 * 1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _canonical_sha256(value: object) -> str: + return hashlib.sha256( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _load_matrix(path: Path) -> dict[str, object]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise TypeError("matrix index must be an object") + if raw.get("schema_version") != "0.1": + raise ValueError("matrix index schema_version mismatch") + records = raw.get("records") + if not isinstance(records, list) or not records: + raise ValueError("matrix index must contain prediction records") + if raw.get("failed_predictions") != 0: + raise ValueError("matrix index contains failed predictions") + return raw + + +def _load_prediction( + matrix_dir: Path, + record: dict[str, object], + spec: LiberoTaskSpec, +) -> tuple[list[list[float]], dict[str, object]]: + cache_key = record.get("cache_key") + if not isinstance(cache_key, str) or len(cache_key) != 64: + raise ValueError("prediction record cache_key is invalid") + path = matrix_dir / "predictions" / f"{cache_key}.json" + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or payload.get("cache_key") != cache_key: + raise ValueError("prediction artifact cache_key mismatch") + prediction = payload.get("prediction") + observation = payload.get("observation") + inference = payload.get("inference") + if not all(isinstance(item, dict) for item in (prediction, observation, inference)): + raise TypeError("prediction artifact sections must be objects") + prediction = prediction # keep the runtime-narrowed type visible to static readers + observation = observation + inference = inference + if observation.get("context_id") != spec.context_id: + raise ValueError("prediction observation context does not match task manifest") + if prediction.get("context_id") != spec.context_id: + raise ValueError("prediction context does not match task manifest") + if prediction.get("seed") != record.get("seed"): + raise ValueError("prediction seed does not match matrix index") + if inference.get("num_inference_steps") != record.get("num_inference_steps"): + raise ValueError("prediction NFE does not match matrix index") + if payload.get("prediction_sha256") != _canonical_sha256(prediction): + raise ValueError("prediction_sha256 mismatch") + actions = prediction.get("actions") + if not isinstance(actions, list) or len(actions) != 32: + raise ValueError("prediction must contain a 32-step action chunk") + parsed: list[list[float]] = [] + for action in actions: + if not isinstance(action, list) or len(action) != 7: + raise ValueError("prediction action shape must be [32, 7]") + row: list[float] = [] + for value in action: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError("prediction actions must be finite numbers") + scalar = float(value) + if not -1.0 <= scalar <= 1.0: + raise ValueError("prediction actions must satisfy LIBERO [-1, 1] bounds") + row.append(scalar) + parsed.append(row) + metadata = { + "prediction_artifact": str(path), + "prediction_artifact_sha256": _sha256_file(path), + "prediction_sha256": payload["prediction_sha256"], + } + return parsed, metadata + + +def _execute_actions( + environment: Any, + snapshot: cf.RuntimeSnapshot, + actions: list[list[float]], + horizons: tuple[int, ...], +) -> dict[str, object]: + import numpy as np + + observation = cf._restore_runtime_snapshot(environment, snapshot) + initial_state_sha256 = cf._array_sha256(cf._integration_state(environment)) + cumulative_return = 0.0 + succeeded = bool(environment.check_success()) + done = bool(environment.env.done) + checkpoints: dict[str, object] = {} + for step_index, action in enumerate(actions[: max(horizons)], start=1): + observation, reward, done, _ = environment.step(np.asarray(action, dtype=np.float64)) + cumulative_return += float(reward) + succeeded = succeeded or bool(environment.check_success()) + if step_index in horizons: + checkpoints[str(step_index)] = { + "eef_position": [float(value) for value in observation["robot0_eef_pos"]], + "object_state": [float(value) for value in observation["object-state"]], + "cumulative_return": cumulative_return, + "success": succeeded, + "done": bool(done), + "simulator_state_sha256": cf._array_sha256(cf._integration_state(environment)), + "observation_sha256": cf._observation_sha256(observation), + } + if done: + break + return { + "initial_state_sha256": initial_state_sha256, + "initial_observation_sha256": cf._observation_sha256(observation), + "initial_eef_position": [float(value) for value in observation["robot0_eef_pos"]], + "initial_object_state": [float(value) for value in observation["object-state"]], + "requested_horizons": list(horizons), + "executed_steps": step_index, + "terminated_early": step_index < max(horizons), + "checkpoints": checkpoints, + } + + +def _task_records(matrix: dict[str, object], task_key: str) -> list[dict[str, object]]: + records = matrix["records"] + if not isinstance(records, list): + raise TypeError("matrix records must be an array") + selected: list[dict[str, object]] = [] + for raw in records: + if not isinstance(raw, dict): + raise TypeError("matrix record must be an object") + if raw.get("task_key") == task_key: + if raw.get("status") not in {"generated", "cache-hit"}: + raise ValueError("matrix record is not a completed prediction") + selected.append(raw) + if not selected: + raise ValueError(f"matrix has no predictions for task: {task_key}") + selected.sort( + key=lambda record: ( + int(record["num_inference_steps"]), + int(record["seed"]), + ) + ) + return selected + + +def _run_task( + matrix_dir: Path, + matrix: dict[str, object], + spec: LiberoTaskSpec, + horizons: tuple[int, ...], + *, + libero_revision: str, + benchmark_id: str, +) -> tuple[list[dict[str, object]], dict[str, object]]: + task_run_dir = matrix_dir / "executions" / spec.key + environment, init_states, _ = cf._prepare_environment( + task_run_dir, + spec, + expected_libero_revision=libero_revision, + benchmark_id=benchmark_id, + ) + records: list[dict[str, object]] = [] + repeatability: dict[str, object] = {} + try: + environment.reset() + environment.set_init_state(init_states[spec.init_state_index]) + for _ in range(spec.wait_steps): + environment.step(cf.DUMMY_ACTION) + snapshot = cf._capture_runtime_snapshot(environment) + snapshot_state_sha256 = cf._array_sha256(snapshot.integration_state) + + for index, prediction_record in enumerate(_task_records(matrix, spec.key)): + actions, metadata = _load_prediction(matrix_dir, prediction_record, spec) + outcome = _execute_actions(environment, snapshot, actions, horizons) + record = { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "seed": prediction_record["seed"], + "num_inference_steps": prediction_record["num_inference_steps"], + "cache_key": prediction_record["cache_key"], + "action_abs_max": max(abs(value) for action in actions for value in action), + "snapshot_state_sha256": snapshot_state_sha256, + **metadata, + "outcome": outcome, + "status": "executed", + } + records.append(record) + if index == 0: + repeated = _execute_actions(environment, snapshot, actions, horizons) + repeatability = { + "cache_key": prediction_record["cache_key"], + "exact": repeated == outcome, + "first": outcome, + "second": repeated, + } + if repeated != outcome: + raise RuntimeError(f"prediction execution is not repeatable: {spec.key}") + finally: + environment.close() + return records, repeatability + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gpu-index", type=int, default=0, help="physical EGL GPU index") + parser.add_argument("--matrix-dir", type=Path, default=DEFAULT_MATRIX_DIR) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument( + "--task-key", + action="append", + default=[], + help="manifest task key; repeat to select several (default: all matrix tasks)", + ) + parser.add_argument( + "--horizon", + action="append", + type=int, + default=[], + help="action prefix checkpoint; repeat to select several (default: 8,16,32)", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.gpu_index < 0: + raise ValueError("gpu-index must be non-negative") + horizons = tuple(sorted(args.horizon or (8, 16, 32))) + if any(horizon <= 0 or horizon > 32 for horizon in horizons): + raise ValueError("execution horizons must be in [1, 32]") + if len(horizons) != len(set(horizons)): + raise ValueError("execution horizons must be unique") + + matrix_dir = args.matrix_dir.expanduser().resolve() + manifest_path = args.manifest.expanduser().resolve() + matrix_path = matrix_dir / "matrix-index.json" + matrix = _load_matrix(matrix_path) + manifest = load_libero_cf_manifest(manifest_path) + matrix_task_keys = matrix.get("task_keys") + if not isinstance(matrix_task_keys, list) or not all( + isinstance(key, str) for key in matrix_task_keys + ): + raise ValueError("matrix task_keys are invalid") + selected_keys = args.task_key or matrix_task_keys + specs = manifest.select(selected_keys) + cf._configure_process(args.gpu_index, matrix_dir / "executions") + + records: list[dict[str, object]] = [] + repeatability: list[dict[str, object]] = [] + failures: list[dict[str, object]] = [] + for spec in specs: + try: + task_records, task_repeatability = _run_task( + matrix_dir, + matrix, + spec, + horizons, + libero_revision=manifest.libero_revision, + benchmark_id=manifest.benchmark_id, + ) + records.extend(task_records) + repeatability.append({"task_key": spec.key, **task_repeatability}) + except Exception as error: + failures.append( + { + "task_key": spec.key, + "status": "failed", + "error_type": type(error).__name__, + "error": str(error), + } + ) + _write_json( + matrix_dir / "execution-index.json", + { + "schema_version": "0.1", + "benchmark_id": manifest.benchmark_id, + "model_id": matrix.get("model_id"), + "source_matrix_sha256": _sha256_file(matrix_path), + "manifest_sha256": _sha256_file(manifest_path), + "horizons": list(horizons), + "expected_executions": sum(len(_task_records(matrix, task.key)) for task in specs), + "completed_executions": len(records), + "failed_tasks": failures, + "repeatability": repeatability, + "records": records, + }, + ) + if failures: + raise RuntimeError( + f"{len(failures)}/{len(specs)} prediction-execution tasks failed; " + f"inspect {matrix_dir / 'execution-index.json'}" + ) + print(f"wrote execution index: {matrix_dir / 'execution-index.json'}") + + +if __name__ == "__main__": + main() diff --git a/environments/starwam/run_libero_smoke.py b/environments/starwam/run_libero_smoke.py index cee8b3d..fe57552 100644 --- a/environments/starwam/run_libero_smoke.py +++ b/environments/starwam/run_libero_smoke.py @@ -20,6 +20,7 @@ import subprocess import sys import time +import uuid from dataclasses import dataclass from pathlib import Path from typing import Any @@ -94,6 +95,9 @@ ResolvedInference, ResolvedPreprocessing, ) +from wamprobe.libero_cf import LiberoTaskSpec, load_libero_cf_manifest # noqa: E402 + +LIBERO_CF_MANIFEST = REPOSITORY_ROOT / "configs" / "benchmarks" / "libero_cf_mini_v0.1.json" @dataclass(frozen=True, slots=True) @@ -257,26 +261,39 @@ def _load_init_states(path: Path) -> Any: return states -def _capture_observation() -> CapturedObservation: +def _capture_observation(spec: LiberoTaskSpec | None = None) -> CapturedObservation: import numpy as np from libero.libero import benchmark, get_libero_path from libero.libero.envs import OffScreenRenderEnv - suite = benchmark.get_benchmark_dict()[TASK_SUITE]() - task = suite.get_task(TASK_ID) - if task.language != TASK: - raise RuntimeError(f"pinned LIBERO task changed: expected {TASK!r}, got {task.language!r}") + task_suite = spec.task_suite if spec is not None else TASK_SUITE + task_id = spec.task_id if spec is not None else TASK_ID + task_text = spec.task if spec is not None else TASK + init_state_index = spec.init_state_index if spec is not None else INIT_STATE_INDEX + wait_steps = spec.wait_steps if spec is not None else WAIT_STEPS + seed = spec.seed if spec is not None else SEED + context_id = spec.context_id if spec is not None else CONTEXT_ID + bddl_sha256 = spec.bddl_sha256 if spec is not None else BDDL_SHA256 + init_states_sha256 = spec.init_states_sha256 if spec is not None else INIT_STATES_SHA256 + expected_shape = spec.init_states_shape if spec is not None else (50, 92) + + suite = benchmark.get_benchmark_dict()[task_suite]() + task = suite.get_task(task_id) + if task.language != task_text: + raise RuntimeError( + f"pinned LIBERO task changed: expected {task_text!r}, got {task.language!r}" + ) bddl_path = Path(suite.get_task_bddl_file_path(TASK_ID)) init_states_path = ( Path(get_libero_path("init_states")) / task.problem_folder / task.init_states_file ) - if _sha256(bddl_path) != BDDL_SHA256: + if _sha256(bddl_path) != bddl_sha256: raise RuntimeError("LIBERO BDDL hash mismatch") - if _sha256(init_states_path) != INIT_STATES_SHA256: + if _sha256(init_states_path) != init_states_sha256: raise RuntimeError("LIBERO init-state hash mismatch") init_states = _load_init_states(init_states_path) - if init_states.shape != (50, 92): + if init_states.shape != expected_shape: raise RuntimeError(f"unexpected LIBERO init-state shape: {init_states.shape}") environment = OffScreenRenderEnv( @@ -284,12 +301,12 @@ def _capture_observation() -> CapturedObservation: camera_heights=256, camera_widths=256, ) - environment.seed(SEED) + environment.seed(seed) done = False try: environment.reset() - observation = environment.set_init_state(init_states[INIT_STATE_INDEX]) - for _ in range(WAIT_STEPS): + observation = environment.set_init_state(init_states[init_state_index]) + for _ in range(wait_steps): observation, _, done, _ = environment.step(LIBERO_DUMMY_ACTION) finally: environment.close() @@ -307,8 +324,8 @@ def _capture_observation() -> CapturedObservation: raise RuntimeError(f"unexpected LIBERO proprio shape: {proprio.shape}") typed_observation = RobotObservation( - context_id=CONTEXT_ID, - task=TASK, + context_id=context_id, + task=task_text, frames=( RGBFrame("agentview", 256, 256, agentview.tobytes()), RGBFrame("wrist", 256, 256, wrist.tobytes()), @@ -339,26 +356,26 @@ def _resolved_config(run_dir: Path) -> Any: return config -def _text_cache_path(config: Any) -> Path: +def _text_cache_path(config: Any, task: str = TASK) -> Path: from starwam.data.lerobot import text_cache_path return text_cache_path( config.data.text_embedding_cache_dir, - TASK, + task, int(config.data.text_len), TEXT_PROMPT_TEMPLATE, str(config.data.text_cache_encoder_id), ) -def _load_text_cache(path: Path, config: Any) -> tuple[Any, Any]: +def _load_text_cache(path: Path, config: Any, task: str = TASK) -> tuple[Any, Any]: import torch payload = torch.load(path, map_location="cpu", weights_only=True) if not isinstance(payload, dict): raise TypeError("text cache must contain a mapping") - if payload.get("task") != TASK or payload.get("prompt") != TEXT_PROMPT_TEMPLATE.format( - task=TASK + if payload.get("task") != task or payload.get("prompt") != TEXT_PROMPT_TEMPLATE.format( + task=task ): raise RuntimeError("text cache task or prompt mismatch") context = payload.get("context") @@ -373,12 +390,12 @@ def _load_text_cache(path: Path, config: Any) -> tuple[Any, Any]: return context, mask -def _text_condition_sha256(context: Any, mask: Any) -> str: +def _text_condition_sha256(context: Any, mask: Any, task: str = TASK) -> str: import torch metadata = { - "task": TASK, - "prompt": TEXT_PROMPT_TEMPLATE.format(task=TASK), + "task": task, + "prompt": TEXT_PROMPT_TEMPLATE.format(task=task), "context_shape": list(context.shape), "context_dtype": str(context.dtype), "mask_shape": list(mask.shape), @@ -392,16 +409,25 @@ def _text_condition_sha256(context: Any, mask: Any) -> str: return digest.hexdigest() -def _prepare_text_cache(config: Any, minimum_free_gib: float) -> Path: +def _prepare_text_caches( + config: Any, + minimum_free_gib: float, + tasks: tuple[str, ...], +) -> tuple[Path, ...]: import torch from starwam.backbone.wan22 import Wan22TextEncoder from starwam.data.lerobot import save_text_cache - cache_path = _text_cache_path(config) - if cache_path.is_file(): - _load_text_cache(cache_path, config) - print(f"text cache already valid: {cache_path}") - return cache_path + cache_paths = tuple(_text_cache_path(config, task) for task in tasks) + missing = [ + (task, path) for task, path in zip(tasks, cache_paths, strict=True) if not path.is_file() + ] + for task, path in zip(tasks, cache_paths, strict=True): + if path.is_file(): + _load_text_cache(path, config, task) + print(f"text cache already valid: {path}") + if not missing: + return cache_paths device = _require_cuda_memory(minimum_free_gib) _set_seed(SEED) @@ -412,21 +438,34 @@ def _prepare_text_cache(config: Any, minimum_free_gib: float) -> Path: device=str(device), dtype=torch.bfloat16, ) - prompt = TEXT_PROMPT_TEMPLATE.format(task=TASK) try: - with torch.inference_mode(): - context, mask = encoder.encode([prompt]) - save_text_cache(cache_path, context[0], mask[0], prompt, TASK) - _load_text_cache(cache_path, config) + for task, cache_path in missing: + prompt = TEXT_PROMPT_TEMPLATE.format(task=task) + with torch.inference_mode(): + contexts, masks = encoder.encode([prompt]) + save_text_cache( + cache_path, + contexts[0], + masks[0], + prompt, + task, + ) + _load_text_cache(cache_path, config, task) + print(f"wrote text cache: {cache_path}") finally: del encoder gc.collect() torch.cuda.empty_cache() - print(f"wrote text cache: {cache_path}") - return cache_path + return cache_paths + + +def _prepare_text_cache(config: Any, minimum_free_gib: float) -> Path: + """Backward-compatible single-task text-cache helper.""" + + return _prepare_text_caches(config, minimum_free_gib, (TASK,))[0] -def _preprocessing(text_cache_sha256: str) -> ResolvedPreprocessing: +def _preprocessing(text_cache_sha256: str, task: str = TASK) -> ResolvedPreprocessing: return ResolvedPreprocessing( camera_order=CAMERA_ORDER, camera_transforms=("rotate_180", "rotate_180"), @@ -438,7 +477,7 @@ def _preprocessing(text_cache_sha256: str) -> ResolvedPreprocessing: tensor_layout="BCHW", camera_concatenation="horizontal", input_value_range=(-1.0, 1.0), - text_prompt=TEXT_PROMPT_TEMPLATE.format(task=TASK), + text_prompt=TEXT_PROMPT_TEMPLATE.format(task=task), text_cache_sha256=text_cache_sha256, proprio_fields=PROPRIO_FIELDS, action_normalization="minmax_then_libero_gripper_binary_inversion", @@ -479,10 +518,18 @@ def _write_observation( captured: CapturedObservation, preprocessing: ResolvedPreprocessing, run_dir: Path, + spec: LiberoTaskSpec | None = None, ) -> Path: import numpy as np from PIL import Image + task_suite = spec.task_suite if spec is not None else TASK_SUITE + task_id = spec.task_id if spec is not None else TASK_ID + init_state_index = spec.init_state_index if spec is not None else INIT_STATE_INDEX + wait_steps = spec.wait_steps if spec is not None else WAIT_STEPS + seed = spec.seed if spec is not None else SEED + bddl_sha256 = spec.bddl_sha256 if spec is not None else BDDL_SHA256 + init_states_sha256 = spec.init_states_sha256 if spec is not None else INIT_STATES_SHA256 output_dir = run_dir / "observations" / captured.observation.context_id output_dir.mkdir(parents=True, exist_ok=True) for frame in captured.observation.frames: @@ -496,16 +543,16 @@ def _write_observation( "preprocessing": preprocessing.to_dict(), "libero": { "revision": LIBERO_REVISION, - "task_suite": TASK_SUITE, - "task_id": TASK_ID, - "init_state_index": INIT_STATE_INDEX, - "wait_steps": WAIT_STEPS, - "seed": SEED, + "task_suite": task_suite, + "task_id": task_id, + "init_state_index": init_state_index, + "wait_steps": wait_steps, + "seed": seed, "done_during_wait": captured.done_during_wait, "bddl_path": str(captured.bddl_path), - "bddl_sha256": BDDL_SHA256, + "bddl_sha256": bddl_sha256, "init_states_path": str(captured.init_states_path), - "init_states_sha256": INIT_STATES_SHA256, + "init_states_sha256": init_states_sha256, "init_states_shape": list(captured.init_states_shape), "init_states_dtype": captured.init_states_dtype, }, @@ -774,15 +821,443 @@ def _run_inference( adapter.close() +def _provenance() -> ModelProvenance: + return ModelProvenance( + adapter_version="0.1.0a0", + upstream_code_revision=STARWAM_REVISION, + model_revision=MODEL_REVISION, + backbone_revision=BACKBONE_REVISION, + checkpoint_sha256=CHECKPOINT_SHA256, + vae_sha256=VAE_SHA256, + text_encoder_sha256=TEXT_ENCODER_SHA256, + ) + + +def _inference_configuration( + config: Any, + *, + num_inference_steps: int, + vae_device: str, +) -> ResolvedInference: + return ResolvedInference( + engine="starwam_mot_infer_action", + num_inference_steps=num_inference_steps, + scheduler="continuous_flow_match_euler", + scheduler_shift=float(config.framework.action_scheduler.infer_shift), + video_conditioning=str(config.framework.action_video_conditioning), + vae_device=vae_device, + sampled_video_frames=9, + ) + + +def _matrix_cache_key( + observation: RobotObservation, + preprocessing: ResolvedPreprocessing, + inference: ResolvedInference, + *, + seed: int, +) -> str: + payload = { + "schema_version": "0.1", + "observation": observation.descriptor(), + "model_id": StarWAMRelease().model_id, + "seed": seed, + "provenance": _provenance().to_dict(), + "preprocessing": preprocessing.to_dict(), + "inference": inference.to_dict(), + } + return hashlib.sha256( + json.dumps( + payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _cached_prediction(path: Path, cache_key: str) -> dict[str, object] | None: + if not path.is_file(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or payload.get("cache_key") != cache_key: + return None + prediction = payload["prediction"] + if not isinstance(prediction, dict): + return None + prediction_sha256 = payload.get("prediction_sha256") + if ( + prediction_sha256 + != hashlib.sha256( + json.dumps( + prediction, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + ): + return None + actions = prediction["actions"] + runtime = prediction["runtime"] + if not isinstance(actions, list) or len(actions) != 32: + return None + if not all( + isinstance(action, list) + and len(action) == 7 + and all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + for value in action + ) + for action in actions + ): + return None + if not isinstance(runtime, dict): + return None + return payload + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def _matrix_record( + *, + spec: LiberoTaskSpec, + seed: int, + num_inference_steps: int, + cache_key: str, + path: Path, + payload: dict[str, object], + status: str, +) -> dict[str, object]: + prediction = payload["prediction"] + if not isinstance(prediction, dict): + raise TypeError("prediction payload must be an object") + runtime = prediction["runtime"] + actions = prediction["actions"] + if not isinstance(runtime, dict) or not isinstance(actions, list): + raise TypeError("prediction runtime and actions have invalid types") + return { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "seed": seed, + "num_inference_steps": num_inference_steps, + "cache_key": cache_key, + "prediction_artifact": str(path), + "status": status, + "action_horizon": len(actions), + "action_dim": len(actions[0]) if actions else 0, + "latency_seconds": runtime.get("latency_seconds"), + "peak_allocated_bytes": runtime.get("peak_allocated_bytes"), + "peak_reserved_bytes": runtime.get("peak_reserved_bytes"), + } + + +def _write_matrix_index( + run_dir: Path, + *, + manifest_path: Path, + task_keys: list[str], + seeds: tuple[int, ...], + inference_steps: tuple[int, ...], + records: list[dict[str, object]], +) -> None: + path = run_dir / "matrix-index.json" + payload = { + "schema_version": "0.1", + "benchmark_id": "libero-cf-mini-v0.1", + "model_id": StarWAMRelease().model_id, + "manifest_uri": str(manifest_path), + "manifest_sha256": _sha256(manifest_path), + "task_keys": task_keys, + "seeds": list(seeds), + "num_inference_steps": list(inference_steps), + "expected_predictions": len(task_keys) * len(seeds) * len(inference_steps), + "completed_predictions": sum( + record.get("status") in {"generated", "cache-hit"} for record in records + ), + "failed_predictions": sum(record.get("status") == "failed" for record in records), + "records": records, + "capability_skips": [ + { + "ablation": "candidate-action-mask", + "status": "skipped", + "reason": ( + "released StarWAM adapter predicts actions from observation/task and " + "does not accept a candidate action as model input" + ), + }, + { + "ablation": "candidate-action-shuffle", + "status": "skipped", + "reason": ("released StarWAM adapter has no action-conditioned future endpoint"), + }, + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text( + json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _run_inference_matrix( + config: Any, + run_dir: Path, + *, + manifest_path: Path, + task_keys: list[str], + seeds: tuple[int, ...], + inference_steps: tuple[int, ...], + minimum_free_gib: float, + vae_device: str, + physical_gpu_index: int, +) -> Path: + import torch + + manifest = load_libero_cf_manifest(manifest_path) + if manifest.libero_revision != LIBERO_REVISION: + raise RuntimeError("LIBERO-CF manifest and StarWAM runner revisions differ") + specs = manifest.select(task_keys) + _prepare_text_caches( + config, + minimum_free_gib, + tuple(spec.task for spec in specs), + ) + + prepared: dict[ + str, + tuple[CapturedObservation, ResolvedPreprocessing, Any, Any], + ] = {} + for spec in specs: + text_path = _text_cache_path(config, spec.task) + context, context_mask = _load_text_cache(text_path, config, spec.task) + preprocessing = _preprocessing( + _text_condition_sha256(context, context_mask, spec.task), + spec.task, + ) + captured = _capture_observation(spec) + _write_observation(captured, preprocessing, run_dir, spec) + prepared[spec.key] = (captured, preprocessing, context, context_mask) + + records: list[dict[str, object]] = [] + jobs: list[ + tuple[ + LiberoTaskSpec, + CapturedObservation, + ResolvedPreprocessing, + Any, + Any, + int, + int, + str, + Path, + ] + ] = [] + for spec in specs: + captured, preprocessing, context, context_mask = prepared[spec.key] + for num_inference_steps in inference_steps: + inference = _inference_configuration( + config, + num_inference_steps=num_inference_steps, + vae_device=vae_device, + ) + for seed in seeds: + cache_key = _matrix_cache_key( + captured.observation, + preprocessing, + inference, + seed=seed, + ) + path = run_dir / "predictions" / f"{cache_key}.json" + cached = _cached_prediction(path, cache_key) + if cached is not None: + records.append( + _matrix_record( + spec=spec, + seed=seed, + num_inference_steps=num_inference_steps, + cache_key=cache_key, + path=path, + payload=cached, + status="cache-hit", + ) + ) + else: + jobs.append( + ( + spec, + captured, + preprocessing, + context, + context_mask, + num_inference_steps, + seed, + cache_key, + path, + ) + ) + _write_matrix_index( + run_dir, + manifest_path=manifest_path, + task_keys=[spec.key for spec in specs], + seeds=seeds, + inference_steps=inference_steps, + records=records, + ) + if not jobs: + return run_dir / "matrix-index.json" + + device = _require_cuda_memory(minimum_free_gib) + _set_seed(seeds[0]) + model = _build_model(config, device, vae_device) + first = jobs[0] + backend = LocalStarWAMBackend( + model=model, + config=config, + context=first[3].unsqueeze(0).to(device=device, dtype=torch.bfloat16), + context_mask=first[4].unsqueeze(0).to(device=device, dtype=torch.bool), + stats=_load_stats(), + preprocessing=first[2], + device=device, + runtime_device=( + f"cuda:0 (physical GPU {physical_gpu_index}, {torch.cuda.get_device_name(device)})" + ), + num_inference_steps=first[5], + ) + adapter = StarWAMAdapter(backend, release=StarWAMRelease()) + try: + active_task_key: str | None = None + for ( + spec, + captured, + preprocessing, + context, + context_mask, + num_inference_steps, + seed, + cache_key, + path, + ) in jobs: + try: + if active_task_key != spec.key: + backend.context = context.unsqueeze(0).to( + device=device, + dtype=torch.bfloat16, + ) + backend.context_mask = context_mask.unsqueeze(0).to( + device=device, + dtype=torch.bool, + ) + active_task_key = spec.key + backend.preprocessing = preprocessing + backend.num_inference_steps = num_inference_steps + prediction = adapter.predict_action(captured.observation, seed=seed) + artifact = ActionPredictionArtifact( + observation=captured.observation, + prediction=prediction, + provenance=_provenance(), + preprocessing=preprocessing, + inference=_inference_configuration( + config, + num_inference_steps=num_inference_steps, + vae_device=vae_device, + ), + ) + if artifact.cache_key != cache_key: + raise RuntimeError("precomputed and generated cache keys differ") + artifact.write_json(path) + payload = artifact.to_dict() + records.append( + _matrix_record( + spec=spec, + seed=seed, + num_inference_steps=num_inference_steps, + cache_key=cache_key, + path=path, + payload=payload, + status="generated", + ) + ) + except Exception as error: + records.append( + { + "task_key": spec.key, + "task_family": spec.task_family, + "context_id": spec.context_id, + "seed": seed, + "num_inference_steps": num_inference_steps, + "cache_key": cache_key, + "status": "failed", + "error_type": type(error).__name__, + "error": str(error), + } + ) + _write_matrix_index( + run_dir, + manifest_path=manifest_path, + task_keys=[task.key for task in specs], + seeds=seeds, + inference_steps=inference_steps, + records=records, + ) + finally: + adapter.close() + failures = sum(record.get("status") == "failed" for record in records) + if failures: + raise RuntimeError(f"{failures}/{len(records)} StarWAM matrix predictions failed") + return run_dir / "matrix-index.json" + + def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--mode", choices=("observe", "text-cache", "infer"), default="infer") + parser.add_argument( + "--mode", + choices=("observe", "text-cache", "infer", "matrix"), + default="infer", + ) parser.add_argument("--gpu-index", type=int, default=0, help="physical CUDA/EGL GPU") parser.add_argument( "--run-dir", type=Path, default=REPOSITORY_ROOT / "runs" / "starwam-libero-smoke" ) parser.add_argument("--minimum-free-gib", type=float, default=18.0) parser.add_argument("--num-inference-steps", type=int, default=1) + parser.add_argument( + "--manifest", + type=Path, + default=LIBERO_CF_MANIFEST, + help="pinned task manifest used by matrix mode", + ) + parser.add_argument( + "--task-key", + action="append", + default=[], + help="matrix task key; repeat to select several (default: all)", + ) + parser.add_argument( + "--matrix-seed", + action="append", + type=int, + default=[], + help="matrix diffusion seed; repeat to select several (default: 0,1,2)", + ) + parser.add_argument( + "--matrix-inference-steps", + action="append", + type=int, + default=[], + help="matrix NFE; repeat to select several (default: 1,4,8)", + ) parser.add_argument("--vae-device", choices=("cpu", "cuda"), default="cpu") parser.add_argument( "--verify-large-hashes", @@ -800,11 +1275,36 @@ def main() -> None: raise ValueError("minimum-free-gib must be positive") if args.num_inference_steps <= 0: raise ValueError("num-inference-steps must be positive") + matrix_seeds = tuple(args.matrix_seed or (0, 1, 2)) + matrix_inference_steps = tuple(args.matrix_inference_steps or (1, 4, 8)) + if any(seed < 0 for seed in matrix_seeds): + raise ValueError("matrix seeds must be non-negative") + if any(steps <= 0 for steps in matrix_inference_steps): + raise ValueError("matrix inference steps must be positive") + if len(matrix_seeds) != len(set(matrix_seeds)): + raise ValueError("matrix seeds must be unique") + if len(matrix_inference_steps) != len(set(matrix_inference_steps)): + raise ValueError("matrix inference steps must be unique") run_dir = args.run_dir.expanduser().resolve() _configure_process(args.gpu_index, run_dir) _verify_inputs(verify_large_hashes=args.verify_large_hashes) config = _resolved_config(run_dir) + if args.mode == "matrix": + index = _run_inference_matrix( + config, + run_dir, + manifest_path=args.manifest.expanduser().resolve(), + task_keys=args.task_key, + seeds=matrix_seeds, + inference_steps=matrix_inference_steps, + minimum_free_gib=args.minimum_free_gib, + vae_device="cuda:0" if args.vae_device == "cuda" else "cpu", + physical_gpu_index=args.gpu_index, + ) + print(f"wrote matrix index: {index}") + return + if args.mode == "text-cache": _prepare_text_cache(config, args.minimum_free_gib) return diff --git a/src/wamprobe/artifacts.py b/src/wamprobe/artifacts.py index 851b877..a94614a 100644 --- a/src/wamprobe/artifacts.py +++ b/src/wamprobe/artifacts.py @@ -19,6 +19,17 @@ def _validate_hex_digest(value: str, length: int, field_name: str) -> None: raise ValueError(f"{field_name} must be a lowercase {length}-character hex digest") +def _stable_sha256(value: object) -> str: + encoded = json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + @dataclass(frozen=True, slots=True) class ModelProvenance: """Immutable code and checkpoint identity for a prediction.""" @@ -229,6 +240,7 @@ def cache_key(self) -> str: def to_dict(self) -> dict[str, object]: """Return the complete JSON-compatible cache artifact.""" + prediction = self.prediction.to_dict() return { "schema_version": self.schema_version, "cache_key": self.cache_key, @@ -236,7 +248,8 @@ def to_dict(self) -> dict[str, object]: "observation": self.observation.descriptor(), "preprocessing": self.preprocessing.to_dict(), "inference": self.inference.to_dict(), - "prediction": self.prediction.to_dict(), + "prediction": prediction, + "prediction_sha256": _stable_sha256(prediction), } def write_json(self, path: Path) -> None: diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index 0ca0d87..d5c8fc3 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -44,6 +44,7 @@ ) from wamprobe.doctor import ModelManifestError, check_model_store, load_manifest from wamprobe.evaluation import EvaluationResult, evaluate +from wamprobe.experiments import analyze_action_experiment, write_action_experiment_report from wamprobe.manipulation_evaluation import evaluate_manipulation from wamprobe.reporting import write_reports from wamprobe.stats import paired_metric_comparison @@ -131,6 +132,14 @@ def _build_parser() -> argparse.ArgumentParser: help="validate checksums and schema for an intervention JSONL file", ) dataset_validate.add_argument("dataset", type=Path) + + experiment_report = subparsers.add_parser( + "experiment-report", + help="analyze a cached real-model prediction/execution matrix", + ) + experiment_report.add_argument("matrix", type=Path, help="matrix-index.json path") + experiment_report.add_argument("execution", type=Path, help="execution-index.json path") + experiment_report.add_argument("--output", type=Path, required=True) return parser @@ -396,6 +405,18 @@ def _run_dataset_validate(args: argparse.Namespace) -> int: return 0 +def _run_experiment_report(args: argparse.Namespace) -> int: + try: + analysis = analyze_action_experiment(args.matrix, args.execution) + json_path, markdown_path = write_action_experiment_report(args.output, analysis) + except (OSError, ValueError) as error: + print(f"WAMProbe experiment report error: {error}", file=sys.stderr) + return 2 + print(f"Experiment JSON: {json_path}") + print(f"Experiment Markdown: {markdown_path}") + return 0 + + def _run_doctor(args: argparse.Namespace) -> int: try: manifest = load_manifest(args.manifest) @@ -443,5 +464,7 @@ def main(argv: Sequence[str] | None = None) -> int: return _run_dataset_export(args) if args.command == "dataset-validate": return _run_dataset_validate(args) + if args.command == "experiment-report": + return _run_experiment_report(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/experiments.py b/src/wamprobe/experiments.py new file mode 100644 index 0000000..2fb346c --- /dev/null +++ b/src/wamprobe/experiments.py @@ -0,0 +1,585 @@ +"""Analysis and reporting for cached real-model action execution matrices.""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import asdict, dataclass +from itertools import combinations +from pathlib import Path +from typing import cast + + +def _mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{field} must be an object") + return cast(dict[str, object], value) + + +def _list(value: object, field: str) -> list[object]: + if not isinstance(value, list): + raise ValueError(f"{field} must be an array") + return cast(list[object], value) + + +def _string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _integer(value: object, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{field} must be an integer") + return value + + +def _number(value: object, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field} must be a finite number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field} must be a finite number") + return result + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(8 * 1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def _read_json(path: Path, field: str) -> dict[str, object]: + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"could not read {field}: {error}") from error + return _mapping(raw, field) + + +@dataclass(frozen=True, slots=True) +class ActionExperimentRecord: + """One predicted action chunk and its simulator prefix outcomes.""" + + task_key: str + seed: int + num_inference_steps: int + cache_key: str + actions: tuple[tuple[float, ...], ...] + latency_seconds: float + peak_allocated_bytes: int + initial_eef_position: tuple[float, float, float] + final_eef_positions: dict[int, tuple[float, float, float]] + cumulative_returns: dict[int, float] + successes: dict[int, bool] + + @property + def action_rms(self) -> float: + """Root-mean-square command magnitude over the complete chunk.""" + + values = [value for action in self.actions for value in action] + return math.sqrt(sum(value * value for value in values) / len(values)) + + @property + def action_smoothness(self) -> float: + """Mean L2 command change between consecutive steps; lower is smoother.""" + + changes = [ + math.dist(left, right) + for left, right in zip(self.actions, self.actions[1:], strict=False) + ] + return sum(changes) / len(changes) + + +@dataclass(frozen=True, slots=True) +class EfficiencySummary: + """Runtime, memory, and command statistics for one NFE setting.""" + + num_inference_steps: int + samples: int + mean_latency_seconds: float + mean_peak_allocated_gib: float + mean_action_rms: float + mean_action_smoothness: float + + +@dataclass(frozen=True, slots=True) +class HorizonSummary: + """Real simulator outcomes at one executed action prefix.""" + + horizon: int + samples: int + success_rate: float + mean_cumulative_return: float + mean_eef_displacement: float + + +@dataclass(frozen=True, slots=True) +class SensitivitySummary: + """Paired output variability along one experimental axis.""" + + axis: str + groups: int + pairs: int + mean_action_rms_difference: float + mean_final_eef_distance: dict[int, float] + + +@dataclass(frozen=True, slots=True) +class ActionExperimentAnalysis: + """Complete metric profile for a StarWAM inference/execution matrix.""" + + model_id: str + benchmark_id: str + predictions: int + failed_predictions: int + executions: int + failed_execution_tasks: int + efficiency: tuple[EfficiencySummary, ...] + horizons: tuple[HorizonSummary, ...] + seed_sensitivity: SensitivitySummary + nfe_sensitivity: SensitivitySummary + correlations: dict[str, float | None] + skipped_analyses: tuple[dict[str, str], ...] + matrix_sha256: str + execution_sha256: str + schema_version: str = "0.1" + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible experiment report.""" + + return { + "schema_version": self.schema_version, + "model_id": self.model_id, + "benchmark_id": self.benchmark_id, + "predictions": self.predictions, + "failed_predictions": self.failed_predictions, + "prediction_failure_rate": self.failed_predictions + / max(self.predictions + self.failed_predictions, 1), + "executions": self.executions, + "failed_execution_tasks": self.failed_execution_tasks, + "efficiency": [asdict(summary) for summary in self.efficiency], + "horizons": [asdict(summary) for summary in self.horizons], + "seed_sensitivity": asdict(self.seed_sensitivity), + "nfe_sensitivity": asdict(self.nfe_sensitivity), + "correlations": self.correlations, + "skipped_analyses": list(self.skipped_analyses), + "matrix_sha256": self.matrix_sha256, + "execution_sha256": self.execution_sha256, + } + + +def _action_chunk(value: object) -> tuple[tuple[float, ...], ...]: + actions: list[tuple[float, ...]] = [] + for action_index, raw_action in enumerate(_list(value, "prediction.actions")): + action = tuple( + _number(item, f"prediction.actions[{action_index}]") + for item in _list(raw_action, f"prediction.actions[{action_index}]") + ) + if not action: + raise ValueError("prediction actions must not be empty") + actions.append(action) + if len(actions) < 2 or len({len(action) for action in actions}) != 1: + raise ValueError("prediction action chunk must have a stable non-empty shape") + return tuple(actions) + + +def _vec3(value: object, field: str) -> tuple[float, float, float]: + items = _list(value, field) + if len(items) != 3: + raise ValueError(f"{field} must contain three coordinates") + return ( + _number(items[0], field), + _number(items[1], field), + _number(items[2], field), + ) + + +def load_action_experiment( + matrix_path: Path, + execution_path: Path, +) -> tuple[dict[str, object], dict[str, object], tuple[ActionExperimentRecord, ...]]: + """Strictly align cached predictions and executed outcomes by cache key.""" + + matrix = _read_json(matrix_path, "matrix index") + execution = _read_json(execution_path, "execution index") + matrix_records = _list(matrix.get("records"), "matrix records") + execution_records = _list(execution.get("records"), "execution records") + execution_by_key: dict[str, dict[str, object]] = {} + for index, raw_record in enumerate(execution_records): + record = _mapping(raw_record, f"execution records[{index}]") + key = _string(record.get("cache_key"), "execution cache_key") + if key in execution_by_key: + raise ValueError(f"duplicate execution cache_key: {key}") + execution_by_key[key] = record + + records: list[ActionExperimentRecord] = [] + for index, raw_matrix_record in enumerate(matrix_records): + matrix_record = _mapping(raw_matrix_record, f"matrix records[{index}]") + status = _string(matrix_record.get("status"), "matrix status") + if status not in {"generated", "cache-hit"}: + continue + cache_key = _string(matrix_record.get("cache_key"), "matrix cache_key") + execution_record = execution_by_key.pop(cache_key, None) + if execution_record is None: + raise ValueError(f"missing execution for cache_key: {cache_key}") + prediction_path = matrix_path.parent / "predictions" / f"{cache_key}.json" + prediction_artifact = _read_json(prediction_path, "prediction artifact") + if prediction_artifact.get("cache_key") != cache_key: + raise ValueError("prediction artifact cache_key mismatch") + prediction = _mapping(prediction_artifact.get("prediction"), "prediction") + runtime = _mapping(prediction.get("runtime"), "prediction.runtime") + outcome = _mapping(execution_record.get("outcome"), "execution outcome") + checkpoints = _mapping(outcome.get("checkpoints"), "outcome.checkpoints") + final_positions: dict[int, tuple[float, float, float]] = {} + returns: dict[int, float] = {} + successes: dict[int, bool] = {} + for raw_horizon, raw_checkpoint in checkpoints.items(): + try: + horizon = int(raw_horizon) + except ValueError as error: + raise ValueError("checkpoint horizon must be an integer string") from error + checkpoint = _mapping(raw_checkpoint, f"checkpoint {horizon}") + final_positions[horizon] = _vec3( + checkpoint.get("eef_position"), + f"checkpoint {horizon} eef_position", + ) + returns[horizon] = _number( + checkpoint.get("cumulative_return"), + f"checkpoint {horizon} cumulative_return", + ) + success = checkpoint.get("success") + if not isinstance(success, bool): + raise ValueError(f"checkpoint {horizon} success must be boolean") + successes[horizon] = success + records.append( + ActionExperimentRecord( + task_key=_string(matrix_record.get("task_key"), "task_key"), + seed=_integer(matrix_record.get("seed"), "seed"), + num_inference_steps=_integer( + matrix_record.get("num_inference_steps"), + "num_inference_steps", + ), + cache_key=cache_key, + actions=_action_chunk(prediction.get("actions")), + latency_seconds=_number( + runtime.get("latency_seconds"), + "runtime.latency_seconds", + ), + peak_allocated_bytes=_integer( + runtime.get("peak_allocated_bytes"), + "runtime.peak_allocated_bytes", + ), + initial_eef_position=_vec3( + outcome.get("initial_eef_position"), + "initial_eef_position", + ), + final_eef_positions=final_positions, + cumulative_returns=returns, + successes=successes, + ) + ) + if execution_by_key: + raise ValueError("execution index contains outcomes without matrix predictions") + if not records: + raise ValueError("experiment contains no aligned completed records") + configuration_keys = [ + (record.task_key, record.seed, record.num_inference_steps) for record in records + ] + if len(configuration_keys) != len(set(configuration_keys)): + raise ValueError("experiment task/seed/NFE configurations must be unique") + return matrix, execution, tuple(records) + + +def _mean(values: list[float]) -> float: + if not values: + raise ValueError("cannot average an empty collection") + return sum(values) / len(values) + + +def _pair_action_rms( + left: ActionExperimentRecord, + right: ActionExperimentRecord, +) -> float: + if len(left.actions) != len(right.actions) or any( + len(left_action) != len(right_action) + for left_action, right_action in zip(left.actions, right.actions, strict=True) + ): + raise ValueError("paired action chunks must have equal shapes") + differences = [ + left_value - right_value + for left_action, right_action in zip(left.actions, right.actions, strict=True) + for left_value, right_value in zip(left_action, right_action, strict=True) + ] + return math.sqrt(sum(value * value for value in differences) / len(differences)) + + +def _sensitivity( + records: tuple[ActionExperimentRecord, ...], + *, + axis: str, + group_key: str, +) -> SensitivitySummary: + grouped: dict[tuple[str, int], list[ActionExperimentRecord]] = {} + for record in records: + varying_value = record.num_inference_steps if group_key == "nfe" else record.seed + grouped.setdefault((record.task_key, varying_value), []).append(record) + action_differences: list[float] = [] + endpoint_differences: dict[int, list[float]] = {} + pair_count = 0 + for values in grouped.values(): + for left, right in combinations(values, 2): + pair_count += 1 + action_differences.append(_pair_action_rms(left, right)) + if left.final_eef_positions.keys() != right.final_eef_positions.keys(): + raise ValueError("paired execution horizons must match") + for horizon in left.final_eef_positions: + endpoint_differences.setdefault(horizon, []).append( + math.dist( + left.final_eef_positions[horizon], + right.final_eef_positions[horizon], + ) + ) + return SensitivitySummary( + axis=axis, + groups=len(grouped), + pairs=pair_count, + mean_action_rms_difference=_mean(action_differences), + mean_final_eef_distance={ + horizon: _mean(values) for horizon, values in sorted(endpoint_differences.items()) + }, + ) + + +def _pearson(left: list[float], right: list[float]) -> float | None: + if len(left) != len(right) or len(left) < 2: + raise ValueError("correlation inputs must have matching lengths of at least two") + left_mean = _mean(left) + right_mean = _mean(right) + left_delta = [value - left_mean for value in left] + right_delta = [value - right_mean for value in right] + denominator = math.sqrt( + sum(value * value for value in left_delta) * sum(value * value for value in right_delta) + ) + if denominator <= 1e-15: + return None + return ( + sum( + left_value * right_value + for left_value, right_value in zip(left_delta, right_delta, strict=True) + ) + / denominator + ) + + +def analyze_action_experiment( + matrix_path: Path, + execution_path: Path, +) -> ActionExperimentAnalysis: + """Compute efficiency, stability, and real control-outcome profiles.""" + + matrix, execution, records = load_action_experiment(matrix_path, execution_path) + nfe_values = sorted({record.num_inference_steps for record in records}) + efficiency = tuple( + EfficiencySummary( + num_inference_steps=nfe, + samples=len(selected), + mean_latency_seconds=_mean([record.latency_seconds for record in selected]), + mean_peak_allocated_gib=_mean( + [record.peak_allocated_bytes / 1024**3 for record in selected] + ), + mean_action_rms=_mean([record.action_rms for record in selected]), + mean_action_smoothness=_mean([record.action_smoothness for record in selected]), + ) + for nfe in nfe_values + if (selected := [record for record in records if record.num_inference_steps == nfe]) + ) + horizon_values = sorted(records[0].final_eef_positions) + if any(sorted(record.final_eef_positions) != horizon_values for record in records): + raise ValueError("all execution records must share horizon checkpoints") + horizons = tuple( + HorizonSummary( + horizon=horizon, + samples=len(records), + success_rate=_mean([float(record.successes[horizon]) for record in records]), + mean_cumulative_return=_mean( + [record.cumulative_returns[horizon] for record in records] + ), + mean_eef_displacement=_mean( + [ + math.dist( + record.initial_eef_position, + record.final_eef_positions[horizon], + ) + for record in records + ] + ), + ) + for horizon in horizon_values + ) + correlations = { + "nfe_vs_latency_pearson": _pearson( + [float(record.num_inference_steps) for record in records], + [record.latency_seconds for record in records], + ), + "nfe_vs_success_pearson": _pearson( + [float(record.num_inference_steps) for record in records], + [float(record.successes[horizon_values[-1]]) for record in records], + ), + } + for horizon in horizon_values: + displacements = [ + math.dist( + record.initial_eef_position, + record.final_eef_positions[horizon], + ) + for record in records + ] + correlations[f"action_rms_vs_eef_displacement_h{horizon}_pearson"] = _pearson( + [record.action_rms for record in records], + displacements, + ) + correlations[f"action_smoothness_vs_eef_displacement_h{horizon}_pearson"] = _pearson( + [record.action_smoothness for record in records], + displacements, + ) + raw_skips = _list(matrix.get("capability_skips", []), "capability_skips") + skipped = tuple( + { + "analysis": _string( + _mapping(item, "capability skip").get("ablation"), + "capability skip ablation", + ), + "reason": _string( + _mapping(item, "capability skip").get("reason"), + "capability skip reason", + ), + } + for item in raw_skips + ) + ( + { + "analysis": "predicted-video-quality-vs-control", + "reason": ( + "the released adapter exposes actions but no predicted video or future-state " + "artifact, so PSNR/SSIM/FVD would be fabricated" + ), + }, + { + "analysis": "success-correlation", + "reason": ( + "all short-horizon fixed-context executions have zero sparse success and " + "return, so the correlation is undefined" + ), + }, + ) + failed_execution_tasks = len(_list(execution.get("failed_tasks", []), "failed execution tasks")) + return ActionExperimentAnalysis( + model_id=_string(matrix.get("model_id"), "model_id"), + benchmark_id=_string(matrix.get("benchmark_id"), "benchmark_id"), + predictions=len(records), + failed_predictions=_integer( + matrix.get("failed_predictions", 0), + "failed_predictions", + ), + executions=len(records), + failed_execution_tasks=failed_execution_tasks, + efficiency=efficiency, + horizons=horizons, + seed_sensitivity=_sensitivity(records, axis="seed", group_key="nfe"), + nfe_sensitivity=_sensitivity(records, axis="nfe", group_key="seed"), + correlations=correlations, + skipped_analyses=skipped, + matrix_sha256=_sha256_file(matrix_path), + execution_sha256=_sha256_file(execution_path), + ) + + +def render_action_experiment_markdown(analysis: ActionExperimentAnalysis) -> str: + """Render a compact human-readable experiment report.""" + + lines = [ + "# WAMProbe StarWAM action experiment", + "", + f"- Model: `{analysis.model_id}`", + f"- Benchmark: `{analysis.benchmark_id}`", + f"- Predictions/executions: {analysis.predictions}/{analysis.executions}", + f"- Failed predictions/tasks: {analysis.failed_predictions}/" + f"{analysis.failed_execution_tasks}", + "", + "## Efficiency by NFE", + "", + "| NFE | Samples | Mean latency (s) | Peak GiB | Action RMS | Step change |", + "|---:|---:|---:|---:|---:|---:|", + ] + for efficiency_summary in analysis.efficiency: + lines.append( + f"| {efficiency_summary.num_inference_steps} | " + f"{efficiency_summary.samples} | " + f"{efficiency_summary.mean_latency_seconds:.4f} | " + f"{efficiency_summary.mean_peak_allocated_gib:.3f} | " + f"{efficiency_summary.mean_action_rms:.4f} | " + f"{efficiency_summary.mean_action_smoothness:.4f} |" + ) + lines.extend( + [ + "", + "## Simulator outcomes", + "", + "| Horizon | Samples | Success | Mean return | Mean EEF displacement |", + "|---:|---:|---:|---:|---:|", + ] + ) + for horizon_summary in analysis.horizons: + lines.append( + f"| {horizon_summary.horizon} | {horizon_summary.samples} | " + f"{horizon_summary.success_rate:.4f} | " + f"{horizon_summary.mean_cumulative_return:.4f} | " + f"{horizon_summary.mean_eef_displacement:.4f} |" + ) + lines.extend( + [ + "", + "## Sensitivity", + "", + f"- Seed action RMS difference: " + f"{analysis.seed_sensitivity.mean_action_rms_difference:.6f}", + f"- NFE action RMS difference: " + f"{analysis.nfe_sensitivity.mean_action_rms_difference:.6f}", + "", + "## Capability-gated and undefined analyses", + "", + ] + ) + lines.extend(f"- `{item['analysis']}`: {item['reason']}" for item in analysis.skipped_analyses) + lines.extend( + [ + "", + "Zero sparse success is reported as a negative fixed-context result. It is not a", + "LIBERO policy success-rate estimate and is not replaced by a composite score.", + "", + ] + ) + return "\n".join(lines) + + +def write_action_experiment_report( + output_dir: Path, + analysis: ActionExperimentAnalysis, +) -> tuple[Path, Path]: + """Write deterministic JSON and Markdown action-experiment reports.""" + + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "experiment.json" + markdown_path = output_dir / "experiment.md" + json_path.write_text( + json.dumps(analysis.to_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + markdown_path.write_text( + render_action_experiment_markdown(analysis), + encoding="utf-8", + ) + return json_path, markdown_path diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index e56450a..39b4564 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -101,6 +101,7 @@ def test_action_artifact_writes_atomic_json_without_raw_rgb(tmp_path: Path) -> N assert payload["provenance"]["vae_sha256"] == "2" * 64 assert payload["inference"]["num_inference_steps"] == 1 assert payload["prediction"]["actions"][1] == [0.1] * 7 + assert len(payload["prediction_sha256"]) == 64 def test_action_artifact_rejects_mismatched_context() -> None: diff --git a/tests/test_experiments.py b/tests/test_experiments.py new file mode 100644 index 0000000..109c0d5 --- /dev/null +++ b/tests/test_experiments.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path + +import pytest + +from wamprobe.cli import main +from wamprobe.experiments import analyze_action_experiment + + +def _write_fixture(tmp_path: Path) -> tuple[Path, Path]: + predictions = tmp_path / "predictions" + predictions.mkdir() + matrix_records: list[dict[str, object]] = [] + execution_records: list[dict[str, object]] = [] + for nfe in (1, 4): + for seed in (0, 1): + cache_key = f"{nfe}{seed}".ljust(64, "a") + action_value = 0.01 * nfe + 0.02 * seed + actions = [ + [action_value, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0], + [action_value * 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0], + ] + (predictions / f"{cache_key}.json").write_text( + json.dumps( + { + "cache_key": cache_key, + "prediction": { + "actions": actions, + "runtime": { + "latency_seconds": float(nfe) / 10.0 + seed / 100.0, + "peak_allocated_bytes": nfe * 1024**3, + }, + }, + } + ), + encoding="utf-8", + ) + matrix_records.append( + { + "task_key": "task-a", + "seed": seed, + "num_inference_steps": nfe, + "cache_key": cache_key, + "status": "generated", + } + ) + displacement = action_value * nfe + checkpoints = { + "1": { + "eef_position": [displacement, 0.0, 0.0], + "cumulative_return": 0.0, + "success": False, + }, + "2": { + "eef_position": [displacement * 2.0, 0.0, 0.0], + "cumulative_return": float(seed == 1 and nfe == 4), + "success": seed == 1 and nfe == 4, + }, + } + execution_records.append( + { + "cache_key": cache_key, + "outcome": { + "initial_eef_position": [0.0, 0.0, 0.0], + "checkpoints": checkpoints, + }, + } + ) + matrix_path = tmp_path / "matrix-index.json" + matrix_path.write_text( + json.dumps( + { + "model_id": "model-a", + "benchmark_id": "benchmark-a", + "failed_predictions": 0, + "capability_skips": [ + {"ablation": "action-mask", "reason": "unsupported capability"} + ], + "records": matrix_records, + } + ), + encoding="utf-8", + ) + execution_path = tmp_path / "execution-index.json" + execution_path.write_text( + json.dumps({"failed_tasks": [], "records": execution_records}), + encoding="utf-8", + ) + return matrix_path, execution_path + + +def test_action_experiment_aligns_matrix_and_reports_sensitivity(tmp_path: Path) -> None: + matrix, execution = _write_fixture(tmp_path) + + analysis = analyze_action_experiment(matrix, execution) + + assert analysis.predictions == 4 + assert [summary.num_inference_steps for summary in analysis.efficiency] == [1, 4] + assert analysis.efficiency[0].mean_latency_seconds == pytest.approx(0.105) + assert analysis.horizons[-1].success_rate == pytest.approx(0.25) + assert analysis.seed_sensitivity.groups == 2 + assert analysis.seed_sensitivity.pairs == 2 + assert analysis.nfe_sensitivity.groups == 2 + assert analysis.nfe_sensitivity.pairs == 2 + assert analysis.correlations["nfe_vs_latency_pearson"] is not None + assert len(analysis.skipped_analyses) == 3 + + +def test_experiment_report_cli_writes_json_and_markdown(tmp_path: Path) -> None: + matrix, execution = _write_fixture(tmp_path) + output = tmp_path / "report" + + assert ( + main( + [ + "experiment-report", + str(matrix), + str(execution), + "--output", + str(output), + ] + ) + == 0 + ) + payload = json.loads((output / "experiment.json").read_text(encoding="utf-8")) + assert payload["predictions"] == 4 + assert "Capability-gated" in (output / "experiment.md").read_text(encoding="utf-8") From 7ccd5d1ccfd83506879ff5e5faab3221f5cbd36c Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 14:09:17 +0000 Subject: [PATCH 14/20] add video fidelity control counterexample --- README.md | 18 +- docs/QUICKSTART.md | 20 ++ docs/WAMProbe_PLAN.md | 9 +- docs/metrics/CORE_METRICS.md | 10 + .../video-control-study.json | 152 ++++++++ .../video-control-study.md | 32 ++ src/wamprobe/cli.py | 33 ++ src/wamprobe/video_control_study.py | 330 ++++++++++++++++++ src/wamprobe/video_metrics.py | 72 ++++ tests/test_video_control_study.py | 58 +++ 10 files changed, 728 insertions(+), 6 deletions(-) create mode 100644 examples/video-control-study/video-control-study.json create mode 100644 examples/video-control-study/video-control-study.md create mode 100644 src/wamprobe/video_control_study.py create mode 100644 src/wamprobe/video_metrics.py create mode 100644 tests/test_video_control_study.py diff --git a/README.md b/README.md index 78576b0..253c131 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ wamprobe report runs/pointmass-demo --output runs/rebuilt-report # Contact and attachment diagnostics use the same report pipeline. wamprobe demo --benchmark blockpush --horizon 6 --output runs/blockpush-demo wamprobe demo --benchmark gripper-catch --horizon 5 --output runs/gripper-catch-demo + +# Contrast rendered-video fidelity with control-grounded metrics. +wamprobe video-control-study --contexts 12 --seed 7 \ + --output runs/video-control-study ``` The command creates: @@ -76,7 +80,10 @@ runs/pointmass-demo/ See the committed [PointMass](examples/pointmass-demo/report.md), [BlockPush](examples/blockpush-demo/report.md), and [Gripper-Catch](examples/gripper-catch-demo/report.md) reports for the expected baseline -profiles. +profiles. The committed +[video/control counterexample](examples/video-control-study/video-control-study.md) shows +that an appearance-corrupted oracle can keep exact state predictions and zero regret while +receiving very low PSNR and global SSIM. Real-model weights are never committed to Git. Before running the StarWAM integration, follow the [model-store layout and download rules](checkpoints/README.md); the first spike requires @@ -115,16 +122,17 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo pinned StarWAM/LIBERO runner with multi-seed/NFE action execution reports; - dependency-free robot action-branch/future contracts plus four task families × four branches × eight steps in LIBERO-CF-Mini, with exact restore and branch-order validation; +- dependency-free RGB PSNR/global SSIM diagnostics and a two-benchmark counterexample study + that keeps traditional video fidelity separate from state accuracy and control value; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap The next milestones are: -1. add a traditional video-quality versus control-value baseline study; -2. expand fixed states and add a minimal replanning/closed-loop experiment; -3. add the Occluded-Object memory diagnostic to the broader Toy tier; -4. prepare a signed v0.1 release candidate and external reproduction smoke. +1. expand fixed states and add a minimal replanning/closed-loop experiment; +2. add the Occluded-Object memory diagnostic to the broader Toy tier; +3. prepare a traceable v0.1 release candidate and external reproduction smoke. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 8219030..33c1cd4 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -72,6 +72,26 @@ WAMProbe intentionally reports a metric profile, not a composite score. A model action-dependent but physically wrong, or visually accurate but useless for candidate selection. +## Compare video fidelity with control value + +Run the deterministic counterexample study on both rendered manipulation benchmarks: + +```bash +wamprobe video-control-study \ + --benchmark all \ + --contexts 12 \ + --seed 7 \ + --output runs/video-control-study +``` + +The command writes `video-control-study.json` and `video-control-study.md`. It reports +PSNR and an explicitly labeled whole-frame global SSIM next to state FDE, Candidate +Ranking Correlation, and Top-1 Regret. The `appearance-corrupted-oracle` deliberately +inverts rendered RGB while preserving exact dynamics: it therefore has FDE `0`, CRC `1`, +and regret `0`, despite PSNR below `1 dB` in the committed run. This is a controlled metric +counterexample, not a claim that visual fidelity is unimportant. See the +[committed report](../examples/video-control-study/video-control-study.md). + Uncertainty is computed at the context level. Each bootstrap draw resamples whole shared initial states, never individual action branches or adjacent frames. Paired comparisons align exact context IDs and report the left-minus-right difference with a 95% interval. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index ad99962..e3b07cb 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1114,6 +1114,13 @@ snapshot 执行全部动作块，并在 horizon 8/16/32 记录状态、回报和 ### Phase 5：控制价值与 v0.1 发布（第 14–16 周） +当前进度（2026-07-15）：已在 BlockPush-2D 与 Gripper-Catch 上运行 12-context 的传统 +视频指标/控制价值反例研究。`appearance-corrupted-oracle` 保持 FDE=0、CRC=1、Regret=0， +但 PSNR 约 0.59 dB；PSNR 与 regret 的跨 profile Pearson 约为 -0.16，且两个 benchmark +分别出现 3/9 和 5/9 个可比较排序冲突。结果和限制已固化在 +[`examples/video-control-study/`](../examples/video-control-study/video-control-study.md)。 +最小 replanning/closed-loop、release artifact 追溯、技术报告和外部复现仍待完成。 + 任务： - candidate ranking 和最小 closed-loop utility； @@ -1293,7 +1300,7 @@ priority:p1 - [x] action shuffle/permutation 产生预期指标下降； - [x] 噪声增加时准确性指标总体退化； - [x] 至少一个 causal/ranking 指标与 simulator return 相关； -- [ ] 报告传统视频指标与控制指标的差异； +- [x] 报告传统视频指标与控制指标的差异； - [x] 公开所有主要失败率和 skipped metrics。 ### 工程质量 diff --git a/docs/metrics/CORE_METRICS.md b/docs/metrics/CORE_METRICS.md index 4a8963e..07f3981 100644 --- a/docs/metrics/CORE_METRICS.md +++ b/docs/metrics/CORE_METRICS.md @@ -18,6 +18,8 @@ are never treated as independent samples. | CRC NDCG | candidate returns | discounted gain after shifting relevance non-negative | higher, [0,1] | shift convention must stay fixed | | CRC pairwise accuracy | candidate returns | correct preference fraction; model ties receive 0.5 | higher, [0,1] | true-return ties are excluded | | Top-1 Regret | candidate returns | oracle return minus return of model-selected candidate | lower, ≥0 | depends on candidate-set coverage | +| RGB PSNR | same-shape rendered RGB frames | `10 log10(255²/MSE)`, averaged over branches and frames | higher, dB; exact matches capped at 100 dB | pixel alignment and appearance can disagree with dynamics/control value | +| Global SSIM diagnostic | same-camera rendered RGB frames | mean per-channel luminance/contrast/structure over each whole frame | higher, typically [-1,1] | not standard windowed SSIM and can hide local/contact errors | | Latency | timed model call | synchronized wall time per prediction | lower, seconds | hardware and warm-up dependent | | Peak allocated/reserved memory | CUDA runtime | maximum allocator bytes during one prediction | lower, GiB | not total board memory or energy | | NFE sensitivity | matched inputs/seeds | paired action RMS and executed endpoint distance across NFE | lower means more stable | stability does not establish correctness | @@ -36,12 +38,20 @@ An undefined correlation caused by a constant outcome is serialized as JSON `nul explained in the report. A zero sparse success rate remains a reported negative result; it is not dropped from aggregation. +PSNR and global SSIM are secondary diagnostics only. They require pixel futures in the +same camera geometry and are never substituted for a missing state, causal, ranking, or +regret metric. The built-in global variant is deliberately named `global_ssim`; reports +must not label it as standard windowed SSIM. Exact PSNR matches use a finite 100 dB cap so +JSON remains portable. + ## Reference-baseline expectations - Oracle dynamics should minimize ADE/FDE and regret while maximizing direction and CRC. - Copy-last and action-agnostic baselines should have weak action dependence and ranking. - Wrong-direction should separate futures but have negative direction correctness. - Seeded noisy dynamics should degrade ADE/FDE and ranking as noise rises. +- Appearance-corrupted oracle futures should preserve exact FDE/CRC/regret while degrading + PSNR/global SSIM, demonstrating that video fidelity cannot replace control grounding. Any new metric must add a failure-mode statement, capability requirement, reference ordering test, aggregation unit and range/direction before it enters a public report. diff --git a/examples/video-control-study/video-control-study.json b/examples/video-control-study/video-control-study.json new file mode 100644 index 0000000..e73a182 --- /dev/null +++ b/examples/video-control-study/video-control-study.json @@ -0,0 +1,152 @@ +{ + "benchmarks": [ + { + "benchmark_id": "blockpush-2d-v0.1", + "comparable_profile_pairs": 9, + "correlations": { + "global_ssim_vs_top1_regret_pearson": 0.10930884736089089, + "psnr_vs_crc_spearman_pearson": 0.15607102279384893, + "psnr_vs_top1_regret_pearson": -0.15802600519981216 + }, + "profiles": [ + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": 1.0, + "mean_psnr_db": 100.0, + "model_id": "oracle-simulator", + "samples": 360, + "state_fde": 0.0, + "top1_regret": 0.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": -0.07902680360133017, + "mean_psnr_db": 0.5881582618368385, + "model_id": "appearance-corrupted-oracle", + "samples": 360, + "state_fde": 0.0, + "top1_regret": 0.0, + "video_transform": "invert-rgb" + }, + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": 0.9058261799885446, + "mean_psnr_db": 39.954938832526466, + "model_id": "noisy-dynamics", + "samples": 360, + "state_fde": 0.03553171008466385, + "top1_regret": 0.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 0.0, + "mean_global_ssim": 0.7489924374301328, + "mean_psnr_db": 38.45470238262374, + "model_id": "copy-last-frame", + "samples": 360, + "state_fde": 0.4368710030764564, + "top1_regret": 0.3, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": -0.25, + "mean_global_ssim": 0.6928435422253573, + "mean_psnr_db": 37.640080722934805, + "model_id": "wrong-direction", + "samples": 360, + "state_fde": 0.7952908435996061, + "top1_regret": 0.3, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 0.0, + "mean_global_ssim": 0.6140902752012081, + "mean_psnr_db": 36.68982810196206, + "model_id": "action-agnostic", + "samples": 360, + "state_fde": 0.9391366999396707, + "top1_regret": 0.3, + "video_transform": "identity" + } + ], + "psnr_regret_order_disagreements": 3 + }, + { + "benchmark_id": "gripper-catch-v0.1", + "comparable_profile_pairs": 9, + "correlations": { + "global_ssim_vs_top1_regret_pearson": 0.17249470782312815, + "psnr_vs_crc_spearman_pearson": 0.16195906182300873, + "psnr_vs_top1_regret_pearson": -0.16195906182300873 + }, + "profiles": [ + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": 1.0, + "mean_psnr_db": 100.0, + "model_id": "oracle-simulator", + "samples": 240, + "state_fde": 0.0, + "top1_regret": 0.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": -0.080095265950554, + "mean_psnr_db": 0.5856883976353673, + "model_id": "appearance-corrupted-oracle", + "samples": 240, + "state_fde": 0.0, + "top1_regret": 0.0, + "video_transform": "invert-rgb" + }, + { + "candidate_ranking_spearman": 1.0, + "mean_global_ssim": 0.9046938602562767, + "mean_psnr_db": 38.21621729123946, + "model_id": "noisy-dynamics", + "samples": 240, + "state_fde": 0.03046040283421649, + "top1_regret": 0.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 0.0, + "mean_global_ssim": 0.5926650960386265, + "mean_psnr_db": 21.484879714071898, + "model_id": "copy-last-frame", + "samples": 240, + "state_fde": 1.0446933677159576, + "top1_regret": 1.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 0.0, + "mean_global_ssim": 0.8321954561324867, + "mean_psnr_db": 45.12254514262558, + "model_id": "wrong-direction", + "samples": 240, + "state_fde": 0.9662883683502264, + "top1_regret": 1.0, + "video_transform": "identity" + }, + { + "candidate_ranking_spearman": 0.0, + "mean_global_ssim": 0.7702514993155695, + "mean_psnr_db": 42.714137158393655, + "model_id": "action-agnostic", + "samples": 240, + "state_fde": 1.155975033047959, + "top1_regret": 1.0, + "video_transform": "identity" + } + ], + "psnr_regret_order_disagreements": 5 + } + ], + "contexts_per_benchmark": 12, + "interpretation": "The appearance-corrupted oracle preserves exact state/control predictions while deliberately lowering RGB fidelity. It is a metric counterexample, not a model quality claim: PSNR/global SSIM cannot replace CRC, regret, or state metrics.", + "schema_version": "0.1", + "seed": 7 +} diff --git a/examples/video-control-study/video-control-study.md b/examples/video-control-study/video-control-study.md new file mode 100644 index 0000000..88a8308 --- /dev/null +++ b/examples/video-control-study/video-control-study.md @@ -0,0 +1,32 @@ +# Traditional video fidelity versus control value + +The appearance-corrupted oracle preserves exact state/control predictions while deliberately lowering RGB fidelity. It is a metric counterexample, not a model quality claim: PSNR/global SSIM cannot replace CRC, regret, or state metrics. + +## blockpush-2d-v0.1 + +| Profile | Transform | PSNR dB | Global SSIM | State FDE | CRC | Regret | +|---|---|---:|---:|---:|---:|---:| +| oracle-simulator | identity | 100.0000 | 1.0000 | 0.0000 | 1.0000 | 0.0000 | +| appearance-corrupted-oracle | invert-rgb | 0.5882 | -0.0790 | 0.0000 | 1.0000 | 0.0000 | +| noisy-dynamics | identity | 39.9549 | 0.9058 | 0.0355 | 1.0000 | 0.0000 | +| copy-last-frame | identity | 38.4547 | 0.7490 | 0.4369 | 0.0000 | 0.3000 | +| wrong-direction | identity | 37.6401 | 0.6928 | 0.7953 | -0.2500 | 0.3000 | +| action-agnostic | identity | 36.6898 | 0.6141 | 0.9391 | 0.0000 | 0.3000 | + +PSNR/regret ordering disagreed on 3/9 comparable profile pairs. + +## gripper-catch-v0.1 + +| Profile | Transform | PSNR dB | Global SSIM | State FDE | CRC | Regret | +|---|---|---:|---:|---:|---:|---:| +| oracle-simulator | identity | 100.0000 | 1.0000 | 0.0000 | 1.0000 | 0.0000 | +| appearance-corrupted-oracle | invert-rgb | 0.5857 | -0.0801 | 0.0000 | 1.0000 | 0.0000 | +| noisy-dynamics | identity | 38.2162 | 0.9047 | 0.0305 | 1.0000 | 0.0000 | +| copy-last-frame | identity | 21.4849 | 0.5927 | 1.0447 | 0.0000 | 1.0000 | +| wrong-direction | identity | 45.1225 | 0.8322 | 0.9663 | 0.0000 | 1.0000 | +| action-agnostic | identity | 42.7141 | 0.7703 | 1.1560 | 0.0000 | 1.0000 | + +PSNR/regret ordering disagreed on 5/9 comparable profile pairs. + +Global SSIM here is an explicitly labeled whole-frame diagnostic, not the +standard windowed implementation. Exact PSNR matches use a finite 100 dB cap. diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index d5c8fc3..8f04709 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -48,6 +48,7 @@ from wamprobe.manipulation_evaluation import evaluate_manipulation from wamprobe.reporting import write_reports from wamprobe.stats import paired_metric_comparison +from wamprobe.video_control_study import run_video_control_study, write_video_control_study def _build_parser() -> argparse.ArgumentParser: @@ -140,6 +141,19 @@ def _build_parser() -> argparse.ArgumentParser: experiment_report.add_argument("matrix", type=Path, help="matrix-index.json path") experiment_report.add_argument("execution", type=Path, help="execution-index.json path") experiment_report.add_argument("--output", type=Path, required=True) + + video_control = subparsers.add_parser( + "video-control-study", + help="compare traditional rendered-video fidelity with control metrics", + ) + video_control.add_argument( + "--benchmark", + choices=("blockpush", "gripper-catch", "all"), + default="all", + ) + video_control.add_argument("--contexts", type=int, default=12) + video_control.add_argument("--seed", type=int, default=7) + video_control.add_argument("--output", type=Path, required=True) return parser @@ -417,6 +431,23 @@ def _run_experiment_report(args: argparse.Namespace) -> int: return 0 +def _run_video_control_study(args: argparse.Namespace) -> int: + names = ("blockpush", "gripper-catch") if args.benchmark == "all" else (args.benchmark,) + try: + study = run_video_control_study( + benchmark_names=names, + contexts=args.contexts, + seed=args.seed, + ) + json_path, markdown_path = write_video_control_study(args.output, study) + except (OSError, ValueError) as error: + print(f"WAMProbe video-control study error: {error}", file=sys.stderr) + return 2 + print(f"Video-control JSON: {json_path}") + print(f"Video-control Markdown: {markdown_path}") + return 0 + + def _run_doctor(args: argparse.Namespace) -> int: try: manifest = load_manifest(args.manifest) @@ -466,5 +497,7 @@ def main(argv: Sequence[str] | None = None) -> int: return _run_dataset_validate(args) if args.command == "experiment-report": return _run_experiment_report(args) + if args.command == "video-control-study": + return _run_video_control_study(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/video_control_study.py b/src/wamprobe/video_control_study.py new file mode 100644 index 0000000..be3fe90 --- /dev/null +++ b/src/wamprobe/video_control_study.py @@ -0,0 +1,330 @@ +"""Toy counterexample study contrasting RGB fidelity with control-value metrics.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Callable +from dataclasses import asdict, dataclass +from itertools import combinations +from pathlib import Path + +from wamprobe.adapters.manipulation import ( + ActionAgnosticManipulationAdapter, + CopyLastManipulationAdapter, + NoisyManipulationAdapter, + OracleManipulationAdapter, + WrongDirectionManipulationAdapter, +) +from wamprobe.api.manipulation import ( + ManipulationAdapter, + ManipulationBenchmark, + ManipulationInterventionSuite, +) +from wamprobe.api.robotics import RGBFrame +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch +from wamprobe.manipulation_evaluation import evaluate_manipulation +from wamprobe.video_metrics import invert_rgb, rgb_global_ssim, rgb_psnr + + +@dataclass(frozen=True, slots=True) +class VideoControlProfile: + """Model-level traditional video and control metrics.""" + + model_id: str + video_transform: str + samples: int + mean_psnr_db: float + mean_global_ssim: float + state_fde: float + candidate_ranking_spearman: float + top1_regret: float + + +@dataclass(frozen=True, slots=True) +class VideoControlBenchmarkResult: + """One benchmark's profiles and cross-profile associations.""" + + benchmark_id: str + profiles: tuple[VideoControlProfile, ...] + correlations: dict[str, float | None] + psnr_regret_order_disagreements: int + comparable_profile_pairs: int + + +@dataclass(frozen=True, slots=True) +class VideoControlStudy: + """Multi-benchmark diagnostic counterexample report.""" + + benchmarks: tuple[VideoControlBenchmarkResult, ...] + contexts_per_benchmark: int + seed: int + interpretation: str + schema_version: str = "0.1" + + def to_dict(self) -> dict[str, object]: + """Return a stable JSON-compatible study.""" + + return { + "schema_version": self.schema_version, + "contexts_per_benchmark": self.contexts_per_benchmark, + "seed": self.seed, + "benchmarks": [asdict(result) for result in self.benchmarks], + "interpretation": self.interpretation, + } + + +@dataclass(frozen=True, slots=True) +class _ProfileSpec: + model_id: str + adapter: Callable[[], ManipulationAdapter] + video_transform: str = "identity" + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) + + +def _pearson(left: list[float], right: list[float]) -> float | None: + left_mean = _mean(left) + right_mean = _mean(right) + left_centered = [value - left_mean for value in left] + right_centered = [value - right_mean for value in right] + denominator = math.sqrt( + sum(value * value for value in left_centered) + * sum(value * value for value in right_centered) + ) + if denominator <= 1e-15: + return None + return ( + sum( + left_value * right_value + for left_value, right_value in zip(left_centered, right_centered, strict=True) + ) + / denominator + ) + + +def _render_frame( + benchmark: ManipulationBenchmark, + group_context: object, + state: object, +) -> RGBFrame: + # Runtime protocols know the concrete manipulation types, while keeping this helper + # intentionally small avoids imposing a rendering method on every future benchmark. + observation = benchmark.observe(group_context, state) # type: ignore[arg-type] + if len(observation.frames) != 1: + raise ValueError("toy video study expects exactly one rendered RGB camera") + return observation.frames[0] + + +def _video_fidelity( + adapter: ManipulationAdapter, + benchmark: ManipulationBenchmark, + suite: ManipulationInterventionSuite, + *, + seed: int, + transform: str, +) -> tuple[float, float, int]: + psnr_values: list[float] = [] + ssim_values: list[float] = [] + try: + for group in suite.groups: + for action in group.actions: + truth = benchmark.rollout(group.context, action) + predicted = adapter.predict_future( + group.context, + action, + horizon=benchmark.horizon, + seed=seed, + ) + for predicted_state, truth_state in zip( + predicted.states, + truth.states, + strict=True, + ): + predicted_frame = _render_frame(benchmark, group.context, predicted_state) + if transform == "invert-rgb": + predicted_frame = invert_rgb(predicted_frame) + truth_frame = _render_frame(benchmark, group.context, truth_state) + psnr_values.append(rgb_psnr(predicted_frame, truth_frame)) + ssim_values.append(rgb_global_ssim(predicted_frame, truth_frame)) + finally: + adapter.close() + return _mean(psnr_values), _mean(ssim_values), len(psnr_values) + + +def _profile_specs(benchmark: ManipulationBenchmark) -> tuple[_ProfileSpec, ...]: + return ( + _ProfileSpec("oracle-simulator", lambda: OracleManipulationAdapter(benchmark)), + _ProfileSpec( + "appearance-corrupted-oracle", + lambda: OracleManipulationAdapter(benchmark), + "invert-rgb", + ), + _ProfileSpec("noisy-dynamics", lambda: NoisyManipulationAdapter(benchmark)), + _ProfileSpec("copy-last-frame", CopyLastManipulationAdapter), + _ProfileSpec( + "wrong-direction", + lambda: WrongDirectionManipulationAdapter(benchmark), + ), + _ProfileSpec( + "action-agnostic", + lambda: ActionAgnosticManipulationAdapter(benchmark), + ), + ) + + +def _run_benchmark( + benchmark: ManipulationBenchmark, + *, + contexts: int, + seed: int, +) -> VideoControlBenchmarkResult: + suite = benchmark.make_suite(contexts=contexts, seed=seed) + profiles: list[VideoControlProfile] = [] + for spec in _profile_specs(benchmark): + control = evaluate_manipulation(spec.adapter(), benchmark, suite, seed=seed) + psnr, ssim, samples = _video_fidelity( + spec.adapter(), + benchmark, + suite, + seed=seed, + transform=spec.video_transform, + ) + profiles.append( + VideoControlProfile( + model_id=spec.model_id, + video_transform=spec.video_transform, + samples=samples, + mean_psnr_db=psnr, + mean_global_ssim=ssim, + state_fde=control.metrics["state_fde"], + candidate_ranking_spearman=control.metrics["candidate_ranking_spearman"], + top1_regret=control.metrics["top1_regret"], + ) + ) + disagreements = 0 + comparable_pairs = 0 + for left, right in combinations(profiles, 2): + regret_difference = left.top1_regret - right.top1_regret + psnr_difference = left.mean_psnr_db - right.mean_psnr_db + if abs(regret_difference) <= 1e-12 or abs(psnr_difference) <= 1e-12: + continue + comparable_pairs += 1 + # Lower regret and higher PSNR agree only when their differences have opposite signs. + if regret_difference * psnr_difference > 0: + disagreements += 1 + return VideoControlBenchmarkResult( + benchmark_id=benchmark.benchmark_id, + profiles=tuple(profiles), + correlations={ + "psnr_vs_top1_regret_pearson": _pearson( + [profile.mean_psnr_db for profile in profiles], + [profile.top1_regret for profile in profiles], + ), + "global_ssim_vs_top1_regret_pearson": _pearson( + [profile.mean_global_ssim for profile in profiles], + [profile.top1_regret for profile in profiles], + ), + "psnr_vs_crc_spearman_pearson": _pearson( + [profile.mean_psnr_db for profile in profiles], + [profile.candidate_ranking_spearman for profile in profiles], + ), + }, + psnr_regret_order_disagreements=disagreements, + comparable_profile_pairs=comparable_pairs, + ) + + +def run_video_control_study( + *, + benchmark_names: tuple[str, ...] = ("blockpush", "gripper-catch"), + contexts: int = 12, + seed: int = 7, +) -> VideoControlStudy: + """Run the diagnostic study on deterministic rendered manipulation benchmarks.""" + + if contexts <= 0: + raise ValueError("contexts must be positive") + benchmarks: list[ManipulationBenchmark] = [] + for name in benchmark_names: + if name == "blockpush": + benchmarks.append(BlockPush2D()) + elif name == "gripper-catch": + benchmarks.append(GripperCatch()) + else: + raise ValueError(f"unsupported video-control benchmark: {name}") + if len(benchmarks) != len({benchmark.benchmark_id for benchmark in benchmarks}): + raise ValueError("video-control benchmark names must be unique") + return VideoControlStudy( + benchmarks=tuple( + _run_benchmark(benchmark, contexts=contexts, seed=seed) for benchmark in benchmarks + ), + contexts_per_benchmark=contexts, + seed=seed, + interpretation=( + "The appearance-corrupted oracle preserves exact state/control predictions while " + "deliberately lowering RGB fidelity. It is a metric counterexample, not a model " + "quality claim: PSNR/global SSIM cannot replace CRC, regret, or state metrics." + ), + ) + + +def render_video_control_markdown(study: VideoControlStudy) -> str: + """Render a human-readable comparison without a composite score.""" + + lines = [ + "# Traditional video fidelity versus control value", + "", + study.interpretation, + "", + ] + for benchmark in study.benchmarks: + lines.extend( + [ + f"## {benchmark.benchmark_id}", + "", + "| Profile | Transform | PSNR dB | Global SSIM | State FDE | CRC | Regret |", + "|---|---|---:|---:|---:|---:|---:|", + ] + ) + for profile in benchmark.profiles: + lines.append( + f"| {profile.model_id} | {profile.video_transform} | " + f"{profile.mean_psnr_db:.4f} | {profile.mean_global_ssim:.4f} | " + f"{profile.state_fde:.4f} | {profile.candidate_ranking_spearman:.4f} | " + f"{profile.top1_regret:.4f} |" + ) + lines.extend( + [ + "", + f"PSNR/regret ordering disagreed on " + f"{benchmark.psnr_regret_order_disagreements}/" + f"{benchmark.comparable_profile_pairs} comparable profile pairs.", + "", + ] + ) + lines.extend( + [ + "Global SSIM here is an explicitly labeled whole-frame diagnostic, not the", + "standard windowed implementation. Exact PSNR matches use a finite 100 dB cap.", + "", + ] + ) + return "\n".join(lines) + + +def write_video_control_study(output_dir: Path, study: VideoControlStudy) -> tuple[Path, Path]: + """Write deterministic JSON and Markdown study reports.""" + + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "video-control-study.json" + markdown_path = output_dir / "video-control-study.md" + json_path.write_text( + json.dumps(study.to_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + markdown_path.write_text(render_video_control_markdown(study), encoding="utf-8") + return json_path, markdown_path diff --git a/src/wamprobe/video_metrics.py b/src/wamprobe/video_metrics.py new file mode 100644 index 0000000..956de1e --- /dev/null +++ b/src/wamprobe/video_metrics.py @@ -0,0 +1,72 @@ +"""Dependency-free diagnostic RGB fidelity metrics.""" + +from __future__ import annotations + +import math + +from wamprobe.api.robotics import RGBFrame + + +def _validate_pair(predicted: RGBFrame, truth: RGBFrame) -> None: + if ( + predicted.height != truth.height + or predicted.width != truth.width + or predicted.camera_name != truth.camera_name + ): + raise ValueError("RGB metric inputs must have matching camera names and shapes") + + +def rgb_psnr(predicted: RGBFrame, truth: RGBFrame, *, exact_cap: float = 100.0) -> float: + """Return RGB PSNR in dB, using a finite cap for exact uint8 matches.""" + + _validate_pair(predicted, truth) + if not math.isfinite(exact_cap) or exact_cap <= 0: + raise ValueError("exact_cap must be finite and positive") + squared_error = sum( + (predicted_value - truth_value) ** 2 + for predicted_value, truth_value in zip(predicted.data, truth.data, strict=True) + ) + if squared_error == 0: + return exact_cap + mse = squared_error / len(predicted.data) + return 10.0 * math.log10(255.0**2 / mse) + + +def _channel_values(frame: RGBFrame, channel: int) -> list[float]: + return [float(value) for value in frame.data[channel::3]] + + +def rgb_global_ssim(predicted: RGBFrame, truth: RGBFrame) -> float: + """Return mean per-channel global SSIM without claiming windowed SSIM parity.""" + + _validate_pair(predicted, truth) + channel_scores: list[float] = [] + c1 = (0.01 * 255.0) ** 2 + c2 = (0.03 * 255.0) ** 2 + for channel in range(3): + left = _channel_values(predicted, channel) + right = _channel_values(truth, channel) + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + left_variance = sum((value - left_mean) ** 2 for value in left) / len(left) + right_variance = sum((value - right_mean) ** 2 for value in right) / len(right) + covariance = sum( + (left_value - left_mean) * (right_value - right_mean) + for left_value, right_value in zip(left, right, strict=True) + ) / len(left) + score = ((2.0 * left_mean * right_mean + c1) * (2.0 * covariance + c2)) / ( + (left_mean**2 + right_mean**2 + c1) * (left_variance + right_variance + c2) + ) + channel_scores.append(score) + return sum(channel_scores) / len(channel_scores) + + +def invert_rgb(frame: RGBFrame) -> RGBFrame: + """Return a deterministic appearance corruption for metric sanity tests.""" + + return RGBFrame( + camera_name=frame.camera_name, + height=frame.height, + width=frame.width, + data=bytes(255 - value for value in frame.data), + ) diff --git a/tests/test_video_control_study.py b/tests/test_video_control_study.py new file mode 100644 index 0000000..1b7a36f --- /dev/null +++ b/tests/test_video_control_study.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest + +from wamprobe.api.robotics import RGBFrame +from wamprobe.cli import main +from wamprobe.video_control_study import run_video_control_study +from wamprobe.video_metrics import invert_rgb, rgb_global_ssim, rgb_psnr + + +def test_rgb_fidelity_metrics_distinguish_exact_and_inverted_frames() -> None: + frame = RGBFrame("camera", 2, 1, bytes((0, 64, 128, 255, 192, 127))) + inverted = invert_rgb(frame) + + assert rgb_psnr(frame, frame) == 100.0 + assert rgb_global_ssim(frame, frame) == pytest.approx(1.0) + assert rgb_psnr(inverted, frame) < 10.0 + assert rgb_global_ssim(inverted, frame) < 0.0 + + +def test_video_control_study_exposes_appearance_control_counterexample() -> None: + study = run_video_control_study( + benchmark_names=("blockpush",), + contexts=4, + seed=5, + ) + result = study.benchmarks[0] + profiles = {profile.model_id: profile for profile in result.profiles} + + oracle = profiles["oracle-simulator"] + corrupted = profiles["appearance-corrupted-oracle"] + assert corrupted.top1_regret == oracle.top1_regret == 0.0 + assert corrupted.state_fde == oracle.state_fde == 0.0 + assert corrupted.mean_psnr_db < oracle.mean_psnr_db + assert result.psnr_regret_order_disagreements > 0 + + +def test_video_control_study_cli_writes_reports(tmp_path: Path) -> None: + assert ( + main( + [ + "video-control-study", + "--benchmark", + "blockpush", + "--contexts", + "4", + "--seed", + "5", + "--output", + str(tmp_path), + ] + ) + == 0 + ) + assert (tmp_path / "video-control-study.json").is_file() + assert "appearance-corrupted-oracle" in (tmp_path / "video-control-study.md").read_text( + encoding="utf-8" + ) From 50713ab622e86c78efb57f2be0b97285ff9ae186 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 14:22:54 +0000 Subject: [PATCH 15/20] add minimal closed-loop utility study --- README.md | 19 +- docs/QUICKSTART.md | 35 +- docs/WAMProbe_PLAN.md | 10 +- docs/benchmarks/TOY_BENCHMARKS.md | 19 +- docs/experiments/TOY_CLOSED_LOOP_V0.1.md | 79 + docs/metrics/CORE_METRICS.md | 12 + .../closed-loop-study/closed-loop-study.json | 4099 +++++++++++++++++ .../closed-loop-study/closed-loop-study.md | 48 + src/wamprobe/cli.py | 39 + src/wamprobe/closed_loop.py | 646 +++ tests/test_closed_loop_study.py | 78 + 11 files changed, 5075 insertions(+), 9 deletions(-) create mode 100644 docs/experiments/TOY_CLOSED_LOOP_V0.1.md create mode 100644 examples/closed-loop-study/closed-loop-study.json create mode 100644 examples/closed-loop-study/closed-loop-study.md create mode 100644 src/wamprobe/closed_loop.py create mode 100644 tests/test_closed_loop_study.py diff --git a/README.md b/README.md index 253c131..d25909d 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ wamprobe demo --benchmark gripper-catch --horizon 5 --output runs/gripper-catch- # Contrast rendered-video fidelity with control-grounded metrics. wamprobe video-control-study --contexts 12 --seed 7 \ --output runs/video-control-study + +# Score a candidate set, execute one true-dynamics step, observe, and replan. +wamprobe closed-loop-study --contexts 12 --seed 7 \ + --output runs/closed-loop-study ``` The command creates: @@ -83,7 +87,10 @@ See the committed [PointMass](examples/pointmass-demo/report.md), profiles. The committed [video/control counterexample](examples/video-control-study/video-control-study.md) shows that an appearance-corrupted oracle can keep exact state predictions and zero regret while -receiving very low PSNR and global SSIM. +receiving very low PSNR and global SSIM. The +[closed-loop study](examples/closed-loop-study/closed-loop-study.md) then runs a real +score-execute-observe loop: oracle/noisy future scorers solve BlockPush and reach at least +91.7% Gripper-Catch success, while action-ignoring scorers receive zero success. Real-model weights are never committed to Git. Before running the StarWAM integration, follow the [model-store layout and download rules](checkpoints/README.md); the first spike requires @@ -124,13 +131,17 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo branches × eight steps in LIBERO-CF-Mini, with exact restore and branch-order validation; - dependency-free RGB PSNR/global SSIM diagnostics and a two-benchmark counterexample study that keeps traditional video fidelity separate from state accuracy and control value; +- a minimal receding-horizon evaluator with random, fixed-policy, simulator-oracle, and + five future-scorer controls, per-context traces, bootstrap intervals, and offline/closed-loop + association analysis; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap The next milestones are: -1. expand fixed states and add a minimal replanning/closed-loop experiment; +1. expand LIBERO initial states and evaluate action-conditioned real-WAM futures when an + adapter exposes that capability; 2. add the Occluded-Object memory diagnostic to the broader Toy tier; 3. prepare a traceable v0.1 release candidate and external reproduction smoke. @@ -141,7 +152,9 @@ The exact toy dynamics and limitations are documented in the [toy benchmark card](docs/benchmarks/TOY_BENCHMARKS.md), [LIBERO-CF-Mini benchmark card](docs/benchmarks/LIBERO_CF_MINI.md), [StarWAM model card](docs/models/STARWAM.md), and -[core metric cards](docs/metrics/CORE_METRICS.md). +[core metric cards](docs/metrics/CORE_METRICS.md). The exact closed-loop protocol and +limitations are recorded in the +[toy closed-loop experiment card](docs/experiments/TOY_CLOSED_LOOP_V0.1.md). ## Development diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 33c1cd4..4ed4c6f 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -92,6 +92,36 @@ and regret `0`, despite PSNR below `1 dB` in the committed run. This is a contro counterexample, not a claim that visual fidelity is unimportant. See the [committed report](../examples/video-control-study/video-control-study.md). +## Run minimal closed-loop replanning + +The closed-loop study uses the same candidate set at every decision, scores candidate +futures, executes only the first true-dynamics step, then rebuilds the context from the +resulting state and replans: + +```bash +wamprobe closed-loop-study \ + --benchmark all \ + --contexts 12 \ + --seed 7 \ + --execute-prefix 1 \ + --resamples 1000 \ + --output runs/closed-loop-study +``` + +`--control-steps` overrides the default episode budget of one benchmark horizon. +`--execute-prefix` controls how many selected chunk steps are executed before observing +again. Candidate scores use at most the remaining episode budget, so a perfect fixed- +horizon predictor is not penalized by a terminal score beyond the steps that can be +executed. + +The report compares five future scorers with a simulator-future scorer, a fixed action-only +policy, and deterministic random candidate selection. It preserves every context's chosen +action sequence, final return, success, and gap to the simulator scorer. The committed +12-context run found CRC/closed-return Pearson values of `0.9855` on BlockPush and `1.0000` +on Gripper-Catch, but these are descriptive associations across only five future-scorer +profiles. See the [report](../examples/closed-loop-study/closed-loop-study.md) and +[experiment card](experiments/TOY_CLOSED_LOOP_V0.1.md). + Uncertainty is computed at the context level. Each bootstrap draw resamples whole shared initial states, never individual action branches or adjacent frames. Paired comparisons align exact context IDs and report the left-minus-right difference with a 95% interval. @@ -155,5 +185,6 @@ subset; verified outputs resume without reopening the simulator. PointMass-2D validates the evaluator and guards against metric bugs. The current LIBERO pilot validates paired data generation across four task families, not policy quality: its diagnostic actions do not solve the tasks. Transfer claims still require more initial -states, WAM future predictions, stochastic generation analysis, and correlation with -closed-loop return. +states, action-conditioned real-WAM future predictions, stochastic generation analysis, +and real-model closed-loop return. The analytic closed-loop study validates evaluator +behavior; it does not fill those transfer gaps. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index e3b07cb..5015c70 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1117,9 +1117,13 @@ snapshot 执行全部动作块，并在 horizon 8/16/32 记录状态、回报和 当前进度（2026-07-15）：已在 BlockPush-2D 与 Gripper-Catch 上运行 12-context 的传统 视频指标/控制价值反例研究。`appearance-corrupted-oracle` 保持 FDE=0、CRC=1、Regret=0， 但 PSNR 约 0.59 dB；PSNR 与 regret 的跨 profile Pearson 约为 -0.16，且两个 benchmark -分别出现 3/9 和 5/9 个可比较排序冲突。结果和限制已固化在 -[`examples/video-control-study/`](../examples/video-control-study/video-control-study.md)。 -最小 replanning/closed-loop、release artifact 追溯、技术报告和外部复现仍待完成。 +分别出现 3/9 和 5/9 个可比较排序冲突。最小 score-execute-observe 闭环也已落地：每次 +只执行 1 步并重规划，oracle future scorer 两任务 success 均为 1，noisy future scorer +分别为 1.0/0.9167，三个 action-ignorance/wrong-direction 对照均为 0；离线 CRC 与闭环 +return 的 5-profile 描述性 Pearson 分别为 0.9855/1.0。全部结果、context-block CI 和限制 +见 [`examples/video-control-study/`](../examples/video-control-study/video-control-study.md) +与 [`docs/experiments/TOY_CLOSED_LOOP_V0.1.md`](experiments/TOY_CLOSED_LOOP_V0.1.md)。 +release artifact 追溯、技术报告和外部复现仍待完成。 任务： diff --git a/docs/benchmarks/TOY_BENCHMARKS.md b/docs/benchmarks/TOY_BENCHMARKS.md index 367d9ff..c93e64c 100644 --- a/docs/benchmarks/TOY_BENCHMARKS.md +++ b/docs/benchmarks/TOY_BENCHMARKS.md @@ -71,4 +71,21 @@ ties are excluded from that denominator. The suite generator accepts `contexts=1000` without external assets or simulator installation. Generated suites are deterministic for a fixed benchmark configuration and -seed; a versioned on-disk intervention dataset loader remains a separate roadmap item. +seed and can be exported as checksummed intervention JSONL. + +## Closed-loop protocol + +`wamprobe closed-loop-study` turns BlockPush and Gripper-Catch into a Tier-2 evaluator. +At each cycle it scores the unchanged legal candidate set, executes only the selected +trajectory prefix under exact benchmark dynamics, creates a new context from the resulting +state, and scores again. The default prefix is one step and the default episode budget is +the benchmark horizon (six BlockPush steps or five Gripper-Catch steps). Candidate scoring +uses `min(prediction_horizon, remaining_episode_steps)`; this avoids rewarding or +penalizing terminal predictions beyond the actual execution budget. + +The fixed comparison set contains five future-prediction adapters, deterministic random +selection, an initial-context fixed action policy, and a greedy simulator-future scorer. +Success means matching the simulator scorer's final return within `1e-9` for these exact +analytic tasks. This equality is an evaluator sanity definition and must not be reused as a +general robot-task success threshold. Full settings, results, and limitations are in the +[experiment card](../experiments/TOY_CLOSED_LOOP_V0.1.md). diff --git a/docs/experiments/TOY_CLOSED_LOOP_V0.1.md b/docs/experiments/TOY_CLOSED_LOOP_V0.1.md new file mode 100644 index 0000000..043985c --- /dev/null +++ b/docs/experiments/TOY_CLOSED_LOOP_V0.1.md @@ -0,0 +1,79 @@ +# Toy closed-loop utility study v0.1 + +## Question + +Does a future scorer with better offline candidate ranking also obtain better return when +the selected action is actually executed and the scorer must replan from the resulting +state? + +This study is an evaluator validation on exact analytic dynamics. It is not evidence that +the released StarWAM checkpoint improves closed-loop LIBERO performance: that adapter does +not expose action-conditioned candidate futures, so WAMProbe does not fabricate them. + +## Fixed protocol + +- benchmarks: BlockPush-2D v0.1 and Gripper-Catch v0.1; +- shared contexts: 12 per benchmark, suite seed 7; +- legal candidate set: the benchmark's full fixed candidate set at every cycle; +- prediction horizon: 6 steps for BlockPush, 5 for Gripper-Catch; +- episode budget: one benchmark horizon; +- execute prefix: one true-dynamics step before observing and replanning; +- score horizon: prediction prefix capped by remaining episode steps; +- ties: stable benchmark candidate order; +- uncertainty: 1,000 context-block bootstrap resamples; +- success: match the greedy simulator-future scorer's final return within `1e-9`. + +Controllers include oracle, seeded noisy, copy-last, wrong-direction and action-agnostic +future scorers, plus a simulator-future scorer, a fixed action-only policy and deterministic +random selection. Offline CRC and Top-1 Regret are intentionally `null` for controllers +that do not emit candidate futures. + +## Reproduce + +```bash +wamprobe closed-loop-study \ + --benchmark all \ + --contexts 12 \ + --seed 7 \ + --execute-prefix 1 \ + --resamples 1000 \ + --output runs/closed-loop-study-v0.1 +``` + +The canonical JSON SHA256 is +`19d2a0108fc19580da4e81f15226a4706a40c9d2c0513857d6bc8dc9000189b3`. + +## Results + +| Benchmark | Future scorer | Offline CRC | Offline regret | Closed return | Success | +|---|---|---:|---:|---:|---:| +| BlockPush | oracle | 1.0000 | 0.0000 | 0.0000 | 1.0000 | +| BlockPush | noisy | 1.0000 | 0.0000 | 0.0000 | 1.0000 | +| BlockPush | copy-last | 0.0000 | 0.3000 | -0.3000 | 0.0000 | +| BlockPush | wrong-direction | -0.2500 | 0.3000 | -0.3000 | 0.0000 | +| BlockPush | action-agnostic | 0.0000 | 0.3000 | -0.3000 | 0.0000 | +| Gripper-Catch | oracle | 1.0000 | 0.0000 | 1.0000 | 1.0000 | +| Gripper-Catch | noisy | 1.0000 | 0.0000 | 0.9967 | 0.9167 | +| Gripper-Catch | copy-last | 0.0000 | 1.0000 | 0.0000 | 0.0000 | +| Gripper-Catch | wrong-direction | 0.0000 | 1.0000 | 0.0000 | 0.0000 | +| Gripper-Catch | action-agnostic | 0.0000 | 1.0000 | 0.0000 | 0.0000 | + +Across the five future-scorer profiles, offline CRC versus mean closed-loop return has +Pearson correlation `0.9855` for BlockPush and `1.0000` for Gripper-Catch. Offline Top-1 +Regret versus return is `-1.0000` for both. No comparable future-scorer pair reverses the +offline-CRC/return ordering. + +## Interpretation and limits + +The result confirms that the evaluator connects offline action ranking to actual repeated +execution on two controlled tasks. It does not establish a population-level correlation: +there are only five deliberately separated reference profiles, three weak profiles tie on +return, and the contexts come from analytic generators rather than independent robot task +distributions. The action-only fixed policy also reaches the task optimum, showing that a +world model is not necessary on these easy generated tasks. + +The first implementation used the full prediction endpoint even when fewer episode steps +remained. That made an exact model overrun the goal near termination. The released protocol +caps the scoring prefix by the remaining budget, and regression tests preserve this MPC +semantics. Larger LIBERO studies must likewise declare prediction horizon, execution prefix, +reward horizon, terminal handling and candidate generator before comparing controllers. diff --git a/docs/metrics/CORE_METRICS.md b/docs/metrics/CORE_METRICS.md index 07f3981..bab12d9 100644 --- a/docs/metrics/CORE_METRICS.md +++ b/docs/metrics/CORE_METRICS.md @@ -20,6 +20,9 @@ are never treated as independent samples. | Top-1 Regret | candidate returns | oracle return minus return of model-selected candidate | lower, ≥0 | depends on candidate-set coverage | | RGB PSNR | same-shape rendered RGB frames | `10 log10(255²/MSE)`, averaged over branches and frames | higher, dB; exact matches capped at 100 dB | pixel alignment and appearance can disagree with dynamics/control value | | Global SSIM diagnostic | same-camera rendered RGB frames | mean per-channel luminance/contrast/structure over each whole frame | higher, typically [-1,1] | not standard windowed SSIM and can hide local/contact errors | +| Closed-loop return | repeated candidate scoring + true transition | task return after score-execute-observe replanning | higher; task-specific units | depends on candidate generator, scorer and execution budget | +| Closed-loop success | shared context and oracle control | fraction matching the simulator-future scorer's return within the fixed tolerance | higher, [0,1] | toy equality threshold is not a general task-success definition | +| Closed-loop oracle gap | same | simulator-future return minus controller return per context | lower; 0 matches reference | a greedy simulator scorer is not a proof of globally optimal control | | Latency | timed model call | synchronized wall time per prediction | lower, seconds | hardware and warm-up dependent | | Peak allocated/reserved memory | CUDA runtime | maximum allocator bytes during one prediction | lower, GiB | not total board memory or energy | | NFE sensitivity | matched inputs/seeds | paired action RMS and executed endpoint distance across NFE | lower means more stable | stability does not establish correctness | @@ -44,6 +47,12 @@ regret metric. The built-in global variant is deliberately named `global_ssim`; must not label it as standard windowed SSIM. Exact PSNR matches use a finite 100 dB cap so JSON remains portable. +Closed-loop metrics additionally require a legal candidate generator, a fixed task scorer, +an executable transition, and an explicit replanning contract. WAMProbe records the total +control steps, executed prefix, candidate tie order and per-context action sequence. The +scoring window is capped by the remaining execution budget. Controllers without predicted +candidate futures retain `null` offline CRC/regret rather than receiving invented values. + ## Reference-baseline expectations - Oracle dynamics should minimize ADE/FDE and regret while maximizing direction and CRC. @@ -52,6 +61,9 @@ JSON remains portable. - Seeded noisy dynamics should degrade ADE/FDE and ranking as noise rises. - Appearance-corrupted oracle futures should preserve exact FDE/CRC/regret while degrading PSNR/global SSIM, demonstrating that video fidelity cannot replace control grounding. +- Oracle/noisy future scorers should outperform copy-last, wrong-direction, and + action-agnostic scorers in the analytic closed loop; random and action-only controls stay + visible rather than being folded into a composite score. Any new metric must add a failure-mode statement, capability requirement, reference ordering test, aggregation unit and range/direction before it enters a public report. diff --git a/examples/closed-loop-study/closed-loop-study.json b/examples/closed-loop-study/closed-loop-study.json new file mode 100644 index 0000000..837eaee --- /dev/null +++ b/examples/closed-loop-study/closed-loop-study.json @@ -0,0 +1,4099 @@ +{ + "benchmarks": [ + { + "benchmark_id": "blockpush-2d-v0.1", + "comparable_future_scorer_pairs": 6, + "control_steps": 6, + "correlations": { + "offline_crc_vs_closed_loop_return_pearson": 0.9855274566525744, + "offline_regret_vs_closed_loop_return_pearson": -1.0 + }, + "execute_prefix": 1, + "offline_return_order_disagreements": 0, + "profiles": [ + { + "controller_id": "oracle-simulator", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + } + ], + "offline_crc_spearman": 1.0, + "offline_top1_regret": 0.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": -0.0, + "quantiles": { + "p05": -0.0, + "p25": -0.0, + "p75": -0.0, + "p95": -0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "noisy-dynamics", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + } + ], + "offline_crc_spearman": 1.0, + "offline_top1_regret": 0.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": -0.0, + "quantiles": { + "p05": -0.0, + "p25": -0.0, + "p75": -0.0, + "p95": -0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "copy-last-frame", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.30000000000000004, + "oracle_gap": 0.30000000000000004, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + } + ], + "offline_crc_spearman": 0.0, + "offline_top1_regret": 0.3, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.3, + "lower": 0.3, + "resamples": 1000, + "unit": "context", + "upper": 0.3 + }, + "contexts": 12, + "mean": 0.3, + "median": 0.3, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": -0.3, + "lower": -0.3, + "resamples": 1000, + "unit": "context", + "upper": -0.3 + }, + "contexts": 12, + "mean": -0.3, + "median": -0.3, + "quantiles": { + "p05": -0.30000000000000004, + "p25": -0.3, + "p75": -0.3, + "p95": -0.3 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "wrong-direction", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.30000000000000004, + "oracle_gap": 0.30000000000000004, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + } + ], + "offline_crc_spearman": -0.25, + "offline_top1_regret": 0.3, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.3, + "lower": 0.3, + "resamples": 1000, + "unit": "context", + "upper": 0.3 + }, + "contexts": 12, + "mean": 0.3, + "median": 0.3, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": -0.3, + "lower": -0.3, + "resamples": 1000, + "unit": "context", + "upper": -0.3 + }, + "contexts": 12, + "mean": -0.3, + "median": -0.3, + "quantiles": { + "p05": -0.30000000000000004, + "p25": -0.3, + "p75": -0.3, + "p95": -0.3 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "action-agnostic", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.30000000000000004, + "oracle_gap": 0.30000000000000004, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + } + ], + "offline_crc_spearman": 0.0, + "offline_top1_regret": 0.3, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.3, + "lower": 0.3, + "resamples": 1000, + "unit": "context", + "upper": 0.3 + }, + "contexts": 12, + "mean": 0.3, + "median": 0.3, + "quantiles": { + "p05": 0.3, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": -0.3, + "lower": -0.3, + "resamples": 1000, + "unit": "context", + "upper": -0.3 + }, + "contexts": 12, + "mean": -0.3, + "median": -0.3, + "quantiles": { + "p05": -0.30000000000000004, + "p25": -0.3, + "p75": -0.3, + "p95": -0.3 + }, + "standard_deviation": 1.6024689053196365e-17 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "simulator-oracle-scorer", + "controller_kind": "simulator-control", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": -0.0, + "quantiles": { + "p05": -0.0, + "p25": -0.0, + "p75": -0.0, + "p95": -0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "fixed-action-policy", + "controller_kind": "action-only-control", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-right", + "push-right", + "push-right", + "push-right", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-left", + "push-left", + "push-left", + "push-left", + "push-left", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-up", + "push-up", + "push-up", + "push-up", + "push-up", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": true + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.0, + "oracle_gap": 0.0, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-down", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": true + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": -0.0, + "quantiles": { + "p05": -0.0, + "p25": -0.0, + "p75": -0.0, + "p95": -0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "random-candidate", + "controller_kind": "random-control", + "episodes": [ + { + "context_id": "blockpush-context-0000", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-left", + "push-up", + "noop", + "push-left", + "push-up" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0001", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-down", + "push-right", + "push-right", + "push-down", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0002", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-down", + "push-up", + "push-left", + "push-left", + "push-down", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0003", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-left", + "push-right", + "push-down", + "push-left", + "push-up", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0004", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-up", + "push-up", + "push-down", + "noop", + "push-up", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0005", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-down", + "push-down", + "push-down", + "push-up", + "noop", + "push-left" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0006", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "push-up", + "noop", + "push-left", + "push-right", + "noop", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0007", + "executed_steps": 6, + "final_return": -0.18, + "oracle_gap": 0.18, + "selected_actions": [ + "push-right", + "push-down", + "push-down", + "noop", + "push-down", + "push-down" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0008", + "executed_steps": 6, + "final_return": -0.24, + "oracle_gap": 0.24, + "selected_actions": [ + "push-right", + "push-up", + "push-down", + "push-right", + "push-left", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0009", + "executed_steps": 6, + "final_return": -0.30000000000000004, + "oracle_gap": 0.30000000000000004, + "selected_actions": [ + "push-right", + "push-down", + "push-right", + "push-left", + "push-right", + "push-right" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0010", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "push-down", + "push-left", + "push-down", + "push-down", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + }, + { + "context_id": "blockpush-context-0011", + "executed_steps": 6, + "final_return": -0.3, + "oracle_gap": 0.3, + "selected_actions": [ + "noop", + "push-right", + "noop", + "push-left", + "push-right", + "noop" + ], + "simulator_oracle_return": -0.0, + "success": false + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.285, + "lower": 0.26, + "resamples": 1000, + "unit": "context", + "upper": 0.3 + }, + "contexts": 12, + "mean": 0.285, + "median": 0.3, + "quantiles": { + "p05": 0.213, + "p25": 0.3, + "p75": 0.3, + "p95": 0.30000000000000004 + }, + "standard_deviation": 0.035707142142714254 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": -0.285, + "lower": -0.3, + "resamples": 1000, + "unit": "context", + "upper": -0.26499999999999996 + }, + "contexts": 12, + "mean": -0.285, + "median": -0.3, + "quantiles": { + "p05": -0.30000000000000004, + "p25": -0.3, + "p75": -0.3, + "p95": -0.21300000000000002 + }, + "standard_deviation": 0.035707142142714254 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + } + ] + }, + { + "benchmark_id": "gripper-catch-v0.1", + "comparable_future_scorer_pairs": 6, + "control_steps": 5, + "correlations": { + "offline_crc_vs_closed_loop_return_pearson": 0.9999976774578939, + "offline_regret_vs_closed_loop_return_pearson": -0.9999976774578939 + }, + "execute_prefix": 1, + "offline_return_order_disagreements": 0, + "profiles": [ + { + "controller_id": "oracle-simulator", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + } + ], + "offline_crc_spearman": 1.0, + "offline_top1_regret": 0.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "noisy-dynamics", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 0.96, + "oracle_gap": 0.040000000000000036, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + } + ], + "offline_crc_spearman": 1.0, + "offline_top1_regret": 0.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.003333333333333336, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.010000000000000009 + }, + "contexts": 12, + "mean": 0.003333333333333336, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.017999999999999988 + }, + "standard_deviation": 0.011055415967851341 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.9966666666666667, + "lower": 0.9899999999999999, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 0.9966666666666667, + "median": 1.0, + "quantiles": { + "p05": 0.982, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.011055415967851341 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.9166666666666666, + "lower": 0.75, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 0.9166666666666666, + "median": 1.0, + "quantiles": { + "p05": 0.55, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.2763853991962833 + } + }, + { + "controller_id": "copy-last-frame", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + } + ], + "offline_crc_spearman": 0.0, + "offline_top1_regret": 1.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "wrong-direction", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + } + ], + "offline_crc_spearman": 0.0, + "offline_top1_regret": 1.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "action-agnostic", + "controller_kind": "future-scorer", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + } + ], + "offline_crc_spearman": 0.0, + "offline_top1_regret": 1.0, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "simulator-oracle-scorer", + "controller_kind": "simulator-control", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "fixed-action-policy", + "controller_kind": "action-only-control", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-left", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-right", + "catch-right", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": true + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.0, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.0 + }, + "contexts": 12, + "mean": 0.0, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.0 + }, + "standard_deviation": 0.0 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 1.0, + "lower": 1.0, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 1.0, + "median": 1.0, + "quantiles": { + "p05": 1.0, + "p25": 1.0, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.0 + } + }, + { + "controller_id": "random-candidate", + "controller_kind": "random-control", + "episodes": [ + { + "context_id": "gripper-catch-context-0000", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-stay", + "catch-right", + "catch-stay", + "catch-right", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0001", + "executed_steps": 5, + "final_return": 1.0, + "oracle_gap": 0.0, + "selected_actions": [ + "catch-right", + "catch-left", + "catch-right", + "catch-stay", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": true + }, + { + "context_id": "gripper-catch-context-0002", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "catch-left", + "noop", + "noop", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0003", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-left", + "noop", + "catch-left", + "noop", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0004", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-stay", + "noop", + "catch-stay", + "catch-stay", + "noop" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0005", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-stay", + "catch-right", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0006", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-right", + "catch-left", + "catch-stay", + "catch-right", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0007", + "executed_steps": 5, + "final_return": 0.92, + "oracle_gap": 0.07999999999999996, + "selected_actions": [ + "catch-stay", + "catch-left", + "catch-right", + "catch-left", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0008", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "noop", + "noop", + "catch-right", + "catch-stay", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0009", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-left", + "catch-right", + "catch-stay", + "catch-left", + "catch-stay" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0010", + "executed_steps": 5, + "final_return": 0.9199999999999999, + "oracle_gap": 0.08000000000000007, + "selected_actions": [ + "catch-left", + "catch-left", + "catch-stay", + "catch-right", + "catch-left" + ], + "simulator_oracle_return": 1.0, + "success": false + }, + { + "context_id": "gripper-catch-context-0011", + "executed_steps": 5, + "final_return": 0.0, + "oracle_gap": 1.0, + "selected_actions": [ + "catch-stay", + "noop", + "noop", + "noop", + "catch-right" + ], + "simulator_oracle_return": 1.0, + "success": false + } + ], + "offline_crc_spearman": null, + "offline_top1_regret": null, + "oracle_gap_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.7633333333333333, + "lower": 0.5133333333333333, + "resamples": 1000, + "unit": "context", + "upper": 1.0 + }, + "contexts": 12, + "mean": 0.7633333333333333, + "median": 1.0, + "quantiles": { + "p05": 0.043999999999999984, + "p25": 0.77, + "p75": 1.0, + "p95": 1.0 + }, + "standard_deviation": 0.4103521522898215 + }, + "return_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.23666666666666666, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.47333333333333333 + }, + "contexts": 12, + "mean": 0.23666666666666666, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.22999999999999998, + "p95": 0.956 + }, + "standard_deviation": 0.4103521522898215 + }, + "success_summary": { + "confidence_interval": { + "confidence_level": 0.95, + "estimate": 0.08333333333333333, + "lower": 0.0, + "resamples": 1000, + "unit": "context", + "upper": 0.25 + }, + "contexts": 12, + "mean": 0.08333333333333333, + "median": 0.0, + "quantiles": { + "p05": 0.0, + "p25": 0.0, + "p75": 0.0, + "p95": 0.4499999999999993 + }, + "standard_deviation": 0.2763853991962833 + } + } + ] + } + ], + "contexts_per_benchmark": 12, + "interpretation": "Each cycle scores the same legal action set, executes only the selected trajectory prefix in true dynamics, then replans from the resulting state; the scoring window is capped by the remaining episode budget. Cross-profile Pearson values are descriptive because there are only five future scorers; context-block intervals remain the primary uncertainty view.", + "resamples": 1000, + "schema_version": "0.1", + "seed": 7 +} diff --git a/examples/closed-loop-study/closed-loop-study.md b/examples/closed-loop-study/closed-loop-study.md new file mode 100644 index 0000000..bcf315c --- /dev/null +++ b/examples/closed-loop-study/closed-loop-study.md @@ -0,0 +1,48 @@ +# Minimal closed-loop replanning utility + +Each cycle scores the same legal action set, executes only the selected trajectory prefix in true dynamics, then replans from the resulting state; the scoring window is capped by the remaining episode budget. Cross-profile Pearson values are descriptive because there are only five future scorers; context-block intervals remain the primary uncertainty view. + +## blockpush-2d-v0.1 + +Control steps: 6; execute prefix: 1. + +| Controller | Kind | Offline CRC | Offline regret | Closed return [95% CI] | Success | Oracle gap | +|---|---|---:|---:|---:|---:|---:| +| oracle-simulator | future-scorer | 1.0000 | 0.0000 | 0.0000 [0.0000, 0.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | future-scorer | 1.0000 | 0.0000 | 0.0000 [0.0000, 0.0000] | 1.0000 | 0.0000 | +| copy-last-frame | future-scorer | 0.0000 | 0.3000 | -0.3000 [-0.3000, -0.3000] | 0.0000 | 0.3000 | +| wrong-direction | future-scorer | -0.2500 | 0.3000 | -0.3000 [-0.3000, -0.3000] | 0.0000 | 0.3000 | +| action-agnostic | future-scorer | 0.0000 | 0.3000 | -0.3000 [-0.3000, -0.3000] | 0.0000 | 0.3000 | +| simulator-oracle-scorer | simulator-control | n/a | n/a | 0.0000 [0.0000, 0.0000] | 1.0000 | 0.0000 | +| fixed-action-policy | action-only-control | n/a | n/a | 0.0000 [0.0000, 0.0000] | 1.0000 | 0.0000 | +| random-candidate | random-control | n/a | n/a | -0.2850 [-0.3000, -0.2650] | 0.0000 | 0.2850 | + +Future-scorer profile correlation: offline CRC vs closed-loop return = 0.9855; offline regret vs closed-loop return = -1.0000. + +Offline-CRC/return ordering disagreed on 0/6 comparable future-scorer pairs. + +## gripper-catch-v0.1 + +Control steps: 5; execute prefix: 1. + +| Controller | Kind | Offline CRC | Offline regret | Closed return [95% CI] | Success | Oracle gap | +|---|---|---:|---:|---:|---:|---:| +| oracle-simulator | future-scorer | 1.0000 | 0.0000 | 1.0000 [1.0000, 1.0000] | 1.0000 | 0.0000 | +| noisy-dynamics | future-scorer | 1.0000 | 0.0000 | 0.9967 [0.9900, 1.0000] | 0.9167 | 0.0033 | +| copy-last-frame | future-scorer | 0.0000 | 1.0000 | 0.0000 [0.0000, 0.0000] | 0.0000 | 1.0000 | +| wrong-direction | future-scorer | 0.0000 | 1.0000 | 0.0000 [0.0000, 0.0000] | 0.0000 | 1.0000 | +| action-agnostic | future-scorer | 0.0000 | 1.0000 | 0.0000 [0.0000, 0.0000] | 0.0000 | 1.0000 | +| simulator-oracle-scorer | simulator-control | n/a | n/a | 1.0000 [1.0000, 1.0000] | 1.0000 | 0.0000 | +| fixed-action-policy | action-only-control | n/a | n/a | 1.0000 [1.0000, 1.0000] | 1.0000 | 0.0000 | +| random-candidate | random-control | n/a | n/a | 0.2367 [0.0000, 0.4733] | 0.0833 | 0.7633 | + +Future-scorer profile correlation: offline CRC vs closed-loop return = 1.0000; offline regret vs closed-loop return = -1.0000. + +Offline-CRC/return ordering disagreed on 0/6 comparable future-scorer pairs. + +Success means matching the greedy simulator-future scorer's final return within +1e-9 on the same context and control budget. Score ties use the benchmark's stable +candidate order. The scoring horizon is capped by the episode steps remaining, +preventing terminal scores beyond the execution budget. These analytic tasks +validate the evaluator; they do not establish real-robot or large-model closed-loop +performance. diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index 8f04709..6fe7271 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -36,6 +36,7 @@ from wamprobe.benchmarks.gripper_catch import GripperCatch from wamprobe.benchmarks.pointmass import PointMass2D from wamprobe.cache import PredictionCache, PredictionCacheRequest +from wamprobe.closed_loop import run_closed_loop_study, write_closed_loop_study from wamprobe.datasets import ( DatasetValidationError, intervention_dataset_sha256, @@ -154,6 +155,22 @@ def _build_parser() -> argparse.ArgumentParser: video_control.add_argument("--contexts", type=int, default=12) video_control.add_argument("--seed", type=int, default=7) video_control.add_argument("--output", type=Path, required=True) + + closed_loop = subparsers.add_parser( + "closed-loop-study", + help="run minimal score-execute-observe replanning diagnostics", + ) + closed_loop.add_argument( + "--benchmark", + choices=("blockpush", "gripper-catch", "all"), + default="all", + ) + closed_loop.add_argument("--contexts", type=int, default=12) + closed_loop.add_argument("--seed", type=int, default=7) + closed_loop.add_argument("--control-steps", type=int, default=None) + closed_loop.add_argument("--execute-prefix", type=int, default=1) + closed_loop.add_argument("--resamples", type=int, default=1000) + closed_loop.add_argument("--output", type=Path, required=True) return parser @@ -448,6 +465,26 @@ def _run_video_control_study(args: argparse.Namespace) -> int: return 0 +def _run_closed_loop_study(args: argparse.Namespace) -> int: + names = ("blockpush", "gripper-catch") if args.benchmark == "all" else (args.benchmark,) + try: + study = run_closed_loop_study( + benchmark_names=names, + contexts=args.contexts, + seed=args.seed, + control_steps=args.control_steps, + execute_prefix=args.execute_prefix, + resamples=args.resamples, + ) + json_path, markdown_path = write_closed_loop_study(args.output, study) + except (OSError, ValueError) as error: + print(f"WAMProbe closed-loop study error: {error}", file=sys.stderr) + return 2 + print(f"Closed-loop JSON: {json_path}") + print(f"Closed-loop Markdown: {markdown_path}") + return 0 + + def _run_doctor(args: argparse.Namespace) -> int: try: manifest = load_manifest(args.manifest) @@ -499,5 +536,7 @@ def main(argv: Sequence[str] | None = None) -> int: return _run_experiment_report(args) if args.command == "video-control-study": return _run_video_control_study(args) + if args.command == "closed-loop-study": + return _run_closed_loop_study(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/closed_loop.py b/src/wamprobe/closed_loop.py new file mode 100644 index 0000000..ab9803a --- /dev/null +++ b/src/wamprobe/closed_loop.py @@ -0,0 +1,646 @@ +"""Dependency-free minimal closed-loop replanning study.""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Callable +from dataclasses import asdict, dataclass +from itertools import combinations +from pathlib import Path +from random import Random + +from wamprobe.adapters.manipulation import ( + ActionAgnosticManipulationAdapter, + CopyLastManipulationAdapter, + NoisyManipulationAdapter, + OracleManipulationAdapter, + WrongDirectionManipulationAdapter, +) +from wamprobe.api.manipulation import ( + ManipulationAction, + ManipulationAdapter, + ManipulationBenchmark, + ManipulationContext, + ManipulationInterventionSuite, + ManipulationState, + ManipulationTrajectory, +) +from wamprobe.benchmarks.blockpush import BlockPush2D +from wamprobe.benchmarks.gripper_catch import GripperCatch +from wamprobe.manipulation_evaluation import evaluate_manipulation +from wamprobe.stats import MetricSummary, summarize + + +@dataclass(frozen=True, slots=True) +class ClosedLoopEpisode: + """One shared-context episode after repeated score-execute-observe cycles.""" + + context_id: str + selected_actions: tuple[str, ...] + executed_steps: int + final_return: float + simulator_oracle_return: float + oracle_gap: float + success: bool + + +@dataclass(frozen=True, slots=True) +class ClosedLoopProfile: + """Offline and closed-loop measurements for one controller.""" + + controller_id: str + controller_kind: str + offline_crc_spearman: float | None + offline_top1_regret: float | None + return_summary: MetricSummary + success_summary: MetricSummary + oracle_gap_summary: MetricSummary + episodes: tuple[ClosedLoopEpisode, ...] + + +@dataclass(frozen=True, slots=True) +class ClosedLoopBenchmarkResult: + """Matched controller results for one analytic benchmark.""" + + benchmark_id: str + control_steps: int + execute_prefix: int + profiles: tuple[ClosedLoopProfile, ...] + correlations: dict[str, float | None] + offline_return_order_disagreements: int + comparable_future_scorer_pairs: int + + +@dataclass(frozen=True, slots=True) +class ClosedLoopStudy: + """Multi-benchmark closed-loop utility report.""" + + benchmarks: tuple[ClosedLoopBenchmarkResult, ...] + contexts_per_benchmark: int + seed: int + resamples: int + interpretation: str + schema_version: str = "0.1" + + def to_dict(self) -> dict[str, object]: + """Return a deterministic JSON-compatible representation.""" + + return { + "schema_version": self.schema_version, + "contexts_per_benchmark": self.contexts_per_benchmark, + "seed": self.seed, + "resamples": self.resamples, + "benchmarks": [asdict(result) for result in self.benchmarks], + "interpretation": self.interpretation, + } + + +@dataclass(frozen=True, slots=True) +class _AdapterSpec: + controller_id: str + adapter: Callable[[], ManipulationAdapter] + + +@dataclass(frozen=True, slots=True) +class _EpisodeOutcome: + context_id: str + selected_actions: tuple[str, ...] + executed_steps: int + final_return: float + + +Selector = Callable[ + [ManipulationContext, tuple[ManipulationAction, ...], int, int], + ManipulationAction, +] + + +def _stable_seed(seed: int, *parts: object) -> int: + material = "\0".join((str(seed), *(str(part) for part in parts))).encode() + return int.from_bytes(hashlib.sha256(material).digest()[:8], "big") + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) + + +def _pearson(left: list[float], right: list[float]) -> float | None: + left_mean = _mean(left) + right_mean = _mean(right) + left_centered = [value - left_mean for value in left] + right_centered = [value - right_mean for value in right] + denominator = math.sqrt( + sum(value * value for value in left_centered) + * sum(value * value for value in right_centered) + ) + if denominator <= 1e-15: + return None + return ( + sum( + left_value * right_value + for left_value, right_value in zip(left_centered, right_centered, strict=True) + ) + / denominator + ) + + +def _adapter_specs(benchmark: ManipulationBenchmark) -> tuple[_AdapterSpec, ...]: + return ( + _AdapterSpec("oracle-simulator", lambda: OracleManipulationAdapter(benchmark)), + _AdapterSpec("noisy-dynamics", lambda: NoisyManipulationAdapter(benchmark)), + _AdapterSpec("copy-last-frame", CopyLastManipulationAdapter), + _AdapterSpec( + "wrong-direction", + lambda: WrongDirectionManipulationAdapter(benchmark), + ), + _AdapterSpec( + "action-agnostic", + lambda: ActionAgnosticManipulationAdapter(benchmark), + ), + ) + + +def _select_predicted_future( + adapter: ManipulationAdapter, + benchmark: ManipulationBenchmark, + context: ManipulationContext, + actions: tuple[ManipulationAction, ...], + *, + seed: int, + score_steps: int, +) -> ManipulationAction: + scores: dict[str, float] = {} + for action in actions: + predicted = adapter.predict_future( + context, + action, + horizon=benchmark.horizon, + seed=seed, + ) + if predicted.context_id != context.context_id: + raise ValueError("adapter returned a mismatched closed-loop context_id") + if predicted.action_name != action.name: + raise ValueError("adapter returned a mismatched closed-loop action_name") + if len(predicted.states) != benchmark.horizon: + raise ValueError("adapter returned a mismatched closed-loop horizon") + scored_prefix = ManipulationTrajectory( + context_id=context.context_id, + action_name=action.name, + states=predicted.states[:score_steps], + ) + scores[action.name] = benchmark.score(context, scored_prefix) + return max(actions, key=lambda action: scores[action.name]) + + +def _select_simulator_future( + benchmark: ManipulationBenchmark, + context: ManipulationContext, + actions: tuple[ManipulationAction, ...], + score_steps: int, +) -> ManipulationAction: + scores: dict[str, float] = {} + for action in actions: + truth = benchmark.rollout(context, action) + scored_prefix = ManipulationTrajectory( + context_id=context.context_id, + action_name=action.name, + states=truth.states[:score_steps], + ) + scores[action.name] = benchmark.score(context, scored_prefix) + return max(actions, key=lambda action: scores[action.name]) + + +def _execute_episode( + benchmark: ManipulationBenchmark, + context: ManipulationContext, + *, + control_steps: int, + execute_prefix: int, + selector: Selector, +) -> _EpisodeOutcome: + state: ManipulationState = context.initial_state + states: list[ManipulationState] = [] + selected_actions: list[str] = [] + actions = benchmark.actions() + remaining = control_steps + replan_index = 0 + while remaining: + step_context = ManipulationContext( + context_id=f"{context.context_id}/replan-{replan_index:04d}", + initial_state=state, + goal=context.goal, + ) + score_steps = min(benchmark.horizon, remaining) + selected = selector(step_context, actions, replan_index, score_steps) + if selected not in actions: + raise ValueError("closed-loop selector returned an unknown action") + truth = benchmark.rollout(step_context, selected) + prefix = min(execute_prefix, remaining) + states.extend(truth.states[:prefix]) + state = truth.states[prefix - 1] + selected_actions.append(selected.name) + remaining -= prefix + replan_index += 1 + trajectory = ManipulationTrajectory( + context_id=context.context_id, + action_name="closed-loop", + states=tuple(states), + ) + return _EpisodeOutcome( + context_id=context.context_id, + selected_actions=tuple(selected_actions), + executed_steps=len(states), + final_return=benchmark.score(context, trajectory), + ) + + +def _episodes_with_oracle( + outcomes: list[_EpisodeOutcome], + oracle_returns: dict[str, float], +) -> tuple[ClosedLoopEpisode, ...]: + episodes: list[ClosedLoopEpisode] = [] + for outcome in outcomes: + oracle_return = oracle_returns[outcome.context_id] + gap = oracle_return - outcome.final_return + episodes.append( + ClosedLoopEpisode( + context_id=outcome.context_id, + selected_actions=outcome.selected_actions, + executed_steps=outcome.executed_steps, + final_return=outcome.final_return, + simulator_oracle_return=oracle_return, + oracle_gap=gap, + success=gap <= 1e-9, + ) + ) + return tuple(episodes) + + +def _profile( + *, + controller_id: str, + controller_kind: str, + offline_crc: float | None, + offline_regret: float | None, + outcomes: list[_EpisodeOutcome], + oracle_returns: dict[str, float], + resamples: int, + seed: int, +) -> ClosedLoopProfile: + episodes = _episodes_with_oracle(outcomes, oracle_returns) + return ClosedLoopProfile( + controller_id=controller_id, + controller_kind=controller_kind, + offline_crc_spearman=offline_crc, + offline_top1_regret=offline_regret, + return_summary=summarize( + [episode.final_return for episode in episodes], + resamples=resamples, + seed=_stable_seed(seed, controller_id, "return"), + ), + success_summary=summarize( + [float(episode.success) for episode in episodes], + resamples=resamples, + seed=_stable_seed(seed, controller_id, "success"), + ), + oracle_gap_summary=summarize( + [episode.oracle_gap for episode in episodes], + resamples=resamples, + seed=_stable_seed(seed, controller_id, "oracle-gap"), + ), + episodes=episodes, + ) + + +def _run_benchmark( + benchmark: ManipulationBenchmark, + *, + contexts: int, + seed: int, + control_steps: int, + execute_prefix: int, + resamples: int, +) -> ClosedLoopBenchmarkResult: + suite: ManipulationInterventionSuite = benchmark.make_suite(contexts=contexts, seed=seed) + simulator_outcomes = [ + _execute_episode( + benchmark, + group.context, + control_steps=control_steps, + execute_prefix=execute_prefix, + selector=lambda current, actions, _index, score_steps: _select_simulator_future( + benchmark, + current, + actions, + score_steps, + ), + ) + for group in suite.groups + ] + oracle_returns = {outcome.context_id: outcome.final_return for outcome in simulator_outcomes} + + profiles: list[ClosedLoopProfile] = [] + for spec in _adapter_specs(benchmark): + offline = evaluate_manipulation(spec.adapter(), benchmark, suite, seed=seed) + adapter = spec.adapter() + try: + + def predicted_selector( + current: ManipulationContext, + actions: tuple[ManipulationAction, ...], + _index: int, + score_steps: int, + current_adapter: ManipulationAdapter = adapter, + ) -> ManipulationAction: + return _select_predicted_future( + current_adapter, + benchmark, + current, + actions, + seed=seed, + score_steps=score_steps, + ) + + outcomes = [ + _execute_episode( + benchmark, + group.context, + control_steps=control_steps, + execute_prefix=execute_prefix, + selector=predicted_selector, + ) + for group in suite.groups + ] + finally: + adapter.close() + profiles.append( + _profile( + controller_id=spec.controller_id, + controller_kind="future-scorer", + offline_crc=offline.metrics["candidate_ranking_spearman"], + offline_regret=offline.metrics["top1_regret"], + outcomes=outcomes, + oracle_returns=oracle_returns, + resamples=resamples, + seed=seed, + ) + ) + + profiles.append( + _profile( + controller_id="simulator-oracle-scorer", + controller_kind="simulator-control", + offline_crc=None, + offline_regret=None, + outcomes=simulator_outcomes, + oracle_returns=oracle_returns, + resamples=resamples, + seed=seed, + ) + ) + fixed_outcomes: list[_EpisodeOutcome] = [] + random_outcomes: list[_EpisodeOutcome] = [] + for group in suite.groups: + fixed_action = benchmark.optimal_action(group.context) + + def fixed_selector( + _current: ManipulationContext, + _actions: tuple[ManipulationAction, ...], + _index: int, + _score_steps: int, + action: ManipulationAction = fixed_action, + ) -> ManipulationAction: + return action + + fixed_outcomes.append( + _execute_episode( + benchmark, + group.context, + control_steps=control_steps, + execute_prefix=execute_prefix, + selector=fixed_selector, + ) + ) + + def random_selector( + current: ManipulationContext, + actions: tuple[ManipulationAction, ...], + index: int, + _score_steps: int, + ) -> ManipulationAction: + rng = Random(_stable_seed(seed, current.context_id, index, "random-controller")) + return actions[rng.randrange(len(actions))] + + random_outcomes.append( + _execute_episode( + benchmark, + group.context, + control_steps=control_steps, + execute_prefix=execute_prefix, + selector=random_selector, + ) + ) + profiles.extend( + ( + _profile( + controller_id="fixed-action-policy", + controller_kind="action-only-control", + offline_crc=None, + offline_regret=None, + outcomes=fixed_outcomes, + oracle_returns=oracle_returns, + resamples=resamples, + seed=seed, + ), + _profile( + controller_id="random-candidate", + controller_kind="random-control", + offline_crc=None, + offline_regret=None, + outcomes=random_outcomes, + oracle_returns=oracle_returns, + resamples=resamples, + seed=seed, + ), + ) + ) + + future_profiles = [ + profile for profile in profiles if profile.controller_kind == "future-scorer" + ] + disagreements = 0 + comparable = 0 + for left, right in combinations(future_profiles, 2): + assert left.offline_crc_spearman is not None + assert right.offline_crc_spearman is not None + offline_difference = left.offline_crc_spearman - right.offline_crc_spearman + return_difference = left.return_summary.mean - right.return_summary.mean + if abs(offline_difference) <= 1e-12 or abs(return_difference) <= 1e-12: + continue + comparable += 1 + if offline_difference * return_difference < 0: + disagreements += 1 + return ClosedLoopBenchmarkResult( + benchmark_id=benchmark.benchmark_id, + control_steps=control_steps, + execute_prefix=execute_prefix, + profiles=tuple(profiles), + correlations={ + "offline_crc_vs_closed_loop_return_pearson": _pearson( + [ + profile.offline_crc_spearman + for profile in future_profiles + if profile.offline_crc_spearman is not None + ], + [profile.return_summary.mean for profile in future_profiles], + ), + "offline_regret_vs_closed_loop_return_pearson": _pearson( + [ + profile.offline_top1_regret + for profile in future_profiles + if profile.offline_top1_regret is not None + ], + [profile.return_summary.mean for profile in future_profiles], + ), + }, + offline_return_order_disagreements=disagreements, + comparable_future_scorer_pairs=comparable, + ) + + +def run_closed_loop_study( + *, + benchmark_names: tuple[str, ...] = ("blockpush", "gripper-catch"), + contexts: int = 12, + seed: int = 7, + control_steps: int | None = None, + execute_prefix: int = 1, + resamples: int = 1000, +) -> ClosedLoopStudy: + """Run score-execute-observe replanning on deterministic manipulation tasks.""" + + if contexts <= 0: + raise ValueError("contexts must be positive") + if control_steps is not None and control_steps <= 0: + raise ValueError("control_steps must be positive") + if execute_prefix <= 0: + raise ValueError("execute_prefix must be positive") + if resamples <= 0: + raise ValueError("resamples must be positive") + if not benchmark_names: + raise ValueError("at least one closed-loop benchmark is required") + + benchmarks: list[ManipulationBenchmark] = [] + for name in benchmark_names: + if name == "blockpush": + benchmarks.append(BlockPush2D()) + elif name == "gripper-catch": + benchmarks.append(GripperCatch()) + else: + raise ValueError(f"unsupported closed-loop benchmark: {name}") + if len(benchmarks) != len({benchmark.benchmark_id for benchmark in benchmarks}): + raise ValueError("closed-loop benchmark names must be unique") + for benchmark in benchmarks: + if execute_prefix > benchmark.horizon: + raise ValueError("execute_prefix must not exceed the prediction horizon") + + return ClosedLoopStudy( + benchmarks=tuple( + _run_benchmark( + benchmark, + contexts=contexts, + seed=seed, + control_steps=benchmark.horizon if control_steps is None else control_steps, + execute_prefix=execute_prefix, + resamples=resamples, + ) + for benchmark in benchmarks + ), + contexts_per_benchmark=contexts, + seed=seed, + resamples=resamples, + interpretation=( + "Each cycle scores the same legal action set, executes only the selected " + "trajectory prefix in true dynamics, then replans from the resulting state; " + "the scoring window is capped by the remaining episode budget. " + "Cross-profile Pearson values are descriptive because there are only five " + "future scorers; context-block intervals remain the primary uncertainty view." + ), + ) + + +def _format_optional(value: float | None) -> str: + return "n/a" if value is None else f"{value:.4f}" + + +def render_closed_loop_markdown(study: ClosedLoopStudy) -> str: + """Render closed-loop results without a composite score.""" + + lines = ["# Minimal closed-loop replanning utility", "", study.interpretation, ""] + for benchmark in study.benchmarks: + lines.extend( + [ + f"## {benchmark.benchmark_id}", + "", + f"Control steps: {benchmark.control_steps}; execute prefix: " + f"{benchmark.execute_prefix}.", + "", + "| Controller | Kind | Offline CRC | Offline regret | " + "Closed return [95% CI] | Success | Oracle gap |", + "|---|---|---:|---:|---:|---:|---:|", + ] + ) + for profile in benchmark.profiles: + interval = profile.return_summary.confidence_interval + lines.append( + f"| {profile.controller_id} | {profile.controller_kind} | " + f"{_format_optional(profile.offline_crc_spearman)} | " + f"{_format_optional(profile.offline_top1_regret)} | " + f"{profile.return_summary.mean:.4f} " + f"[{interval.lower:.4f}, {interval.upper:.4f}] | " + f"{profile.success_summary.mean:.4f} | " + f"{profile.oracle_gap_summary.mean:.4f} |" + ) + crc_correlation = benchmark.correlations["offline_crc_vs_closed_loop_return_pearson"] + regret_correlation = benchmark.correlations["offline_regret_vs_closed_loop_return_pearson"] + lines.extend( + [ + "", + "Future-scorer profile correlation: offline CRC vs closed-loop return = " + f"{_format_optional(crc_correlation)}; offline regret vs closed-loop " + f"return = {_format_optional(regret_correlation)}.", + "", + "Offline-CRC/return ordering disagreed on " + f"{benchmark.offline_return_order_disagreements}/" + f"{benchmark.comparable_future_scorer_pairs} comparable future-scorer pairs.", + "", + ] + ) + lines.extend( + [ + "Success means matching the greedy simulator-future scorer's final return within", + "1e-9 on the same context and control budget. Score ties use the benchmark's stable", + "candidate order. The scoring horizon is capped by the episode steps remaining,", + "preventing terminal scores beyond the execution budget. These analytic tasks", + "validate the evaluator; they do not establish real-robot or large-model closed-loop", + "performance.", + "", + ] + ) + return "\n".join(lines) + + +def write_closed_loop_study(output_dir: Path, study: ClosedLoopStudy) -> tuple[Path, Path]: + """Write deterministic JSON and Markdown closed-loop reports.""" + + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "closed-loop-study.json" + markdown_path = output_dir / "closed-loop-study.md" + json_path.write_text( + json.dumps(study.to_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + markdown_path.write_text(render_closed_loop_markdown(study), encoding="utf-8") + return json_path, markdown_path diff --git a/tests/test_closed_loop_study.py b/tests/test_closed_loop_study.py new file mode 100644 index 0000000..8746b15 --- /dev/null +++ b/tests/test_closed_loop_study.py @@ -0,0 +1,78 @@ +from pathlib import Path + +import pytest + +from wamprobe.cli import main +from wamprobe.closed_loop import run_closed_loop_study + + +def test_closed_loop_replanning_separates_oracle_from_action_ignorance() -> None: + study = run_closed_loop_study( + benchmark_names=("blockpush",), + contexts=4, + seed=5, + execute_prefix=1, + resamples=64, + ) + result = study.benchmarks[0] + profiles = {profile.controller_id: profile for profile in result.profiles} + + oracle = profiles["oracle-simulator"] + copy_last = profiles["copy-last-frame"] + assert oracle.success_summary.mean == 1.0 + assert oracle.return_summary.mean > copy_last.return_summary.mean + assert oracle.offline_crc_spearman == 1.0 + assert copy_last.offline_crc_spearman == 0.0 + assert all(len(episode.selected_actions) == result.control_steps for episode in oracle.episodes) + assert result.correlations["offline_crc_vs_closed_loop_return_pearson"] is not None + assert result.correlations["offline_crc_vs_closed_loop_return_pearson"] > 0.0 + + +def test_closed_loop_study_is_deterministic_and_validates_execution_budget() -> None: + first = run_closed_loop_study( + benchmark_names=("gripper-catch",), + contexts=3, + seed=11, + execute_prefix=1, + resamples=32, + ) + second = run_closed_loop_study( + benchmark_names=("gripper-catch",), + contexts=3, + seed=11, + execute_prefix=1, + resamples=32, + ) + assert first.to_dict() == second.to_dict() + + with pytest.raises(ValueError, match="execute_prefix"): + run_closed_loop_study( + benchmark_names=("blockpush",), + contexts=2, + execute_prefix=7, + ) + + +def test_closed_loop_study_cli_writes_structured_reports(tmp_path: Path) -> None: + assert ( + main( + [ + "closed-loop-study", + "--benchmark", + "blockpush", + "--contexts", + "2", + "--seed", + "5", + "--resamples", + "32", + "--output", + str(tmp_path), + ] + ) + == 0 + ) + assert (tmp_path / "closed-loop-study.json").is_file() + report = (tmp_path / "closed-loop-study.md").read_text(encoding="utf-8") + assert "oracle-simulator" in report + assert "offline CRC" in report From 248bd3d9203fb378c7e16434e6ac6fb6a6636b14 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 14:44:53 +0000 Subject: [PATCH 16/20] prepare reproducible v0.1 release candidate --- .github/dependabot.yml | 12 + .github/workflows/ci.yml | 19 ++ .github/workflows/publish-pypi.yml | 52 ++++ .github/workflows/release-candidate.yml | 41 +++ .gitignore | 8 + CHANGELOG.md | 18 +- CITATION.cff | 3 +- CONTRIBUTING.md | 7 +- README.md | 12 +- docs/QUICKSTART.md | 16 + docs/WAMProbe_PLAN.md | 7 +- .../EXTERNAL_REPRODUCTION_TEMPLATE.md | 18 ++ docs/reproducibility/REPRODUCIBILITY.md | 65 ++++ docs/research/ADAPTER_SELECTION.md | 6 +- paper/CLAIMS_CHECKLIST.md | 15 + paper/README.md | 19 ++ paper/main.tex | 294 ++++++++++++++++++ paper/references.bib | 42 +++ pyproject.toml | 2 +- release/README.md | 46 +++ release/evidence-manifest-v0.1.json | 107 +++++++ release/evidence-manifest-v0.1.schema.json | 54 ++++ scripts/build_release_candidate.py | 208 +++++++++++++ src/wamprobe/__init__.py | 2 +- src/wamprobe/cli.py | 32 ++ src/wamprobe/release_audit.py | 289 +++++++++++++++++ tests/test_import.py | 2 +- tests/test_release_audit.py | 130 ++++++++ uv.lock | 2 +- 29 files changed, 1515 insertions(+), 13 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/publish-pypi.yml create mode 100644 .github/workflows/release-candidate.yml create mode 100644 docs/reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md create mode 100644 docs/reproducibility/REPRODUCIBILITY.md create mode 100644 paper/CLAIMS_CHECKLIST.md create mode 100644 paper/README.md create mode 100644 paper/main.tex create mode 100644 paper/references.bib create mode 100644 release/README.md create mode 100644 release/evidence-manifest-v0.1.json create mode 100644 release/evidence-manifest-v0.1.schema.json create mode 100644 scripts/build_release_candidate.py create mode 100644 src/wamprobe/release_audit.py create mode 100644 tests/test_release_audit.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e5e3023 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 5 + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6921f8..13fdd34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,22 @@ jobs: run: mypy - name: Test with coverage run: pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 + + package-candidate: + needs: quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@v6 + with: + version: "0.11.6" + enable-cache: true + - name: Install locked development environment + run: uv sync --extra dev --locked + - name: Build, reproduce, audit, and smoke candidate + run: .venv/bin/python scripts/build_release_candidate.py --dist-dir dist/ci-candidate diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 0000000..4964c54 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,52 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + inputs: + tag_confirmation: + description: Must exactly match the selected Git tag (for example v0.1.0rc1) + required: true + type: string + +permissions: + contents: read + +jobs: + publish: + if: ${{ github.ref_type == 'tag' && inputs.tag_confirmation == github.ref_name }} + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/wamprobe/ + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@v6 + with: + version: "0.11.6" + enable-cache: true + - name: Require tag and package version to match + run: | + package_version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + test "v${package_version}" = "${GITHUB_REF_NAME}" + - name: Install locked development environment + run: uv sync --extra dev --locked + - name: Rebuild and verify candidate from the clean tag + run: .venv/bin/python scripts/build_release_candidate.py + - name: Stage distributions without the audit sidecar + run: | + mkdir -p dist/publish + cp dist/release-candidate/*.whl dist/publish/ + cp dist/release-candidate/*.tar.gz dist/publish/ + - name: Publish with PyPI Trusted Publishing + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/publish/ + verbose: true diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml new file mode 100644 index 0000000..1e81206 --- /dev/null +++ b/.github/workflows/release-candidate.yml @@ -0,0 +1,41 @@ +name: Release candidate + +on: + workflow_dispatch: + +permissions: + contents: read + id-token: write + attestations: write + +jobs: + build-audit-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@v6 + with: + version: "0.11.6" + enable-cache: true + - name: Install locked development environment + run: uv sync --extra dev --locked + - name: Build twice, audit evidence, and smoke clean wheel + run: .venv/bin/python scripts/build_release_candidate.py + - name: Attest wheel and source distribution provenance + uses: actions/attest-build-provenance@v2 + with: + subject-path: | + dist/release-candidate/*.whl + dist/release-candidate/*.tar.gz + - name: Upload candidate bundle + uses: actions/upload-artifact@v4 + with: + name: wamprobe-release-candidate-${{ github.sha }} + if-no-files-found: error + retention-days: 30 + path: dist/release-candidate/ diff --git a/.gitignore b/.gitignore index ce3af87..bf3ecc1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,14 @@ vendor/ *.pth *.safetensors +# Local technical-report builds +paper/*.aux +paper/*.bbl +paper/*.blg +paper/*.log +paper/*.out +paper/*.pdf + # Editors and operating systems .DS_Store .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0ffca..479b04f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,10 @@ All notable changes to WAMProbe will be documented here. -## [Unreleased] +## [0.1.0rc1] - 2026-07-15 + +This candidate is prepared for reproducibility review and is not yet published to PyPI or +as a final GitHub Release. ### Added @@ -20,3 +23,16 @@ All notable changes to WAMProbe will be documented here. output for local model artifacts. - Reproducible StarWAM environment preflight and a narrow patch that restricts verified inference checkpoint loads to PyTorch's weights-only deserializer. +- Checksummed intervention JSONL, corruption-detecting content-addressed prediction cache, + exact-context comparison, and report-only regeneration commands. +- Contact-aware BlockPush-2D and attachment-aware Gripper-Catch analytic benchmarks. +- Four-family LIBERO-CF-Mini generation with exact snapshot restore, repeat, branch-order, + provenance, and cache validation. +- A pinned StarWAM 36-prediction matrix and 36 action-chunk simulator executions with + explicit unsupported-metric skips and retained zero-success results. +- Dependency-free PSNR/global-SSIM diagnostics and a traditional-video/control-value + counterexample study. +- Minimal score-execute-observe closed-loop replanning with context-block intervals and + offline-ranking/return association analysis. +- Release artifact audit, candidate workflow, clean-wheel smoke test, evidence manifest, + and technical-report draft. diff --git a/CITATION.cff b/CITATION.cff index eb1c8c3..d347401 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,7 @@ cff-version: 1.2.0 message: "If you use WAMProbe, please cite the software release." title: "WAMProbe: Counterfactual Evaluation for World Action Models" type: software -version: 0.1.0-alpha -date-released: 2026-07-15 +version: 0.1.0-rc.1 license: Apache-2.0 repository-code: "https://github.com/myheart521/WAMProbe" authors: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b462031..c8b9b87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,9 +16,14 @@ Before opening a pull request, run: ruff format --check . ruff check . mypy -pytest --cov=wamprobe --cov-report=term-missing +pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 ``` +Changes to public schemas, committed evidence, package metadata, or release tooling must +also run `.venv/bin/python scripts/build_release_candidate.py` from a clean commit. Do not +use `--allow-dirty` as release evidence; that switch exists only for local development of +the audit itself. + ## Contribution requirements - Follow test-driven development for behavior changes. diff --git a/README.md b/README.md index d25909d..41a9339 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ the same initial state └── expert act → predicted future / true future ``` -The project is in an early alpha. Its dependency-free core includes an analytic +The project is preparing a `0.1.0rc1` source candidate. Its dependency-free core includes an analytic PointMass-2D benchmark, while opt-in isolated runners now cover one released StarWAM observation-to-action path and a four-family paired LIBERO-CF-Mini simulator pilot. @@ -134,6 +134,9 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - a minimal receding-horizon evaluator with random, fixed-policy, simulator-oracle, and five future-scorer controls, per-context traces, bootstrap intervals, and offline/closed-loop association analysis; +- a versioned evidence manifest, byte-reproducible wheel/sdist builder, archive audit, + offline clean-wheel demo smoke, manual provenance-attestation workflow, and LaTeX + technical-report draft; - Python 3.11–3.13 CI with linting, strict typing, and coverage. ## Roadmap @@ -143,7 +146,8 @@ The next milestones are: 1. expand LIBERO initial states and evaluate action-conditioned real-WAM futures when an adapter exposes that capability; 2. add the Occluded-Object memory diagnostic to the broader Toy tier; -3. prepare a traceable v0.1 release candidate and external reproduction smoke. +3. obtain an independent reproduction smoke, then review the candidate for explicit + maintainer-approved PyPI/GitHub publication. See the [detailed Chinese project plan](docs/WAMProbe_PLAN.md), [quick-start notes](docs/QUICKSTART.md), [failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md), @@ -155,6 +159,10 @@ The exact toy dynamics and limitations are documented in the [core metric cards](docs/metrics/CORE_METRICS.md). The exact closed-loop protocol and limitations are recorded in the [toy closed-loop experiment card](docs/experiments/TOY_CLOSED_LOOP_V0.1.md). +Release reviewers can start with the +[reproducibility guide](docs/reproducibility/REPRODUCIBILITY.md), +[candidate procedure](release/README.md), and +[technical report source](paper/main.tex). ## Development diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 4ed4c6f..d2f6938 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -122,6 +122,22 @@ on Gripper-Catch, but these are descriptive associations across only five future profiles. See the [report](../examples/closed-loop-study/closed-loop-study.md) and [experiment card](experiments/TOY_CLOSED_LOOP_V0.1.md). +## Audit a release candidate + +From a clean commit, the candidate command builds twice with the commit timestamp as +`SOURCE_DATE_EPOCH`, verifies identical wheel/sdist hashes and evidence identities, then +installs the wheel offline in a fresh environment and runs a two-context demo: + +```bash +uv sync --extra dev --locked +.venv/bin/python scripts/build_release_candidate.py +``` + +The command writes ignored artifacts and `release-manifest.json` below +`dist/release-candidate/`. It does not publish or tag anything. See the +[release procedure](../release/README.md) and +[reproducibility guide](reproducibility/REPRODUCIBILITY.md). + Uncertainty is computed at the context level. Each bootstrap draw resamples whole shared initial states, never individual action branches or adjacent frames. Paired comparisons align exact context IDs and report the left-minus-right difference with a 95% interval. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 5015c70..90f06ad 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -1123,7 +1123,10 @@ snapshot 执行全部动作块，并在 horizon 8/16/32 记录状态、回报和 return 的 5-profile 描述性 Pearson 分别为 0.9855/1.0。全部结果、context-block CI 和限制 见 [`examples/video-control-study/`](../examples/video-control-study/video-control-study.md) 与 [`docs/experiments/TOY_CLOSED_LOOP_V0.1.md`](experiments/TOY_CLOSED_LOOP_V0.1.md)。 -release artifact 追溯、技术报告和外部复现仍待完成。 +`0.1.0rc1` 候选版本现已具备双构建 SHA 一致性、归档安全/metadata 审计、证据 manifest、 +离线干净 wheel 安装 smoke、手动 provenance attestation workflow，并提供可在 Overleaf +直接编译的 5 页技术报告初稿。正式 PyPI/GitHub Release、上游 PR 和外部用户复现仍需 +维护者或第三方授权，未被自动触发。 任务： @@ -1311,7 +1314,7 @@ priority:p1 - [x] 核心 coverage ≥ 85%； - [x] schema 有版本和兼容策略； -- [ ] 所有 release artifact 可追溯； +- [x] 所有 release artifact 可追溯； - [x] README 15 分钟 quickstart 可执行； - [x] benchmark/model/metric cards 完整； - [ ] 至少一个外部用户复现 smoke test。 diff --git a/docs/reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md b/docs/reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md new file mode 100644 index 0000000..6c97f67 --- /dev/null +++ b/docs/reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md @@ -0,0 +1,18 @@ +# External reproduction report + +- Reproducer (name or handle): +- Date and timezone: +- WAMProbe Git tag/commit: +- Operating system and architecture: +- Python version: +- Installation command: +- Smoke command: +- Exit status: +- Elapsed time: +- Output `summary.json` SHA256: +- Deviations from the documented procedure: +- Result: PASS / FAIL +- Optional notes or issue URL: + +The reproducer should attach this report to a GitHub issue or Discussion without tokens, +browser sessions, private paths, model files, or other secrets. diff --git a/docs/reproducibility/REPRODUCIBILITY.md b/docs/reproducibility/REPRODUCIBILITY.md new file mode 100644 index 0000000..28568bb --- /dev/null +++ b/docs/reproducibility/REPRODUCIBILITY.md @@ -0,0 +1,65 @@ +# Reproducibility guide + +## Levels of reproduction + +The dependency-free core, the simulator tier, and the real-model tier have different +requirements and must not be conflated. + +| Level | Hardware | External assets | Expected evidence | +|---|---|---|---| +| CPU core | Python 3.11–3.13 CPU | none | 72+ tests, toy reports, video/control and closed-loop studies | +| LIBERO-CF-Mini | Linux GPU renderer | pinned LIBERO checkout/assets | exact restore, repeated/order-reversed branches, checksummed images/states | +| StarWAM | RTX-class CUDA GPU | pinned StarWAM, Wan2.2 and LIBERO files | cached action matrix, execution index, structured report | + +## CPU reproduction + +```bash +git clone https://github.com/myheart521/WAMProbe.git +cd WAMProbe +git checkout +python -m venv .venv +source .venv/bin/activate +python -m pip install -e '.[dev]' + +ruff format --check . +ruff check . +mypy +pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 + +wamprobe demo --benchmark pointmass --contexts 12 --seed 7 \ + --output runs/pointmass-reproduction +wamprobe video-control-study --contexts 12 --seed 7 \ + --output runs/video-control-reproduction +wamprobe closed-loop-study --contexts 12 --seed 7 --execute-prefix 1 \ + --resamples 1000 --output runs/closed-loop-reproduction +``` + +Compare generated video/control and closed-loop JSON files byte-for-byte with the committed +examples. The canonical SHA256 values are `1de2009f...cf4` and `19d2a010...89b3`. + +## Simulator and real-model reproduction + +Follow the isolated environment recipes in `environments/libero/README.md` and +`environments/starwam/README.md`. Run `wamprobe doctor --verify-hashes` before importing +upstream model code. The release evidence manifest pins upstream revisions, configuration +files, committed reports, and the locally retained prediction/execution index hashes. + +Generated RGB, simulator state, model weights and raw predictions remain outside Git. +Their reports contain hashes and explicit availability labels; absence is an error, not a +reason to synthesize substitute results. Candidate-action metrics remain skipped for the +released StarWAM interface because it accepts observations and emits an action chunk, not +an action-conditioned future for each proposed candidate. + +## Candidate package reproduction + +Run `scripts/build_release_candidate.py` from a clean commit. A valid candidate has two +byte-reproducible distributions, a `release-manifest.json`, a passing offline clean-wheel +install, and matching evidence hashes. See [`release/README.md`](../../release/README.md). + +## Independent reproduction report + +An external reproducer should record the Git commit/tag, operating system, Python version, +install command, exact smoke command, exit status, output JSON SHA256, elapsed time, and any +deviation. A template is provided in +[`EXTERNAL_REPRODUCTION_TEMPLATE.md`](EXTERNAL_REPRODUCTION_TEMPLATE.md). Maintainers must +not mark the external-smoke release gate complete from a run on their own machine. diff --git a/docs/research/ADAPTER_SELECTION.md b/docs/research/ADAPTER_SELECTION.md index 2675de4..89b69e3 100644 --- a/docs/research/ADAPTER_SELECTION.md +++ b/docs/research/ADAPTER_SELECTION.md @@ -122,7 +122,11 @@ The next coding slice is deliberately narrow: 5. capability declaration and mocked CPU contract tests: **complete**; 6. opt-in fixed LIBERO GPU smoke runner: **complete**; 7. paired simulator snapshot/restore and counterfactual scoring: **complete**; -8. execute cached StarWAM candidate branches and expand LIBERO-CF-Mini tasks: **next**. +8. execute cached StarWAM action chunks and expand LIBERO-CF-Mini to four task families: + **complete**; +9. evaluate action-conditioned real-WAM candidate futures: **capability-blocked for the + released StarWAM adapter**, which accepts observations and emits actions but does not + accept candidate actions or return one future per candidate. ## Sources checked diff --git a/paper/CLAIMS_CHECKLIST.md b/paper/CLAIMS_CHECKLIST.md new file mode 100644 index 0000000..929b827 --- /dev/null +++ b/paper/CLAIMS_CHECKLIST.md @@ -0,0 +1,15 @@ +# Technical report claim audit + +| Claim | Evidence | Current status | +|---|---|---| +| Reference metrics separate known failure baselines | committed PointMass, BlockPush and Gripper-Catch reports/tests | supported on analytic tasks | +| Traditional pixel fidelity cannot replace control metrics | 12-context two-task appearance-corruption study, JSON SHA `1de2009f...cf4` | supported as a constructed counterexample | +| Offline CRC tracks analytic closed-loop return | five future scorers, two 12-context tasks, JSON SHA `19d2a010...89b3` | descriptive only | +| LIBERO branches restore the same simulator state | four tasks, repeat/order checks, maximum error 0 | supported for the fixed pilot | +| StarWAM inference and action execution are reproducible | 36 predictions, 36 executions, verified cache/index hashes | supported for pinned short-horizon inputs | +| StarWAM improves LIBERO task success | all tested sparse success values are 0 | not supported and not claimed | +| WAMProbe metrics predict real-model closed-loop gain | StarWAM exposes no candidate-conditioned future in this adapter | not yet tested | + +Any revision of the abstract, introduction, or conclusion must remain consistent with this +table. New claims require a structured artifact, an applicability statement, and a failure +or uncertainty analysis. diff --git a/paper/README.md b/paper/README.md new file mode 100644 index 0000000..754a882 --- /dev/null +++ b/paper/README.md @@ -0,0 +1,19 @@ +# Technical report source + +`main.tex` is an Overleaf-ready draft. It deliberately separates analytic evidence, +paired simulator validation, and the real StarWAM action-only experiment. + +Compile locally with: + +```bash +cd paper +pdflatex main.tex +bibtex main +pdflatex main.tex +pdflatex main.tex +``` + +Before submission, replace corporate `et al.` bibliography placeholders with complete +author metadata from the pinned records, add publication-quality figures, expand the +real-model candidate-future experiment, and obtain an independent reproduction. Generated +PDF and LaTeX auxiliary files are not committed. diff --git a/paper/main.tex b/paper/main.tex new file mode 100644 index 0000000..9bfc89d --- /dev/null +++ b/paper/main.tex @@ -0,0 +1,294 @@ +\documentclass[10pt]{article} + +\usepackage[T1]{fontenc} +\usepackage{amsmath} +\usepackage{booktabs} +\usepackage[margin=1in]{geometry} +\usepackage[hidelinks]{hyperref} +\usepackage[expansion=false]{microtype} + +\title{WAMProbe: Capability-Aware Counterfactual Evaluation\\for World Action Models} +\author{WAMProbe Contributors} +\date{Technical Report Draft, July 2026} + +\begin{document} +\maketitle + +\begin{abstract} +World Action Models (WAMs) combine future prediction with robot action modeling, but the +role of the predicted future is difficult to identify from task success or video fidelity +alone. We present WAMProbe, an open-source evaluation toolkit organized around paired +action interventions from a shared initial state. The toolkit routes metrics through an +explicit capability declaration and reports causal response, state accuracy, candidate +ranking, control regret, and compute cost as a profile rather than a composite score. +The current release candidate contains three analytic benchmarks, a four-family paired LIBERO +simulator pilot, and a pinned StarWAM action-prediction adapter. Reference baselines show +the intended failure signatures: a wrong-direction model responds to action while +predicting reversed dynamics, and action-ignoring models fail candidate ranking. A +controlled appearance corruption preserves exact state prediction and zero regret while +reducing PSNR to approximately 0.59 dB, which shows that pixel fidelity cannot replace +control-grounded measures. In an analytic score-execute-observe study, offline Candidate +Ranking Correlation has descriptive Pearson association 0.9855 and 1.0000 with mean +closed-loop return on BlockPush and Gripper-Catch, respectively, across five deliberately +separated future scorers. A separate 36-sample StarWAM matrix executes every predicted +action chunk successfully in LIBERO but obtains zero sparse task success at the tested +short horizons. We retain this negative result and do not infer real-model closed-loop +utility from the analytic study. +\end{abstract} + +\section{Introduction} + +Task success is necessary for robot policy evaluation, but it does not identify why a +system succeeds or fails. Recent WAM diagnostics report differences in motion +consistency, target progress, distractor motion, and runtime that disappear in a single +success number \cite{mai2026beyond}. A second ambiguity arises when a predicted video is +plausible but incompatible with the action that supposedly caused it +\cite{ruan2026compatible}. Explicit future generation can also add inference cost without +providing a corresponding control gain \cite{yuan2026fastwam}. These findings motivate a +more direct question: under the same observation, do different proposed actions lead to +the correct different futures, and do those futures select actions that improve control? + +WAMProbe treats a shared initial state with multiple action branches as the primary +evaluation unit. This design supports paired comparisons and prevents context variation +from being mistaken for action sensitivity. It also exposes simple failure modes that a +single rollout hides. A copy-last predictor ignores action. A wrong-direction predictor +is action-dependent but causally incorrect. An action-agnostic success prior predicts the +same attractive outcome for every candidate and therefore cannot rank them. + +The toolkit makes three practical choices. First, an adapter declares whether it predicts +pixels, states, latents, actions, candidate scores, or world features. Unsupported metrics +are skipped with a reason. Second, uncertainty resamples complete contexts rather than +correlated branches or frames. Third, reports retain metric profiles and negative results; +WAMProbe does not publish a single total score. + +\section{Related Evaluation Problems} + +WAMs cover several interfaces, including imagine-then-execute systems, joint video-action +models, feature-conditioned policies, and action predictors trained with future-prediction +objectives. Their outputs are not interchangeable. Dynamic-consistency work argues for +checking whether predicted state changes agree with actions and warns that static +backgrounds can make failed trajectories appear consistent \cite{ruan2026compatible}. +Adaptive execution work identifies a separate deployment problem: a fixed action chunk +can continue after the observed state has diverged from the imagined future +\cite{wang2026trust}. WAMProbe addresses these concerns with action branches, explicit +no-op and transition-magnitude diagnostics, prefix execution, and replanning. + +LIBERO provides task suites, simulator states, and standardized manipulation settings for +lifelong robot learning \cite{liu2023libero}. WAMProbe uses pinned LIBERO tasks as a paired +data-generation tier, not as a new policy leaderboard. The current real-model integration +targets the released StarWAM observation-to-action path \cite{pan2026starwam}. Because +that interface does not accept candidate actions or emit action-conditioned future states +or videos, candidate-action and future-fidelity metrics are structurally unavailable for +this adapter. + +\section{Counterfactual Evaluation} + +Let $c$ denote a context containing the observation, task, initial simulator state, and +goal. A candidate generator provides actions $\mathcal{A}(c)=\{a_1,\ldots,a_K\}$. For each +candidate, an adapter predicts a future $\hat{\tau}_i$, while a simulator or paired dataset +provides the reference future $\tau_i$. WAMProbe stores every branch under the same context +identifier and verifies that simulator branches begin from an identical snapshot. + +\subsection{Causal and dynamics profile} + +Action Dependence measures separation among predicted branch endpoints relative to the +separation among reference endpoints. It detects response to action but not correctness. +Counterfactual Direction compares predicted and true displacement directions. State ADE +and FDE measure trajectory and terminal error after semantic state alignment. No-op +fidelity checks the explicit no-op reference; it does not assume that a passive world, +such as a falling object, should freeze. + +The use of separate measures is important. If $\hat{\tau}(a)$ follows the negative of the +true action effect, its branches remain well separated and Action Dependence can be high, +while direction and state error correctly identify the failure. + +\subsection{Candidate ranking and regret} + +A fixed task scorer $R(c,\tau)$ converts each future into a candidate value. Candidate +Ranking Correlation (CRC) reports Spearman correlation, Kendall $\tau_b$, NDCG, and +pairwise preference accuracy with explicit tie handling. Top-1 Regret is + +\begin{equation} +\operatorname{Regret}(c)= +\max_{a_i\in\mathcal{A}(c)}R(c,\tau_i) +- R(c,\tau_{\hat\imath}), +\quad +\hat\imath=\arg\max_i R(c,\hat\tau_i). +\end{equation} + +This definition keeps prediction and control semantics separate: the adapter predicts a +future, a fixed scorer values it, and regret is evaluated with the true branch outcome. + +\subsection{Closed-loop utility} + +The minimal closed-loop evaluator repeats four operations: score the legal candidate set, +select the highest-scoring future, execute only a prefix under true dynamics, and construct +the next context from the resulting state. The scoring prefix is capped by the remaining +episode budget. Without this cap, an exact fixed-horizon model can be penalized for a +terminal state beyond the steps that will actually be executed. The report records the +candidate order, execution prefix, total control steps, selected action sequence, final +return, success, and gap to a greedy simulator-future scorer for every context. + +\section{Benchmarks and Implementation} + +The dependency-free tier contains PointMass-2D, BlockPush-2D, and Gripper-Catch. PointMass +provides exact planar motion. BlockPush adds a pre-contact phase followed by directed +object motion. Gripper-Catch requires spatial alignment and an explicit close command +before a falling object becomes attached. The latter two benchmarks expose exact state +and a deterministic $64\times64$ RGB observation. + +LIBERO-CF-Mini fixes four tasks spanning spatial, object, articulated-goal, and long-horizon +families. Each context branches into no-op, positive and negative end-effector motion, and +gripper closure for eight steps from the same simulator snapshot. The snapshot includes +MuJoCo integration state and Python-side controller, observable, gripper, clock, and random +state. Generation verifies two independent restores, a repeated branch, and forward versus +reverse branch order. + +The package contains deterministic JSONL intervention data, content-addressed prediction +caches with payload checksums, context-level bootstrap intervals, exact-context paired +comparisons, and JSON, Markdown, and HTML reports. Python 3.11--3.13 continuous integration +runs formatting, linting, strict type checking, tests, and an 85\% coverage floor. + +\section{Experiments} + +\subsection{Reference baseline sanity checks} + +Table~\ref{tab:pointmass} shows the PointMass profile. Oracle dynamics attain exact state +prediction and ranking. The wrong-direction baseline has maximal Action Dependence but +direction score $-1$, CRC $-0.686$, and the largest regret. Copy-last and action-agnostic +baselines have zero Action Dependence and zero CRC. BlockPush and Gripper-Catch reproduce +the same qualitative ordering while testing contact and attachment semantics. + +\begin{table}[t] +\centering +\small +\caption{PointMass-2D reference profile over 12 shared contexts. Higher ADS, direction, +and CRC are better; lower FDE and regret are better.} +\label{tab:pointmass} +\begin{tabular}{lrrrrr} +\toprule +Model & ADS & Direction & FDE & CRC & Regret \\ +\midrule +Oracle & 1.000 & 1.000 & 0.000 & 1.000 & 0.000 \\ +Noisy linear & 0.989 & 0.997 & 0.093 & 0.977 & 0.000 \\ +Copy last & 0.000 & 0.000 & 0.800 & 0.000 & 1.000 \\ +Wrong direction & 1.000 & -1.000 & 1.600 & -0.686 & 2.000 \\ +Action agnostic & 0.000 & 0.000 & 1.166 & 0.000 & 1.000 \\ +\bottomrule +\end{tabular} +\end{table} + +\subsection{Paired LIBERO data generation} + +The LIBERO-CF-Mini pilot contains four tasks, four branches per task, eight post-action +steps, and two camera views. All independent restore, repeat, and branch-order checks pass; +the maximum integration-state error is 0.0. A second complete invocation validates the +snapshot sidecars, structured records, and every PNG by size and SHA256, then returns four +cache hits without opening the simulator. All diagnostic branch returns and sparse success +values are zero. This tier therefore validates paired data generation and branch +separation, not policy quality. + +\subsection{StarWAM action prediction and execution} + +The StarWAM matrix contains four tasks, three diffusion seeds, and action NFE values 1, 4, +and 8, for 36 predictions. Every inference succeeds, a repeated run obtains 36 verified +cache hits, and all 36 action chunks execute from their paired LIBERO snapshots. Mean +latencies are 0.780, 0.972, and 1.216 seconds for NFE 1, 4, and 8; peak allocated GPU memory +is approximately 11.39 GiB. Mean end-effector displacement for executed prefixes 8, 16, +and 32 is 0.1716, 0.1121, and 0.0000 meters. Longer execution is not monotonically more +useful in this diagnostic. Sparse return and success remain zero for every sample, so a +success correlation is undefined and no LIBERO task-solving claim is made. + +\subsection{Video fidelity is not control value} + +We evaluate six profiles on 12 BlockPush and 12 Gripper-Catch contexts. An +appearance-corrupted oracle inverts every predicted RGB value but preserves the exact +state future. It retains FDE 0, CRC 1, and regret 0 while obtaining PSNR 0.588 and 0.586 dB +on the two benchmarks. PSNR and regret ordering disagree on 3 of 9 and 5 of 9 comparable +profile pairs. The Pearson association between PSNR and regret is approximately $-0.16$ +in each benchmark profile. This constructed counterexample does not imply that appearance +is irrelevant. It establishes the narrower claim that PSNR and a whole-frame global SSIM +diagnostic cannot substitute for state accuracy, ranking, or regret. + +\subsection{Offline ranking and analytic closed-loop return} + +Table~\ref{tab:closedloop} reports score-execute-observe evaluation with one executed step +per replan and 1,000 context-block bootstrap resamples. Oracle and noisy future scorers +solve BlockPush; the noisy scorer reaches 91.7\% exact-reference success on Gripper-Catch +with mean return 0.9967. Copy-last, wrong-direction, and action-agnostic future scorers have +zero success on both tasks. CRC and mean closed-loop return have Pearson association 0.9855 +for BlockPush and 1.0000 for Gripper-Catch across the five future scorers. Regret and return +have association $-1.0000$ in both cases. + +\begin{table}[t] +\centering +\small +\caption{Analytic closed-loop results over 12 contexts. Correlations use only the five +future-scorer rows; they are descriptive rather than population estimates.} +\label{tab:closedloop} +\begin{tabular}{llrrr} +\toprule +Task & Future scorer & Offline CRC & Mean return & Success \\ +\midrule +BlockPush & Oracle & 1.000 & 0.000 & 1.000 \\ +BlockPush & Noisy & 1.000 & 0.000 & 1.000 \\ +BlockPush & Copy last & 0.000 & -0.300 & 0.000 \\ +BlockPush & Wrong direction & -0.250 & -0.300 & 0.000 \\ +BlockPush & Action agnostic & 0.000 & -0.300 & 0.000 \\ +Gripper-Catch & Oracle & 1.000 & 1.000 & 1.000 \\ +Gripper-Catch & Noisy & 1.000 & 0.997 & 0.917 \\ +Gripper-Catch & Copy last & 0.000 & 0.000 & 0.000 \\ +Gripper-Catch & Wrong direction & 0.000 & 0.000 & 0.000 \\ +Gripper-Catch & Action agnostic & 0.000 & 0.000 & 0.000 \\ +\bottomrule +\end{tabular} +\end{table} + +The result confirms the evaluator path from offline ranking to repeated execution on two +controlled tasks. It does not show that a learned WAM improves control: the five profiles +are intentionally separated analytic baselines, three weak profiles tie on return, and a +fixed action-only policy also reaches the task optimum. + +\section{Reproducibility and Release Discipline} + +Model weights, raw RGB, simulator states, and generated predictions remain outside Git. +Committed manifests pin upstream revisions and configuration hashes, while reports retain +the hashes of local evidence indexes. A release-candidate command fixes the source date to +the commit timestamp, builds wheel and source distributions twice, requires byte-identical +SHA256 values, rejects unsafe or forbidden archive paths, checks committed evidence, and +installs the wheel without network access in a fresh virtual environment. Public release +is separate from candidate construction and requires independent reproduction and +maintainer approval. + +\section{Limitations} + +The analytic benchmarks have known low-dimensional dynamics and cannot establish visual, +physical, or task-distribution transfer. The closed-loop correlations contain five +designed profiles and should not be interpreted as statistical evidence over trained +models. LIBERO-CF-Mini has one initial state per task family and diagnostic actions rather +than expert candidates. The StarWAM slice exposes actions but no action-conditioned future +artifact, preventing the central candidate-future metrics on that model. The real-model +matrix contains short open-loop executions with constant sparse return. A paper-level +claim about WAM control utility requires more initial states and at least one adapter that +can score the same candidate set through predicted futures. + +The current whole-frame global SSIM implementation is a dependency-free diagnostic, not +the standard windowed metric. Pixel metrics are secondary. The toolkit also does not +provide real-robot safety certification, and it does not normalize all robot action spaces +into a universal representation. + +\section{Conclusion} + +WAMProbe turns action conditioning from an informal expectation into a paired evaluation +contract. Its current evidence supports three limited conclusions: reference metrics +separate known causal failures, pixel fidelity cannot replace control-grounded evaluation, +and offline candidate ranking tracks closed-loop return in two analytic sanity tasks. The +real StarWAM experiment establishes a reproducible action-prediction and execution path but +does not yet establish task success or candidate-future utility. Preserving these boundaries +is central to the project: unsupported metrics are skipped, negative results remain visible, +and every public claim is linked to structured evidence. + +\bibliographystyle{plain} +\bibliography{references} + +\end{document} diff --git a/paper/references.bib b/paper/references.bib new file mode 100644 index 0000000..2fc260f --- /dev/null +++ b/paper/references.bib @@ -0,0 +1,42 @@ +@article{mai2026beyond, + author = {{Mai et al.}}, + title = {Beyond Task Success: Behavioral and Representational Diagnostics for WAM and VLA}, + journal = {arXiv preprint arXiv:2606.01095v1}, + year = {2026} +} + +@article{ruan2026compatible, + author = {{Ruan et al.}}, + title = {Is the Future Compatible? Diagnosing Dynamic Consistency in World Action Models}, + journal = {arXiv preprint arXiv:2605.07514v1}, + year = {2026} +} + +@article{wang2026trust, + author = {{Wang et al.}}, + title = {When to Trust Imagination: Adaptive Action Execution for World Action Models}, + journal = {arXiv preprint arXiv:2605.06222v2}, + year = {2026} +} + +@article{yuan2026fastwam, + author = {{Yuan et al.}}, + title = {Fast-WAM: Do World Action Models Need Test-time Future Imagination?}, + journal = {arXiv preprint arXiv:2603.16666v2}, + year = {2026} +} + +@inproceedings{liu2023libero, + author = {{Liu et al.}}, + title = {LIBERO: Benchmarking Knowledge Transfer for Lifelong Robot Learning}, + booktitle = {Advances in Neural Information Processing Systems}, + year = {2023} +} + +@misc{pan2026starwam, + author = {{Pan et al.}}, + title = {StarWAM Official Repository}, + year = {2026}, + howpublished = {\url{https://github.com/shaohua-pan/StarWAM}}, + note = {Revision f6c771fc3be0a9bc271ea4f1531d8ea35efb0ec7} +} diff --git a/pyproject.toml b/pyproject.toml index dce7837..cc8aa7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "wamprobe" -version = "0.1.0a0" +version = "0.1.0rc1" description = "Causal consistency and control-utility evaluation for World Action Models" readme = "README.md" requires-python = ">=3.11" diff --git a/release/README.md b/release/README.md new file mode 100644 index 0000000..cb4e4f7 --- /dev/null +++ b/release/README.md @@ -0,0 +1,46 @@ +# Release candidate procedure + +WAMProbe separates candidate verification from public release. The candidate workflow +builds and attests artifacts but cannot publish to PyPI or create a GitHub Release. + +## Local candidate + +Run from a clean commit with the locked development environment: + +```bash +uv sync --extra dev --locked +.venv/bin/python scripts/build_release_candidate.py +``` + +The script fixes `SOURCE_DATE_EPOCH` to the source commit timestamp, builds the wheel and +sdist twice, requires byte-identical SHA256 values, validates archive paths and package +metadata, checks every committed evidence hash, installs the wheel without network or +dependencies in a fresh virtual environment, and runs the CLI/demo smoke. Outputs remain +below ignored `dist/release-candidate/`: + +```text +wamprobe--py3-none-any.whl +wamprobe-.tar.gz +release-manifest.json +``` + +The GitHub `Release candidate` workflow repeats this process and adds GitHub build +provenance attestations. It is manual-dispatch only. + +## Public release boundary + +Before publishing, a maintainer must review the candidate manifest, confirm the version +and changelog, obtain one independent clean-environment smoke report, configure the PyPI +Trusted Publisher for the repository/environment, and approve both the Git tag and GitHub +Release text. Public `v0.1.0` publication is intentionally not automated from this Draft PR. + +The guarded `Publish to PyPI` workflow is manual-dispatch only, runs only when the selected +ref is a tag, requires the typed confirmation to equal that tag, verifies that +`v` equals the tag, rebuilds the clean candidate, and uses short-lived +GitHub OIDC through the protected `pypi` environment. It contains no API token and does not +create a GitHub Release. Repository administrators must configure the PyPI Trusted +Publisher and require environment approval before the first dispatch. + +Rollback for a bad pre-release means yanking the affected PyPI file (without deleting its +audit record), marking the GitHub Release as a pre-release with a warning, and preparing a +new version. Published filenames and tags are immutable and must never be overwritten. diff --git a/release/evidence-manifest-v0.1.json b/release/evidence-manifest-v0.1.json new file mode 100644 index 0000000..70241e6 --- /dev/null +++ b/release/evidence-manifest-v0.1.json @@ -0,0 +1,107 @@ +{ + "schema_version": "0.1", + "release_version": "0.1.0rc1", + "committed_evidence": [ + { + "path": "release/evidence-manifest-v0.1.schema.json", + "kind": "release-schema", + "sha256": "5d4d7deaef2c67b42c0c440370a89d7bed09b7fefce322040f18f1d33aa66603" + }, + { + "path": "configs/models/upstream_models.json", + "kind": "upstream-model-pins", + "sha256": "169ff5e6767f47dfd49eef15f5bc7fec6494da82392fbbc8c08e42878686d1cf" + }, + { + "path": "configs/benchmarks/libero_cf_mini_v0.1.json", + "kind": "benchmark-manifest", + "sha256": "1660ab49ec0e5aac58f4801f0ec9a12326053f21f5f1974dd3b6ea63260f78c3" + }, + { + "path": "schemas/capability-v0.1.schema.json", + "kind": "public-schema", + "sha256": "ba6dc045b74ac526b35299186952a44580e8602f697b42685c70c8b9711a89cd" + }, + { + "path": "schemas/intervention-v0.1.schema.json", + "kind": "public-schema", + "sha256": "0dfad48358608b523d2512b0e4f65111b4f0f07f106908463471f8d156702e60" + }, + { + "path": "schemas/result-v0.1.schema.json", + "kind": "public-schema", + "sha256": "a5edc7d1e06e7471f71fbf4f7323f84bf3be0d28228808a539ab78b420caf52f" + }, + { + "path": "examples/pointmass-demo/summary.json", + "kind": "analytic-result", + "sha256": "26ae2d8288414ce321badd699d91726e9459fbe369eec2f0b7f8269089676735" + }, + { + "path": "examples/blockpush-demo/summary.json", + "kind": "analytic-result", + "sha256": "28a52c8c85b4150552878b05ac55babd995538a9304f591dc8d8897f330f786e" + }, + { + "path": "examples/gripper-catch-demo/summary.json", + "kind": "analytic-result", + "sha256": "52772e0abec6c249eeef4532657a94eea9b6484146667666f523303bb47312fc" + }, + { + "path": "examples/video-control-study/video-control-study.json", + "kind": "video-control-result", + "sha256": "1de2009fa09a025db1f16ff55b07220ee679ba228f01971e08a9808e717b6cf4" + }, + { + "path": "examples/closed-loop-study/closed-loop-study.json", + "kind": "closed-loop-result", + "sha256": "19d2a0108fc19580da4e81f15226a4706a40c9d2c0513857d6bc8dc9000189b3" + }, + { + "path": "docs/benchmarks/libero_cf_mini_verified_v0.1.json", + "kind": "paired-simulator-verification", + "sha256": "8d2009d1ceb48da88858764ecda231df532cbf96eaeac054758db2d1f8baed04" + }, + { + "path": "docs/experiments/STARWAM_LIBERO_CF_MINI_V0.1.md", + "kind": "real-model-experiment-report", + "sha256": "e86b9d171a75f4bd34f4e2a6a14ce6f838c0230ef7a19e414950491bf58c8a96" + }, + { + "path": "paper/main.tex", + "kind": "technical-report-draft", + "sha256": "a5f78a0deec7d0f90c1a13c9e648001b6ecdf33933a4d6701af85a24942c9360" + }, + { + "path": "paper/CLAIMS_CHECKLIST.md", + "kind": "claim-evidence-audit", + "sha256": "b5711af4ed8ae1b0074824d2465700f8b46683055849f4904a60f80747f44d0c" + } + ], + "external_evidence": [ + { + "artifact_id": "starwam-prediction-matrix-v0.1", + "kind": "real-model-prediction-index", + "sha256": "61fd988ea648922652ef2ab23f08b14a9849d37353d2e4894eb839fe3dfdb12d", + "location": "runs/starwam-libero-cf-mini-v0.1/matrix-index.json", + "availability": "release-bundle-pending", + "reason_not_committed": "Index refers to ignored model predictions and host-local artifact paths." + }, + { + "artifact_id": "starwam-execution-index-v0.1", + "kind": "real-model-simulator-execution-index", + "sha256": "3bb9d5bdab9437cf07b0ed6567a8ddb146191447931a59a098b853e2fd3e9846", + "location": "runs/starwam-libero-cf-mini-v0.1/execution-index.json", + "availability": "release-bundle-pending", + "reason_not_committed": "Index refers to ignored simulator traces and host-local snapshots." + }, + { + "artifact_id": "libero-cf-mini-generation-index-v0.1", + "kind": "paired-simulator-raw-index", + "sha256": "70f2523ebb86151c687d2c4c9768df78b39e65b6e6490252d2115fd5489eb042", + "location": "runs/libero-cf-mini-v0.1/index.json", + "availability": "local-only", + "reason_not_committed": "Raw RGB/state branches are generated data and remain outside Git." + } + ] +} diff --git a/release/evidence-manifest-v0.1.schema.json b/release/evidence-manifest-v0.1.schema.json new file mode 100644 index 0000000..0a0358a --- /dev/null +++ b/release/evidence-manifest-v0.1.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/myheart521/WAMProbe/release/evidence-manifest-v0.1.schema.json", + "title": "WAMProbe release evidence manifest v0.1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "release_version", + "committed_evidence", + "external_evidence" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "release_version": {"type": "string", "minLength": 1}, + "committed_evidence": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "kind", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "kind": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + }, + "external_evidence": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "artifact_id", + "kind", + "sha256", + "location", + "availability", + "reason_not_committed" + ], + "properties": { + "artifact_id": {"type": "string", "minLength": 1}, + "kind": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "location": {"type": "string", "minLength": 1}, + "availability": {"enum": ["local-only", "release-bundle-pending"]}, + "reason_not_committed": {"type": "string", "minLength": 1} + } + } + } + } +} diff --git a/scripts/build_release_candidate.py b/scripts/build_release_candidate.py new file mode 100644 index 0000000..ea65706 --- /dev/null +++ b/scripts/build_release_candidate.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Build, reproduce, audit, and smoke-test a WAMProbe release candidate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import cast + + +def _run( + command: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + +def _git(root: Path, *arguments: str) -> str: + return _run(["git", *arguments], cwd=root).stdout.strip() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _artifact_hashes(directory: Path) -> dict[str, str]: + artifacts = sorted((*directory.glob("*.whl"), *directory.glob("*.tar.gz"))) + if len(artifacts) != 2: + raise RuntimeError("candidate build must produce exactly one wheel and one sdist") + return {artifact.name: _sha256(artifact) for artifact in artifacts} + + +def _project_version(root: Path) -> str: + payload = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8")) + project = payload.get("project") + if not isinstance(project, dict) or not isinstance(project.get("version"), str): + raise RuntimeError("pyproject.toml has no project version") + return cast(str, project["version"]) + + +def _build(uv: str, root: Path, output: Path, env: dict[str, str]) -> None: + _run( + [ + uv, + "build", + "--clear", + "--no-create-gitignore", + "--out-dir", + str(output), + ], + cwd=root, + env=env, + ) + + +def _clean_install_smoke(root: Path, wheel: Path) -> None: + with tempfile.TemporaryDirectory(prefix="wamprobe-install-") as raw_directory: + directory = Path(raw_directory) + environment = directory / "venv" + _run([sys.executable, "-m", "venv", str(environment)], cwd=directory) + binary_dir = environment / ("Scripts" if os.name == "nt" else "bin") + python = binary_dir / ("python.exe" if os.name == "nt" else "python") + wamprobe = binary_dir / ("wamprobe.exe" if os.name == "nt" else "wamprobe") + _run( + [str(python), "-m", "pip", "install", "--no-deps", "--no-index", str(wheel)], + cwd=directory, + ) + clean_env = os.environ.copy() + clean_env.pop("PYTHONPATH", None) + _run([str(wamprobe), "--help"], cwd=directory, env=clean_env) + output = directory / "smoke-run" + _run( + [ + str(wamprobe), + "demo", + "--contexts", + "2", + "--seed", + "7", + "--output", + str(output), + ], + cwd=directory, + env=clean_env, + ) + if not (output / "summary.json").is_file(): + raise RuntimeError("clean wheel smoke did not create summary.json") + installed_version = _run( + [str(python), "-c", "import wamprobe; print(wamprobe.__version__)"], + cwd=directory, + env=clean_env, + ).stdout.strip() + if installed_version != _project_version(root): + raise RuntimeError("clean wheel version does not match pyproject.toml") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dist-dir", + type=Path, + default=Path("dist/release-candidate"), + ) + parser.add_argument( + "--evidence-manifest", + type=Path, + default=Path("release/evidence-manifest-v0.1.json"), + ) + parser.add_argument("--allow-dirty", action="store_true") + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + dist_dir = (root / args.dist_dir).resolve() + evidence_manifest = (root / args.evidence_manifest).resolve() + if not dist_dir.is_relative_to(root / "dist"): + raise RuntimeError("release candidate output must stay below repository dist/") + if _git(root, "status", "--porcelain", "--untracked-files=all") and not args.allow_dirty: + raise RuntimeError("release candidate requires a clean Git worktree") + + uv = shutil.which("uv") + if uv is None: + raise RuntimeError("uv is required to build the release candidate") + source_commit = _git(root, "rev-parse", "HEAD") + source_date_epoch = int(_git(root, "show", "-s", "--format=%ct", "HEAD")) + env = os.environ.copy() + env["SOURCE_DATE_EPOCH"] = str(source_date_epoch) + + _build(uv, root, dist_dir, env) + first_hashes = _artifact_hashes(dist_dir) + with tempfile.TemporaryDirectory(prefix="wamprobe-rebuild-") as raw_rebuild: + rebuild = Path(raw_rebuild) + _build(uv, root, rebuild, env) + second_hashes = _artifact_hashes(rebuild) + if first_hashes != second_hashes: + raise RuntimeError("release artifacts are not reproducible with SOURCE_DATE_EPOCH") + + manifest = dist_dir / "release-manifest.json" + _run( + [ + sys.executable, + "-m", + "wamprobe", + "release-audit", + "--dist", + str(dist_dir), + "--evidence-manifest", + str(evidence_manifest), + "--repository-root", + str(root), + "--source-commit", + source_commit, + "--source-date-epoch", + str(source_date_epoch), + "--output", + str(manifest), + ], + cwd=root, + env=env, + ) + wheel = next(dist_dir.glob("*.whl")) + _clean_install_smoke(root, wheel) + + payload = json.loads(manifest.read_text(encoding="utf-8")) + payload["checks"].extend( + ["double-build-sha256-reproducibility", "clean-wheel-install-and-demo"] + ) + payload["reproducible_build"] = { + "source_date_epoch": source_date_epoch, + "matched": True, + "artifact_sha256": first_hashes, + } + payload["clean_install_smoke"] = { + "network_disabled_for_install": True, + "dependencies": "none", + "commands": ["wamprobe --help", "wamprobe demo --contexts 2 --seed 7"], + "passed": True, + } + manifest.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"WAMProbe {_project_version(root)} release candidate verified") + print(f"Source commit: {source_commit}") + print(f"Manifest: {manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/wamprobe/__init__.py b/src/wamprobe/__init__.py index 52b2f4e..dba4eca 100644 --- a/src/wamprobe/__init__.py +++ b/src/wamprobe/__init__.py @@ -1,3 +1,3 @@ """WAMProbe: counterfactual evaluation for World Action Models.""" -__version__ = "0.1.0a0" +__version__ = "0.1.0rc1" diff --git a/src/wamprobe/cli.py b/src/wamprobe/cli.py index 6fe7271..062d336 100644 --- a/src/wamprobe/cli.py +++ b/src/wamprobe/cli.py @@ -47,6 +47,7 @@ from wamprobe.evaluation import EvaluationResult, evaluate from wamprobe.experiments import analyze_action_experiment, write_action_experiment_report from wamprobe.manipulation_evaluation import evaluate_manipulation +from wamprobe.release_audit import audit_release from wamprobe.reporting import write_reports from wamprobe.stats import paired_metric_comparison from wamprobe.video_control_study import run_video_control_study, write_video_control_study @@ -171,6 +172,17 @@ def _build_parser() -> argparse.ArgumentParser: closed_loop.add_argument("--execute-prefix", type=int, default=1) closed_loop.add_argument("--resamples", type=int, default=1000) closed_loop.add_argument("--output", type=Path, required=True) + + release_audit = subparsers.add_parser( + "release-audit", + help="verify built distributions and reproducibility evidence", + ) + release_audit.add_argument("--dist", type=Path, required=True) + release_audit.add_argument("--evidence-manifest", type=Path, required=True) + release_audit.add_argument("--repository-root", type=Path, default=Path(".")) + release_audit.add_argument("--source-commit", required=True) + release_audit.add_argument("--source-date-epoch", type=int, required=True) + release_audit.add_argument("--output", type=Path, required=True) return parser @@ -485,6 +497,24 @@ def _run_closed_loop_study(args: argparse.Namespace) -> int: return 0 +def _run_release_audit(args: argparse.Namespace) -> int: + try: + report = audit_release( + dist_dir=args.dist, + evidence_manifest=args.evidence_manifest, + repository_root=args.repository_root, + source_commit=args.source_commit, + source_date_epoch=args.source_date_epoch, + output_path=args.output, + ) + except (OSError, ValueError) as error: + print(f"WAMProbe release audit error: {error}", file=sys.stderr) + return 2 + print(f"Release audit: {args.output}") + print(f"Artifacts: {len(report.artifacts)}; evidence: {len(report.committed_evidence)}") + return 0 + + def _run_doctor(args: argparse.Namespace) -> int: try: manifest = load_manifest(args.manifest) @@ -538,5 +568,7 @@ def main(argv: Sequence[str] | None = None) -> int: return _run_video_control_study(args) if args.command == "closed-loop-study": return _run_closed_loop_study(args) + if args.command == "release-audit": + return _run_release_audit(args) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/wamprobe/release_audit.py b/src/wamprobe/release_audit.py new file mode 100644 index 0000000..9d75810 --- /dev/null +++ b/src/wamprobe/release_audit.py @@ -0,0 +1,289 @@ +"""Release artifact and reproducibility-evidence auditing.""" + +from __future__ import annotations + +import hashlib +import json +import re +import tarfile +import zipfile +from dataclasses import asdict, dataclass +from email.parser import Parser +from pathlib import Path, PurePosixPath +from typing import cast + +from wamprobe import __version__ + +_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}") +_SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +_FORBIDDEN_PARTS = { + ".env", + ".git", + "checkpoints", + "runs", + "vendor", +} + + +@dataclass(frozen=True, slots=True) +class ReleaseArtifact: + """One built distribution with its immutable identity.""" + + name: str + kind: str + size_bytes: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class CommittedEvidence: + """One repository evidence file verified against the release declaration.""" + + path: str + kind: str + size_bytes: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class ReleaseAudit: + """Machine-readable link from source commit to distributions and evidence.""" + + release_version: str + source_commit: str + source_date_epoch: int + artifacts: tuple[ReleaseArtifact, ...] + evidence_manifest: str + evidence_manifest_sha256: str + committed_evidence: tuple[CommittedEvidence, ...] + external_evidence: tuple[dict[str, object], ...] + checks: tuple[str, ...] + schema_version: str = "0.1" + + def to_dict(self) -> dict[str, object]: + """Return a stable JSON-compatible audit record.""" + + return { + "schema_version": self.schema_version, + "release_version": self.release_version, + "source_commit": self.source_commit, + "source_date_epoch": self.source_date_epoch, + "artifacts": [asdict(artifact) for artifact in self.artifacts], + "evidence_manifest": self.evidence_manifest, + "evidence_manifest_sha256": self.evidence_manifest_sha256, + "committed_evidence": [asdict(evidence) for evidence in self.committed_evidence], + "external_evidence": list(self.external_evidence), + "checks": list(self.checks), + } + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _json_object(path: Path) -> dict[str, object]: + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"could not read release evidence manifest: {error}") from error + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise ValueError("release evidence manifest must be a JSON object") + return cast(dict[str, object], raw) + + +def _safe_archive_path(name: str) -> PurePosixPath: + path = PurePosixPath(name) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError(f"release archive contains an unsafe path: {name}") + if any(part in _FORBIDDEN_PARTS for part in path.parts): + raise ValueError(f"release archive contains a forbidden path: {name}") + return path + + +def _verify_wheel(path: Path, version: str) -> None: + try: + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + archive_paths = [_safe_archive_path(name) for name in names] + metadata_names = [name for name in names if name.endswith(".dist-info/METADATA")] + entry_names = [name for name in names if name.endswith(".dist-info/entry_points.txt")] + if len(metadata_names) != 1 or len(entry_names) != 1: + raise ValueError("wheel must contain one METADATA and one entry_points.txt") + if not any(path.parts[0] == "wamprobe" for path in archive_paths): + raise ValueError("wheel does not contain the wamprobe package") + metadata = Parser().parsestr(archive.read(metadata_names[0]).decode("utf-8")) + if metadata.get("Name") != "wamprobe" or metadata.get("Version") != version: + raise ValueError("wheel package name/version does not match the release") + if metadata.get("License-Expression") != "Apache-2.0": + raise ValueError("wheel does not declare the Apache-2.0 license expression") + entry_points = archive.read(entry_names[0]).decode("utf-8") + if "wamprobe = wamprobe.cli:main" not in entry_points: + raise ValueError("wheel does not contain the wamprobe console entry point") + except (OSError, UnicodeError, zipfile.BadZipFile) as error: + raise ValueError(f"could not verify wheel: {error}") from error + + +def _verify_sdist(path: Path, version: str) -> None: + required = {"LICENSE", "README.md", "pyproject.toml"} + root = f"wamprobe-{version}" + found: set[str] = set() + try: + with tarfile.open(path, "r:gz") as archive: + for member in archive.getmembers(): + member_path = _safe_archive_path(member.name) + if member.issym() or member.islnk(): + raise ValueError("sdist must not contain symbolic or hard links") + if member_path.parts[0] != root: + raise ValueError("sdist contains a path outside its versioned root") + if len(member_path.parts) == 2: + found.add(member_path.parts[1]) + except (OSError, tarfile.TarError) as error: + raise ValueError(f"could not verify sdist: {error}") from error + missing = required - found + if missing: + raise ValueError(f"sdist is missing required files: {sorted(missing)}") + + +def _manifest_evidence( + manifest: Path, + *, + repository_root: Path, + version: str, +) -> tuple[tuple[CommittedEvidence, ...], tuple[dict[str, object], ...]]: + payload = _json_object(manifest) + if payload.get("schema_version") != "0.1": + raise ValueError("release evidence manifest uses an unsupported schema_version") + if payload.get("release_version") != version: + raise ValueError("release evidence manifest version does not match the package") + raw_committed = payload.get("committed_evidence") + raw_external = payload.get("external_evidence") + if not isinstance(raw_committed, list) or not raw_committed: + raise ValueError("release evidence manifest needs committed_evidence") + if not isinstance(raw_external, list): + raise ValueError("release evidence manifest external_evidence must be an array") + + root = repository_root.resolve(strict=True) + evidence: list[CommittedEvidence] = [] + seen_paths: set[str] = set() + for item in raw_committed: + if not isinstance(item, dict): + raise ValueError("committed evidence entries must be JSON objects") + path_value = item.get("path") + kind = item.get("kind") + expected_sha256 = item.get("sha256") + if ( + not isinstance(path_value, str) + or not path_value + or not isinstance(kind, str) + or not kind + or not isinstance(expected_sha256, str) + or _SHA256_PATTERN.fullmatch(expected_sha256) is None + ): + raise ValueError("committed evidence entry fields are invalid") + relative = PurePosixPath(path_value) + if relative.is_absolute() or ".." in relative.parts or path_value in seen_paths: + raise ValueError("committed evidence paths must be unique safe relative paths") + seen_paths.add(path_value) + candidate = root.joinpath(*relative.parts) + if candidate.is_symlink() or not candidate.is_file(): + raise ValueError(f"committed evidence is missing or unsafe: {path_value}") + resolved = candidate.resolve(strict=True) + if not resolved.is_relative_to(root): + raise ValueError(f"committed evidence escapes the repository: {path_value}") + actual_sha256 = _sha256(resolved) + if actual_sha256 != expected_sha256: + raise ValueError(f"committed evidence checksum mismatch: {path_value}") + evidence.append( + CommittedEvidence( + path=path_value, + kind=kind, + size_bytes=resolved.stat().st_size, + sha256=actual_sha256, + ) + ) + + external: list[dict[str, object]] = [] + for item in raw_external: + if not isinstance(item, dict) or not all(isinstance(key, str) for key in item): + raise ValueError("external evidence entries must be JSON objects") + external.append(cast(dict[str, object], item)) + return tuple(evidence), tuple(external) + + +def audit_release( + *, + dist_dir: Path, + evidence_manifest: Path, + repository_root: Path, + source_commit: str, + source_date_epoch: int, + output_path: Path | None = None, + version: str = __version__, +) -> ReleaseAudit: + """Verify distributions and evidence, then optionally write their traceability record.""" + + if _COMMIT_PATTERN.fullmatch(source_commit) is None: + raise ValueError("source_commit must be a lowercase 40-character Git SHA") + if source_date_epoch <= 0: + raise ValueError("source_date_epoch must be positive") + if not dist_dir.is_dir(): + raise ValueError("release dist directory does not exist") + + wheels = sorted(dist_dir.glob("*.whl")) + sdists = sorted(dist_dir.glob("*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError("release dist must contain exactly one wheel and one sdist") + expected_wheel = f"wamprobe-{version}-py3-none-any.whl" + expected_sdist = f"wamprobe-{version}.tar.gz" + if wheels[0].name != expected_wheel or sdists[0].name != expected_sdist: + raise ValueError("release artifact filenames do not match the package version") + + _verify_wheel(wheels[0], version) + _verify_sdist(sdists[0], version) + committed, external = _manifest_evidence( + evidence_manifest, + repository_root=repository_root, + version=version, + ) + artifacts = ( + ReleaseArtifact( + name=wheels[0].name, + kind="wheel", + size_bytes=wheels[0].stat().st_size, + sha256=_sha256(wheels[0]), + ), + ReleaseArtifact( + name=sdists[0].name, + kind="sdist", + size_bytes=sdists[0].stat().st_size, + sha256=_sha256(sdists[0]), + ), + ) + audit = ReleaseAudit( + release_version=version, + source_commit=source_commit, + source_date_epoch=source_date_epoch, + artifacts=artifacts, + evidence_manifest=evidence_manifest.name, + evidence_manifest_sha256=_sha256(evidence_manifest), + committed_evidence=committed, + external_evidence=external, + checks=( + "artifact-count-and-filenames", + "archive-path-safety", + "wheel-metadata-and-entry-point", + "sdist-required-files", + "committed-evidence-sha256", + ), + ) + if output_path is not None: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(audit.to_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return audit diff --git a/tests/test_import.py b/tests/test_import.py index 257f50b..5407422 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -2,4 +2,4 @@ def test_version_is_exposed() -> None: - assert wamprobe.__version__ == "0.1.0a0" + assert wamprobe.__version__ == "0.1.0rc1" diff --git a/tests/test_release_audit.py b/tests/test_release_audit.py new file mode 100644 index 0000000..c08d842 --- /dev/null +++ b/tests/test_release_audit.py @@ -0,0 +1,130 @@ +import hashlib +import io +import json +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from wamprobe import __version__ +from wamprobe.cli import main +from wamprobe.release_audit import audit_release + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_release_fixture(root: Path) -> tuple[Path, Path]: + dist = root / "dist" + dist.mkdir() + wheel = dist / f"wamprobe-{__version__}-py3-none-any.whl" + dist_info = f"wamprobe-{__version__}.dist-info" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("wamprobe/__init__.py", f'__version__ = "{__version__}"\n') + archive.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.4\n" + "Name: wamprobe\n" + f"Version: {__version__}\n" + "License-Expression: Apache-2.0\n", + ) + archive.writestr( + f"{dist_info}/entry_points.txt", + "[console_scripts]\nwamprobe = wamprobe.cli:main\n", + ) + + sdist = dist / f"wamprobe-{__version__}.tar.gz" + prefix = f"wamprobe-{__version__}" + with tarfile.open(sdist, "w:gz") as archive: + for name, payload in ( + ("LICENSE", b"Apache-2.0\n"), + ("README.md", b"# WAMProbe\n"), + ("pyproject.toml", f'[project]\nversion = "{__version__}"\n'.encode()), + ): + info = tarfile.TarInfo(f"{prefix}/{name}") + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + + evidence = root / "evidence.json" + evidence.write_text('{"result": "verified"}\n', encoding="utf-8") + manifest = root / "evidence-manifest.json" + manifest.write_text( + json.dumps( + { + "schema_version": "0.1", + "release_version": __version__, + "committed_evidence": [ + { + "path": "evidence.json", + "kind": "test-evidence", + "sha256": _sha256(evidence), + } + ], + "external_evidence": [], + } + ), + encoding="utf-8", + ) + return dist, manifest + + +def test_release_audit_verifies_archives_and_committed_evidence(tmp_path: Path) -> None: + dist, manifest = _write_release_fixture(tmp_path) + output = tmp_path / "release-audit.json" + + audit = audit_release( + dist_dir=dist, + evidence_manifest=manifest, + repository_root=tmp_path, + source_commit="a" * 40, + source_date_epoch=1_700_000_000, + output_path=output, + ) + + assert audit.release_version == __version__ + assert {artifact.kind for artifact in audit.artifacts} == {"wheel", "sdist"} + assert audit.committed_evidence[0].path == "evidence.json" + assert json.loads(output.read_text(encoding="utf-8"))["source_commit"] == "a" * 40 + + +def test_release_audit_rejects_tampered_evidence(tmp_path: Path) -> None: + dist, manifest = _write_release_fixture(tmp_path) + (tmp_path / "evidence.json").write_text("tampered\n", encoding="utf-8") + + with pytest.raises(ValueError, match="checksum"): + audit_release( + dist_dir=dist, + evidence_manifest=manifest, + repository_root=tmp_path, + source_commit="b" * 40, + source_date_epoch=1_700_000_000, + ) + + +def test_release_audit_cli_writes_manifest(tmp_path: Path) -> None: + dist, manifest = _write_release_fixture(tmp_path) + output = tmp_path / "release-audit.json" + + assert ( + main( + [ + "release-audit", + "--dist", + str(dist), + "--evidence-manifest", + str(manifest), + "--repository-root", + str(tmp_path), + "--source-commit", + "c" * 40, + "--source-date-epoch", + "1700000000", + "--output", + str(output), + ] + ) + == 0 + ) + assert output.is_file() diff --git a/uv.lock b/uv.lock index d67bc57..b3dbe99 100644 --- a/uv.lock +++ b/uv.lock @@ -446,7 +446,7 @@ wheels = [ [[package]] name = "wamprobe" -version = "0.1.0a0" +version = "0.1.0rc1" source = { editable = "." } [package.optional-dependencies] From 0133ac900dec6cb32906db5032b1d7893a5d8195 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 14:52:50 +0000 Subject: [PATCH 17/20] make sdist selection checkout invariant --- pyproject.toml | 10 ++++++++++ release/README.md | 4 ++++ scripts/build_release_candidate.py | 22 +++++++++++++--------- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cc8aa7b..727265a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,16 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["src/wamprobe"] +[tool.hatch.build.targets.sdist] +include = [ + "/CHANGELOG.md", + "/CITATION.cff", + "/LICENSE", + "/README.md", + "/pyproject.toml", + "/src/wamprobe", +] + [tool.hatch.build] exclude = [ "/.coverage", diff --git a/release/README.md b/release/README.md index cb4e4f7..fa63dec 100644 --- a/release/README.md +++ b/release/README.md @@ -18,6 +18,10 @@ metadata, checks every committed evidence hash, installs the wheel without netwo dependencies in a fresh virtual environment, and runs the CLI/demo smoke. Outputs remain below ignored `dist/release-candidate/`: +The sdist uses an explicit source/package-metadata allowlist instead of Hatch's default VCS +file discovery, so a normal clone, shallow Actions checkout, and Git worktree select the +same content. + ```text wamprobe--py3-none-any.whl wamprobe-.tar.gz diff --git a/scripts/build_release_candidate.py b/scripts/build_release_candidate.py index ea65706..e812d58 100644 --- a/scripts/build_release_candidate.py +++ b/scripts/build_release_candidate.py @@ -22,15 +22,19 @@ def _run( cwd: Path, env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=cwd, - env=env, - check=True, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) + try: + return subprocess.run( + command, + cwd=cwd, + env=env, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as error: + rendered = " ".join(command) + raise RuntimeError(f"command failed ({rendered}):\n{error.stdout}") from error def _git(root: Path, *arguments: str) -> str: From 38a440deb2cec50f5636b743a3cc2a2c25558a15 Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 15:17:45 +0000 Subject: [PATCH 18/20] add repository contract and docs gates --- .github/ISSUE_TEMPLATE/benchmark_proposal.yml | 51 ++ .github/PULL_REQUEST_TEMPLATE.md | 2 + .github/workflows/ci.yml | 6 + .github/workflows/docs.yml | 47 ++ .gitignore | 1 + CHANGELOG.md | 2 + CONTRIBUTING.md | 2 + README.md | 7 +- configs/models/starwam-capabilities-v0.1.json | 10 + docs/QUICKSTART.md | 12 +- docs/WAMProbe_PLAN.md | 7 +- docs/benchmarks/LIBERO_CF_MINI.md | 4 +- docs/index.md | 30 ++ docs/models/STARWAM.md | 6 +- docs/reproducibility/REPRODUCIBILITY.md | 3 +- environments/starwam/run_libero_smoke.py | 5 +- mkdocs.yml | 40 ++ pyproject.toml | 2 + release/evidence-manifest-v0.1.json | 7 +- .../intervention-manipulation-v0.1.json | 21 + .../examples/intervention-pointmass-v0.1.json | 15 + schemas/intervention-v0.1.schema.json | 134 +++-- scripts/validate_repository.py | 229 +++++++++ tests/test_repository_validation.py | 18 + uv.lock | 467 ++++++++++++++++++ 25 files changed, 1080 insertions(+), 48 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/benchmark_proposal.yml create mode 100644 .github/workflows/docs.yml create mode 100644 configs/models/starwam-capabilities-v0.1.json create mode 100644 docs/index.md create mode 100644 mkdocs.yml create mode 100644 schemas/examples/intervention-manipulation-v0.1.json create mode 100644 schemas/examples/intervention-pointmass-v0.1.json create mode 100644 scripts/validate_repository.py create mode 100644 tests/test_repository_validation.py diff --git a/.github/ISSUE_TEMPLATE/benchmark_proposal.yml b/.github/ISSUE_TEMPLATE/benchmark_proposal.yml new file mode 100644 index 0000000..488ae0f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/benchmark_proposal.yml @@ -0,0 +1,51 @@ +name: Benchmark proposal +description: Propose a paired intervention benchmark or benchmark extension +title: "[Benchmark]: " +labels: ["area:benchmark", "needs-rfc"] +body: + - type: textarea + id: diagnostic_scope + attributes: + label: Diagnostic scope + description: Name the WAM failure modes and capability tier this benchmark evaluates. + validations: + required: true + - type: textarea + id: intervention_contract + attributes: + label: Intervention and restore contract + description: Describe shared contexts, legal action branches, snapshot restore, and branch-order checks. + validations: + required: true + - type: textarea + id: data_license + attributes: + label: Data, simulator, and asset licenses + description: Include upstream versions, licenses, redistribution limits, and required hashes. + validations: + required: true + - type: textarea + id: splits + attributes: + label: Splits and leakage controls + description: Define ID/OOD splits and how train/evaluation overlap is prevented or reported. + validations: + required: true + - type: textarea + id: smoke + attributes: + label: Smallest reproducible smoke path + description: Give one bounded command, hardware requirements, runtime, and expected structured output. + validations: + required: true + - type: checkboxes + id: evidence + attributes: + label: Required evidence + options: + - label: Generation or conversion code is included. + required: true + - label: A benchmark card states supported and unsupported claims. + required: true + - label: Oracle and failure baselines have expected-ordering tests. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3f6588f..2ebd1e4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,6 +11,8 @@ - [ ] `ruff format --check .` - [ ] `ruff check .` - [ ] `mypy` +- [ ] `python scripts/validate_repository.py` +- [ ] `mkdocs build --strict` - [ ] `pytest --cov=wamprobe --cov-report=term-missing` ## Metric/adapter/benchmark checklist diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13fdd34..48777e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,12 @@ jobs: run: ruff check . - name: Type check run: mypy + - name: Validate JSON Schemas and Markdown links + if: matrix.python-version == '3.11' + run: python scripts/validate_repository.py + - name: Build documentation site + if: matrix.python-version == '3.11' + run: mkdocs build --strict - name: Test with coverage run: pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..4875595 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: Documentation + +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: github-pages + cancel-in-progress: true + +jobs: + build-and-deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install documentation environment + run: python -m pip install -e '.[dev]' + - name: Validate repository documentation + run: python scripts/validate_repository.py + - name: Build documentation site + run: mkdocs build --strict + - uses: actions/configure-pages@v5 + - uses: actions/upload-pages-artifact@v3 + with: + path: site + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index bf3ecc1..ede4259 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ .venv/ build/ dist/ +site/ # Local data, model weights, generated predictions, and reports data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 479b04f..9a35230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ as a final GitHub Release. ### Added +- Strict MkDocs documentation, GitHub Pages deployment, repository-local Markdown link + validation, and Draft 2020-12 validation of canonical public-schema instances. - Typed alpha `WAMAdapter` protocol and capability declaration. - Analytic PointMass-2D paired counterfactual benchmark. - Oracle, copy-last-frame, wrong-direction, and action-agnostic reference baselines. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8b9b87..fbff90d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,6 +16,8 @@ Before opening a pull request, run: ruff format --check . ruff check . mypy +python scripts/validate_repository.py +mkdocs build --strict pytest --cov=wamprobe --cov-report=term-missing --cov-fail-under=85 ``` diff --git a/README.md b/README.md index 41a9339..3de4122 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ **Counterfactual evaluation for World Action Models.** +Documentation: [myheart521.github.io/WAMProbe](https://myheart521.github.io/WAMProbe/) + WAMProbe tests whether a World Action Model (WAM) predicts futures that are causally controlled by the input action and useful for choosing robot actions. It complements task-success and video-quality benchmarks with paired interventions: @@ -137,7 +139,8 @@ PYTHONPATH=src python -m wamprobe demo --output runs/pointmass-demo - a versioned evidence manifest, byte-reproducible wheel/sdist builder, archive audit, offline clean-wheel demo smoke, manual provenance-attestation workflow, and LaTeX technical-report draft; -- Python 3.11–3.13 CI with linting, strict typing, and coverage. +- Python 3.11–3.13 CI with linting, strict typing, coverage, public JSON Schema validation, + repository-local Markdown link checking, and a strict documentation build. ## Roadmap @@ -171,6 +174,8 @@ python -m pip install -e '.[dev]' ruff format --check . ruff check . mypy +python scripts/validate_repository.py +mkdocs build --strict pytest --cov=wamprobe --cov-report=term-missing ``` diff --git a/configs/models/starwam-capabilities-v0.1.json b/configs/models/starwam-capabilities-v0.1.json new file mode 100644 index 0000000..b335673 --- /dev/null +++ b/configs/models/starwam-capabilities-v0.1.json @@ -0,0 +1,10 @@ +{ + "model_id": "starwam-wan22-5b-mot-libero", + "future_representation": "none", + "predicts_actions": true, + "scores_actions": false, + "exposes_world_features": false, + "stochastic": true, + "deterministic_seed": true, + "supports_batching": false +} diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index d2f6938..0a0f492 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -90,7 +90,7 @@ Ranking Correlation, and Top-1 Regret. The `appearance-corrupted-oracle` deliber inverts rendered RGB while preserving exact dynamics: it therefore has FDE `0`, CRC `1`, and regret `0`, despite PSNR below `1 dB` in the committed run. This is a controlled metric counterexample, not a claim that visual fidelity is unimportant. See the -[committed report](../examples/video-control-study/video-control-study.md). +[committed report](https://github.com/myheart521/WAMProbe/blob/main/examples/video-control-study/video-control-study.md). ## Run minimal closed-loop replanning @@ -119,7 +119,7 @@ policy, and deterministic random candidate selection. It preserves every context action sequence, final return, success, and gap to the simulator scorer. The committed 12-context run found CRC/closed-return Pearson values of `0.9855` on BlockPush and `1.0000` on Gripper-Catch, but these are descriptive associations across only five future-scorer -profiles. See the [report](../examples/closed-loop-study/closed-loop-study.md) and +profiles. See the [report](https://github.com/myheart521/WAMProbe/blob/main/examples/closed-loop-study/closed-loop-study.md) and [experiment card](experiments/TOY_CLOSED_LOOP_V0.1.md). ## Audit a release candidate @@ -135,7 +135,7 @@ uv sync --extra dev --locked The command writes ignored artifacts and `release-manifest.json` below `dist/release-candidate/`. It does not publish or tag anything. See the -[release procedure](../release/README.md) and +[release procedure](https://github.com/myheart521/WAMProbe/blob/main/release/README.md) and [reproducibility guide](reproducibility/REPRODUCIBILITY.md). Uncertainty is computed at the context level. Each bootstrap draw resamples whole shared @@ -145,7 +145,7 @@ align exact context IDs and report the left-minus-right difference with a 95% in ## Validate real-model artifacts The first real adapter uses pinned StarWAM and Wan2.2 files stored outside Git. Follow -[`checkpoints/README.md`](../checkpoints/README.md), then run: +[`checkpoints/README.md`](https://github.com/myheart521/WAMProbe/blob/main/checkpoints/README.md), then run: ```bash wamprobe doctor @@ -159,7 +159,7 @@ code `2` means the manifest itself could not be read safely. Use `--json` for au ## Run the opt-in StarWAM smoke probe After reproducing the isolated environment in -[`environments/starwam/README.md`](../environments/starwam/README.md), select a physical +[`environments/starwam/README.md`](https://github.com/myheart521/WAMProbe/blob/main/environments/starwam/README.md), select a physical GPU with enough free memory and run the text encoder and action model in separate processes: @@ -192,7 +192,7 @@ and state descriptors for all eight future steps, repeats no-op, and executes all branches in reverse order. A successful run requires byte-identical MuJoCo integration states, force-refreshed initial observations, repeated rollouts, and order-independent branch results. See -[`environments/libero/README.md`](../environments/libero/README.md) for the output layout +[`environments/libero/README.md`](https://github.com/myheart521/WAMProbe/blob/main/environments/libero/README.md) for the output layout and the required Python-side gripper-state restoration. Repeat `--task-key` to select a subset; verified outputs resume without reopening the simulator. diff --git a/docs/WAMProbe_PLAN.md b/docs/WAMProbe_PLAN.md index 90f06ad..4ba4623 100644 --- a/docs/WAMProbe_PLAN.md +++ b/docs/WAMProbe_PLAN.md @@ -989,8 +989,9 @@ CLI 验收要求： 当前进度（2026-07-15）：15 个文献失败模式和首个 Adapter 审计已经完成，见 [`docs/research/WAM_VLA_FAILURE_CASES.md`](research/WAM_VLA_FAILURE_CASES.md) 与 -[`docs/research/ADAPTER_SELECTION.md`](research/ADAPTER_SELECTION.md)。外部研究者 README -理解度检查仍需人工完成。 +[`docs/research/ADAPTER_SELECTION.md`](research/ADAPTER_SELECTION.md)。MkDocs 文档站、 +严格站点构建、仓库内 Markdown 链接检查和公开 JSON Schema/规范实例验证也已进入 CI； +外部研究者 README 理解度检查仍需人工完成。 任务： @@ -1121,7 +1122,7 @@ snapshot 执行全部动作块，并在 horizon 8/16/32 记录状态、回报和 只执行 1 步并重规划，oracle future scorer 两任务 success 均为 1，noisy future scorer 分别为 1.0/0.9167，三个 action-ignorance/wrong-direction 对照均为 0；离线 CRC 与闭环 return 的 5-profile 描述性 Pearson 分别为 0.9855/1.0。全部结果、context-block CI 和限制 -见 [`examples/video-control-study/`](../examples/video-control-study/video-control-study.md) +见 [`examples/video-control-study/`](https://github.com/myheart521/WAMProbe/blob/main/examples/video-control-study/video-control-study.md) 与 [`docs/experiments/TOY_CLOSED_LOOP_V0.1.md`](experiments/TOY_CLOSED_LOOP_V0.1.md)。 `0.1.0rc1` 候选版本现已具备双构建 SHA 一致性、归档安全/metadata 审计、证据 manifest、 离线干净 wheel 安装 smoke、手动 provenance attestation workflow，并提供可在 Overleaf diff --git a/docs/benchmarks/LIBERO_CF_MINI.md b/docs/benchmarks/LIBERO_CF_MINI.md index 5f4bcfc..901e1ef 100644 --- a/docs/benchmarks/LIBERO_CF_MINI.md +++ b/docs/benchmarks/LIBERO_CF_MINI.md @@ -18,7 +18,7 @@ The fixed task manifest covers four intentionally different families: | `long-horizon-task0` | `libero_10` | long-horizon composition | two objects → basket | The source selection, task text, BDDL/init-state filenames and hashes are pinned in -[`configs/benchmarks/libero_cf_mini_v0.1.json`](../../configs/benchmarks/libero_cf_mini_v0.1.json). +[`configs/benchmarks/libero_cf_mini_v0.1.json`](https://github.com/myheart521/WAMProbe/blob/main/configs/benchmarks/libero_cf_mini_v0.1.json). The manifest's SHA256 is `1660ab49ec0e5aac58f4801f0ec9a12326053f21f5f1974dd3b6ea63260f78c3`. @@ -69,7 +69,7 @@ reported four cache hits without opening a simulator. ## Reproduction Follow the isolated environment instructions in -[`environments/libero/README.md`](../../environments/libero/README.md), then run: +[`environments/libero/README.md`](https://github.com/myheart521/WAMProbe/blob/main/environments/libero/README.md), then run: ```bash environments/starwam/.venv/bin/python environments/libero/generate_cf_pilot.py \ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..44e7adb --- /dev/null +++ b/docs/index.md @@ -0,0 +1,30 @@ +# WAMProbe + +WAMProbe is a counterfactual evaluation toolkit for World Action Models. It asks whether +different candidate actions produce different predicted futures, whether those differences +agree with the real dynamics, and whether the predictions improve action selection. + +The same initial state is restored before every branch: + +```text +shared context +├── no-op → predicted future / true future +├── move left → predicted future / true future +├── move right → predicted future / true future +└── expert act → predicted future / true future +``` + +WAMProbe reports action dependence, direction correctness, physical accuracy, candidate +ranking, regret, closed-loop return, uncertainty, and compute cost as separate profiles. +It intentionally does not publish a single composite score. + +## Start here + +- Follow the [15-minute quick start](QUICKSTART.md) for dependency-free CPU experiments. +- Read the [scope and capability RFC](rfcs/0001-scope-and-capabilities.md) before adding an adapter. +- Use the [core metric cards](metrics/CORE_METRICS.md) to select capability-compatible metrics. +- Review the [reproducibility guide](reproducibility/REPRODUCIBILITY.md) before comparing results. +- Inspect the [StarWAM model card](models/STARWAM.md) for the first real-model integration. + +Committed example reports and release artifacts remain available in the +[GitHub repository](https://github.com/myheart521/WAMProbe). diff --git a/docs/models/STARWAM.md b/docs/models/STARWAM.md index 27f0cc9..8768ed2 100644 --- a/docs/models/STARWAM.md +++ b/docs/models/STARWAM.md @@ -1,5 +1,9 @@ # StarWAM adapter model card +The machine-readable capability declaration is pinned in +[`configs/models/starwam-capabilities-v0.1.json`](https://github.com/myheart521/WAMProbe/blob/main/configs/models/starwam-capabilities-v0.1.json) +and validated against the public Draft 2020-12 capability schema in CI. + ## Model and adapter identity | Field | Value | @@ -9,7 +13,7 @@ | Released model | `panshaohua/starwam@7d4bfe3ec76172ca17169fa959d21da099d386fe` | | Checkpoint SHA256 | `d24edea01579880327cfd9dc84d24adab82e420dca9652e614ad697bc8cc5378` | | Backbone | Wan2.2 TI2V-5B revision `921dbaf3f1674a56f47e83fb80a34bac8a8f203e` | -| Adapter version | `0.1.0a0` | +| Adapter version | `0.1.0rc1` | | Upstream code/model license | Apache-2.0 as declared by upstream repositories | The WAMProbe adapter is an isolated integration target, not a reimplementation or an diff --git a/docs/reproducibility/REPRODUCIBILITY.md b/docs/reproducibility/REPRODUCIBILITY.md index 28568bb..bc97b53 100644 --- a/docs/reproducibility/REPRODUCIBILITY.md +++ b/docs/reproducibility/REPRODUCIBILITY.md @@ -54,7 +54,8 @@ an action-conditioned future for each proposed candidate. Run `scripts/build_release_candidate.py` from a clean commit. A valid candidate has two byte-reproducible distributions, a `release-manifest.json`, a passing offline clean-wheel -install, and matching evidence hashes. See [`release/README.md`](../../release/README.md). +install, and matching evidence hashes. See +[`release/README.md`](https://github.com/myheart521/WAMProbe/blob/main/release/README.md). ## Independent reproduction report diff --git a/environments/starwam/run_libero_smoke.py b/environments/starwam/run_libero_smoke.py index fe57552..03c12f0 100644 --- a/environments/starwam/run_libero_smoke.py +++ b/environments/starwam/run_libero_smoke.py @@ -83,6 +83,7 @@ if str(SOURCE_ROOT) not in sys.path: sys.path.insert(0, str(SOURCE_ROOT)) +from wamprobe import __version__ # noqa: E402 from wamprobe.adapters.starwam import ( # noqa: E402 StarWAMAdapter, StarWAMBackendResult, @@ -779,7 +780,7 @@ def _run_inference( observation=captured.observation, prediction=prediction, provenance=ModelProvenance( - adapter_version="0.1.0a0", + adapter_version=__version__, upstream_code_revision=STARWAM_REVISION, model_revision=MODEL_REVISION, backbone_revision=BACKBONE_REVISION, @@ -823,7 +824,7 @@ def _run_inference( def _provenance() -> ModelProvenance: return ModelProvenance( - adapter_version="0.1.0a0", + adapter_version=__version__, upstream_code_revision=STARWAM_REVISION, model_revision=MODEL_REVISION, backbone_revision=BACKBONE_REVISION, diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..3f0857c --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,40 @@ +site_name: WAMProbe +site_description: Counterfactual evaluation for World Action Models +site_url: https://myheart521.github.io/WAMProbe/ +repo_url: https://github.com/myheart521/WAMProbe +repo_name: myheart521/WAMProbe +docs_dir: docs +site_dir: site +strict: true + +theme: + name: readthedocs + +markdown_extensions: + - tables + - toc: + permalink: true + +nav: + - Home: index.md + - Quick start: QUICKSTART.md + - Design: + - Scope and capabilities: rfcs/0001-scope-and-capabilities.md + - Counterfactual metrics: rfcs/0002-counterfactual-metrics.md + - Detailed project plan: WAMProbe_PLAN.md + - Research: + - WAM/VLA failure cases: research/WAM_VLA_FAILURE_CASES.md + - Adapter selection: research/ADAPTER_SELECTION.md + - Benchmarks: + - Toy benchmarks: benchmarks/TOY_BENCHMARKS.md + - LIBERO-CF-Mini: benchmarks/LIBERO_CF_MINI.md + - Metrics: metrics/CORE_METRICS.md + - Models: + - StarWAM: models/STARWAM.md + - Experiments: + - StarWAM on LIBERO-CF-Mini: experiments/STARWAM_LIBERO_CF_MINI_V0.1.md + - Toy closed loop: experiments/TOY_CLOSED_LOOP_V0.1.md + - Reproducibility: + - Guide: reproducibility/REPRODUCIBILITY.md + - External report template: reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md + - GPU nightly: reproducibility/GPU_NIGHTLY.md diff --git a/pyproject.toml b/pyproject.toml index 727265a..0d7bac3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,8 @@ wamprobe = "wamprobe.cli:main" [project.optional-dependencies] dev = [ + "jsonschema>=4.25,<5", + "mkdocs>=1.6,<2", "mypy", "pytest", "pytest-cov", diff --git a/release/evidence-manifest-v0.1.json b/release/evidence-manifest-v0.1.json index 70241e6..9fd555c 100644 --- a/release/evidence-manifest-v0.1.json +++ b/release/evidence-manifest-v0.1.json @@ -12,6 +12,11 @@ "kind": "upstream-model-pins", "sha256": "169ff5e6767f47dfd49eef15f5bc7fec6494da82392fbbc8c08e42878686d1cf" }, + { + "path": "configs/models/starwam-capabilities-v0.1.json", + "kind": "model-capability-manifest", + "sha256": "736bb77b1edc87760b4a1e6640d8df15db3693c1fe1331821873bb359e41ea49" + }, { "path": "configs/benchmarks/libero_cf_mini_v0.1.json", "kind": "benchmark-manifest", @@ -25,7 +30,7 @@ { "path": "schemas/intervention-v0.1.schema.json", "kind": "public-schema", - "sha256": "0dfad48358608b523d2512b0e4f65111b4f0f07f106908463471f8d156702e60" + "sha256": "c842e6c3de0b9fa07347081e6dd4c90aaa5edc7c1abcfa7cb852ad30474eae52" }, { "path": "schemas/result-v0.1.schema.json", diff --git a/schemas/examples/intervention-manipulation-v0.1.json b/schemas/examples/intervention-manipulation-v0.1.json new file mode 100644 index 0000000..b5b0db1 --- /dev/null +++ b/schemas/examples/intervention-manipulation-v0.1.json @@ -0,0 +1,21 @@ +{ + "schema_version": "0.1", + "benchmark_id": "blockpush-2d-v0.1", + "state_kind": "manipulation-2d", + "context": { + "context_id": "schema-example-manipulation", + "initial_state": { + "effector_position": [-0.5, 0.0], + "object_position": [0.0, 0.0], + "gripper_closed": false, + "object_attached": false, + "in_contact": false + }, + "goal": [0.5, 0.0] + }, + "actions": [ + {"name": "noop", "effector_delta": [0.0, 0.0], "close_gripper": false}, + {"name": "push-right", "effector_delta": [0.25, 0.0], "close_gripper": false} + ], + "record_sha256": "46d0bd0e235afd6b992e04bffaa65564a14d63876b2afa306ca4c8b38a65b95c" +} diff --git a/schemas/examples/intervention-pointmass-v0.1.json b/schemas/examples/intervention-pointmass-v0.1.json new file mode 100644 index 0000000..20dad2d --- /dev/null +++ b/schemas/examples/intervention-pointmass-v0.1.json @@ -0,0 +1,15 @@ +{ + "schema_version": "0.1", + "benchmark_id": "pointmass-2d-v0.1", + "state_kind": "pointmass-2d", + "context": { + "context_id": "schema-example-pointmass", + "position": [0.0, 0.0], + "goal": [1.0, 0.0] + }, + "actions": [ + {"name": "noop", "delta": [0.0, 0.0]}, + {"name": "right", "delta": [0.25, 0.0]} + ], + "record_sha256": "707e01c332591ddf20f0218673744a5259be09282639ec7750379ba21e213c0d" +} diff --git a/schemas/intervention-v0.1.schema.json b/schemas/intervention-v0.1.schema.json index 4f47032..42dd5b3 100644 --- a/schemas/intervention-v0.1.schema.json +++ b/schemas/intervention-v0.1.schema.json @@ -1,44 +1,116 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/myheart521/WAMProbe/schemas/intervention-v0.1.schema.json", - "title": "WAMProbe 2D intervention suite v0.1", - "type": "object", - "additionalProperties": false, - "required": ["schema_version", "benchmark_id", "groups"], - "properties": { - "schema_version": {"const": "0.1"}, - "benchmark_id": {"type": "string", "minLength": 1}, - "groups": { + "title": "WAMProbe intervention JSONL record v0.1", + "oneOf": [ + {"$ref": "#/$defs/pointmass_record"}, + {"$ref": "#/$defs/manipulation_record"} + ], + "$defs": { + "vec2": { "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": ["context_id", "position", "goal", "actions"], - "properties": { - "context_id": {"type": "string", "minLength": 1}, - "position": {"$ref": "#/$defs/vec2"}, - "goal": {"$ref": "#/$defs/vec2"}, - "actions": { - "type": "array", - "minItems": 2, - "items": { + "prefixItems": [{"type": "number"}, {"type": "number"}], + "items": false + }, + "pointmass_record": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "benchmark_id", + "state_kind", + "context", + "actions", + "record_sha256" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "state_kind": {"const": "pointmass-2d"}, + "context": { + "type": "object", + "additionalProperties": false, + "required": ["context_id", "position", "goal"], + "properties": { + "context_id": {"type": "string", "minLength": 1}, + "position": {"$ref": "#/$defs/vec2"}, + "goal": {"$ref": "#/$defs/vec2"} + } + }, + "actions": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "delta"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "delta": {"$ref": "#/$defs/vec2"} + } + } + }, + "record_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + }, + "manipulation_record": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "benchmark_id", + "state_kind", + "context", + "actions", + "record_sha256" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "benchmark_id": {"type": "string", "minLength": 1}, + "state_kind": {"const": "manipulation-2d"}, + "context": { + "type": "object", + "additionalProperties": false, + "required": ["context_id", "initial_state", "goal"], + "properties": { + "context_id": {"type": "string", "minLength": 1}, + "initial_state": { "type": "object", - "required": ["name", "delta"], + "additionalProperties": false, + "required": [ + "effector_position", + "object_position", + "gripper_closed", + "object_attached", + "in_contact" + ], "properties": { - "name": {"type": "string", "minLength": 1}, - "delta": {"$ref": "#/$defs/vec2"} + "effector_position": {"$ref": "#/$defs/vec2"}, + "object_position": {"$ref": "#/$defs/vec2"}, + "gripper_closed": {"type": "boolean"}, + "object_attached": {"type": "boolean"}, + "in_contact": {"type": "boolean"} } + }, + "goal": {"$ref": "#/$defs/vec2"} + } + }, + "actions": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "effector_delta", "close_gripper"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "effector_delta": {"$ref": "#/$defs/vec2"}, + "close_gripper": {"type": "boolean"} } } - } + }, + "record_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} } } - }, - "$defs": { - "vec2": { - "type": "array", - "prefixItems": [{"type": "number"}, {"type": "number"}], - "items": false - } } } diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py new file mode 100644 index 0000000..c699bf2 --- /dev/null +++ b/scripts/validate_repository.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Validate committed JSON Schema instances and repository-local Markdown links.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import unicodedata +from collections import defaultdict +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError + +ROOT = Path(__file__).resolve().parents[1] + +SCHEMA_CASES: dict[str, tuple[str, ...]] = { + "schemas/capability-v0.1.schema.json": ("configs/models/starwam-capabilities-v0.1.json",), + "schemas/intervention-v0.1.schema.json": ( + "schemas/examples/intervention-pointmass-v0.1.json", + "schemas/examples/intervention-manipulation-v0.1.json", + ), + "schemas/result-v0.1.schema.json": ( + "examples/pointmass-demo/summary.json", + "examples/blockpush-demo/summary.json", + "examples/gripper-catch-demo/summary.json", + ), + "release/evidence-manifest-v0.1.schema.json": ("release/evidence-manifest-v0.1.json",), +} + +INLINE_LINK = re.compile( + r"!?\[[^\]\n]*\]\(\s*(?P<[^>\n]+>|[^\s)]+)", +) +REFERENCE_LINK = re.compile( + r"^\s*\[[^\]\n]+\]:\s*(?P<[^>\n]+>|\S+)", + flags=re.MULTILINE, +) +HEADING = re.compile(r"^ {0,3}#{1,6}\s+(?P.+?)\s*#*\s*$", flags=re.MULTILINE) + + +class RepositoryValidationError(RuntimeError): + """Raised when a committed repository contract is invalid.""" + + +def _load_json(path: Path) -> object: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def validate_schemas(root: Path) -> tuple[int, int]: + """Check every public schema and its committed canonical instances.""" + + failures: list[str] = [] + instance_count = 0 + for schema_name, instance_names in SCHEMA_CASES.items(): + schema_path = root / schema_name + try: + schema = _load_json(schema_path) + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + except (OSError, json.JSONDecodeError, SchemaError) as error: + failures.append(f"{schema_name}: invalid schema: {error}") + continue + + for instance_name in instance_names: + instance_count += 1 + try: + instance = _load_json(root / instance_name) + except (OSError, json.JSONDecodeError) as error: + failures.append(f"{instance_name}: invalid JSON: {error}") + continue + errors = sorted(validator.iter_errors(instance), key=lambda item: list(item.path)) + for error in errors: + location = "/".join(str(part) for part in error.absolute_path) or "" + failures.append(f"{instance_name}:{location}: {error.message}") + + if failures: + raise RepositoryValidationError("JSON Schema validation failed:\n" + "\n".join(failures)) + return len(SCHEMA_CASES), instance_count + + +def _without_fenced_code(markdown: str) -> str: + output: list[str] = [] + fence: str | None = None + for line in markdown.splitlines(): + marker = re.match(r"^\s*(`{3,}|~{3,})", line) + if marker is not None: + current = marker.group(1)[0] + if fence is None: + fence = current + elif fence == current: + fence = None + output.append("") + elif fence is None: + output.append(line) + else: + output.append("") + return "\n".join(output) + + +def _markdown_files(root: Path) -> list[Path]: + command = [ + "git", + "-C", + str(root), + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + "*.md", + ] + try: + result = subprocess.run(command, check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError): + excluded = { + ".git", + ".venv", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + "dist", + "runs", + "site", + "vendor", + } + return sorted( + path + for path in root.rglob("*.md") + if not any(part in excluded for part in path.relative_to(root).parts) + ) + return sorted(root / Path(name.decode("utf-8")) for name in result.stdout.split(b"\0") if name) + + +def _slug(text: str) -> str: + text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) + text = re.sub(r"<[^>]+>", "", text).strip().lower() + characters = [ + character + for character in text + if character in {" ", "-", "_"} or unicodedata.category(character).startswith(("L", "N")) + ] + return "".join(characters).replace(" ", "-") + + +def _anchors(path: Path) -> set[str]: + markdown = _without_fenced_code(path.read_text(encoding="utf-8")) + occurrences: defaultdict[str, int] = defaultdict(int) + anchors: set[str] = set() + for match in HEADING.finditer(markdown): + base = _slug(match.group("text")) + count = occurrences[base] + occurrences[base] += 1 + anchors.add(base if count == 0 else f"{base}-{count}") + return anchors + + +def _targets(markdown: str) -> list[str]: + content = _without_fenced_code(markdown) + matches = list(INLINE_LINK.finditer(content)) + list(REFERENCE_LINK.finditer(content)) + return [match.group("target").strip("<>") for match in matches] + + +def validate_markdown_links(root: Path) -> tuple[int, int]: + """Check repository-local file targets and Markdown heading fragments.""" + + failures: list[str] = [] + links_checked = 0 + root_resolved = root.resolve() + for source in _markdown_files(root): + relative_source = source.relative_to(root) + try: + targets = _targets(source.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError) as error: + failures.append(f"{relative_source}: cannot read Markdown: {error}") + continue + for target in targets: + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or target.startswith("//"): + continue + links_checked += 1 + local_name = unquote(parsed.path) + if local_name: + if local_name.startswith("/"): + failures.append( + f"{relative_source}: repository-local link is absolute: {target}" + ) + continue + destination = (source.parent / local_name).resolve() + else: + destination = source.resolve() + try: + destination.relative_to(root_resolved) + except ValueError: + failures.append(f"{relative_source}: link escapes repository: {target}") + continue + if not destination.exists(): + failures.append(f"{relative_source}: missing link target: {target}") + continue + if parsed.fragment and destination.suffix.lower() == ".md": + fragment = unquote(parsed.fragment).removeprefix("user-content-") + if fragment not in _anchors(destination): + failures.append(f"{relative_source}: missing Markdown anchor: {target}") + + if failures: + raise RepositoryValidationError("Markdown link validation failed:\n" + "\n".join(failures)) + return len(_markdown_files(root)), links_checked + + +def main() -> int: + """Run all repository-level validation gates.""" + + try: + schema_count, instance_count = validate_schemas(ROOT) + markdown_count, link_count = validate_markdown_links(ROOT) + except RepositoryValidationError as error: + print(error, file=sys.stderr) + return 1 + print(f"Validated {instance_count} instances against {schema_count} Draft 2020-12 JSON Schemas") + print(f"Checked {link_count} repository-local links across {markdown_count} Markdown files") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_repository_validation.py b/tests/test_repository_validation.py new file mode 100644 index 0000000..c618452 --- /dev/null +++ b/tests/test_repository_validation.py @@ -0,0 +1,18 @@ +import subprocess +import sys +from pathlib import Path + + +def test_repository_schemas_and_markdown_links_are_valid() -> None: + root = Path(__file__).resolve().parents[1] + result = subprocess.run( + [sys.executable, "scripts/validate_repository.py"], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "Draft 2020-12 JSON Schemas" in result.stdout + assert "repository-local links" in result.stdout diff --git a/uv.lock b/uv.lock index b3dbe99..0a34ce0 100644 --- a/uv.lock +++ b/uv.lock @@ -47,6 +47,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -145,6 +166,18 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -154,6 +187,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "librt" version = "0.13.0" @@ -229,6 +301,136 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, ] +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, +] + [[package]] name = "mypy" version = "2.3.0" @@ -308,6 +510,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -356,6 +567,222 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + [[package]] name = "ruff" version = "0.15.21" @@ -381,6 +808,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -451,6 +887,8 @@ source = { editable = "." } [package.optional-dependencies] dev = [ + { name = "jsonschema" }, + { name = "mkdocs" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -459,9 +897,38 @@ dev = [ [package.metadata] requires-dist = [ + { name = "jsonschema", marker = "extra == 'dev'", specifier = ">=4.25,<5" }, + { name = "mkdocs", marker = "extra == 'dev'", specifier = ">=1.6,<2" }, { name = "mypy", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'" }, ] provides-extras = ["dev"] + +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" }, + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] From d58b02336ba5e28709cecb16e8f4d3e65fae3a1c Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 15:22:15 +0000 Subject: [PATCH 19/20] update workflows for Node 24 actions --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/docs.yml | 10 +++++----- .github/workflows/gpu-nightly.yml | 4 ++-- .github/workflows/publish-pypi.yml | 6 +++--- .github/workflows/release-candidate.yml | 10 +++++----- CHANGELOG.md | 5 +++++ 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48777e3..6736abd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,8 @@ jobs: python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -43,13 +43,13 @@ jobs: needs: quality runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v8 with: version: "0.11.6" enable-cache: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4875595..f557b64 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,8 +27,8 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.11" cache: pip @@ -38,10 +38,10 @@ jobs: run: python scripts/validate_repository.py - name: Build documentation site run: mkdocs build --strict - - uses: actions/configure-pages@v5 - - uses: actions/upload-pages-artifact@v3 + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 with: path: site - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/gpu-nightly.yml b/.github/workflows/gpu-nightly.yml index 89470a0..18fffed 100644 --- a/.github/workflows/gpu-nightly.yml +++ b/.github/workflows/gpu-nightly.yml @@ -25,7 +25,7 @@ jobs: timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: clean: false - name: Validate provisioned runner @@ -69,7 +69,7 @@ jobs: --output runs/gpu-nightly/report - name: Upload structured evidence if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: wamprobe-gpu-nightly-${{ github.run_id }} if-no-files-found: warn diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 4964c54..82bf37c 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -22,13 +22,13 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v8 with: version: "0.11.6" enable-cache: true diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index 1e81206..2450349 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -12,13 +12,13 @@ jobs: build-audit-smoke: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v8 with: version: "0.11.6" enable-cache: true @@ -27,13 +27,13 @@ jobs: - name: Build twice, audit evidence, and smoke clean wheel run: .venv/bin/python scripts/build_release_candidate.py - name: Attest wheel and source distribution provenance - uses: actions/attest-build-provenance@v2 + uses: actions/attest-build-provenance@v4 with: subject-path: | dist/release-candidate/*.whl dist/release-candidate/*.tar.gz - name: Upload candidate bundle - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: wamprobe-release-candidate-${{ github.sha }} if-no-files-found: error diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a35230..0910d0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,3 +38,8 @@ as a final GitHub Release. offline-ranking/return association analysis. - Release artifact audit, candidate workflow, clean-wheel smoke test, evidence manifest, and technical-report draft. + +### Changed + +- GitHub workflows now use the current Node 24-compatible major releases of official + checkout, Python setup, artifact, Pages, provenance, and uv setup actions. From 00104868a1122efa4a29b445eb48b29a7d4591cf Mon Sep 17 00:00:00 2001 From: EvoVerse Date: Wed, 15 Jul 2026 15:25:18 +0000 Subject: [PATCH 20/20] pin resolvable setup-uv action tag --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-pypi.yml | 2 +- .github/workflows/release-candidate.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6736abd..9df7cee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.3.2 with: version: "0.11.6" enable-cache: true diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 82bf37c..450d4a8 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.3.2 with: version: "0.11.6" enable-cache: true diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index 2450349..c0ab9f9 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.11" - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.3.2 with: version: "0.11.6" enable-cache: true