diff --git a/.github/ISSUE_TEMPLATE/experiment_result.yml b/.github/ISSUE_TEMPLATE/experiment_result.yml new file mode 100644 index 0000000..35c2761 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/experiment_result.yml @@ -0,0 +1,70 @@ +name: Experiment result +description: Submit a reproducible WAMProbe result for review +title: "[Result]: " +labels: ["experiment-result", "reproduction"] +body: + - type: markdown + attributes: + value: >- + Submit structured evidence, not only a screenshot. Do not include tokens, + credentials, private datasets, browser sessions, or model weights. + - type: input + id: wamprobe_revision + attributes: + label: WAMProbe version and commit + placeholder: 0.1.0rc1 / full Git commit SHA + validations: + required: true + - type: textarea + id: model_provenance + attributes: + label: Adapter and model provenance + description: Include adapter revision, upstream repository revision, checkpoint ID/hash, license, and preprocessing configuration. + validations: + required: true + - type: textarea + id: benchmark + attributes: + label: Benchmark configuration + description: Include benchmark ID, context count/IDs, action set, seed, horizon, split, and any deviations. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment and compute + description: Include operating system, Python, accelerator/driver, dependency lock or image digest, peak memory, and elapsed time. + validations: + required: true + - type: textarea + id: commands + attributes: + label: Exact commands + render: shell + validations: + required: true + - type: textarea + id: artifacts + attributes: + label: Artifacts and SHA256 hashes + description: Link or attach summary.json, results.jsonl, report.md/report.html, evidence manifest, and hashes where public redistribution is allowed. + validations: + required: true + - type: textarea + id: interpretation + attributes: + label: Result and limitations + description: Report the metric profile, unsupported metrics, uncertainty unit, failures, and the narrow claim supported by this run. + validations: + required: true + - type: checkboxes + id: integrity + attributes: + label: Evidence integrity + options: + - label: Every action branch uses the same restored initial context. + required: true + - label: The submitted result was generated by the revisions and commands above. + required: true + - label: No secret, private data, checkpoint, or unverifiable substituted output is attached. + required: true diff --git a/.github/ISSUE_TEMPLATE/external_reproduction.yml b/.github/ISSUE_TEMPLATE/external_reproduction.yml new file mode 100644 index 0000000..21757cd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/external_reproduction.yml @@ -0,0 +1,82 @@ +name: External reproduction +description: Report an independent clean-install WAMProbe smoke reproduction +title: "[External reproduction]: " +labels: ["reproduction"] +body: + - type: markdown + attributes: + value: >- + Thank you for independently checking WAMProbe. This form records the + evidence needed for the external-smoke release gate. Failed attempts are + useful too. Do not post credentials or private machine paths. + - type: input + id: reproducer + attributes: + label: Reproducer name or handle + validations: + required: true + - type: input + id: date + attributes: + label: Date and timezone + placeholder: 2026-07-16 UTC + validations: + required: true + - type: input + id: revision + attributes: + label: WAMProbe tag or full commit SHA + validations: + required: true + - type: textarea + id: environment + attributes: + label: Clean environment + description: Include OS/architecture, Python version, and whether this was a new venv, container, or machine. + validations: + required: true + - type: textarea + id: commands + attributes: + label: Installation and smoke commands + render: shell + validations: + required: true + - type: dropdown + id: result + attributes: + label: Result + options: + - PASS + - FAIL + validations: + required: true + - type: input + id: exit_status + attributes: + label: Exit status and elapsed time + placeholder: exit 0; 42 seconds + validations: + required: true + - type: input + id: output_hash + attributes: + label: Output summary.json SHA256 + validations: + required: true + - type: textarea + id: deviations + attributes: + label: Deviations, errors, or notes + description: Write "None" when the documented procedure was followed exactly. For a failure, include the smallest non-sensitive error output. + validations: + required: true + - type: checkboxes + id: independence + attributes: + label: Independence confirmation + options: + - label: I performed this install and smoke test independently; this is not a maintainer-run substitute for an external user. + required: true + - label: I reviewed the form and removed tokens, credentials, private paths, and other secrets. + required: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fbff90d..a98b88b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,27 @@ Thank you for helping make WAM evaluation more causal, reproducible, and control-grounded. +## Choose a contribution path + +- **Connect a model:** run the + [custom adapter starter](docs/adapters/CUSTOM_ADAPTER.md), then open an + [Adapter proposal](https://github.com/myheart521/WAMProbe/issues/new?template=adapter_proposal.yml). +- **Add a paired diagnostic:** open a + [Benchmark proposal](https://github.com/myheart521/WAMProbe/issues/new?template=benchmark_proposal.yml) + before generating a large dataset. +- **Add or change a metric:** open a + [Metric proposal](https://github.com/myheart521/WAMProbe/issues/new?template=metric_proposal.yml) + and state how the metric can be gamed. +- **Share a run:** use the + [Experiment result](https://github.com/myheart521/WAMProbe/issues/new?template=experiment_result.yml) + form with standard artifacts and hashes. +- **Reproduce independently:** use the + [External reproduction](https://github.com/myheart521/WAMProbe/issues/new?template=external_reproduction.yml) + form. A maintainer-controlled run cannot satisfy the external-user gate. +- **Pick a bounded task:** see the + [roadmap](docs/NEXT_STEPS.md) and issues labeled + [`good first issue`](https://github.com/myheart521/WAMProbe/labels/good%20first%20issue). + ## Development setup ```bash @@ -10,6 +31,15 @@ source .venv/bin/activate python -m pip install -e '.[dev]' ``` +Verify the adapter starter before replacing its backend: + +```bash +python -m examples.custom_adapter.run \ + --output runs/custom-adapter \ + --contexts 8 \ + --seed 7 +``` + Before opening a pull request, run: ```bash @@ -31,11 +61,14 @@ the audit itself. - Follow test-driven development for behavior changes. - Keep the core package CPU-only and avoid heavy mandatory dependencies. - New adapters must publish a capability declaration and pin an upstream revision used by - their smoke test. + their smoke test. Preserve context/action identities and reject malformed model outputs + at the adapter boundary. - New benchmarks must document generation, splits, licenses, and shared-state guarantees. - New metrics must include a metric card, expected baseline ordering, and at least one failure/anti-gaming test. - Do not commit datasets, checkpoints, generated runs, tokens, or browser sessions. +- Result submissions must identify the WAMProbe, adapter, model, checkpoint, preprocessing, + and benchmark revisions; include exact commands and public artifact hashes. Please use focused commits and explain both the scientific reason and user-visible impact of a change. diff --git a/README.md b/README.md index f072284..a71cbf8 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,15 @@ Start with the [scope and capability RFC](docs/rfcs/0001-scope-and-capabilities. [counterfactual metrics RFC](docs/rfcs/0002-counterfactual-metrics.md), and the [adapter selection record](docs/research/ADAPTER_SELECTION.md). +For a working integration instead of pseudocode, run the tested +[custom adapter starter kit](examples/custom_adapter/README.md) and follow the +[step-by-step adapter guide](docs/adapters/CUSTOM_ADAPTER.md): + +```bash +python -m examples.custom_adapter.run \ + --output runs/custom-adapter --contexts 8 --seed 7 +``` + ## Real-model artifacts Model weights are never committed to Git. The first StarWAM spike requires approximately @@ -280,11 +289,14 @@ Review the [reproducibility guide](docs/reproducibility/REPRODUCIBILITY.md) and | Run the CPU demo | [Quick start](docs/QUICKSTART.md) | | Understand the scientific motivation | [WAM/VLA failure-case evidence map](docs/research/WAM_VLA_FAILURE_CASES.md) | | Interpret a score correctly | [Core metric cards](docs/metrics/CORE_METRICS.md) | -| Add a model integration | [Scope and capability RFC](docs/rfcs/0001-scope-and-capabilities.md) | +| Add a model integration | [Custom adapter starter guide](docs/adapters/CUSTOM_ADAPTER.md) | +| Submit a structured experiment | [Experiment result form](https://github.com/myheart521/WAMProbe/issues/new?template=experiment_result.yml) | +| Reproduce independently | [External reproduction form](https://github.com/myheart521/WAMProbe/issues/new?template=external_reproduction.yml) | | Reproduce LIBERO-CF-Mini | [Benchmark card](docs/benchmarks/LIBERO_CF_MINI.md) | | Inspect the closed-loop protocol | [Experiment card](docs/experiments/TOY_CLOSED_LOOP_V0.1.md) | | Audit the release | [Reproducibility guide](docs/reproducibility/REPRODUCIBILITY.md) | | Read the full implementation plan | [Detailed Chinese project plan](docs/WAMProbe_PLAN.md) | +| Choose the next milestone | [Dependency-aware roadmap](docs/NEXT_STEPS.md) | | Browse all documentation | [Published documentation site](https://myheart521.github.io/WAMProbe/) | ## Project status and roadmap @@ -293,7 +305,8 @@ The `v0.1.0rc1` engineering scope is implemented and published. The remaining fo acceptance item is one genuine independent clean-install smoke report in [Issue #2](https://github.com/myheart521/WAMProbe/issues/2). -Next research milestones: +The [dependency-aware roadmap](docs/NEXT_STEPS.md) tracks evidence gates separately from +repository work. Next research milestones: 1. expand LIBERO initial-state coverage; 2. evaluate action-conditioned real-WAM futures when an adapter exposes that capability; @@ -306,6 +319,11 @@ Contributions are welcome—especially adapters, paired benchmark generators, me anti-gaming tests, and independent reproductions. New metrics must document the failure they detect and include a sanity check against reference baselines. +Start with a labeled +[`good first issue`](https://github.com/myheart521/WAMProbe/labels/good%20first%20issue), +an [Adapter proposal](https://github.com/myheart521/WAMProbe/issues/new?template=adapter_proposal.yml), +or the [Experiment result form](https://github.com/myheart521/WAMProbe/issues/new?template=experiment_result.yml). + ```bash python -m pip install -e '.[dev]' ruff format --check . diff --git a/README.zh-CN.md b/README.zh-CN.md index 96b4ab1..963436b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -235,6 +235,15 @@ class MyWorldModel: [反事实指标 RFC](docs/rfcs/0002-counterfactual-metrics.md)和 [适配器选择记录](docs/research/ADAPTER_SELECTION.md)。 +如果希望从可运行代码开始,而不是照着伪代码重新搭建,请运行已经测试过的 +[自定义 Adapter Starter Kit](examples/custom_adapter/README.md),并按照 +[适配器接入指南](docs/adapters/CUSTOM_ADAPTER.md)逐步替换后端: + +```bash +python -m examples.custom_adapter.run \ + --output runs/custom-adapter --contexts 8 --seed 7 +``` + ## 真实模型文件 模型权重永远不会提交到 Git。第一个 StarWAM 验证需要约 46.3 GB 固定版本的 StarWAM 和 @@ -271,11 +280,14 @@ StarWAM 和 LIBERO 的隔离环境、GPU 预检、预处理来源及 smoke 命 | 运行 CPU 演示 | [快速开始](docs/QUICKSTART.md) | | 理解项目的研究动机 | [WAM/VLA 失败案例证据图谱](docs/research/WAM_VLA_FAILURE_CASES.md) | | 正确解释指标 | [核心指标卡](docs/metrics/CORE_METRICS.md) | -| 接入一个新模型 | [范围与能力 RFC](docs/rfcs/0001-scope-and-capabilities.md) | +| 接入一个新模型 | [自定义适配器接入指南](docs/adapters/CUSTOM_ADAPTER.md) | +| 提交结构化实验结果 | [实验结果表单](https://github.com/myheart521/WAMProbe/issues/new?template=experiment_result.yml) | +| 独立复现项目 | [外部复现表单](https://github.com/myheart521/WAMProbe/issues/new?template=external_reproduction.yml) | | 复现 LIBERO-CF-Mini | [基准卡](docs/benchmarks/LIBERO_CF_MINI.md) | | 检查闭环实验协议 | [实验卡](docs/experiments/TOY_CLOSED_LOOP_V0.1.md) | | 审计发行版 | [可复现性指南](docs/reproducibility/REPRODUCIBILITY.md) | | 阅读完整实施计划 | [WAMProbe 中文规划文档](docs/WAMProbe_PLAN.md) | +| 选择下一阶段任务 | [带依赖关系的后续路线图](docs/NEXT_STEPS.md) | | 浏览全部文档 | [在线文档站](https://myheart521.github.io/WAMProbe/) | ## 项目状态与路线图 @@ -283,7 +295,7 @@ StarWAM 和 LIBERO 的隔离环境、GPU 预检、预处理来源及 smoke 命 `v0.1.0rc1` 工程范围已经实现并发布。正式验收目前只剩一项:由真实独立用户在 [Issue #2](https://github.com/myheart521/WAMProbe/issues/2) 提交干净环境安装复现报告。 -下一阶段研究计划: +[后续路线图](docs/NEXT_STEPS.md)会把外部证据门槛和仓库内工程任务分开跟踪。下一阶段研究计划: 1. 扩展 LIBERO 初始状态覆盖; 2. 在适配器真正暴露对应能力后,评测动作条件真实 WAM 未来; @@ -295,6 +307,11 @@ StarWAM 和 LIBERO 的隔离环境、GPU 预检、预处理来源及 smoke 命 欢迎提交适配器、配对基准生成器、指标反作弊测试和独立复现报告。新增指标必须说明它要 发现的失败模式,并包含相对参考基线的 sanity check。 +可以先认领带有 +[`good first issue`](https://github.com/myheart521/WAMProbe/labels/good%20first%20issue) 标签的任务, +提交 [Adapter 提案](https://github.com/myheart521/WAMProbe/issues/new?template=adapter_proposal.yml), +或使用[实验结果表单](https://github.com/myheart521/WAMProbe/issues/new?template=experiment_result.yml)。 + ```bash python -m pip install -e '.[dev]' ruff format --check . diff --git a/docs/NEXT_STEPS.md b/docs/NEXT_STEPS.md new file mode 100644 index 0000000..6ab577c --- /dev/null +++ b/docs/NEXT_STEPS.md @@ -0,0 +1,112 @@ +# Next steps / 后续路线图 + +This roadmap separates work that can be automated in the repository from gates +that require independent people or new scientific evidence. A checked code task +must never be used as a substitute for an unmet evidence gate. + +本路线图区分“仓库内可以自动完成的工程任务”和“必须由独立用户或新实验提供证据的验收 +门槛”。代码完成不代表外部证据已经满足。 + +## Dependency path / 依赖关系 + +```text +P0 independent reproduction ──► P1 final v0.1.0 + +P2 contributor readiness ──► P3 broader paired benchmarks + └──► P4 action-conditioned real WAM + +P1 stable release + public evidence ──► P5 DOI and outreach +``` + +## Milestones / 里程碑 + +| Priority | Milestone | Current state | Completion evidence | +|---|---|---|---| +| P0 | Independent clean-install reproduction / 独立干净环境复现 | **Blocked by an independent user** | A non-maintainer posts commit, environment, commands, exit status, elapsed time, and output hash in Issue #2 | +| P1 | Promote `v0.1.0rc1` to `v0.1.0` / 发布正式版 | **Blocked by P0** | P0 accepted, release audit green, signed/tagged release artifacts and provenance available | +| P2 | Contributor readiness / 贡献者接入能力 | **In progress** | Runnable adapter starter, contribution/result forms, roadmap, labeled starter issues, green CI | +| P3 | Broader paired diagnostics / 扩展配对诊断 | **Planned** | More LIBERO initial states plus Occluded-Object memory benchmark with restore/order checks | +| P4 | Real action-conditioned WAM evidence / 真实动作条件 WAM | **Planned; model capability required** | Public model exposes candidate-conditioned futures, adapter passes semantic tests, bounded report is reproducible | +| P5 | Archival and outreach / 归档与传播 | **After stable release** | Zenodo DOI, updated citation metadata, release notes, public result call | + +Tracked work: + +- P0: [independent reproduction gate #2](https://github.com/myheart521/WAMProbe/issues/2) +- P3: [LIBERO initial-state coverage #10](https://github.com/myheart521/WAMProbe/issues/10) + and [Occluded-Object RFC #7](https://github.com/myheart521/WAMProbe/issues/7) +- P4: [public action-conditioned WAM audit #9](https://github.com/myheart521/WAMProbe/issues/9) +- Starter contributions: [adapter troubleshooting #11](https://github.com/myheart521/WAMProbe/issues/11) + and [malformed-output tests #12](https://github.com/myheart521/WAMProbe/issues/12) +- P5: [Zenodo DOI preparation #8](https://github.com/myheart521/WAMProbe/issues/8) + +## P0 — independent reproduction + +The maintainer can improve instructions and debug a failed report, but cannot +perform this acceptance item on the maintainer's own machine. The external user +should use the **External reproduction** issue form. Issue #2 remains the source +of truth. + +维护者可以完善说明、协助排错,但不能在自己的机器上代替外部用户完成验收。真实独立用户 +应通过 **External reproduction** 表单提交,Issue #2 是唯一验收记录。 + +## P1 — stable release gate + +After P0 is accepted: + +1. confirm the candidate commit is clean and all required CI/security checks pass; +2. run the documented reproducible build and archive audit; +3. update version, changelog, citation, and release notes consistently; +4. publish the final tag and artifacts through the release workflow; +5. verify a clean install from the published package. + +Do not publish `v0.1.0` before P0. + +## P2 — contributor readiness + +Repository-owned deliverables: + +- a tested `examples/custom_adapter` integration path; +- a detailed adapter guide with semantic and provenance checks; +- structured experiment-result and external-reproduction issue forms; +- visible, labeled, bounded issues suitable for first-time contributors; +- README and documentation-site links to each contribution path. + +## P3 — benchmark expansion + +Two bounded tracks should remain separate: + +- **LIBERO-CF-Mini coverage:** add initial states and perturbations while preserving + snapshot restore, shared-context pairing, deterministic action order reversal, + asset licensing, and split/leakage documentation. +- **Occluded-Object diagnostic:** test whether action-conditioned futures retain a + task-relevant object after it becomes unobserved. Define oracle, copy-last, + memoryless, and wrong-memory baselines before adding a learned model. + +Every addition needs a benchmark card, generation or conversion code, hashes, +and expected-ordering tests. + +## P4 — real WAM integration + +Select a public model only when its actual interface can produce a future for +each proposed action from the same context. A model that emits only an action +chunk may be useful for another WAMProbe capability tier, but it does not provide +candidate-conditioned future evidence. + +The first integration should be intentionally small: one pinned checkpoint, one +configuration, one paired context group, a documented GPU budget, cached +prediction provenance, and an explicit list of unavailable metrics. Expand only +after the semantic smoke test passes. + +## P5 — DOI and outreach + +Prepare Zenodo metadata and communication material after the stable release so +the DOI points to immutable public artifacts. Outreach should ask for independent +reproductions and comparable structured reports, not only stars or screenshots. + +## Maintainer decision rules / 维护规则 + +- Never mark P0 complete from a maintainer-controlled machine. +- Never release a metric without an anti-gaming or failure-baseline test. +- Never label a proxy as a capability the model does not expose. +- Never commit checkpoints, datasets, credentials, or generated private evidence. +- Prefer small reproducible issues with an explicit acceptance checklist. diff --git a/docs/adapters/CUSTOM_ADAPTER.md b/docs/adapters/CUSTOM_ADAPTER.md new file mode 100644 index 0000000..af7ad55 --- /dev/null +++ b/docs/adapters/CUSTOM_ADAPTER.md @@ -0,0 +1,132 @@ +# Connect a custom state-future model + +This guide turns a model-specific inference function into WAMProbe's typed +`WAMAdapter` contract. Start from the runnable code in +[`examples/custom_adapter`](https://github.com/myheart521/WAMProbe/tree/main/examples/custom_adapter) +rather than +building the evaluation loop yourself. + +## What the starter kit proves + +The example deliberately separates model inference from evaluation: + +```text +shared Context2D + candidate Action2D + │ + ▼ + PredictionRequest + │ + ▼ + your StateFutureBackend + │ + ▼ +validated horizon-length state future + │ + ▼ + WAMProbe metrics and standard report artifacts +``` + +The included `LinearStateBackend` is only a wiring oracle. It uses the action +delta directly and should achieve zero Top-1 Regret on PointMass2D. It is not a +learned model and its result is not research evidence. + +## Run the unchanged example + +From a development checkout: + +```bash +python -m pip install -e '.[dev]' +python -m examples.custom_adapter.run \ + --output runs/custom-adapter \ + --contexts 8 \ + --seed 7 +``` + +The output directory contains: + +```text +runs/custom-adapter/ +├── summary.json # aggregate result, uncertainty, and schema version +├── results.jsonl # stable context-level records +├── report.md # reviewable metric table +└── report.html # standalone human-readable report +``` + +## Implement the backend boundary + +Copy `examples/custom_adapter` into a model-specific module and replace +`LinearStateBackend` with a class implementing this narrow protocol: + +```python +class MyBackend: + def predict_positions( + self, + request: PredictionRequest, + ) -> tuple[tuple[float, float], ...]: + model_input = preprocess(request) + prediction = self.model.generate(model_input, seed=request.seed) + return decode_positions(prediction, horizon=request.horizon) +``` + +Every request field has evaluation semantics and must be mapped intentionally: + +| Field | Meaning | Integration check | +|---|---|---| +| `context_id` | Identity of the shared initial state | Equal across every action branch in a group | +| `action_name` | Stable candidate-action identity | Preserved in the returned trajectory | +| `position_xy` | Current observed state | Never replaced with a post-action state | +| `goal_xy` | Task goal used by the toy scorer | Keep separate from the action input | +| `action_delta_xy` | Counterfactual intervention | Must affect the model input or conditioning path | +| `horizon` | Required number of future states | Return exactly this many states | +| `seed` | Reproduction seed | Forward to every stochastic sampler | + +The adapter rejects non-positive horizons, missing or extra time steps, malformed +coordinates, and non-finite values before metrics see them. Keep these checks when +replacing the backend. + +## Declare only real capabilities + +`ModelCapabilities` controls which evidence can be interpreted. The starter +declares deterministic state futures. If repeated calls with the same seed are +not deterministic, set `deterministic_seed=False`. Set `stochastic=True` when the +model represents a distribution or samples futures. Do not claim pixels, +latents, action scores, or action prediction unless the adapter exposes those +outputs directly. + +The current starter targets models that can decode a two-dimensional state +future. Pixel- or latent-only models require a capability-specific adapter and +metrics; do not silently convert an unrelated proxy into a state trajectory. + +## Required model-specific tests + +Before proposing an adapter, add tests that demonstrate: + +1. two different actions from one context reach the model as different inputs; +2. context and action identities survive preprocessing and decoding; +3. the same seed reproduces the same result, or nondeterminism is declared; +4. wrong-length, NaN, and infinite outputs fail loudly; +5. the smallest public checkpoint/configuration completes one bounded smoke run; +6. expected oracle and broken-baseline ordering remains intact. + +Run the repository checks listed in +[`CONTRIBUTING.md`](https://github.com/myheart521/WAMProbe/blob/main/CONTRIBUTING.md), +then open an **Adapter proposal** before a large integration. Pin the upstream +repository revision, checkpoint hash, preprocessing version, license, hardware, +and exact smoke command. + +## Submit a result responsibly + +Use the **Experiment result** issue form and attach or link the standard report +artifacts. A valid result identifies the WAMProbe commit, adapter and upstream +revisions, benchmark configuration, hardware/software environment, exact command, +artifact hashes, and unsupported metrics. Generated evidence must not contain +tokens, private data, browser sessions, local credentials, or model weights. + +## 中文速览 + +这个 Starter Kit 的作用,是把模型自己的推理函数接到 WAMProbe 的类型化评测接口上。 +你通常只需要复制 `examples/custom_adapter`,替换 `LinearStateBackend`,并把 +`PredictionRequest` 中的上下文、动作、预测长度和随机种子完整传给模型。后端必须返回与 +`horizon` 等长的有限二维状态序列;不要丢弃动作条件,也不要把像素或潜变量偷偷换成 +不相关的状态代理。提交适配器前,需要固定上游版本、权重哈希、预处理规则、环境和 smoke +命令,并为动作条件、确定性和异常输出补测试。 diff --git a/docs/index.md b/docs/index.md index 9cd1547..ab5c246 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,10 +21,12 @@ It intentionally does not publish a single composite score. ## Start here - Follow the [15-minute quick start](QUICKSTART.md) for dependency-free CPU experiments. +- Connect a model with the tested [custom adapter starter guide](adapters/CUSTOM_ADAPTER.md). - Read the [scope and capability RFC](rfcs/0001-scope-and-capabilities.md) before adding an adapter. - Use the [core metric cards](metrics/CORE_METRICS.md) to select capability-compatible metrics. - Review the [reproducibility guide](reproducibility/REPRODUCIBILITY.md) before comparing results. - Inspect the [StarWAM model card](models/STARWAM.md) for the first real-model integration. +- Check [Next steps](NEXT_STEPS.md) for milestone dependencies and contribution-sized tasks. Install the dependency-free core with `pip install wamprobe`. Committed example reports and audited release artifacts remain available on diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..5d95b54 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Runnable examples shipped with the WAMProbe source distribution.""" diff --git a/examples/custom_adapter/README.md b/examples/custom_adapter/README.md new file mode 100644 index 0000000..b7f467f --- /dev/null +++ b/examples/custom_adapter/README.md @@ -0,0 +1,33 @@ +# Custom adapter starter kit + +This runnable example shows the smallest safe bridge between a custom +state-future model and WAMProbe. The adapter preserves the context ID and action +name, passes an explicit action intervention to the backend, validates the +prediction horizon and finite coordinates, and declares machine-readable model +capabilities. + +Run it from the repository root: + +```bash +python -m examples.custom_adapter.run \ + --output runs/custom-adapter \ + --contexts 8 \ + --seed 0 +``` + +The command writes `summary.json`, `results.jsonl`, `report.md`, and +`report.html`. The included linear backend should achieve zero Top-1 Regret on +PointMass2D; this is a wiring check, not a research result. + +To connect a model: + +1. Implement `StateFutureBackend.predict_positions()` in `adapter.py`. +2. Map every field in `PredictionRequest` to your model input without dropping + the action intervention or context identity. +3. Return exactly one `(x, y)` state per requested horizon step. +4. Update `ModelCapabilities` if your backend is stochastic or has different + behavior. +5. Run this example, then add model-specific tests before submitting results. + +See [`docs/adapters/CUSTOM_ADAPTER.md`](../../docs/adapters/CUSTOM_ADAPTER.md) +for the full integration and submission guide. diff --git a/examples/custom_adapter/__init__.py b/examples/custom_adapter/__init__.py new file mode 100644 index 0000000..3091376 --- /dev/null +++ b/examples/custom_adapter/__init__.py @@ -0,0 +1,9 @@ +"""Minimal, runnable state-future adapter starter kit.""" + +from examples.custom_adapter.adapter import ( + LinearStateBackend, + PredictionRequest, + StarterWAMAdapter, +) + +__all__ = ["LinearStateBackend", "PredictionRequest", "StarterWAMAdapter"] diff --git a/examples/custom_adapter/adapter.py b/examples/custom_adapter/adapter.py new file mode 100644 index 0000000..9467984 --- /dev/null +++ b/examples/custom_adapter/adapter.py @@ -0,0 +1,113 @@ +"""A small reference bridge from an arbitrary backend to ``WAMAdapter``. + +Replace ``LinearStateBackend`` with a wrapper around your model. Keep the +``StarterWAMAdapter`` validation boundary so benchmark inputs and model outputs +remain semantically aligned. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Protocol + +from wamprobe.api.capabilities import FutureRepresentation, ModelCapabilities +from wamprobe.api.types import Action2D, Context2D, Trajectory2D, Vec2 + + +@dataclass(frozen=True, slots=True) +class PredictionRequest: + """Backend-neutral prediction request with explicit intervention semantics.""" + + context_id: str + action_name: str + position_xy: tuple[float, float] + goal_xy: tuple[float, float] + action_delta_xy: tuple[float, float] + horizon: int + seed: int + + +class StateFutureBackend(Protocol): + """Narrow interface that a custom state-future model must implement.""" + + def predict_positions( + self, + request: PredictionRequest, + ) -> tuple[tuple[float, float], ...]: + """Return one predicted ``(x, y)`` state for every requested step.""" + + ... + + +class LinearStateBackend: + """Dependency-free example backend using the supplied action intervention.""" + + def predict_positions( + self, + request: PredictionRequest, + ) -> tuple[tuple[float, float], ...]: + x, y = request.position_xy + dx, dy = request.action_delta_xy + return tuple((x + dx * step, y + dy * step) for step in range(1, request.horizon + 1)) + + +class StarterWAMAdapter: + """Validate and expose a custom state-future backend as a WAMProbe adapter.""" + + def __init__( + self, + backend: StateFutureBackend, + *, + model_id: str = "starter-linear-state", + ) -> None: + self._backend = backend + self._capabilities = ModelCapabilities( + model_id=model_id, + future_representation=FutureRepresentation.STATES, + deterministic_seed=True, + ) + + @property + def capabilities(self) -> ModelCapabilities: + """Declare the output representation and deterministic seed behavior.""" + + return self._capabilities + + def predict_future( + self, + context: Context2D, + action: Action2D, + *, + horizon: int, + seed: int, + ) -> Trajectory2D: + """Translate typed inputs, call the backend, and validate its state future.""" + + if horizon <= 0: + raise ValueError("horizon must be positive") + request = PredictionRequest( + context_id=context.context_id, + action_name=action.name, + position_xy=(context.position.x, context.position.y), + goal_xy=(context.goal.x, context.goal.y), + action_delta_xy=(action.delta.x, action.delta.y), + horizon=horizon, + seed=seed, + ) + positions = self._backend.predict_positions(request) + if len(positions) != horizon: + raise ValueError(f"backend must return exactly {horizon} states") + + states: list[Vec2] = [] + for index, position in enumerate(positions): + if len(position) != 2: + raise ValueError(f"backend state {index} must contain exactly two coordinates") + x, y = position + if not math.isfinite(x) or not math.isfinite(y): + raise ValueError(f"backend state {index} coordinates must be finite") + states.append(Vec2(float(x), float(y))) + return Trajectory2D(context.context_id, action.name, tuple(states)) + + def close(self) -> None: + """Release resources; the starter backend owns none.""" diff --git a/examples/custom_adapter/run.py b/examples/custom_adapter/run.py new file mode 100644 index 0000000..9f0d028 --- /dev/null +++ b/examples/custom_adapter/run.py @@ -0,0 +1,43 @@ +"""Run the custom-adapter starter kit against PointMass2D.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from examples.custom_adapter.adapter import LinearStateBackend, StarterWAMAdapter +from wamprobe.benchmarks.pointmass import PointMass2D +from wamprobe.evaluation import evaluate +from wamprobe.reporting import write_reports + + +def run(output: Path, *, contexts: int, seed: int) -> tuple[Path, Path, Path]: + """Evaluate the starter adapter and write WAMProbe's standard artifacts.""" + + benchmark = PointMass2D() + suite = benchmark.make_suite(contexts=contexts, seed=seed) + adapter = StarterWAMAdapter(LinearStateBackend()) + result = evaluate(adapter, benchmark, suite, seed=seed) + return write_reports( + output, + benchmark=benchmark.benchmark_id, + results=[result], + seed=seed, + ) + + +def main() -> None: + """Parse CLI arguments and run the starter evaluation.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=Path("runs/custom-adapter")) + parser.add_argument("--contexts", type=int, default=8) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + paths = run(args.output, contexts=args.contexts, seed=args.seed) + for path in paths: + print(path) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 3f0857c..e36c8c7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,8 @@ nav: - Research: - WAM/VLA failure cases: research/WAM_VLA_FAILURE_CASES.md - Adapter selection: research/ADAPTER_SELECTION.md + - Adapters: + - Connect a custom model: adapters/CUSTOM_ADAPTER.md - Benchmarks: - Toy benchmarks: benchmarks/TOY_BENCHMARKS.md - LIBERO-CF-Mini: benchmarks/LIBERO_CF_MINI.md @@ -38,3 +40,4 @@ nav: - Guide: reproducibility/REPRODUCIBILITY.md - External report template: reproducibility/EXTERNAL_REPRODUCTION_TEMPLATE.md - GPU nightly: reproducibility/GPU_NIGHTLY.md + - Next steps: NEXT_STEPS.md diff --git a/pyproject.toml b/pyproject.toml index 85b3e68..fa28503 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,16 @@ packages = ["src/wamprobe"] include = [ "/CHANGELOG.md", "/CITATION.cff", + "/CONTRIBUTING.md", "/LICENSE", "/README.md", "/README.zh-CN.md", + "/docs/**/*.md", "/docs/assets/*.png", + "/examples/__init__.py", + "/examples/custom_adapter/*.md", + "/examples/custom_adapter/*.py", + "/mkdocs.yml", "/pyproject.toml", "/src/wamprobe", ] @@ -62,6 +68,7 @@ exclude = [ [tool.pytest.ini_options] testpaths = ["tests"] +pythonpath = ["."] addopts = [ "--strict-markers", ] diff --git a/tests/test_custom_adapter_example.py b/tests/test_custom_adapter_example.py new file mode 100644 index 0000000..9c5ed09 --- /dev/null +++ b/tests/test_custom_adapter_example.py @@ -0,0 +1,70 @@ +import json +from pathlib import Path + +import pytest + +from examples.custom_adapter.adapter import ( + LinearStateBackend, + PredictionRequest, + StarterWAMAdapter, +) +from examples.custom_adapter.run import run +from wamprobe.api.capabilities import FutureRepresentation +from wamprobe.api.model import WAMAdapter +from wamprobe.api.types import Action2D, Context2D, Vec2 + + +def test_starter_adapter_is_typed_seeded_and_action_conditioned() -> None: + adapter = StarterWAMAdapter(LinearStateBackend(), model_id="starter-test") + context = Context2D("context-0", position=Vec2(0.0, 0.0), goal=Vec2(1.0, 0.0)) + action = Action2D("right", Vec2(0.25, 0.0)) + + first = adapter.predict_future(context, action, horizon=4, seed=7) + second = adapter.predict_future(context, action, horizon=4, seed=7) + + assert isinstance(adapter, WAMAdapter) + assert adapter.capabilities.model_id == "starter-test" + assert adapter.capabilities.future_representation is FutureRepresentation.STATES + assert adapter.capabilities.deterministic_seed is True + assert first == second + assert first.context_id == context.context_id + assert first.action_name == action.name + assert first.states == ( + Vec2(0.25, 0.0), + Vec2(0.5, 0.0), + Vec2(0.75, 0.0), + Vec2(1.0, 0.0), + ) + + +def test_starter_adapter_rejects_a_backend_with_the_wrong_horizon() -> None: + class ShortBackend: + def predict_positions( + self, + request: PredictionRequest, + ) -> tuple[tuple[float, float], ...]: + return ((request.position_xy[0], request.position_xy[1]),) + + adapter = StarterWAMAdapter(ShortBackend()) + context = Context2D("context-0", position=Vec2(0.0, 0.0), goal=Vec2(1.0, 0.0)) + action = Action2D("right", Vec2(0.25, 0.0)) + + with pytest.raises(ValueError, match="exactly 4 states"): + adapter.predict_future(context, action, horizon=4, seed=7) + + +def test_starter_runner_writes_the_standard_report_contract(tmp_path: Path) -> None: + paths = run(tmp_path, contexts=4, seed=7) + + assert {path.name for path in paths} == {"summary.json", "report.md", "report.html"} + assert {path.name for path in tmp_path.iterdir()} == { + "summary.json", + "results.jsonl", + "report.md", + "report.html", + } + summary = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) + assert summary["benchmark"] == "pointmass-2d-v0.1" + assert summary["results"][0]["model_id"] == "starter-linear-state" + assert summary["results"][0]["contexts"] == 4 + assert summary["results"][0]["metrics"]["top1_regret"] == 0.0