From b3e8cbf899e237333eeb4b6f67043a4ef9dc96cb Mon Sep 17 00:00:00 2001 From: Endofthestars <85158737+Endofthestars@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:55:17 +0800 Subject: [PATCH] fix(mcp): let a Task Capsule record its acting Host benchwork_open_task minted every Task Capsule with a hardcoded codex Host and exposed no way for a non-Codex Tier 1 Host to describe itself, so a truthful claude-code provenance was refused with VALIDATION_REJECTED. Athanor was right to refuse; the defect was in the MCP boundary, in a module whose own docstring claims to be host-neutral. open_task now accepts the same optional host_session argument that complete_task and fail_task already took. The default remains codex so existing clients are unchanged, and an unregistered Host is refused at the boundary instead of being minted and later rejected by Ward. This is an additive optional field under the MCP API Policy; the 38 frozen tool names are unchanged. Found by the Claude Code CLI Tier 1 Host trial, which is recorded under examples/phase2-final/host-claude-code together with the repaired loop: Capsule TK-B280B5160682 with host claude-code, Ward PASS, and Receipt RC-47D74468807A. Claude Code CLI moves from PENDING_HOST_VALIDATION to PASS. --- CHANGELOG.md | 5 + docs/en/HOST_SUPPORT_MATRIX.md | 36 +++++- docs/en/PHASE2_ACCEPTANCE.md | 29 ++++- docs/en/ROADMAP.md | 12 +- .../en/plugins/acceptance-exception-policy.md | 6 +- docs/en/releases/0.3.0rc2.md | 22 +++- .../phase2-final/host-claude-code/README.md | 99 ++++++++++++++ .../phase2-final/host-claude-code/SHA256SUMS | 2 + .../agent-result-provenance.json | 25 ++++ .../host-claude-code/trial-log.json | 121 ++++++++++++++++++ hosts/claude-code/README.md | 10 +- .../assets/host-capability-matrix.yaml | 4 +- src/benchwork/mcp/runtime.py | 21 ++- tests/mcp/test_runtime.py | 90 +++++++++++++ tests/plugin/test_plugin_package.py | 2 +- 15 files changed, 462 insertions(+), 22 deletions(-) create mode 100644 examples/phase2-final/host-claude-code/README.md create mode 100644 examples/phase2-final/host-claude-code/SHA256SUMS create mode 100644 examples/phase2-final/host-claude-code/agent-result-provenance.json create mode 100644 examples/phase2-final/host-claude-code/trial-log.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cd05a..72dd0fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ versioned schemas plus explicit migration notes for compatibility. ## 0.3.0rc2 — The Invitation +- Fixed `benchwork_open_task` minting every Task Capsule for the `codex` Host. + It now accepts the optional `host_session` argument the completion tools + already took, so a non-Codex Host can record truthful provenance. The default + remains `codex` and an unregistered Host is refused. +- Recorded the Claude Code CLI Tier 1 Host PASS from its 2026-07-31 trial. - Added the auditable POSIX installer and exact release/channel manifests. - Added uv-first isolated installation, pipx fallback, plugin staging, installer state, Host diagnostics, repair, rollback, and uninstall. diff --git a/docs/en/HOST_SUPPORT_MATRIX.md b/docs/en/HOST_SUPPORT_MATRIX.md index 968c4d0..a58d1a6 100644 --- a/docs/en/HOST_SUPPORT_MATRIX.md +++ b/docs/en/HOST_SUPPORT_MATRIX.md @@ -20,7 +20,8 @@ tier. Tier 1 is evaluated per declared release Host. Codex CLI is the primary Phase 2 Host. Host-neutral compatibility with another CLI is not the same as an -interactive PASS for that Host. +interactive PASS for that Host; Claude Code CLI earned its own PASS from its +own trial, not from the shared contract. ## Current acceptance @@ -29,7 +30,7 @@ interactive PASS for that Host. | Core Runtime | 0 | PASS | Required gate satisfied | | MCP STDIO | MCP | PASS | Required gate satisfied | | Codex CLI | 1 | PASS | Primary CLI gate satisfied | -| Claude Code CLI | 1 | PENDING_HOST_VALIDATION | Host-neutral contract only; not a claimed interactive PASS | +| Claude Code CLI | 1 | PASS | Second CLI gate satisfied by the 2026-07-31 trial | | Codex IDE extension | 2 | BLOCKED_BY_ENVIRONMENT | Accepted exception `HOST-IDE-001` | | Claude Desktop | 2 | OPTIONAL_NOT_RUN | No Phase 2 release impact | | External Review | disclosure | WAITING_FOR_DISCLOSURE_AUTHORIZATION | Correctly denied; not a failure | @@ -57,12 +58,37 @@ Reproduction and closure: 6. Record Host version, plugin version, tool count, Receipt identifier, and any exception without copying secrets or conversation reasoning. +## 2026-07-31 Claude Code CLI trial + +Fresh ephemeral Claude Code CLI `2.1.220` conversations loaded the Benchwork +MCP server with `--strict-mcp-config` against a temporary project outside the +source repository. The server reported `connected` and the Host discovered all +38 registry tools. `benchwork_status` and `benchwork_next_actions` each +returned `ok: true` with an `mcp-tool-result/1.0` envelope, both from an +explicit request and from the broad request "What should we do next for this +study?" with 26 non-MCP tools still available. + +The bounded Task loop initially failed. `benchwork_open_task` minted every Task +Capsule with a hardcoded `codex` Host, so a truthful `claude-code` provenance +was refused with `VALIDATION_REJECTED`. Athanor was correct; the MCP boundary +gave a non-Codex Host no way to describe itself. After `benchwork_open_task` +was given the same optional `host_session` argument the completion tools +already accepted, the loop completed: Capsule `TK-B280B5160682` recorded host +`claude-code`, Ward returned `PASS`, and Athanor issued Receipt +`RC-47D74468807A` for the accepted `code-inspection-result/1.0` output. The +source repository never acquired a `.benchwork/` directory. + +Evidence is published under +[`examples/phase2-final/host-claude-code`](../../examples/phase2-final/host-claude-code/README.md). + ## Evidence policy PASS requires a dated, reproducible Host trial. Contract inspection or shared -configuration alone is insufficient. `BLOCKED_BY_ENVIRONMENT` requires an -identifier, owner, reason, impact, reproduction steps, and the boundary before -which it must be closed. +configuration alone is insufficient. The Claude Code CLI trial demonstrates +why: the adapter, the Capsule schema, and Ward all already admitted +`claude-code`, and only a live Host reached the one line that did not. +`BLOCKED_BY_ENVIRONMENT` requires an identifier, owner, reason, impact, +reproduction steps, and the boundary before which it must be closed. See the [Phase 2 acceptance matrix](PHASE2_ACCEPTANCE.md) and [Acceptance Exception Policy](plugins/acceptance-exception-policy.md). diff --git a/docs/en/PHASE2_ACCEPTANCE.md b/docs/en/PHASE2_ACCEPTANCE.md index 3abd784..0328525 100644 --- a/docs/en/PHASE2_ACCEPTANCE.md +++ b/docs/en/PHASE2_ACCEPTANCE.md @@ -27,7 +27,7 @@ states. | Tier 0 — Kernel | PASS | Required | Chronicle, Schema, Ward, Capability, replay, and deterministic suites | | MCP | PASS | Required | in-memory and spawned-STDIO discovery, envelopes, pagination, Task and canonical tool tests | | Tier 1 — Codex CLI Host | PASS | Required before Phase release | fresh Codex CLI explicit/implicit Skill trials and MCP calls | -| Tier 1 — Claude Code CLI | PENDING_HOST_VALIDATION | Not a claimed Phase 2 Host PASS | host-neutral contract is implemented; independent interactive evidence is pending | +| Tier 1 — Claude Code CLI | PASS | Second CLI Host gate satisfied | fresh Claude Code CLI MCP discovery, read-only calls, and a bounded Task loop with an Athanor Receipt | | Tier 2 — IDE Host | BLOCKED_BY_ENVIRONMENT (accepted) | Does not block Core/MCP/CLI | `HOST-IDE-001`; extension installed, no graphical extension Host available | | External Review | WAITING_FOR_DISCLOSURE_AUTHORIZATION | Does not block local review or Core/MCP/CLI | Disclosure Gate denied an unapproved diff review | @@ -79,6 +79,33 @@ an accepted `bench.study.audit` proposal, replacement Protocol preview/commit, registered implementation and Runs, Alembic analysis, Assessment, and a human-sealed `REPAIR` Decision through Chronicle replay. +## 2026-07-31 Claude Code CLI Tier 1 trial + +- Fresh ephemeral Claude Code CLI `2.1.220` conversations loaded the Benchwork + MCP server with `--strict-mcp-config` against a temporary project outside the + source repository. The server reported `connected` and all 38 registry tools + were discovered. +- `benchwork_status` and `benchwork_next_actions` both returned `ok=true` with + `mcp-tool-result/1.0`, from an explicit request and from the broad request + "What should we do next for this study?" while 26 non-MCP tools remained + available. +- The first bounded Task loop was blocked. `benchwork_open_task` minted the + Capsule with a hardcoded `codex` Host, so truthful `claude-code` provenance + was refused with `VALIDATION_REJECTED`. The Host reported the rejection + rather than resubmitting a false provenance. +- `benchwork_open_task` now accepts the optional `host_session` argument that + the completion tools already took, defaulting to `codex` and refusing an + unregistered Host. The repaired loop completed `bench.code.inspect` as + `TK-B280B5160682` with Capsule host `claude-code`, Ward `PASS`, and Receipt + `RC-47D74468807A`. +- The Host used only `Read` alongside the Benchwork tools, matching the Capsule + Circle. Deep Doctor reported 2 verified Chronicle events with an intact + Receipt chain, and the source repository had no `.benchwork/` directory. +- Evidence is published under + [`examples/phase2-final/host-claude-code`](../../examples/phase2-final/host-claude-code/README.md). +- IDE execution and External Review were not exercised; `HOST-IDE-001` and + `WAITING_FOR_DISCLOSURE_AUTHORIZATION` are unchanged. + ## 2026-07-30 M17 rc1 trial - The Python wheel and source distribution built as `0.3.0rc1`; Twine accepted diff --git a/docs/en/ROADMAP.md b/docs/en/ROADMAP.md index fb147a3..e0ff431 100644 --- a/docs/en/ROADMAP.md +++ b/docs/en/ROADMAP.md @@ -127,10 +127,14 @@ audit-to-`REPAIR` acceptance; migration and release documentation; and release-gated CI coverage. Phase 2 is sealed with Kernel, MCP, Plugin, and Codex CLI gates passing. -`HOST-IDE-001` remains an accepted `BLOCKED_BY_ENVIRONMENT` Tier 2 exception, -External Review remains `WAITING_FOR_DISCLOSURE_AUTHORIZATION`, and Claude Code -CLI remains `PENDING_HOST_VALIDATION`. These states are not reclassified as -failures or fabricated PASS results. +`HOST-IDE-001` remains an accepted `BLOCKED_BY_ENVIRONMENT` Tier 2 exception +and External Review remains `WAITING_FOR_DISCLOSURE_AUTHORIZATION`. These +states are not reclassified as failures or fabricated PASS results. + +Claude Code CLI was `PENDING_HOST_VALIDATION` at the rc1 seal and recorded its +own Tier 1 PASS on 2026-07-31, after its trial found and repaired a Task +Capsule Host defect. See the +[Host Support Matrix](HOST_SUPPORT_MATRIX.md). ## M18: The Invitation diff --git a/docs/en/plugins/acceptance-exception-policy.md b/docs/en/plugins/acceptance-exception-policy.md index 2234381..7474203 100644 --- a/docs/en/plugins/acceptance-exception-policy.md +++ b/docs/en/plugins/acceptance-exception-policy.md @@ -22,8 +22,10 @@ An unavailable Tier 2 environment is recorded as `BLOCKED_BY_ENVIRONMENT`, not the release boundary before which validation becomes required. Tier 1 is accepted per Host. Codex CLI is the Phase 2 primary Host and has a -recorded PASS. Claude Code CLI remains `PENDING_HOST_VALIDATION`; the -host-neutral tool contract does not manufacture an interactive Host result. +recorded PASS. Claude Code CLI recorded its own PASS on 2026-07-31 from its own +interactive trial; the host-neutral tool contract does not manufacture an +interactive Host result, and that trial found and repaired a Task Capsule Host +defect the shared contract had hidden. The current IDE exception is: diff --git a/docs/en/releases/0.3.0rc2.md b/docs/en/releases/0.3.0rc2.md index 1baca1c..0919215 100644 --- a/docs/en/releases/0.3.0rc2.md +++ b/docs/en/releases/0.3.0rc2.md @@ -5,8 +5,9 @@ canonical: true # Benchwork 0.3.0rc2 — The Invitation -`0.3.0rc2` adds the auditable macOS/Linux/WSL installer without changing the -M17 scientific control-plane contracts. +`0.3.0rc2` adds the auditable macOS/Linux/WSL installer. The M17 scientific +control-plane contracts are unchanged apart from one additive optional field, +recorded under Fixed below. ## Added @@ -19,6 +20,23 @@ M17 scientific control-plane contracts. - installer-owned repair, rollback, PATH blocks, and uninstall - release assets, SBOM, provenance, GitHub Pages publication, and installer CI +## Fixed + +`benchwork_open_task` minted every Task Capsule for the `codex` Host, so a +non-Codex Tier 1 Host could not record truthful provenance: Athanor correctly +refused the Agent Result with `VALIDATION_REJECTED`. The tool now accepts the +same optional `host_session` argument `benchwork_complete_task` and +`benchwork_fail_task` already took, and mints the Capsule for that Host. + +This is an additive optional field under the +[MCP API Policy](../MCP_API_POLICY.md). The 38 frozen tool names are unchanged, +the default remains `codex` so existing clients behave exactly as before, and +an unregistered Host is now refused at the boundary instead of being minted and +later rejected by Ward. + +The Claude Code CLI Tier 1 trial that found this is published under +[`examples/phase2-final/host-claude-code`](../../../examples/phase2-final/host-claude-code/README.md). + ## Safety The installer does not require root, mutate system Python, create research diff --git a/examples/phase2-final/host-claude-code/README.md b/examples/phase2-final/host-claude-code/README.md new file mode 100644 index 0000000..40ec453 --- /dev/null +++ b/examples/phase2-final/host-claude-code/README.md @@ -0,0 +1,99 @@ +# Claude Code CLI Tier 1 Host trial + +This directory records the 2026-07-31 Tier 1 Host validation for Claude Code +CLI, which closed the `PENDING_HOST_VALIDATION` state left open by the Phase 2 +acceptance. It contains exported, non-canonical evidence only. No `.benchwork/` +state file and no conversation reasoning is included here. + +## Method + +Each trial was a fresh, ephemeral Claude Code CLI `2.1.220` conversation +started with `--strict-mcp-config`, so the Benchwork MCP server was the only +one loaded. Every trial ran against a temporary project outside this +repository; the source repository never acquired a `.benchwork/` directory. + +Transcripts were captured as `stream-json` and distilled into +[`trial-log.json`](trial-log.json). The Host session identifier was pinned with +`--session-id`, so the `invocation_id` recorded in Chronicle can be checked +against the transcript that produced it. + +## What the trial found + +Read-only discovery passed immediately. The MCP server reported `connected` +and the Host discovered all 38 registry tools. `benchwork_status` and +`benchwork_next_actions` each returned `ok: true` with an +`mcp-tool-result/1.0` envelope, both when the tools were named explicitly and +when the Host was given only the broad request "What should we do next for this +study?" with 26 non-MCP tools still available. + +The bounded Task loop did not pass. `benchwork_open_task` minted every Task +Capsule with a hardcoded `codex` Host and exposed no parameter for the acting +Host, so a truthful `claude-code` provenance was refused: + +```text +VALIDATION_REJECTED: Agent Result provenance Host does not match its Task Capsule +``` + +Athanor was right to refuse. The defect was that the MCP boundary gave a +non-Codex Host no way to describe itself, in a module whose own docstring +claims to be host-neutral. The Host reported the rejection rather than +resubmitting a false `codex` provenance to force a green run. + +This is what the evidence policy means when it says contract inspection is +insufficient. `hosts.py` already exported a `ClaudeCodeHostAdapter`, the Task +Capsule schema already enumerated `claude-code`, and Ward already admitted it. +Only a live Host trial reached the one line that did not. + +## The repair + +`benchwork_open_task` now accepts the same optional `host_session` argument +that `benchwork_complete_task` and `benchwork_fail_task` already took, and +mints the Capsule for that Host. The default remains `codex`, so existing +Codex flows are unchanged, and an unregistered Host is refused rather than +minted and later rejected by Ward. + +## Result + +The repaired loop completed end to end. Chronicle recorded the Capsule and the +accepted Agent Result under the acting Host: + +| Item | Value | +| --- | --- | +| Program | `RP-001`, Receipt `RC-3913CD0AC6A6` | +| Task | `TK-B280B5160682`, capability `bench.code.inspect` | +| Capsule Host | `claude-code` | +| Circle | `read` only, no network, 900 s | +| Ward | `PASS` | +| Completion | `ok: true`, Receipt `RC-47D74468807A` | +| Output | `code-inspection-result/1.0` | + +The Host used only `Read` alongside the Benchwork tools, matching the Capsule +Circle. Deep Doctor reported 2 verified Chronicle events with an intact Receipt +chain. The canonical provenance excerpt is in +[`agent-result-provenance.json`](agent-result-provenance.json). + +## Verify + +```bash +sha256sum --check examples/phase2-final/host-claude-code/SHA256SUMS + +jq -e '.verdict.tier_1_claude_code_cli == "PASS"' \ + examples/phase2-final/host-claude-code/trial-log.json + +jq -e '.agent_result.provenance.host == "claude-code"' \ + examples/phase2-final/host-claude-code/agent-result-provenance.json + +python3 -m unittest \ + tests.mcp.test_runtime.MCPRuntimeTest.test_declared_host_owns_its_capsule_and_completes_the_task \ + tests.mcp.test_runtime.MCPRuntimeTest.test_capsule_defaults_to_codex_and_rejects_a_foreign_host_result \ + tests.mcp.test_runtime.MCPRuntimeTest.test_unknown_task_host_is_refused +``` + +## Scope + +This trial closes Tier 1 for Claude Code CLI only. `HOST-IDE-001` remains +`BLOCKED_BY_ENVIRONMENT` and External Review remains +`WAITING_FOR_DISCLOSURE_AUTHORIZATION`; neither was exercised here. The +inspection output also recorded a residual risk worth carrying forward: the +Circle preservation requirement in the Host adapter README is documentation +prose, not an enforced mechanism. diff --git a/examples/phase2-final/host-claude-code/SHA256SUMS b/examples/phase2-final/host-claude-code/SHA256SUMS new file mode 100644 index 0000000..01a19de --- /dev/null +++ b/examples/phase2-final/host-claude-code/SHA256SUMS @@ -0,0 +1,2 @@ +f032311827150fa5a728df72ea1842b0aa32f6c02e978d4b0b7f038a4e144f45 examples/phase2-final/host-claude-code/agent-result-provenance.json +56e8b4a0b59e5d4eb22606fd7a09aff6237d24bcb7755f4c461de836d0bf250a examples/phase2-final/host-claude-code/trial-log.json diff --git a/examples/phase2-final/host-claude-code/agent-result-provenance.json b/examples/phase2-final/host-claude-code/agent-result-provenance.json new file mode 100644 index 0000000..4c7cbfc --- /dev/null +++ b/examples/phase2-final/host-claude-code/agent-result-provenance.json @@ -0,0 +1,25 @@ +{ + "agent_result": { + "outputs": [ + { + "blob_sigil": "sha256:f718355fa44902306d2b0680e940769b99c19a91b3f42af56ae4a74faea10860", + "schema": "code-inspection-result/1.0", + "uri": ".benchwork/mcp/task-results/TK-B280B5160682-f718355fa449.json" + } + ], + "provenance": { + "host": "claude-code", + "invocation_id": "44444444-4444-4444-8444-444444444444", + "model": "claude-opus-5", + "provider": "anthropic", + "runtime": "claude-code-cli/2.1.220" + }, + "schema_version": "agent-result/1.1", + "status": "COMPLETED" + }, + "capsule_host": "claude-code", + "event_id": "CE-8E61DDA3265E", + "exported_from": "Chronicle event payload of the round B Task completion", + "object_id": "TK-B280B5160682", + "occurred_at": "2026-07-31T08:46:31.240296+00:00" +} diff --git a/examples/phase2-final/host-claude-code/trial-log.json b/examples/phase2-final/host-claude-code/trial-log.json new file mode 100644 index 0000000..ad6dbd4 --- /dev/null +++ b/examples/phase2-final/host-claude-code/trial-log.json @@ -0,0 +1,121 @@ +{ + "schema_version": "host-trial-log/1.0", + "host": "claude-code", + "tier": "tier_1_cli", + "date": "2026-07-31", + "host_version": "claude-code-cli/2.1.220", + "benchwork_version": "0.3.0rc2", + "mcp_server": "benchwork 0.3.0rc2", + "registry_tool_count": 38, + "method": "Fresh ephemeral Claude Code CLI conversations started with --strict-mcp-config against a temporary Benchwork project outside the source repository. Transcripts were captured as stream-json and distilled to tool names, envelope headers and status fields.", + "rounds": [ + { + "round": "A", + "state": "pre-fix", + "trials": [ + { + "id": "trial1-explicit-readonly", + "session_id": "11111111-1111-4111-8111-111111111111", + "intent": "Explicit read-only MCP discovery and calls", + "mcp_server_status": "connected", + "benchwork_tools_discovered": 38, + "tool_calls": [ + "mcp__benchwork__benchwork_status", + "mcp__benchwork__benchwork_next_actions" + ], + "envelopes": [ + {"schema_version": "mcp-tool-result/1.0", "ok": true}, + {"schema_version": "mcp-tool-result/1.0", "ok": true} + ], + "shell_or_file_edit_tools_used": [], + "outcome": "PASS" + }, + { + "id": "trial2-implicit-selection", + "session_id": "22222222-2222-4222-8222-222222222222", + "intent": "Implicit selection from the broad request 'What should we do next for this study?'", + "mcp_server_status": "connected", + "benchwork_tools_discovered": 38, + "non_mcp_tools_available": 26, + "tool_calls": [ + "mcp__benchwork__benchwork_status", + "mcp__benchwork__benchwork_next_actions" + ], + "envelopes": [ + {"schema_version": "mcp-tool-result/1.0", "ok": true}, + {"schema_version": "mcp-tool-result/1.0", "ok": true} + ], + "shell_or_file_edit_tools_used": [], + "notes": "The Host reached the control plane on its own while 26 non-MCP tools remained available. Read was denied by the trial's own restriction and the Host continued within bounds.", + "outcome": "PASS" + }, + { + "id": "trial3-task-loop", + "session_id": "33333333-3333-4333-8333-333333333333", + "intent": "Bounded bench.code.inspect Task loop with truthful Host provenance", + "program_receipt": "RC-0DD15A0C7771", + "task_id": "TK-B8F27F61AD04", + "capsule_host": "codex", + "declared_host_session": "claude-code", + "completion_ok": false, + "completion_receipt": null, + "error_code": "VALIDATION_REJECTED", + "error_message": "Agent Result provenance Host does not match its Task Capsule", + "outcome": "BLOCKED", + "notes": "benchwork_open_task minted the Capsule with a hardcoded codex Host and exposed no way to declare the acting Host. The Host reported the rejection instead of resubmitting a false codex provenance." + } + ] + }, + { + "round": "B", + "state": "post-fix", + "trials": [ + { + "id": "trial4-task-loop-repaired", + "session_id": "44444444-4444-4444-8444-444444444444", + "intent": "Same bounded Task loop after benchwork_open_task accepted host_session", + "mcp_server_status": "connected", + "benchwork_tools_discovered": 38, + "tool_calls": [ + "mcp__benchwork__benchwork_create_program", + "mcp__benchwork__benchwork_open_task", + "mcp__benchwork__benchwork_get_task", + "Read", + "mcp__benchwork__benchwork_complete_task" + ], + "program_id": "RP-001", + "program_receipt": "RC-3913CD0AC6A6", + "task_id": "TK-B280B5160682", + "capability": "bench.code.inspect", + "capsule_host": "claude-code", + "capsule_circle": {"tools": ["read"], "network": false, "time_budget_seconds": 900}, + "ward_status": "PASS", + "completion_ok": true, + "completion_receipt": "RC-47D74468807A", + "output_schema": "code-inspection-result/1.0", + "shell_or_file_edit_tools_used": [], + "outcome": "PASS" + } + ] + } + ], + "integrity": { + "chronicle_events": 2, + "chronicle_verified": true, + "receipt_chain_intact": true, + "source_repository_benchwork_directory": "absent" + }, + "verdict": { + "tier_1_claude_code_cli": "PASS", + "requires": { + "interactive_session": "PASS", + "mcp": "PASS", + "tool_calls": "PASS", + "receipts": "PASS" + }, + "accepted_exceptions": [ + "HOST-IDE-001", + "EXTERNAL_REVIEW_DISCLOSURE_PENDING" + ] + } +} diff --git a/hosts/claude-code/README.md b/hosts/claude-code/README.md index eda929a..9d5ddc0 100644 --- a/hosts/claude-code/README.md +++ b/hosts/claude-code/README.md @@ -9,5 +9,11 @@ bwork host propose claude-code bench.evidence.discover \ --input-sigil sha256: --tool read --tool web --time-budget 300 --network ``` -The future Claude Code integration must preserve the Capsule's Circle rather -than expanding its tools or authority. +A Claude Code integration must preserve the Capsule's Circle rather than +expanding its tools or authority. + +Claude Code CLI holds a Tier 1 PASS from its 2026-07-31 trial, recorded in +[`examples/phase2-final/host-claude-code`](../../examples/phase2-final/host-claude-code/README.md). +A Host that opens its own Task Capsule declares itself with `host_session` so +Chronicle records the acting Host; Athanor refuses an Agent Result whose +provenance Host does not match its Capsule. diff --git a/plugins/benchwork/assets/host-capability-matrix.yaml b/plugins/benchwork/assets/host-capability-matrix.yaml index d259a8c..1f9490c 100644 --- a/plugins/benchwork/assets/host-capability-matrix.yaml +++ b/plugins/benchwork/assets/host-capability-matrix.yaml @@ -35,7 +35,7 @@ "tier": "tier_1_cli", "mcp": true, "gui": false, - "acceptance": "PENDING_HOST_VALIDATION", + "acceptance": "PASS", "external_review": "REQUIRES_DISCLOSURE_APPROVAL" }, "codex_ide": { @@ -69,7 +69,7 @@ "cli_host": "PASS", "cli_hosts": { "codex_cli": "PASS", - "claude_code_cli": "PENDING_HOST_VALIDATION" + "claude_code_cli": "PASS" }, "ide_host": { "status": "BLOCKED_BY_ENVIRONMENT", diff --git a/src/benchwork/mcp/runtime.py b/src/benchwork/mcp/runtime.py index b6de6dc..e5c39c4 100644 --- a/src/benchwork/mcp/runtime.py +++ b/src/benchwork/mcp/runtime.py @@ -14,6 +14,7 @@ from ..athanor import Athanor, AthanorError, _exclusive_lock, content_sigil from ..circle import CapsuleStore, CapabilityRegistry, Ward from ..doctor import deep_doctor +from ..hosts import HOSTS from ..project import ProjectContext, discover_project_root from ..schema_validation import _schema_directory, validate_instance from ..tasks import TaskService @@ -21,6 +22,8 @@ from .pagination import page +DEFAULT_HOST = "codex" + CODEX_HUMAN_ACTOR = { "actor_id": "interactive-user", "actor_type": "human", @@ -392,8 +395,9 @@ def benchwork_open_task( network: bool | None = None, approval_reason: str | None = None, review_id: str | None = None, + host_session: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Open a bounded Codex Task Capsule and evaluate it through Ward.""" + """Open a bounded Host Task Capsule and evaluate it through Ward.""" tool = "benchwork_open_task" def operation() -> dict[str, Any]: @@ -453,7 +457,7 @@ def operation() -> dict[str, Any]: ), "network": network if network is not None else contract["network"], }, - host="codex", + host=self._acting_host(host_session), bindings=bindings, ) receipt = None @@ -493,10 +497,21 @@ def operation() -> dict[str, Any]: return self._run(tool, operation) + @staticmethod + def _acting_host(host_session: dict[str, Any] | None) -> str: + """Resolve the Host driving a Task, defaulting to the primary CLI Host.""" + if host_session is None: + return DEFAULT_HOST + validate_instance("host-session-provenance-1.0.json", host_session) + host = str(host_session["host"]) + if host not in HOSTS: + raise AthanorError(f"unknown Task Host: {host}") + return host + @staticmethod def _agent_provenance(host_session: dict[str, Any] | None) -> dict[str, str]: if host_session is None: - return {"host": "codex", "runtime": "interactive-session"} + return {"host": DEFAULT_HOST, "runtime": "interactive-session"} validate_instance("host-session-provenance-1.0.json", host_session) provenance = { "host": host_session["host"], diff --git a/tests/mcp/test_runtime.py b/tests/mcp/test_runtime.py index 49a6192..6316e27 100644 --- a/tests/mcp/test_runtime.py +++ b/tests/mcp/test_runtime.py @@ -112,6 +112,96 @@ def test_task_completion_builds_and_accepts_semantic_blob(self) -> None: task = self.tools.benchwork_get_task(task_id) self.assertEqual(task["data"]["result"]["status"], "COMPLETED") + def _claude_code_session(self) -> dict[str, str]: + return { + "schema_version": "host-session-provenance/1.0", + "host": "claude-code", + "session_id": "session-claude-code", + "provider": "anthropic", + "runtime": "claude-code-cli", + } + + def _inspection_output(self) -> dict[str, list[str]]: + return { + "inspected_files": ["hosts/claude-code/README.md"], + "findings": ["The adapter preserves the Capsule Circle."], + "tests_considered": ["tests/test_hosts.py"], + "residual_risks": ["Synthetic fixture rather than a live Host."], + } + + def test_declared_host_owns_its_capsule_and_completes_the_task(self) -> None: + program = self.tools.benchwork_create_program( + "claude-code-host", + "Claude Code Host loop", + {"statement": "A non-Codex Host completes a bounded Task."}, + )["data"]["program_id"] + session = self._claude_code_session() + + opened = self.tools.benchwork_open_task( + "bench.code.inspect", + program, + "Inspect the Claude Code Host adapter.", + host_session=session, + ) + self.assertTrue(opened["ok"]) + self.assertEqual(opened["data"]["task"]["host"], "claude-code") + task_id = opened["data"]["task"]["task_id"] + + completed = self.tools.benchwork_complete_task( + task_id, + "Inspected the Claude Code Host adapter.", + self._inspection_output(), + session, + ) + self.assertTrue(completed["ok"]) + self.assertIsNotNone(completed["receipt"]) + + def test_capsule_defaults_to_codex_and_rejects_a_foreign_host_result(self) -> None: + program = self.tools.benchwork_create_program( + "host-mismatch", + "Host mismatch", + {"statement": "Provenance must match the Task Capsule."}, + )["data"]["program_id"] + + opened = self.tools.benchwork_open_task( + "bench.code.inspect", + program, + "Inspect without declaring a Host.", + ) + self.assertEqual(opened["data"]["task"]["host"], "codex") + + rejected = self.tools.benchwork_complete_task( + opened["data"]["task"]["task_id"], + "Inspected under a mismatched Host.", + self._inspection_output(), + self._claude_code_session(), + ) + self.assertFalse(rejected["ok"]) + self.assertNotIn("receipt", rejected) + self.assertEqual(rejected["error"]["code"], "VALIDATION_REJECTED") + self.assertIn("does not match its Task Capsule", rejected["error"]["message"]) + + def test_unknown_task_host_is_refused(self) -> None: + program = self.tools.benchwork_create_program( + "unknown-host", + "Unknown Host", + {"statement": "Only registered Hosts may own a Capsule."}, + )["data"]["program_id"] + + refused = self.tools.benchwork_open_task( + "bench.code.inspect", + program, + "Inspect from an unregistered Host.", + host_session={ + "schema_version": "host-session-provenance/1.0", + "host": "unregistered-host", + "session_id": "session-unknown", + }, + ) + self.assertFalse(refused["ok"]) + self.assertEqual(refused["error"]["code"], "NOT_FOUND") + self.assertEqual(refused["error"]["message"], "unregistered-host") + def test_seal_preview_is_idempotent_and_stale_state_fails_closed(self) -> None: program_id = self._program_with_hypothesis() preview_result = self.tools.benchwork_preview_rq_seal( diff --git a/tests/plugin/test_plugin_package.py b/tests/plugin/test_plugin_package.py index 3701754..a9cf09d 100644 --- a/tests/plugin/test_plugin_package.py +++ b/tests/plugin/test_plugin_package.py @@ -79,7 +79,7 @@ def test_acceptance_assets_are_machine_readable_and_fail_closed(self) -> None: ) self.assertEqual( matrix["current_acceptance"]["cli_hosts"]["claude_code_cli"], - "PENDING_HOST_VALIDATION", + "PASS", ) self.assertEqual( matrix["current_acceptance"]["external_review"],