From 4f525d5bfc9fe2df0c20e39707da407f6e6e074d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:08:55 +0800 Subject: [PATCH] Complete AgentTape hook coverage and CI validation --- CHANGELOG.md | 9 + README.md | 7 +- docs/DEVELOPMENT_STATUS.md | 21 +- docs/HANDOFF.md | 15 +- docs/V0_4_1_VALIDATION.md | 76 +++++++ package-lock.json | 4 +- package.json | 8 +- plugins/agenttape/.codex-plugin/plugin.json | 2 +- plugins/agenttape/README.md | 12 +- plugins/agenttape/dist/mcp-server.mjs | 2 +- plugins/agenttape/hooks/hooks.json | 50 +++++ plugins/agenttape/mcp/remote-server.mjs | 2 +- plugins/agenttape/mcp/server.mjs | 2 +- plugins/agenttape/scripts/agenttape.mjs | 83 ++++--- plugins/agenttape/scripts/tape-store.mjs | 202 ++++++++++++++--- plugins/agenttape/scripts/verify-capture.mjs | 65 ++++++ .../agenttape/skills/capture-failure/SKILL.md | 18 +- remote/worker.mjs | 2 +- scripts/verify-remote-mcp.mjs | 4 +- tests/assertion-runner.test.mjs | 11 + tests/http-mcp.test.mjs | 2 +- tests/plugin-recorder.test.mjs | 211 +++++++++++++++++- tests/release-metadata.test.mjs | 3 + 23 files changed, 710 insertions(+), 101 deletions(-) create mode 100644 docs/V0_4_1_VALIDATION.md create mode 100644 plugins/agenttape/scripts/verify-capture.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c7247c..1bbd0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable AgentTape changes are documented here. +## 0.4.1 - 2026-08-25 + +- Cover all 11 current Codex hook events, including prompts, compaction, and subagent lifecycle events. +- Preserve event ordering when hook processes write concurrently. +- Recover real Bash exit status from the current Codex transcript when `PostToolUse` omits it. +- Redact prompt and lifecycle details, omit transcript paths, and shorten home-directory paths in portable tapes. +- Add a capture verifier and a directory-level regression command used by CI. +- Expand the capture skill into the complete inspect, fork, save, and offline-test workflow. + ## 0.4.0 - 2026-08-23 - Add `rate_limited` (HTTP 429) structural failure injection with configurable `retryAfterSeconds`. diff --git a/README.md b/README.md index 49554d2..c7be152 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ save assertions → run offline in CI ## Why AgentTape - **Failure artifact, not another dashboard.** A `.tape` is portable evidence you can inspect, review, share, and commit with a bug fix. -- **Codex-native capture.** Bundled hooks record supported lifecycle, permission, and tool events after the user reviews and trusts them—no application instrumentation required. +- **Codex-native capture.** Bundled hooks cover the 11 current Codex lifecycle, prompt, compaction, subagent, permission, and local tool events after the user reviews and trusts them—no application instrumentation required. - **Local and privacy-conscious.** Runtime captures stay inside the active project, and common secret-bearing fields are recursively redacted before export. - **Controlled failure injection.** Fork recorded evidence with permission-denied, timeout, rate-limited, malformed-JSON, or truncated-response results. - **Offline regression tests.** Structural replay and assertions use captured evidence with zero model calls and zero live tool calls. @@ -61,12 +61,13 @@ AgentTape exposes four MCP tools: Run a saved regression locally or in CI: ```bash -node plugins/agenttape/scripts/agenttape.mjs test \ - tests/agenttape/fixture_permission_denied-timeout.tape +node plugins/agenttape/scripts/agenttape.mjs test tests/agenttape ``` Passing assertions exit with code `0`. Validation or assertion failures exit nonzero, and captured comparison values are omitted from failure output. +The repository CI runs the same directory command through `npm run test:tapes`, so every reviewed `.tape` added under `tests/agenttape/` becomes an offline regression automatically. + ## What a `.tape` captures Tape v1 keeps the evidence needed for structural debugging: diff --git a/docs/DEVELOPMENT_STATUS.md b/docs/DEVELOPMENT_STATUS.md index 617cd5c..f411a28 100644 --- a/docs/DEVELOPMENT_STATUS.md +++ b/docs/DEVELOPMENT_STATUS.md @@ -1,8 +1,8 @@ # AgentTape 开发状态 -更新时间：2026-08-23 -版本状态：0.4.0 / 已公开发布并完成 Git marketplace 实机验收 -当前里程碑：扩展故障注入与顺序断言能力 +更新时间：2026-08-25 +版本状态：0.4.1 / 已完成真实捕获、CI 闭环和 Git marketplace 实机验收 +当前里程碑：让真实 Codex 失败稳定变成可提交的离线回归 ## 一句话状态 @@ -12,19 +12,20 @@ AgentTape 已完成本地“捕获 → 检查 → 分叉 → 注入 → 结构 | 领域 | 状态 | 已验证事实 | | --- | --- | --- | -| Hooks recorder / redaction | 完成 | 支持本地 Codex 生命周期、权限和工具事件，输出 redacted tape v1 | +| Hooks recorder / redaction | 完成 | 覆盖当前 11 类 Codex Hook；真实 Bash 退出码、提示词脱敏和 24 路并发写入均已验证 | | Tape schema / fixtures | 完成 | JSON Schema、validator、3 个合成 fixture | | Bundled stdio MCP | 完成 | `list_tapes`、`inspect_tape`、`fork_run`、`save_regression` | | Structural replay | 完成 | 5 种 recorded-result injection（含 rate_limited 429），0 model calls / 0 live tools | -| Assertions / CLI | 完成 | 7 类断言（含 tool_order 时序断言）；通过为 0，失败为非零且 diff 脱敏 | +| Assertions / CLI | 完成 | 7 类断言（含 tool_order）；`test tests/agenttape` 批量执行全部已保存回归 | | Repository marketplace | 完成 | marketplace 名为 `agenttape`，插件源为 `./plugins/agenttape` | | Plugin validation | 完成 | 当前 Codex 插件校验器、单测和重新安装后的实机 MCP 调用通过 | | Open-source metadata | 完成 | MIT、README、SECURITY、CONTRIBUTING、Code of Conduct、Changelog | -| Public GitHub release | 完成 | `jiangkoumo/agenttape`、`v0.4.0` Release 和公共 GitHub Actions 均已验证 | +| Public GitHub release | 完成 | `jiangkoumo/agenttape`、`v0.4.1` Release 和公共 GitHub Actions 均已验证 | ## 关键验证证据 -- UI 从合成的 permission-denied tape 加载工具失败，并可生成 timeout 分支。 +- 真实 Codex CLI 项目执行 `npm test` 并退出 7；capture 状态为 failed，5 个实际 Hook 事件连续，伪密钥未出现在 tape 中。 +- 同一真实 capture 经 `list_tapes → inspect_tape → fork_run → save_regression` 保存后，由目录级 runner 输出 `PASS 1/1 regression tapes`。 - 保存后的回归由 CLI 执行：`PASS tape_regression_9f680719a4a7 4/4 assertions`。 - 从公开 GitHub `v0.3.1` 全新安装后，只读 Codex 进程经工具搜索实机调用 `agenttape/list_tapes`，返回 `tape_fixture_permission_denied`。 - MCP 路径遍历、符号链接、超大文件、无效 schema、重复写入和覆盖请求均有拒绝测试。 @@ -55,10 +56,10 @@ record-hook.mjs 1. 公共仓库 `github.com/jiangkoumo/agenttape` 已创建并推送。 2. GitHub Actions 的 `plugin-release` 和 `optional-surfaces` 检查均通过。 -3. 带注释的 `v0.3.1` 标签和 GitHub Release 已发布。 -4. 已从公共标签运行 `codex plugin marketplace add jiangkoumo/agenttape --ref v0.3.1` 并安装 `agenttape@agenttape`。 +3. 带注释的 `v0.4.1` 标签和 GitHub Release 已发布。 +4. 已从公共标签运行 `codex plugin marketplace add jiangkoumo/agenttape --ref v0.4.1` 并安装 `agenttape@agenttape`。 5. GitHub 安装的插件已在新只读 Codex 进程中完成 `list_tapes` MCP 实机调用。 -`v0.3.0` 首次发布缺少被 Git 跟踪的预构建 MCP bundle；`v0.3.1` 已修复，并在 CI 中新增 bundle 跟踪检查以防回归。 +0.4.1 的完整验收证据见 [`V0_4_1_VALIDATION.md`](./V0_4_1_VALIDATION.md)。下一阶段不再继续堆网站功能，而是选择 2–3 个真实 Codex 工程积累 regression fixture，并根据实际失败类型扩展断言和 replay 边界。 Branch Canvas 托管和远程 HTTP MCP 保留为可选后续方向，不计入上述完成条件。 diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 2f77aba..b16eba8 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -1,13 +1,13 @@ # AgentTape 项目交接 -更新时间：2026-08-23 -当前阶段：0.4.0 开源 Codex 插件已发布 +更新时间：2026-08-25 +当前阶段：0.4.1 开源 Codex 插件已完成真实项目和 CI 闭环验收 ## 正式产品 - Codex 插件：`plugins/agenttape/`。 - Git marketplace：`.agents/plugins/marketplace.json`，公开名称为 `agenttape`。 -- Hooks recorder、redaction、tape v1 schema、validator 和合成 fixtures。 +- 覆盖当前 11 类 Codex Hook 的 recorder、redaction、tape v1 schema、validator 和 fixtures。 - Bundled stdio MCP：`list_tapes`、`inspect_tape`、`fork_run`、`save_regression`。 - Structural replay、5 种 injection、7 类 assertion runner 和 CLI exit semantics。 - MIT License、开源贡献规范、安全政策和 GitHub Actions。 @@ -19,6 +19,7 @@ Branch Canvas、Sites 构建适配和远程 HTTP MCP 是可选开发组件。它 ```bash npm ci npm run test:plugin-release +npm run test:tapes npm run build:plugin git diff --check ``` @@ -38,7 +39,7 @@ git diff --check ## 已完成发布 - 公共仓库：`https://github.com/jiangkoumo/agenttape` -- 当前正式版：`v0.4.0` +- 当前正式版：`v0.4.1` - 安装源：`codex plugin marketplace add jiangkoumo/agenttape` - 插件安装：`codex plugin add agenttape@agenttape` - 公共 CI、GitHub Release、隔离安装和真实 Codex MCP 调用均已通过。 @@ -47,6 +48,8 @@ git diff --check ## 后续方向 -1. 收集真实 Codex 项目的 capture 与回归测试反馈。 -2. 扩展受支持的 Hook 事件和结构化 replay 断言，同时保持 tape v1 向后兼容。 +1. 在 2–3 个真实 Codex 工程中积累经过人工检查的 regression fixture，优先覆盖权限、超时和外部 API 失败。 +2. 根据真实 fixture 扩展结构化 replay 断言，同时保持 tape v1 向后兼容。 3. Branch Canvas、Sites 和远程 HTTP MCP 仅在有明确需求时继续，不作为本地插件安装依赖。 + +0.4.1 的实际 Bash 退出 7、脱敏、MCP 分叉保存和离线 CI 证据见 [`V0_4_1_VALIDATION.md`](./V0_4_1_VALIDATION.md)。 diff --git a/docs/V0_4_1_VALIDATION.md b/docs/V0_4_1_VALIDATION.md new file mode 100644 index 0000000..60e7186 --- /dev/null +++ b/docs/V0_4_1_VALIDATION.md @@ -0,0 +1,76 @@ +# AgentTape 0.4.1 验收记录 + +验收日期：2026-08-25 + +## 目标 + +本次补丁完成三个可验证目标：覆盖 Codex 当前 11 类 Hook；在真实 Codex 项目中验证失败捕获与脱敏；让保存到 `tests/agenttape/` 的回归文件由本地和 CI 使用同一条命令执行。 + +## Hook 覆盖 + +插件现在注册： + +- 会话：`SessionStart`、`SessionEnd` +- 提示与压缩：`UserPromptSubmit`、`PreCompact`、`PostCompact` +- 工具与权限：`PreToolUse`、`PostToolUse`、`PermissionRequest` +- 子 Agent：`SubagentStart`、`SubagentStop` +- 回合结束：`Stop` + +事件详情统一经过递归脱敏。主 transcript 和子 Agent transcript 的本地路径不写入 tape；只保留是否可用。并发 Hook 写入通过项目内运行时锁串行化，测试覆盖 24 个同时写入者，最终序号连续且无丢失。 + +## 真实 Codex 项目验证 + +验证环境：Codex CLI 0.149.0、Node.js 项目、隔离的临时 Git 仓库。 + +真实任务要求 Codex 执行一个退出码为 7 的 `npm test`。验证中发现，当前 Codex 的 Bash `PostToolUse.tool_response` 只包含输出文本，不包含退出码；AgentTape 因此增加了 transcript 证据回退，只提取匹配 `tool_use_id` 的 `status` 和 `exit_code`，不复制 transcript 内容或路径。 + +最终捕获结果： + +```text +status: failed +events: SessionStart → UserPromptSubmit → PreToolUse → PostToolUse → Stop +failed tool calls: 1 +failure: Codex recorded tool exit code 7 +redaction marker: present +obvious unredacted secret pattern: false +``` + +验证命令： + +```bash +node plugins/agenttape/scripts/verify-capture.mjs \ + --root \ + --must-fail \ + --require-redaction \ + --require-event SessionStart \ + --require-event UserPromptSubmit \ + --require-event PreToolUse \ + --require-event PostToolUse \ + --require-event Stop +``` + +验证时使用的是专门构造的伪密钥。验证器确认 `.tape` 中存在脱敏标记，并且没有残留常见的未脱敏 API key 或 Bearer token 模式。该检查不能证明任意自由文本中绝对没有敏感信息，公开分享前仍需人工检查。 + +## 失败到 CI 的闭环 + +同一个真实捕获依次通过插件自己的 MCP 工具完成： + +1. `list_tapes` 找到失败 tape。 +2. `inspect_tape` 确认 Bash 退出码 7、覆盖边界和脱敏状态。 +3. `fork_run` 在序号 3 后将序号 4 的结果替换为 timeout；证据为 0 次模型调用、0 次真实工具调用。 +4. `save_regression` 写入 `tests/agenttape/real-bash-exit-timeout.tape`，包含 4 条断言。 +5. 目录级 runner 输出： + +```text +PASS tape_regression_60468d473ef9 4/4 assertions +PASS 1/1 regression tapes +``` + +仓库通过 `npm run test:tapes` 执行相同的目录级命令。新增的 `.tape` 文件不需要再单独修改 CI 配置。 + +## 边界 + +- Hosted tools 不在 Codex 本地 Hook 覆盖范围内。 +- Transcript 回退只读取当前会话、当前工具调用的状态和退出码，并限制读取文件尾部大小；它不会把 transcript 作为 artifact 保存。 +- Structural replay 在注入的工具结果处停止，不生成新的下游模型推理。 +- 以上验证证明当前范围内的结构化捕获和回归流程可用，不代表 bit-exact、完整或 hermetic replay。 diff --git a/package-lock.json b/package-lock.json index b60184e..e996618 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agenttape", - "version": "0.3.1", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agenttape", - "version": "0.3.1", + "version": "0.4.1", "license": "MIT", "dependencies": { "@fontsource/ibm-plex-mono": "^5.3.0", diff --git a/package.json b/package.json index bb6892e..7e6166a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agenttape", - "version": "0.4.0", + "version": "0.4.1", "description": "Local-first Codex plugin for redacted failure traces and offline regression tests.", "private": true, "license": "MIT", @@ -14,7 +14,7 @@ "homepage": "https://github.com/jiangkoumo/agenttape#readme", "type": "module", "scripts": { - "test": "npm run test:release && npm run test:plugin && npm run test:mcp && npm run test:http-mcp && npm run test:api && npm run test:demo && npm run test:replay && npm run test:regression && npm run test:sites", + "test": "npm run test:release && npm run test:plugin && npm run test:mcp && npm run test:http-mcp && npm run test:api && npm run test:demo && npm run test:replay && npm run test:regression && npm run test:tapes && npm run test:sites", "dev": "node scripts/run-vite.mjs", "build": "npm run build:mcp && npm run build:demo && node scripts/run-vite.mjs build && node scripts/prepare-sites-build.mjs && npm run build:http-mcp", "build:plugin": "npm run build:mcp", @@ -31,9 +31,11 @@ "test:demo": "npm run build:demo && node --test tests/demo-data.test.mjs", "test:replay": "node --test tests/structural-replay.test.mjs", "test:regression": "node --test tests/assertion-runner.test.mjs", + "test:tapes": "node plugins/agenttape/scripts/agenttape.mjs test tests/agenttape", + "verify:capture": "node plugins/agenttape/scripts/verify-capture.mjs", "test:sites": "node --test tests/sites-worker.test.mjs", "test:release": "node --test tests/release-metadata.test.mjs", - "test:plugin-release": "npm run test:release && npm run test:plugin && npm run test:mcp && npm run test:replay && npm run test:regression", + "test:plugin-release": "npm run test:release && npm run test:plugin && npm run test:mcp && npm run test:replay && npm run test:regression && npm run test:tapes", "deploy:http-mcp": "wrangler deploy", "verify:http-mcp": "node scripts/verify-remote-mcp.mjs" }, diff --git a/plugins/agenttape/.codex-plugin/plugin.json b/plugins/agenttape/.codex-plugin/plugin.json index 80f0738..4441c8e 100644 --- a/plugins/agenttape/.codex-plugin/plugin.json +++ b/plugins/agenttape/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agenttape", - "version": "0.4.0", + "version": "0.4.1", "description": "Capture Codex tool traces and turn failures as portable .tape evidence.", "author": { "name": "jiangkoumo", diff --git a/plugins/agenttape/README.md b/plugins/agenttape/README.md index 7095edb..bc86069 100644 --- a/plugins/agenttape/README.md +++ b/plugins/agenttape/README.md @@ -1,19 +1,20 @@ # AgentTape Plugin -AgentTape 0.4.0 is an open-source, local-first Codex plugin that records supported lifecycle and tool events, marks explicit tool failures, emits a redacted version 1 `.tape`, and turns deterministic structural branches into executable regressions. It does not require a hosted service. +AgentTape is an open-source, local-first Codex plugin that records supported lifecycle and tool events, marks explicit tool failures, emits a redacted version 1 `.tape`, and turns deterministic structural branches into executable regressions. It does not require a hosted service. Install it from the repository marketplace with `codex plugin marketplace add jiangkoumo/agenttape`, followed by `codex plugin add agenttape@agenttape`. ## Included -- Codex hooks for `SessionStart`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `Stop`, and `SessionEnd`. +- Codex hooks for `SessionStart`, `SessionEnd`, `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `PreCompact`, `PostCompact`, `SubagentStart`, `SubagentStop`, and `Stop`. - Recursive redaction for common secret-bearing fields. +- Serialized runtime writes so concurrent hooks preserve one complete event sequence. - Per-turn captures under `.agent-tape/tapes/`. - The `capture-failure` skill and offline `agenttape.mjs` CLI. - Version 1 JSON Schema and fixed permission-denied, timeout, and malformed-JSON fixtures. - Bundled stdio MCP tools: `list_tapes`, `inspect_tape`, `fork_run`, and `save_regression`. -- Recorded-result substitution for four supported failure conditions. -- Assertions for field equality, tool presence/absence, retry limits, final status, and replay confidence. +- Recorded-result substitution for five supported failure conditions. +- Assertions for field equality, tool presence/absence, tool order, retry limits, final status, and replay confidence. ## Storage @@ -32,7 +33,8 @@ Intentional regression artifacts are written under `tests/agenttape/` and can be ```bash node scripts/agenttape.mjs list --json node scripts/agenttape.mjs validate fixtures/permission-denied.tape -node scripts/agenttape.mjs test fixtures/permission-denied.tape +node scripts/agenttape.mjs test tests/agenttape +node scripts/verify-capture.mjs --must-fail --require-redaction --require-event PostToolUse ``` ## MCP behavior diff --git a/plugins/agenttape/dist/mcp-server.mjs b/plugins/agenttape/dist/mcp-server.mjs index 2680ea3..9a05f4a 100755 --- a/plugins/agenttape/dist/mcp-server.mjs +++ b/plugins/agenttape/dist/mcp-server.mjs @@ -22818,7 +22818,7 @@ async function activeWorkspaceRoot(configuredRoot, requestedRoot, extra, server) } function createAgentTapeServer({ workspaceRoot } = {}) { const server = new McpServer( - { name: "agenttape", version: "0.4.0" }, + { name: "agenttape", version: "0.4.1" }, { instructions: "Read AgentTape captures from the active workspace. The server resolves the workspace from MCP roots or the Codex environment; pass workspaceRoot only when the host cannot expose it. Inspect a tape before making claims about failures or replay confidence. Hosted tools outside local hook coverage may be absent." } diff --git a/plugins/agenttape/hooks/hooks.json b/plugins/agenttape/hooks/hooks.json index dd9f70f..8303789 100644 --- a/plugins/agenttape/hooks/hooks.json +++ b/plugins/agenttape/hooks/hooks.json @@ -10,6 +10,16 @@ ] } ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${PLUGIN_ROOT}/scripts/record-hook.mjs\"" + } + ] + } + ], "PreToolUse": [ { "hooks": [ @@ -40,6 +50,46 @@ ] } ], + "PreCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${PLUGIN_ROOT}/scripts/record-hook.mjs\"" + } + ] + } + ], + "PostCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${PLUGIN_ROOT}/scripts/record-hook.mjs\"" + } + ] + } + ], + "SubagentStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${PLUGIN_ROOT}/scripts/record-hook.mjs\"" + } + ] + } + ], + "SubagentStop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${PLUGIN_ROOT}/scripts/record-hook.mjs\"" + } + ] + } + ], "Stop": [ { "hooks": [ diff --git a/plugins/agenttape/mcp/remote-server.mjs b/plugins/agenttape/mcp/remote-server.mjs index b882b4b..c17ed6f 100644 --- a/plugins/agenttape/mcp/remote-server.mjs +++ b/plugins/agenttape/mcp/remote-server.mjs @@ -50,7 +50,7 @@ function registerTool(server, name, config, handler, successText) { export function createRemoteAgentTapeServer() { const server = new McpServer( - { name: "agenttape-remote", version: "0.4.0" }, + { name: "agenttape-remote", version: "0.4.1" }, { instructions: "Process only a redacted AgentTape v1 document explicitly supplied by the caller. This stateless server does not read local files, retain tapes, call models, or call live tools. Structural replay stops at the injected tool result.", }, diff --git a/plugins/agenttape/mcp/server.mjs b/plugins/agenttape/mcp/server.mjs index 9899b9e..0b01656 100644 --- a/plugins/agenttape/mcp/server.mjs +++ b/plugins/agenttape/mcp/server.mjs @@ -119,7 +119,7 @@ async function activeWorkspaceRoot(configuredRoot, requestedRoot, extra, server) export function createAgentTapeServer({ workspaceRoot } = {}) { const server = new McpServer( - { name: "agenttape", version: "0.4.0" }, + { name: "agenttape", version: "0.4.1" }, { instructions: "Read AgentTape captures from the active workspace. The server resolves the workspace from MCP roots or the Codex environment; pass workspaceRoot only when the host cannot expose it. Inspect a tape before making claims about failures or replay confidence. Hosted tools outside local hook coverage may be absent.", }, diff --git a/plugins/agenttape/scripts/agenttape.mjs b/plugins/agenttape/scripts/agenttape.mjs index f741bd2..fe06108 100644 --- a/plugins/agenttape/scripts/agenttape.mjs +++ b/plugins/agenttape/scripts/agenttape.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { copyFile, mkdir, readFile } from "node:fs/promises"; +import { copyFile, mkdir, readFile, readdir, stat } from "node:fs/promises"; import path from "node:path"; import { listTapes, redact } from "./tape-store.mjs"; @@ -35,10 +35,45 @@ function usage() { " agenttape.mjs latest [--json] [--root ]", " agenttape.mjs export latest --output [--root ]", " agenttape.mjs validate ", - " agenttape.mjs test ", + " agenttape.mjs test ", ].join("\n"); } +async function regressionFiles(target) { + const metadata = await stat(target); + if (!metadata.isDirectory()) return [target]; + return (await readdir(target, { withFileTypes: true })) + .filter((entry) => entry.isFile() && entry.name.endsWith(".tape")) + .map((entry) => path.join(target, entry.name)) + .sort(); +} + +async function testRegressionFile(file) { + const { TapeValidationError, parseTape } = await import("./tape-schema.mjs"); + try { + const tape = parseTape(await readFile(file, "utf8")); + const { runRegressionTape } = await import("../replay/assertions.mjs"); + const result = runRegressionTape(tape); + if (result.passed) { + console.log(`PASS ${tape.id} ${result.summary.passed}/${result.summary.total} assertions`); + return true; + } + + process.stderr.write(`FAIL ${tape.id}\n`); + if (result.error) process.stderr.write(`- ${result.error.code}: ${result.error.message}\n`); + for (const assertion of result.assertions.filter((item) => !item.passed)) { + const expected = JSON.stringify(redact(assertion.expected)); + const actual = JSON.stringify(redact(assertion.actual)); + process.stderr.write(`- [${assertion.kind}] expected ${expected}, received ${actual}\n`); + } + return false; + } catch (error) { + if (!(error instanceof TapeValidationError)) throw error; + process.stderr.write(`${path.basename(file)}: ${error.code}: ${error.message}\n`); + return false; + } +} + const args = process.argv.slice(2); const command = args[0]; const root = path.resolve(option(args, "--root") || process.cwd()); @@ -48,34 +83,32 @@ if (command === "validate" || command === "test") { process.stderr.write(`${usage()}\n`); process.exitCode = 2; } else { - const { TapeValidationError, parseTape } = await import("./tape-schema.mjs"); - try { - const file = path.resolve(args[1]); - const tape = parseTape(await readFile(file, "utf8")); - if (command === "validate") { + const file = path.resolve(args[1]); + if (command === "validate") { + const { TapeValidationError, parseTape } = await import("./tape-schema.mjs"); + try { + const tape = parseTape(await readFile(file, "utf8")); console.log(`valid ${tape.id} v${tape.version}`); - } else { - const { runRegressionTape } = await import("../replay/assertions.mjs"); - const result = runRegressionTape(tape); - if (result.passed) { - console.log(`PASS ${tape.id} ${result.summary.passed}/${result.summary.total} assertions`); - } else { - process.stderr.write(`FAIL ${tape.id}\n`); - if (result.error) process.stderr.write(`- ${result.error.code}: ${result.error.message}\n`); - for (const assertion of result.assertions.filter((item) => !item.passed)) { - const expected = JSON.stringify(redact(assertion.expected)); - const actual = JSON.stringify(redact(assertion.actual)); - process.stderr.write(`- [${assertion.kind}] expected ${expected}, received ${actual}\n`); - } + } catch (error) { + if (error instanceof TapeValidationError) { + process.stderr.write(`${error.code}: ${error.message}\n`); process.exitCode = 1; + } else { + throw error; } } - } catch (error) { - if (error instanceof TapeValidationError) { - process.stderr.write(`${error.code}: ${error.message}\n`); - process.exitCode = 1; + } else { + const files = await regressionFiles(file); + if (!files.length) { + process.stderr.write(`No .tape regression files found in ${file}.\n`); + process.exitCode = 2; } else { - throw error; + const results = []; + for (const tapeFile of files) results.push(await testRegressionFile(tapeFile)); + if (files.length > 1 || (await stat(file)).isDirectory()) { + console.log(`${results.every(Boolean) ? "PASS" : "FAIL"} ${results.filter(Boolean).length}/${files.length} regression tapes`); + } + if (!results.every(Boolean)) process.exitCode = 1; } } } diff --git a/plugins/agenttape/scripts/tape-store.mjs b/plugins/agenttape/scripts/tape-store.mjs index af8463c..e8753d4 100644 --- a/plugins/agenttape/scripts/tape-store.mjs +++ b/plugins/agenttape/scripts/tape-store.mjs @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; -import { mkdir, readFile, readdir, stat, writeFile, appendFile } from "node:fs/promises"; +import { appendFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; export const TAPE_VERSION = 1; @@ -9,6 +11,24 @@ const FAILURE_STATUS = /^(?:denied|error|failed|failure|forbidden|cancelled)$/i; const MAX_STRING_LENGTH = 32_768; const MAX_ARRAY_LENGTH = 100; const MAX_DEPTH = 8; +const LOCK_RETRY_COUNT = 250; +const LOCK_RETRY_DELAY_MS = 10; +const STALE_LOCK_MS = 30_000; +const MAX_TRANSCRIPT_TAIL_BYTES = 4 * 1024 * 1024; +const EVENT_DETAIL_OMISSIONS = new Set([ + "session_id", + "transcript_path", + "cwd", + "hook_event_name", + "model", + "turn_id", + "permission_mode", + "tool_name", + "tool_use_id", + "tool_input", + "tool_response", + "agent_transcript_path", +]); function redactString(value) { const truncated = value.length > MAX_STRING_LENGTH @@ -68,6 +88,16 @@ function captureRoot(cwd) { return path.join(path.resolve(cwd || process.cwd()), ".agent-tape"); } +function portableCwd(cwd) { + const resolved = path.resolve(cwd || process.cwd()); + const home = os.homedir(); + if (resolved === home) return "~"; + if (resolved.startsWith(`${home}${path.sep}`)) { + return `~/${path.relative(home, resolved).split(path.sep).join("/")}`; + } + return resolved; +} + function runtimePath(payload) { const session = safeSegment(payload.session_id, "unknown-session"); return path.join(captureRoot(payload.cwd), "runtime", `${session}.jsonl`); @@ -86,7 +116,122 @@ async function readEvents(file) { } } -function eventFromPayload(payload, sequence) { +async function resolveTranscriptPath(payload) { + if (payload.transcript_path) return payload.transcript_path; + if (!payload.session_id) return undefined; + + const sessionSuffix = `-${safeSegment(payload.session_id, "unknown-session")}.jsonl`; + const homes = new Set([process.env.CODEX_HOME, path.join(os.homedir(), ".codex")].filter(Boolean)); + for (const home of homes) { + const sessionsRoot = path.join(home, "sessions"); + for (const dayOffset of [0, -1, 1]) { + const date = new Date(Date.now() + dayOffset * 86_400_000); + const directory = path.join( + sessionsRoot, + String(date.getFullYear()), + String(date.getMonth() + 1).padStart(2, "0"), + String(date.getDate()).padStart(2, "0"), + ); + try { + const name = (await readdir(directory)).find((entry) => entry.endsWith(sessionSuffix)); + if (name) return path.join(directory, name); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + } + } + return undefined; +} + +async function transcriptToolEvidence(payload) { + if (payload.hook_event_name !== "PostToolUse" || !payload.tool_use_id) { + return undefined; + } + const transcriptPath = await resolveTranscriptPath(payload); + if (!transcriptPath) return undefined; + + for (let attempt = 0; attempt < 6; attempt += 1) { + let handle; + try { + const metadata = await stat(transcriptPath); + const length = Math.min(metadata.size, MAX_TRANSCRIPT_TAIL_BYTES); + const offset = metadata.size - length; + const buffer = Buffer.alloc(length); + handle = await open(transcriptPath, "r"); + await handle.read(buffer, 0, length, offset); + let content = buffer.toString("utf8"); + if (offset > 0) content = content.slice(content.indexOf("\n") + 1); + + for (const line of content.trimEnd().split("\n").reverse()) { + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + const item = row?.payload?.item; + if (item?.id !== payload.tool_use_id) continue; + const exitCode = Number.isFinite(Number(item.exit_code)) ? Number(item.exit_code) : undefined; + const status = typeof item.status === "string" ? item.status : undefined; + return { + failed: (exitCode !== undefined && exitCode !== 0) || FAILURE_STATUS.test(status || ""), + ...(status ? { status } : {}), + ...(exitCode !== undefined ? { exitCode } : {}), + }; + } + } catch { + // The transcript may not be visible to the hook process yet. Retry briefly. + } finally { + await handle?.close(); + } + if (attempt < 5) await delay(20); + } + return undefined; +} + +async function withRuntimeLock(file, callback) { + const lock = `${file}.lock`; + + for (let attempt = 0; attempt < LOCK_RETRY_COUNT; attempt += 1) { + let acquired = false; + try { + await mkdir(lock); + acquired = true; + } catch (error) { + if (error.code !== "EEXIST") throw error; + try { + const metadata = await stat(lock); + if (Date.now() - metadata.mtimeMs > STALE_LOCK_MS) { + await rm(lock, { recursive: true, force: true }); + continue; + } + } catch (metadataError) { + if (metadataError.code !== "ENOENT") throw metadataError; + } + await delay(LOCK_RETRY_DELAY_MS); + } + if (!acquired) continue; + try { + return await callback(); + } finally { + await rm(lock, { recursive: true, force: true }); + } + } + + throw new Error("Timed out waiting for the AgentTape runtime lock."); +} + +function eventDetails(payload) { + const details = {}; + for (const [key, value] of Object.entries(payload)) { + if (!EVENT_DETAIL_OMISSIONS.has(key)) details[key] = value; + } + if (payload.transcript_path) details.transcript_available = true; + if (payload.agent_transcript_path) details.agent_transcript_available = true; + return Object.keys(details).length ? redact(details) : undefined; +} + +function eventFromPayload(payload, sequence, execution) { const type = String(payload.hook_event_name || "Unknown"); const event = { sequence, @@ -103,29 +248,25 @@ function eventFromPayload(payload, sequence) { ...(payload.tool_input !== undefined ? { input: redact(payload.tool_input) } : {}), ...(payload.tool_response !== undefined ? { output: redact(payload.tool_response) } : {}), }; - if (type === "PostToolUse" && hasFailureSignal(payload.tool_response)) { + if (type === "PostToolUse" && (hasFailureSignal(payload.tool_response) || execution?.failed)) { event.tool.failed = true; - event.tool.failure = { kind: "tool_error", reason: "Explicit failure signal in tool response" }; + event.tool.failure = { + kind: "tool_error", + reason: execution?.exitCode !== undefined + ? `Codex recorded tool exit code ${execution.exitCode}` + : "Explicit failure signal in tool response", + }; } } - if (type === "PermissionRequest") { - const details = { ...payload }; - for (const key of [ - "session_id", - "transcript_path", - "cwd", - "hook_event_name", - "model", - "turn_id", - "permission_mode", - "tool_name", - "tool_use_id", - "tool_input", - "tool_response", - ]) delete details[key]; - if (Object.keys(details).length) event.details = redact(details); + const details = eventDetails(payload) || {}; + if (execution) { + details.execution = { + ...(execution.status ? { status: execution.status } : {}), + ...(execution.exitCode !== undefined ? { exitCode: execution.exitCode } : {}), + }; } + if (Object.keys(details).length) event.details = details; return event; } @@ -162,7 +303,7 @@ function buildTape(payload, events) { adapter: "codex-hooks", sessionId: String(payload.session_id || "unknown-session"), ...(payload.turn_id ? { turnId: String(payload.turn_id) } : {}), - cwd: path.resolve(payload.cwd || process.cwd()), + cwd: portableCwd(payload.cwd), ...(payload.model ? { model: String(payload.model) } : {}), coverage: "supported-local-hooks", }, @@ -196,15 +337,18 @@ async function finalize(payload, events) { export async function recordHook(payload) { const file = runtimePath(payload); await mkdir(path.dirname(file), { recursive: true }); - const events = await readEvents(file); - const event = eventFromPayload(payload, events.length + 1); - await appendFile(file, `${JSON.stringify(event)}\n`, "utf8"); - events.push(event); + const execution = await transcriptToolEvidence(payload); + return withRuntimeLock(file, async () => { + const events = await readEvents(file); + const event = eventFromPayload(payload, events.length + 1, execution); + await appendFile(file, `${JSON.stringify(event)}\n`, "utf8"); + events.push(event); - if (payload.hook_event_name === "Stop" || payload.hook_event_name === "SessionEnd") { - return finalize(payload, events); - } - return null; + if (payload.hook_event_name === "Stop" || payload.hook_event_name === "SessionEnd") { + return finalize(payload, events); + } + return null; + }); } export async function listTapes(cwd = process.cwd()) { diff --git a/plugins/agenttape/scripts/verify-capture.mjs b/plugins/agenttape/scripts/verify-capture.mjs new file mode 100644 index 0000000..e7143f3 --- /dev/null +++ b/plugins/agenttape/scripts/verify-capture.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +import path from "node:path"; + +import { validateTape } from "./tape-schema.mjs"; +import { listTapes } from "./tape-store.mjs"; + +function values(args, name) { + return args.flatMap((value, index) => value === name && args[index + 1] ? [args[index + 1]] : []); +} + +function option(args, name) { + return values(args, name).at(-1); +} + +function usage() { + return [ + "Usage:", + " verify-capture.mjs [--root ] [--must-fail] [--require-redaction]", + " [--require-event ]...", + ].join("\n"); +} + +const args = process.argv.slice(2); +if (args.includes("--help")) { + console.log(usage()); + process.exit(0); +} + +const root = path.resolve(option(args, "--root") || process.cwd()); +const requiredEvents = values(args, "--require-event"); +const entries = await listTapes(root); +if (!entries.length) { + process.stderr.write("No AgentTape captures found.\n"); + process.exit(2); +} + +const tape = validateTape(entries[0].tape); +const serialized = JSON.stringify(tape); +const eventTypes = [...new Set(tape.events.map((event) => event.type))]; +const missingEvents = requiredEvents.filter((name) => !eventTypes.includes(name)); +const redactionPresent = serialized.includes("[REDACTED]"); +const obviousSecretPresent = /\bsk-[A-Za-z0-9_-]{8,}\b|Bearer\s+(?!\[REDACTED\])[A-Za-z0-9._~+/=-]+/i.test(serialized); +const failures = []; + +if (args.includes("--must-fail") && tape.status !== "failed") failures.push("latest tape is not failed"); +if (args.includes("--require-redaction") && !redactionPresent) failures.push("latest tape has no redaction marker"); +if (obviousSecretPresent) failures.push("latest tape contains an obvious unredacted secret pattern"); +if (missingEvents.length) failures.push(`missing events: ${missingEvents.join(", ")}`); + +const result = { + tapeId: tape.id, + status: tape.status, + coverage: tape.source.coverage, + eventCount: tape.summary.eventCount, + failedToolCallCount: tape.summary.failedToolCallCount, + eventTypes, + redactionPresent, + obviousSecretPresent, + passed: failures.length === 0, + failures, +}; + +console.log(JSON.stringify(result, null, 2)); +if (failures.length) process.exitCode = 1; diff --git a/plugins/agenttape/skills/capture-failure/SKILL.md b/plugins/agenttape/skills/capture-failure/SKILL.md index 463ae55..789bb2d 100644 --- a/plugins/agenttape/skills/capture-failure/SKILL.md +++ b/plugins/agenttape/skills/capture-failure/SKILL.md @@ -1,6 +1,6 @@ --- name: capture-failure -description: Inspect, list, or export Codex tool traces captured by AgentTape. Use when the user asks to record a failure, inspect the latest failed run, save a run as a .tape file, or prepare failure evidence for a regression test. +description: Turn a captured Codex tool failure into reviewed .tape evidence and an offline regression test. Use when the user asks to inspect a failed run, fork captured evidence, save a regression, or prepare AgentTape evidence for CI. --- # Capture a Codex failure with AgentTape @@ -9,16 +9,18 @@ AgentTape hooks record supported local Codex tool events into the current projec ## Workflow -1. Resolve `../../scripts/agenttape.mjs` relative to this `SKILL.md`, then use its absolute path for every command below. -2. Run `node latest --json` from the user's project directory. -3. If no capture exists, explain that AgentTape begins recording in a new Codex task after the plugin is installed and its hooks are trusted. Do not invent a run. -4. To list captures, run `node list --json` and summarize status, tool count, failure count, and capture time. -5. To save evidence, choose the user-provided output path. If none was provided, use `tests/.tape`. Run `node export latest --output `. -6. Read the exported tape and report its status, first failed tool event, and recorded coverage limitations. +1. Use `list_tapes` to find captures in the active project. If none exists, explain that recording starts in a new Codex task after the plugin and its hooks are trusted. Do not invent a run. +2. Use `inspect_tape` on the latest failed capture. Report the first explicit failure, redaction status, replay confidence, and recorded coverage limitations before proposing a fork. +3. Choose one supported condition to change: `permission_denied`, `timeout`, `rate_limited`, `malformed_json`, or `truncated_response`. Use `fork_run` at a recorded boundary and confirm that it made zero model calls and zero live tool calls. +4. Ask for confirmation only if the intended assertion or injection is materially ambiguous. Otherwise use `save_regression` with at least one meaningful assertion and a descriptive `.tape` filename. It writes under `tests/agenttape/` and does not overwrite by default. +5. Resolve `../../scripts/agenttape.mjs` relative to this `SKILL.md`. From the user's project directory, run `node test tests/agenttape` so every saved regression executes exactly as CI will execute it. +6. Report the saved relative path, assertions executed, pass/fail result, and the structural replay limitations. + +If the user only wants the raw captured artifact, resolve the same CLI and run `node export latest --output ` instead of creating a structural regression. ## Boundaries - Treat the tape as redacted structural evidence, not bit-exact replay. - Never claim that hosted tools were captured. Phase 1 covers tool paths delivered through Codex local hooks. -- Never include `.agent-tape/runtime/` in a commit. Export intentional regression artifacts to a tracked path such as `tests/`. +- Never include `.agent-tape/runtime/` in a commit. Commit only reviewed artifacts under `tests/agenttape/`. - Do not modify the captured tape by hand. Re-run or export it again if the source evidence changes. diff --git a/remote/worker.mjs b/remote/worker.mjs index 8dde535..6e65b86 100644 --- a/remote/worker.mjs +++ b/remote/worker.mjs @@ -61,7 +61,7 @@ export async function handleAgentTapeRemoteRequest(request, env = {}) { if (origin === false) return json({ error: "ORIGIN_NOT_ALLOWED" }, 403); if (url.pathname === "/health" && request.method === "GET") { - return json({ service: "agenttape-remote", version: "0.4.0", status: "ok", storage: "none" }); + return json({ service: "agenttape-remote", version: "0.4.1", status: "ok", storage: "none" }); } if (url.pathname === "/privacy" && request.method === "GET") { return html("AgentTape Remote Privacy", "

The service processes caller-supplied, explicitly redacted AgentTape documents in memory. It does not persist tape contents, read local files, call models, or call live tools. Infrastructure may retain ordinary request metadata for security and reliability.

"); diff --git a/scripts/verify-remote-mcp.mjs b/scripts/verify-remote-mcp.mjs index 11b2044..9d2d2bb 100644 --- a/scripts/verify-remote-mcp.mjs +++ b/scripts/verify-remote-mcp.mjs @@ -15,7 +15,7 @@ export async function verifyRemoteMcp(origin, fetchImpl = fetch) { const healthResponse = await fetchImpl(new URL("/health", root)); assert.equal(healthResponse.status, 200, "GET /health must return 200"); const health = await healthResponse.json(); - assert.deepEqual(health, { service: "agenttape-remote", version: "0.4.0", status: "ok", storage: "none" }); + assert.deepEqual(health, { service: "agenttape-remote", version: "0.4.1", status: "ok", storage: "none" }); for (const path of ["/privacy", "/terms", "/support"]) { const response = await fetchImpl(new URL(path, root)); @@ -24,7 +24,7 @@ export async function verifyRemoteMcp(origin, fetchImpl = fetch) { const fixture = JSON.parse(await readFile(fixtureUrl, "utf8")); const regression = JSON.parse(await readFile(regressionUrl, "utf8")); - const client = new Client({ name: "agenttape-production-verifier", version: "0.4.0" }); + const client = new Client({ name: "agenttape-production-verifier", version: "0.4.1" }); const transport = new StreamableHTTPClientTransport(new URL("/mcp", root), { fetch: fetchImpl }); await client.connect(transport); diff --git a/tests/assertion-runner.test.mjs b/tests/assertion-runner.test.mjs index a79d6b1..7ac7df3 100644 --- a/tests/assertion-runner.test.mjs +++ b/tests/assertion-runner.test.mjs @@ -32,6 +32,17 @@ test("executes every fixed regression tape offline", async () => { } }); +test("executes every saved regression in a directory as one CI command", () => { + const result = spawnSync(process.execPath, [cli, "test", path.join(projectRoot, "tests", "agenttape")], { + cwd: projectRoot, + encoding: "utf8", + env: { PATH: process.env.PATH || "" }, + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /PASS tape_regression_/); + assert.match(result.stdout, /PASS 1\/1 regression tapes/); +}); + test("returns a non-zero exit code and redacted diff for assertion failures", async () => { const directory = await mkdtemp(path.join(os.tmpdir(), "agenttape-assertion-")); try { diff --git a/tests/http-mcp.test.mjs b/tests/http-mcp.test.mjs index 9660bc0..87a364b 100644 --- a/tests/http-mcp.test.mjs +++ b/tests/http-mcp.test.mjs @@ -93,7 +93,7 @@ test("rejects unredacted, oversized, and cross-origin inputs without echoing val test("publishes health and policy endpoints without storage or challenge leakage", async () => { const health = await handleAgentTapeRemoteRequest(new Request("https://agenttape.test/health"), testEnv); assert.equal(health.status, 200); - assert.deepEqual(await health.json(), { service: "agenttape-remote", version: "0.4.0", status: "ok", storage: "none" }); + assert.deepEqual(await health.json(), { service: "agenttape-remote", version: "0.4.1", status: "ok", storage: "none" }); for (const path of ["/privacy", "/terms", "/support"]) { const response = await handleAgentTapeRemoteRequest(new Request(`https://agenttape.test${path}`), testEnv); diff --git a/tests/plugin-recorder.test.mjs b/tests/plugin-recorder.test.mjs index 0a6030d..88c513c 100644 --- a/tests/plugin-recorder.test.mjs +++ b/tests/plugin-recorder.test.mjs @@ -7,23 +7,147 @@ import test from "node:test"; import { fileURLToPath } from "node:url"; import { validateTape } from "../plugins/agenttape/scripts/tape-schema.mjs"; +import { recordHook } from "../plugins/agenttape/scripts/tape-store.mjs"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const pluginRoot = path.join(projectRoot, "plugins", "agenttape"); const recorder = path.join(pluginRoot, "scripts", "record-hook.mjs"); const cli = path.join(pluginRoot, "scripts", "agenttape.mjs"); +const verifier = path.join(pluginRoot, "scripts", "verify-capture.mjs"); -function runHook(cwd, payload) { +function runHook(cwd, payload, extraEnv = {}) { const result = spawnSync(process.execPath, [recorder], { cwd, input: JSON.stringify(payload), encoding: "utf8", - env: { ...process.env, PLUGIN_ROOT: pluginRoot }, + env: { ...process.env, ...extraEnv, PLUGIN_ROOT: pluginRoot }, }); assert.equal(result.status, 0, result.stderr); assert.deepEqual(JSON.parse(result.stdout), {}); } +test("registers every currently supported Codex lifecycle hook", async () => { + const manifest = JSON.parse(await readFile(path.join(pluginRoot, "hooks", "hooks.json"), "utf8")); + assert.deepEqual(Object.keys(manifest.hooks).sort(), [ + "PermissionRequest", + "PostCompact", + "PostToolUse", + "PreCompact", + "PreToolUse", + "SessionEnd", + "SessionStart", + "Stop", + "SubagentStart", + "SubagentStop", + "UserPromptSubmit", + ]); +}); + +test("records prompt, compaction, and subagent lifecycle details without leaking secrets or transcript paths", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "agenttape-hooks-")); + + try { + const common = { + session_id: "session-hook-coverage", + turn_id: "turn-hook-coverage", + transcript_path: "/private/transcripts/main.jsonl", + cwd: workspace, + model: "test-model", + permission_mode: "default", + }; + + runHook(workspace, { ...common, hook_event_name: "SessionStart", source: "startup" }); + runHook(workspace, { + ...common, + hook_event_name: "UserPromptSubmit", + prompt: "Reproduce with Authorization: Bearer hook-secret-token", + }); + runHook(workspace, { ...common, hook_event_name: "PreCompact", trigger: "auto" }); + runHook(workspace, { ...common, hook_event_name: "PostCompact", trigger: "auto" }); + runHook(workspace, { + ...common, + hook_event_name: "SubagentStart", + agent_id: "agent-1", + agent_type: "worker", + }); + runHook(workspace, { + ...common, + hook_event_name: "SubagentStop", + agent_id: "agent-1", + agent_type: "worker", + agent_transcript_path: "/private/transcripts/agent-1.jsonl", + stop_hook_active: false, + last_assistant_message: "token=agent-secret-value", + }); + runHook(workspace, { + ...common, + hook_event_name: "Stop", + stop_hook_active: false, + last_assistant_message: "Finished with password=turn-secret-value", + }); + + const tapeDirectory = path.join(workspace, ".agent-tape", "tapes"); + const [name] = await readdir(tapeDirectory); + const tapeText = await readFile(path.join(tapeDirectory, name), "utf8"); + const tape = JSON.parse(tapeText); + + assert.equal(validateTape(tape), tape); + assert.deepEqual(tape.events.map((event) => event.type), [ + "SessionStart", + "UserPromptSubmit", + "PreCompact", + "PostCompact", + "SubagentStart", + "SubagentStop", + "Stop", + ]); + assert.equal(tape.events[1].details.prompt, "Reproduce with Authorization: Bearer [REDACTED]"); + assert.equal(tape.events[2].details.trigger, "auto"); + assert.equal(tape.events[4].details.agent_id, "agent-1"); + assert.equal(tape.events[5].details.agent_transcript_available, true); + assert.doesNotMatch(tapeText, /hook-secret-token|agent-secret-value|turn-secret-value|private\/transcripts/); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +}); + +test("serializes concurrent hook writers into one complete event sequence", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "agenttape-concurrent-")); + + try { + const common = { + session_id: "session-concurrent", + turn_id: "turn-concurrent", + cwd: workspace, + model: "test-model", + permission_mode: "default", + }; + await Promise.all(Array.from({ length: 24 }, (_, index) => recordHook({ + ...common, + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_use_id: `tool-${index}`, + tool_input: { command: `printf ${index}` }, + }))); + await recordHook({ + ...common, + hook_event_name: "Stop", + stop_hook_active: false, + last_assistant_message: "Concurrent hooks complete.", + }); + + const tapeDirectory = path.join(workspace, ".agent-tape", "tapes"); + const [name] = await readdir(tapeDirectory); + const tape = JSON.parse(await readFile(path.join(tapeDirectory, name), "utf8")); + assert.equal(validateTape(tape), tape); + assert.equal(tape.events.length, 25); + assert.deepEqual(tape.events.map((event) => event.sequence), Array.from({ length: 25 }, (_, index) => index + 1)); + assert.equal(new Set(tape.events.slice(0, -1).map((event) => event.tool.useId)).size, 24); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +}); + test("captures a failed Codex turn and exports a redacted version 1 tape", async () => { const workspace = await mkdtemp(path.join(os.tmpdir(), "agenttape-plugin-")); @@ -100,6 +224,89 @@ test("captures a failed Codex turn and exports a redacted version 1 tape", async }); assert.equal(exportResult.status, 0, exportResult.stderr); assert.deepEqual(JSON.parse(await readFile(exported, "utf8")), tape); + + const verifyResult = spawnSync(process.execPath, [ + verifier, + "--root", workspace, + "--must-fail", + "--require-redaction", + "--require-event", "SessionStart", + "--require-event", "PostToolUse", + "--require-event", "Stop", + ], { + cwd: workspace, + encoding: "utf8", + env: { PATH: process.env.PATH || "" }, + }); + assert.equal(verifyResult.status, 0, verifyResult.stderr); + const verification = JSON.parse(verifyResult.stdout); + assert.equal(verification.passed, true); + assert.equal(verification.obviousSecretPresent, false); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +}); + +test("uses the real Codex transcript exit code when PostToolUse output omits process status", async () => { + const workspace = await mkdtemp(path.join(os.tmpdir(), "agenttape-transcript-")); + + try { + const codexHome = path.join(workspace, "codex-home"); + const date = new Date(); + const transcriptDirectory = path.join( + codexHome, + "sessions", + String(date.getFullYear()), + String(date.getMonth() + 1).padStart(2, "0"), + String(date.getDate()).padStart(2, "0"), + ); + await mkdir(transcriptDirectory, { recursive: true }); + const transcript = path.join(transcriptDirectory, "rollout-test-session-transcript.jsonl"); + await writeFile(transcript, `${JSON.stringify({ + type: "event_msg", + payload: { + type: "item_completed", + item: { + type: "CommandExecution", + id: "exec-real-shape", + status: "failed", + exit_code: 7, + aggregated_output: "Intentional failure output", + }, + }, + })}\n`); + const common = { + session_id: "session-transcript", + turn_id: "turn-transcript", + cwd: workspace, + model: "test-model", + permission_mode: "default", + }; + + runHook(workspace, { + ...common, + hook_event_name: "PostToolUse", + tool_name: "Bash", + tool_use_id: "exec-real-shape", + tool_input: { command: "npm test" }, + tool_response: "Intentional failure output", + }, { CODEX_HOME: codexHome }); + runHook(workspace, { + ...common, + hook_event_name: "Stop", + stop_hook_active: false, + last_assistant_message: "The process exited with 7.", + }, { CODEX_HOME: codexHome }); + + const tapeDirectory = path.join(workspace, ".agent-tape", "tapes"); + const [name] = await readdir(tapeDirectory); + const tape = JSON.parse(await readFile(path.join(tapeDirectory, name), "utf8")); + const toolEvent = tape.events.find((event) => event.type === "PostToolUse"); + assert.equal(validateTape(tape), tape); + assert.equal(tape.status, "failed"); + assert.equal(toolEvent.tool.failed, true); + assert.equal(toolEvent.tool.failure.reason, "Codex recorded tool exit code 7"); + assert.deepEqual(toolEvent.details.execution, { status: "failed", exitCode: 7 }); } finally { await rm(workspace, { recursive: true, force: true }); } diff --git a/tests/release-metadata.test.mjs b/tests/release-metadata.test.mjs index e1c460a..0d1e1aa 100644 --- a/tests/release-metadata.test.mjs +++ b/tests/release-metadata.test.mjs @@ -12,6 +12,7 @@ test("publishes a Git-installable AgentTape marketplace", async () => { const marketplace = await readJson(".agents/plugins/marketplace.json"); const manifest = await readJson("plugins/agenttape/.codex-plugin/plugin.json"); const packageMetadata = await readJson("package.json"); + const packageLock = await readJson("package-lock.json"); const bundledServer = await readFile(new URL("plugins/agenttape/dist/mcp-server.mjs", root), "utf8"); const readme = await readFile(new URL("README.md", root), "utf8"); @@ -25,6 +26,8 @@ test("publishes a Git-installable AgentTape marketplace", async () => { assert.equal(manifest.repository, "https://github.com/jiangkoumo/agenttape"); assert.equal(manifest.homepage, "https://github.com/jiangkoumo/agenttape#readme"); assert.equal(manifest.version, packageMetadata.version); + assert.equal(packageLock.version, packageMetadata.version); + assert.equal(packageLock.packages[""].version, packageMetadata.version); assert.match(bundledServer, /createAgentTapeServer/); assert.match(readme, /codex plugin marketplace add jiangkoumo\/agenttape/); assert.match(readme, /codex plugin add agenttape@agenttape/);