diff --git a/README.md b/README.md index b6bc643..55fb895 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,9 @@ Rule of thumb: if you build and operate your own agent in production, use a trac ## Features +- **Automatic session health** — Opens with factual failure, repetition, follow-up and last-recorded call-state summaries. Missing/running/unknown results have evidence links; no human labels or model calls required. Manual notes and transfers are opt-in and never hide automatic facts. [Scope and offline checks](docs/diagnostics.md#automatic-session-health). +- **Modification/check chronology** — Distinguish checks before an edit, checks overlapping it and later outcomes. A passed earlier check or a successful output pipeline is not post-change validation; ambiguous command fragments remain unknown. [Recognition and coverage limits](docs/diagnostics.md#modification-and-verification-chronology). + - **Per-turn ledger** — In the session summary, from two user turns on: one row per user turn with wall-clock time, tokens (input + output + cache) and cost, bars scaled to the session maximum, tool-call counts inline (error counts in the row tooltip), click to jump. Answers "why did this take 40 minutes / cost $3" without reading the transcript. - **Multi-platform** — Unified view across OpenClaw, Codex, Claude Code, Hermes, OMP, DeepSeek Harness and Gemini CLI sessions (dsh's multi-frame zstd session logs are decompressed transparently; Gemini CLI's `/rewind` checkpoints are folded so rewound history never renders twice) - **Session browser** — Browse agents, filter/search sessions, view message history diff --git a/README.zh-CN.md b/README.zh-CN.md index cb11d14..b08baae 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -54,6 +54,9 @@ LangSmith、Langfuse 这类观测平台面向的是*你自己写的* agent：接 ## 功能特性 +- **自动会话体检** — 默认自动整理失败、重复操作、后续候选和调用最后记录状态；执行中、未知及未记录结果可追溯证据。不依赖人工标注或模型调用，笔记与迁移改为可选，不影响自动事实展示。[口径与离线验证](docs/diagnostics.md#自动体检无需人工标注)。 +- **修改—检查时序** — 区分修改前成功的检查、与修改重叠的检查及后续最新结果；不把先前通过或管道整体成功当作修改后的验证。复杂命令片段执行状态保持未知。[识别边界](docs/diagnostics.md#修改与验证的先后顺序)。 + - **有证据的失败事件（React UI）** — 将同一调用所在用户轮次、同工具、完整同参数的待复查失败分组，重复最多的操作优先展示；可跳转首末及每条原始证据。同参成功切断分组，缺少参数不合并。执行成功采用明确零退出码或 OMP 原生完成证据；事件不等于根因或任务失败。本地规则，无 LLM。[合成演示与判定边界](docs/diagnostics.md#中文使用指南)。 - **后续相关操作** — 展示仅 `i` 参数不同的调用，以及同轮次、明确同文件的后续修改；标明关系依据、五类结果状态并可跳转证据。候选不自动关闭事件、不代表原问题已修复。[匹配边界](docs/diagnostics.md#后续相关操作)。 - **本机复核队列** — 用必填依据标记“需跟进”“预期失败”“其他验证已通过”，仅存当前浏览器；新证据使旧标记失效，人工判断不改写自动结果。无需账号或复核后端。[使用方式与存储边界](docs/diagnostics.md#本机复核闭环)。 diff --git a/claims.json b/claims.json index bf59505..7e81c20 100644 --- a/claims.json +++ b/claims.json @@ -102,16 +102,16 @@ }, { "id": "test-count", - "claim": "255 tests pass on Node's built-in test runner, the count docs/ROADMAP.md records for `npm test`.", - "value": "255", - "metric": "passing node:test cases (# tests 255 / # pass 255 / # fail 0)", - "method": "npm test → node --test test/*.test.js, run in the claims job after npm ci, and the TAP summary is asserted. The roadmap sentence ('255 tests on Node's built-in runner (`npm test`, 2026-09-23)') is verified by the run, not read back from the prose.", + "claim": "290 tests pass on Node's built-in test runner, the count docs/ROADMAP.md records for `npm test`.", + "value": "290", + "metric": "passing node:test cases (# tests 290 / # pass 290 / # fail 0)", + "method": "npm test → node --test test/*.test.js, run in the claims job after npm ci, and the TAP summary is asserted. The roadmap sentence ('290 tests on Node's built-in runner (`npm test`, 2026-09-23)') is verified by the run, not read back from the prose.", "repro": "npm test 2>&1 | grep -E '^# (tests|pass|fail)'", "evidence": "docs/ROADMAP.md", "as_of": "2026-09-13", "check": { "cmd": "npm test 2>&1 | grep -E '^# (tests|pass|fail)'", - "expect": { "contains": ["# tests 255", "# pass 255", "# fail 0"] }, + "expect": { "contains": ["# tests 290", "# pass 290", "# fail 0"] }, "timeout": 120 } }, @@ -238,7 +238,7 @@ "check": { "cmd": "node scripts/claims-receipts.mjs tests-node-only", "expect": { - "equals": "18 files in test/ · 14 distinct requires: 11 node builtins, 3 relative, 0 third-party" + "equals": "20 files in test/ · 14 distinct requires: 11 node builtins, 3 relative, 0 third-party" }, "timeout": 60 } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index dc445a8..ecc2f68 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -8,18 +8,20 @@ - **Session browser** with tool-call inspection, trace/waterfall view, spawn tracking and message timeline - **Prompt tooling** — extraction (noise filtered), template clustering with outcome attribution, Claude-powered rewrites, and a prompt library that installs entries as native slash commands - **Global search** across all platforms, insights dashboard, incremental session backup -- **React + Vite frontend** served by an Express backend; 255 tests on Node's built-in runner (`npm test`, 2026-09-23), CI on Node 22 +- **React + Vite frontend** served by an Express backend; 290 tests on Node's built-in runner (`npm test`, 2026-09-23), CI on Node 22 - **Evidence-backed failure events and local review** with full-result invalidation, evidence navigation and narrow-screen session layout ## Current priorities -The product direction is **review the coding-agent sessions you already have, with evidence**. First prove that a useful review can be completed locally; then make it easy to try and share. Stars, downloads and screenshots alone do not establish usefulness. This order replaces the earlier feature-first ordering below. +The product direction is **local execution evidence for the coding agents you already use**: what actually ran, which results support a conclusion, and what the log cannot establish. Automatic facts come first; human labels are optional. Native agents increasingly include dashboards, usage metrics and workflow progress, so generic statistics are not the differentiator. Stars, downloads and screenshots alone do not establish usefulness. + +External grounding: official guidance emphasizes [executable verification](https://code.claude.com/docs/en/best-practices#give-claude-a-way-to-verify-its-work), [background subagent workflows](https://code.claude.com/docs/en/workflows) and [coordination/resume limits](https://code.claude.com/docs/en/agent-teams). These are capabilities and engineering constraints, not market-size evidence. User issue reports and vendor-fixed bugs are research inputs, not proof of permanent product gaps. | Priority | Outcome | Acceptance, not a promise | | --- | --- | --- | | P0 | Make the new workflow immediately testable | A demo-only entry opens a clearly synthetic case: 7 pending records in 2 events, all evidence accessible, local review does not rewrite automatic results. Preserve the existing default demo and samples. | | P1 | Current-session review portability implemented | Preview-only import and explicit plaintext download; exact identity/evidence matching, no overwrites, bounded schema and partial-failure reporting. Validate with synthetic migration and publish after CI; whole-history backup and path remapping remain out of scope. | -| P2 | Explain what happened after a failure, then measure usefulness | Show only-i and same-turn/same-file follow-up candidates with explicit status and evidence, never automatic recovery. Validate unchanged diagnostic outcomes on frozen logs. Then measure owner review tasks using a fixed rubric; separate unknowns and stale labels, and do not infer precision or time saved from candidate counts. | +| P2 | Execution/verification evidence, not another statistics dashboard | Automatic health and modification/check chronology: distinguish before/overlap/after and latest outcomes, show unsupported command coverage, keep optional notes independent. Deterministic transformations and frozen-log regressions verify association rules, not file coverage, human time saved or universal accuracy. Next investigate explicit per-step check outcomes and task/child linkage before relaxing ambiguous shell assumptions. | | P3 | Make releases reproducible for contributors | Keep clean-install tests, generated fixtures, documentation claims and release/package verification aligned. Add browser regression automation when it can run deterministically without personal logs. | No launch dates or star-count targets are promised. Progress is gated on these observable outcomes. Physical-device/keyboard coverage and complex Trace/analytics layouts remain separate work, not implied by the session-screen checks. diff --git a/docs/diagnostics-verification.md b/docs/diagnostics-verification.md index 742133c..95f4a83 100644 --- a/docs/diagnostics-verification.md +++ b/docs/diagnostics-verification.md @@ -95,3 +95,26 @@ A fixed, previously inspected set of 15 real sessions (five each from OMP, Codex Storage keys stayed stable. Only the three events with added candidates acquired new review fingerprints; no-candidate events retained their previous fingerprint contract. No real review labels were persisted. Raw logs, parameters and detailed sample identifiers remain private; the regression is not an independent benchmark, global accuracy estimate or time-saving measurement. The hosted synthetic sample now includes one later successful edit of the same absolute file path with different arguments. It remains **2 events / 7 pending records / 1 recovered record**. Browser checks verified the relation caveat, source result, desktop/mobile rendering and zero static-demo API requests. + +## Automatic health and chronology acceptance + +The next increment makes automatic facts the default. Saved manual labels do not hide automatic events, and the review store is not read or hashed until **人工笔记与迁移（可选）** is opened. Existing notes and transfers remain available. Refresh or returning to automatic mode shows all events again. + +The test suite now has **290 passing Node tests**. `test/session-health.test.js` contains 11 cases including **200 fixed transformations** over 20 synthetic bases; `test/verification-chronology.test.js` contains 24 cases including **120 temporal transformations** over 20 bases. All known outcomes, evidence partitions and call-state accounting invariants pass without a model judge or human labels. These counts are regression coverage, not measured real-world accuracy or user time saved. + +Health pairs only unique, ordered call IDs; it summarizes the last recorded state and separates unknown/no-result/running from failure and cancellation. Reused or absent IDs do not supply success evidence. One candidate referenced by multiple events is counted once by result position. Tests verify source messages are not modified. + +Chronology distinguishes successful modification-tool results, prior successful checks, overlapping checks and later attempts. A later failure stays visible even if an earlier attempt passed; simultaneously initiated checks do not get an invented uniquely latest order. Known-different explicit directories are excluded and missing directory context is explicit. Tool success does not prove bytes changed; conventional script names do not prove contents or test coverage. + +The initial direct-command recognizer found **zero checks** in the frozen real-log set. A bounded lexer was added for explicit command segments instead of hiding that coverage gap. It recognizes **59 calls containing checks** (52 OMP, 7 Claude Code), all with **unknown check outcome** because the commands are compound; a pipeline's successful exit is not assigned to its test fragment. Substitutions, heredocs, control flow and quoted mentions are not interpreted as executed checks. The Codex subset had no recognized checks or explicit edit/write records under this scope, so no conclusion about its verification quality follows. + +The same 15 frozen sessions / 2,024 tool results retained exactly the v1.20.0 diagnostic output: 142 failures, 141 pending records, 76 events and 1 recovery. Explicit modification tools returned successfully 240 times; 112 had later recognized check-call/fragment records and 128 did not. **These are temporal relations only**—none establish that the modified file was tested, and unclassified commands may contain validation. The dataset was previously inspected, includes old sessions and is not a blind benchmark. Raw data and detailed sample IDs remain private. + +Browser checks used synthetic fixtures only: + +- Automatic mode works with review-storage reads denied; zero review-key reads occur before opt-in. Optional manual filtering returns to the complete event set on exit, and notes persist without becoming mandatory. +- A no-result call gains a recorded result without a reload; gaps fall from 3 to 2 while unrelated failures remain unchanged. +- A check passes before a modification; a second check overlaps it and a third uses another directory. None is reported as post-change validation. Appending a failed later check then a successful one updates the latest result and retains both attempts. +- Call/result jumps resolve to actual source records without moving outer page navigation. At 390px the chronology panel has no horizontal overflow; static demo uses zero backend API requests. + +Use `node scripts/demo-verification.cjs` to replay the temporal example and `node scripts/demo-follow-up.cjs` for missing-result updates. The public hosted demo exposes the earlier-test/later-edit example under automatic health. Physical-device keyboards and every complex view are not certified. Existing lint findings (91 warnings / 159 infos) and bundle-size warnings remain. diff --git a/docs/diagnostics.md b/docs/diagnostics.md index 2c6bdb9..522c093 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -1,12 +1,64 @@ # Evidence-backed failure events -**Find repeated failed operations without losing the evidence.** AgentXRay groups recorded failures for review; it does not decide whether the agent finished your task correctly. +**Get an automatic, evidence-backed session check without labelling events.** AgentXRay summarizes recorded failures, related operations and result gaps; it does not decide whether the agent finished your task correctly. Human notes remain optional. + +## Automatic session health + +The default **自动会话体检** view requires no human labels, review storage access or model calls. It displays all automatic events even if you previously marked some as expected or verified elsewhere. + +- **Failure follow-up evidence:** failure-record count, remaining event count and recorded same-argument successful retries. Candidate success does not count as automatic recovery. +- **Repeated operations:** unresolved events with multiple failures, plus the number of failure records in those events. This does not include already recovered episodes or estimate time wasted. +- **Later related results:** candidate result positions are deduplicated across events, with separate success/failure/running/cancelled/unknown counts. Event links and original result jumps remain available. +- **Call-result gaps:** each recorded call is associated with its last recorded result by ID and order. Running, unknown and missing-result calls are listed with evidence. A missing result does not mean the process is still running; an old running record is not a live status check. Cancelled/stopped calls are counted separately, not treated as gaps. +- Missing/reused call IDs are ambiguous, not guessed. Orphan or pre-call results and unassignable results are counted separately. No detected failures is not a task-success verdict. + +Open **人工笔记与迁移（可选）** only if you want to save notes, filter by manual labels or transfer reviews. This enables the existing local-review workflow without deleting prior notes. **返回自动体检** restores every automatic event; refresh returns to the automatic view. Review-storage denial does not block the default view. + +### Offline evaluation without human labels + +`node --test test/session-health.test.js` checks 200 deterministic transformations across 20 fixed synthetic bases: append/remove a matching success, change command/cwd/i, insert a pre-started parallel call, and supply running/cancelled/unknown results. Each uses explicit expected recovery counts, complete event-evidence membership and a call-state partition invariant. Separate cases cover latest-result updates, missing/orphan/reused IDs, embedded calls and shared candidate deduplication. + +These are **known-case tests**, not human judgments or a real-world precision benchmark. Computation time is not time saved by a person. The existing frozen-log regression is also checked against the preceding implementation without changing its diagnostic output. + +Try `node scripts/demo-follow-up.cjs`: the fixture includes running, unknown and no-result calls. Enter `c` to append the missing result; the gap count falls without changing unrelated failures. No labels are needed. Enter `q` to clean up the isolated server. + +![Synthetic automatic session health](../screenshots/automatic-health.png) + +## Modification and verification chronology + +The **修改—检查时序** section asks a narrower, evidence-based question than “did the task pass?”: + +> Did a check return successfully before a modification tool returned successfully, and what check commands are recorded afterward? + +It shows each explicit modification-tool success, the nearest earlier successful check, all overlapping checks, the later check calls and the latest later outcome. A passed check from before an edit cannot certify that edit. A check started before the edit completed is not post-change validation, even if its result arrived later. A failed later check is not hidden because an earlier check passed. + +**Recognition scope:** + +- Modifications are `edit`, `Edit`, `write`, `Write` and `MultiEdit` with an unambiguous literal `path`/`file_path` and a uniquely paired successful tool result. This is a tool-result fact, not proof that bytes changed. Failed/incomplete mutation calls and unsupported patch/parameter formats are counted separately. +- Direct test runners include `pytest`, `python[3] -m pytest` and `node --test`. `npm test` and named `test/build/lint/typecheck` scripts under npm/pnpm/yarn are recognized as **script-name conventions**, not verified script contents. Help/watch/list-only/dry-run modes are excluded. +- A sole literal absolute `cd ... && check` can provide directory context. If that whole command fails, the failure may be in `cd`, so the check outcome is unknown. +- A bounded lexer recognizes check command segments in more complex `&&`, `;`, newline or pipeline commands without executing anything. A fragment inside `npm test | tail ...` is **not known to have run or passed** from the wrapper exit code, even when the wrapper succeeds. Quoted text inside echo/python arguments is not treated as an executable command. Substitutions, heredocs and shell control syntax are not interpreted. +- Only same-transcript record order is compared. Known-different/conflicting explicit directories are excluded. Missing directory context is labelled unknown. No filesystem lookup, implicit directory tracking, source coverage or cross-session causality is inferred. + +Every displayed modification/check links to its original call and, when present, result. Unclassified execution calls remain counted, so “no later recognized check” never means “no testing occurred”. The original failure/candidate rules and human-note fingerprints remain unchanged. + +Try the controlled sequence in a source checkout: + +```sh +node scripts/demo-verification.cjs +``` + +The OMP synthetic session begins with a passed check, a completed edit, an overlapping check, and a different-directory check. No relevant later check is recorded. Enter `f` to append a failed post-edit check, then `s` for a successful check: the UI updates automatically and retains both attempts. Enter `q` or Ctrl-C to clean up. No command from the synthetic transcript is executed. + +The static hosted sample likewise has an earlier passed test followed by a later same-file edit. Its background `verify-config` script is intentionally outside the current naming allowlist; do not confuse “unclassified” with “failed”. + +![Synthetic modification/check chronology](../screenshots/verification-chronology.png) This guide covers the React UI. The synthetic terminal walkthrough uses a source checkout; package installation and published versions are listed in GitHub Releases. ## Try it without sharing your logs -For a no-install tour, open the [hosted demo](https://alloevil.github.io/AgentXRay/) and click **体验失败复核 / Try diagnostics**. Its synthetic sample has 7 pending failure records in 2 events (6 repeated edits and 1 search failure), plus an earlier automatically recovered test failure. Open the first/last/all evidence and record a browser-local review note. This is a static sample: no live results are appended and no actual tool commands run. The terminal walkthrough below is a separate, larger fixture for live updates. +For a no-install tour, open the [hosted demo](https://alloevil.github.io/AgentXRay/) and click **体验自动体检 / Try diagnostics**. Its synthetic sample has 7 pending failure records in 2 events (6 repeated edits and 1 search failure), plus an earlier automatically recovered test failure. Open the first/last/all evidence or optionally record a browser-local review note. This is a static sample: no live results are appended and no actual tool commands run. The terminal walkthrough below is a separate, larger fixture for live updates. From the repository root, after installing the root and frontend dependencies: @@ -28,7 +80,7 @@ The synthetic “66 → 1” example is a fixture, not an accuracy or productivi ## Use your existing logs -Run `npm start` from the built checkout. Select a session and open its **消息** view. The “失败后验证” panel uses the currently loaded transcript; selecting a child session analyzes the child, not the parent's combined history. No SDK, new log instrumentation or model call is required for these rules. +Run `npm start` from the built checkout. Select a session and open its **消息** view. The “自动会话体检” panel uses the currently loaded transcript; selecting a child session analyzes the child, not the parent's combined history. No SDK, new log instrumentation or model call is required for these rules. Start with repeated operations, open the latest failure, and inspect the surrounding calls. A card is a reason to review the transcript—not an instruction to rerun a possibly destructive command. Equivalent commands, alternate verification and external fixes still require your judgment. @@ -65,7 +117,7 @@ The hosted **Try diagnostics** sample also includes a successful same-file edit ## Local review workflow -The automatic result and your judgment are separate. The header always reports automatic events and pending failure records. The review filters show which of those events you have inspected: +Human review is optional: first open **人工笔记与迁移（可选）**. The automatic result and your judgment are separate. The header always reports automatic events and pending failure records. In optional mode, the review filters show which events you have inspected: | Queue | Meaning | | --- | --- | @@ -95,7 +147,7 @@ Storage limits: ## Transfer reviews between browsers -Open **迁移当前会话复核 / Export & import** in the diagnostics panel. This is a local, explicit file transfer—not a server backup or authenticated proof that a task passed. +Open **人工笔记与迁移（可选）**, then **迁移当前会话复核 / Export & import** in the diagnostics panel. This is a local, explicit file transfer—not a server backup or authenticated proof that a task passed. 1. On the source browser, open the session you reviewed and select **预览导出**. The full JSON is shown before downloading. Only reviews whose identity and full-evidence fingerprint still match the **currently loaded automatic events** are included; the selected review filter does not limit the export. 2. Inspect the notes, acknowledge the plaintext warning, then select **下载复核 JSON**. The filename is always `agentxray-reviews.json`, without a session ID or path. Store it safely: notes you typed may include secrets even though the app does not copy logs into the file. @@ -154,6 +206,8 @@ node --test test/diagnostic-events.test.js test/diagnostics.test.js test/omp-out node --test test/diagnostic-reviews.test.js node --test test/review-transfer.test.js node --test test/follow-up-evidence.test.js +node --test test/session-health.test.js +node --test test/verification-chronology.test.js npm test npm run build:ui npm run lint @@ -163,9 +217,29 @@ The local frozen regression set contained 30 sessions and 9,076 tool results. Gr ## 中文使用指南 +### 自动体检无需人工标注 + +默认进入“自动会话体检”，直接展示失败后完成证据、重复失败操作、去重后的后续相关结果，以及调用最后记录状态。全部自动事件都可见，不因旧人工标签而隐藏；不开启笔记时不会读取复核存储。 + +“调用结果缺口”列出记录为执行中、未知、未记录结果的调用，并可跳转证据。这里说的是日志最后状态，不是实时进程；未知不等于失败，未记录结果也不代表任务未完成。重复/候选数字都有明确单位，不提供笼统健康分。 + +人工笔记保留为可选：点击“人工笔记与迁移（可选）”才打开旧队列及迁移功能，旧笔记不删除；“返回自动体检”恢复所有事件。刷新默认回到自动模式。 + +无需人工参与的验收：`node --test test/session-health.test.js` 在 20 个固定合成基例上运行 200 个规则变换，检查已知恢复结果、证据分组和调用状态总数，不靠模型自评或人打标签。它证明规则对这些用例的行为，不等同真实准确率或节省时间。 + +### 修改与验证的先后顺序 + +自动体检中的“修改—检查时序”展示：修改工具何时成功返回、此前最近的成功检查、重叠检查、之后出现的检查，以及最新结果。**测试在修改之前通过，不代表修改之后通过；并行检查晚返回，也不算修改后的验证。** + +直接测试运行器和约定命名的 test/build/lint/typecheck 脚本分开标识。有限解析器能看见 `cd ... && npm test` 等命令片段，但复合命令的整体退出码不能证明测试片段执行或通过，尤其是接 `tail` 等管道时。未知范围和未归类命令都会显示，不把“未识别”解释成“没测试”。 + +这里比较当前会话的记录顺序，已知不同工作目录不关联、目录缺失显示未知；不判断文件测试覆盖、不执行日志命令、不宣判任务成功。所有原始调用/结果可以点击查看。修改仅识别明确 edit/write/MultiEdit 工具，shell/patch 修改等暂不解析。 + +运行 `node scripts/demo-verification.cjs`，在合成会话中输入 `f` / `s` 分别追加修改后的失败/成功检查，观察结果自动更新；`q` 退出。另有 120 个固定时序变换测试，覆盖先后、重叠、缺结果等反例，无需人工参与。 + **目标：先找到值得复查的重复操作，再追溯证据，而不是把几百条失败强行解释成几个根因。** -免安装体验：[在线 Demo](https://alloevil.github.io/AgentXRay/)，点击“体验失败复核 / Try diagnostics”。这个独立合成案例将 7 条待复查记录聚为 2 个事件（6 次同参 edit 失败、1 次搜索失败），并展示一次早先测试失败的自动恢复。可查看全部证据、填写本浏览器的人工复核；它是静态示例，不追加真实结果。下面的本机终端演示则使用更大的合成日志来验证实时变化。 +免安装体验：[在线 Demo](https://alloevil.github.io/AgentXRay/)，点击“体验自动体检 / Try diagnostics”。这个独立合成案例将 7 条待复查记录聚为 2 个事件（6 次同参 edit 失败、1 次搜索失败），并展示一次早先测试失败的自动恢复。可查看全部证据、填写本浏览器的人工复核；它是静态示例，不追加真实结果。下面的本机终端演示则使用更大的合成日志来验证实时变化。 从源码仓库运行 `npm run build:ui`，再运行 `node scripts/demo-diagnostics.cjs`。打开终端打印的地址，选择 OMP 的 `[Synthetic]` 会话： @@ -174,7 +248,7 @@ The local frozen regression set contained 30 sessions and 9,076 tool results. Gr - 保持自动刷新，在终端输入 `r` 并回车，追加合成成功结果；页面自动变为 **6 个事件、6 条待复查记录、67 条已有同参成功记录**。 - 输入 `q` 或 Ctrl-C 退出并清理。演示使用临时 HOME 和合成日志，不读取你的真实会话、不执行日志里的命令。 -日常使用：构建后运行 `npm start`，选择你的会话，在“消息”视图查看“失败后验证”。子会话独立分析；优先读最新失败和前后操作，不要不加判断地重跑原命令。 +日常使用：构建后运行 `npm start`，选择你的会话，在“消息”视图查看“自动会话体检”。子会话独立分析；优先读最新失败和前后操作，不要不加判断地重跑原命令。 **边界：**只合并同调用轮次、同工具、完整同参数的记录，不忽略 `i` 或工作目录；成功结果切断分组，缺少参数不合并。事件可含并行调用和不同错误，不等于同一根因或串行重试。首末间隔不是耗时或浪费时间。执行中、取消和未知不算成功验证；其他方式修复、等价命令、后台任务链仍需人工判断。 @@ -184,7 +258,7 @@ The local frozen regression set contained 30 sessions and 9,076 tool results. Gr ## 本机复核闭环 -先从“待复核”开始，打开事件证据，点击“记录人工复核”，选择： +这是可选功能，不是使用前提。先打开“人工笔记与迁移（可选）”，再查看事件证据、点击“记录人工复核”，选择： - **需跟进**：写清楚下一步检查什么、哪里仍不确定。 - **预期失败**：写清楚为什么这个失败是合理的，例如负向探测无匹配。 @@ -200,7 +274,7 @@ The local frozen regression set contained 30 sessions and 9,076 tool results. Gr ## 迁移复核记录 -在诊断面板展开“迁移当前会话复核 / Export & import”： +先打开“人工笔记与迁移（可选）”，再展开“迁移当前会话复核 / Export & import”： 1. **源浏览器**打开已有复核的会话，点击“预览导出”，检查将下载的完整 JSON，勾选明文提醒后下载。仅导出当前自动事件中仍有效的复核，筛选队列不影响范围。 2. **目标浏览器**打开相同平台、配置目录、会话及子会话，选择文件。预览逐条展示依据、时间与处理决定，尚不写入数据。 diff --git a/docs/releases/v1.21.0.md b/docs/releases/v1.21.0.md new file mode 100644 index 0000000..c35120e --- /dev/null +++ b/docs/releases/v1.21.0.md @@ -0,0 +1,27 @@ +# v1.21.0 — Automatic execution evidence, without mandatory review + +AgentXRay now opens with an automatic factual check rather than a human review queue. It also distinguishes checks run before a modification, overlapping it and afterward—without claiming that tests cover the file or that a task passed. + +## What changed + +- **Automatic health first:** failure/recovery records, repeated unresolved operations, unique follow-up results and last-recorded tool-call states. Missing, ambiguous, running and unknown records have original evidence links. +- **Human notes are optional:** open “人工笔记与迁移（可选）” for saved notes, filters or transfers. Default mode does not read review storage or hide events based on labels. No notes are deleted. +- **Modification/check chronology:** see the earlier successful check, successful edit/write result, overlapping checks, subsequent attempts and latest outcome. Never replace a later failure with an earlier passing result; same-message parallel starts have no uniquely latest order. +- **Bounded command recognition:** direct common test runners and test/build/lint/typecheck script names are distinguished. A literal absolute `cd ... && check` carries explicit directory evidence. Check fragments in other compound/pipeline commands stay outcome-unknown, even if the wrapper succeeds. +- **Hands-on synthetic demo:** `node scripts/demo-verification.cjs` accepts `f` and `s` to append failed/successful post-edit checks. No commands from the fixture are executed. + +## Evidence and limits + +290 Node tests passed, including 320 deterministic transformations without human labels or model judges. Normal/demo builds, lint and claims checks pass. Frozen-log output is identical to the prior diagnostic rules; browser tests cover optional storage, source jumps, live result updates, chronology and narrow screens. + +Real logs exposed why simple command-name matching is insufficient: 59 check-containing calls in the frozen set were compound and correctly remain outcome-unknown. Unrecognized shell commands, patch modifications, hidden working-directory changes, cross-session checks and actual test coverage are not inferred. “No later recognized check” does not mean “no testing occurred”. Unknown is not failure, and last-recorded running status is not a live process check. + +No new runtime dependencies, backend endpoints, telemetry, model calls or automatic logged-command execution. Existing review identities and diagnostic failure/candidate/recovery rules are preserved. + +[Usage and limitations](https://github.com/alloevil/AgentXRay/blob/master/docs/diagnostics.md#modification-and-verification-chronology) · [Verification receipt](https://github.com/alloevil/AgentXRay/blob/master/docs/diagnostics-verification.md) + +```sh +npx @alloevil/agent-xray@1.21.0 +``` + +Requires Node 22.13+ (22.15+ for compressed DeepSeek Harness logs). diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 709a063..d0d2fb6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -83,7 +83,7 @@ export default function App() { store.setView('sessions'); returnToContent(); }}> - 体验失败复核 / Try diagnostics + 体验自动体检 / Try diagnostics + {manual ? <>

人工复核仅存当前浏览器，未加密、不上传。换端口、浏览器或清除站点数据后可能不可见；证据变化会重新待复核。

@@ -150,9 +163,10 @@ export function SessionDiagnostics({ messages, onScrollToMessage, reviewScope }: ) : null} {notice?.events === report.events ?

{notice.text}

: null} + : null} {report.events.length ? ( <> -
+ {manual ?
{Object.entries(REVIEW_FILTERS).map(([value, label]) => { const count = value === 'all' ? report.events.length : report.events.filter((event) => reviewState(reviews.entries[event.id]) === value).length; return ( @@ -163,10 +177,10 @@ export function SessionDiagnostics({ messages, onScrollToMessage, reviewScope }: ); })} -
+
: null}
    {matching.slice(0, visibleCount).map((event) => ( - { reviews.update(event.id, status, note); @@ -176,7 +190,7 @@ export function SessionDiagnostics({ messages, onScrollToMessage, reviewScope }:
{!matching.length ?

此复核队列为空；自动诊断事件仍可在“全部”中查看，不代表任务通过。

: null}
- 当前队列已展示 {Math.min(visibleCount, matching.length)} / {matching.length} 个事件 · 共 {report.failureCount} 条失败记录 + {manual ? '当前复核队列' : '自动事件'}已展示 {Math.min(visibleCount, matching.length)} / {matching.length} 个事件 · 共 {report.failureCount} 条失败记录 {hiddenCount ? ( :

日志缺少定位标识

} + + ); + })} + + )} + {visible < health.gaps.length ? : null} + +
+ ); +} diff --git a/frontend/src/views/sessions/VerificationChronology.tsx b/frontend/src/views/sessions/VerificationChronology.tsx new file mode 100644 index 0000000..a351e6f --- /dev/null +++ b/frontend/src/views/sessions/VerificationChronology.tsx @@ -0,0 +1,84 @@ +import { useState } from 'react'; +import type { analyzeVerificationChronology, ChronologyCheck, ExecutionEvidence } from './diagnostics'; +import { messageAnchorId } from './lib'; + +const STATES = { success: '成功返回', failure: '失败', running: '记录为执行中', cancelled: '取消 / 停止', unknown: '状态未知', 'no-result': '未记录结果' }; + +function OperationEvidence({ operation, onJump }: { operation: ExecutionEvidence; onJump: (id: string) => void }) { + const anchor = operation.resultMessage ? messageAnchorId(operation.resultMessage) : null; + const callAnchor = operation.toolCallId || messageAnchorId(operation.callMessage); + return ( +
+

{operation.command || operation.target || operation.toolName}

+

+ {STATES[operation.state]} · 调用 #{operation.callIndex + 1} + {operation.resultIndex !== null ? ` → 结果 #${operation.resultIndex + 1}` : ''} + {operation.basis ? ` · ${operation.basis === 'runner-command' ? '直接测试运行器' : '约定脚本名，未读取脚本内容'}` : ''} +

+ {operation.commandMode ?

{operation.commandMode === 'compound-fragment' + ? '复合命令中的检查片段；执行与通过状态未知' : operation.commandMode === 'directory-prefix' ? '显式 cd 目录前缀 + 检查命令' : '直接检查命令'}

: null} + 
{operation.evidence}
+
+ {callAnchor ? : null} + {anchor ? : null} +
+
+ ); +} + +function CheckEvidence({ check, onJump }: { check: ChronologyCheck; onJump: (id: string) => void }) { + return ( +
+

{check.scope === 'same-recorded-directory' ? '显式记录工作目录相同；仍不证明文件覆盖' : '目录关联未知；仅同会话记录顺序，不证明文件覆盖'}

+ +
+ ); +} + +export function VerificationChronology({ chronology, onJump }: { + chronology: ReturnType; + onJump: (id: string) => void; +}) { + const [visible, setVisible] = useState(5); + const [checksVisible, setChecksVisible] = useState(5); + return ( +
+ 修改—检查时序 · {chronology.successfulModifications} 次修改工具成功返回 · {chronology.withoutLaterCheck} 次其后未记录检查 +

+ {chronology.changedAfterLastPassedCheck} 次在已成功检查后又有修改、其后未记录可识别检查。这里只核对调用/结果顺序，不证明文件内容实际变化、测试覆盖或任务通过。 + 并行/重叠检查不算修改后的检查，已知不同目录不关联，缺少目录只显示时序。跨会话验证及隐含文件修改不在范围内。 +

+

+ 识别到 {chronology.modificationCalls} 次明确目标的修改调用、{chronology.checks.length} 次检查调用； + {chronology.unconfirmedModifications} 次修改未有明确成功结果，{chronology.unclassifiedModificationCalls} 次修改工具参数/格式不支持， + {chronology.unclassifiedShellCalls} 次执行调用未识别检查片段。复合命令的片段是否执行或通过保持未知；test/build/lint/typecheck 脚本名不是脚本语义验证。 +

+ {!chronology.modifications.length ?

未记录到本规则可确认的修改工具成功结果，不代表没有修改。

: ( +
    + {chronology.modifications.slice(0, visible).map((row) => ( +
  1. +

    {row.latestOrderAmbiguous ? '修改之后有多次检查；最近调用位于同一消息，顺序不唯一' : row.latestLater ? `修改之后有检查调用 · 最近${STATES[row.latestLater.operation.state]}` + : row.priorSuccess ? '检查成功后又有修改；其后未记录可识别检查' : '修改之后未记录可识别检查'}

    + + {row.priorSuccess ?
    修改之前的最近成功检查
    : null} + {row.latestLater ?

    修改之后的最近检查（不是验收结论）

    : null} + {row.laterChecks.length > 1 ?
    全部 {row.laterChecks.length} 次后续检查（含失败和未知） +
      {row.laterChecks.map((check, index) =>
      2. )}
    +
    : null} + {row.overlappingChecks.length ?
    {row.overlappingChecks.length} 次与修改重叠 / 已在运行的检查，不算后续验证 +
      {row.overlappingChecks.map((check, index) =>
      2. )}
    +
    : null} + {row.excludedScopeChecks ?

    {row.excludedScopeChecks} 次检查因目录不同或冲突未建立关系。

    : null} +
    2. + ))} +
+ )} + {visible < chronology.modifications.length ? : null} +
+ 全部可识别检查 · {chronology.checks.length} 次 +
    {chronology.checks.slice(0, checksVisible).map((operation, index) =>
    2. )}
+ {checksVisible < chronology.checks.length ? : null} +
+
+ ); +} diff --git a/frontend/src/views/sessions/diagnostics.ts b/frontend/src/views/sessions/diagnostics.ts index ed7b626..6d97f0f 100644 --- a/frontend/src/views/sessions/diagnostics.ts +++ b/frontend/src/views/sessions/diagnostics.ts @@ -297,3 +297,299 @@ export function diagnoseSession(messages: SessionMessage[]) { events, }; } + +export interface CompletionGap { + message: SessionMessage; + index: number; + toolCallId: string | null; + toolName: string; + state: 'running' | 'unknown' | 'no-result'; + reason: 'ambiguous-id' | 'last-result' | 'no-result'; + evidence: string; +} + +interface CallEvidence { + id: string | null; + message: SessionMessage; + call: RecordedCall; + result?: { message: SessionMessage; index: number }; +} + +function collectCallEvidence(messages: SessionMessage[]) { + const calls: CallEvidence[] = []; + const byId = new Map(); + let userTurn = 0; + messages.forEach((message, index) => { + if (message.role === 'user') userTurn++; + function record(id: string | null | undefined, name: string | null | undefined, value: unknown) { + const entry = { id: id || null, message, call: { name: name || '未知工具', args: parseArguments(value), + key: null, onlyIKey: null, fileKey: null, index, userTurn } }; + calls.push(entry); + if (id) { const list = byId.get(id) || []; list.push(entry); byId.set(id, list); } + } + if (message.role === 'toolCall') record(message.toolCallId || message.id, message.toolName, message.details); + if (message.role === 'assistant') for (const part of message.content || []) { + if (part.type === 'toolCall') record(part.id, part.name, part.arguments ?? part.input); + } + }); + let orphanResults = 0; + let unassignedResults = 0; + let toolResultCount = 0; + messages.forEach((message, index) => { + if (message.role !== 'toolResult') return; + toolResultCount++; + const matches = message.toolCallId ? byId.get(message.toolCallId) : undefined; + if (!matches?.length || matches.every((entry) => entry.call.index >= index)) { orphanResults++; return; } + if (matches.length !== 1) { unassignedResults++; return; } + matches[0].result = { message, index }; + }); + return { calls, byId, orphanResults, unassignedResults, toolResultCount }; +} + +export function summarizeSessionHealth(messages: SessionMessage[], report = diagnoseSession(messages)) { + const { calls, byId, orphanResults, unassignedResults, toolResultCount } = collectCallEvidence(messages); + const callStates = { success: 0, failure: 0, running: 0, cancelled: 0, unknown: 0, 'no-result': 0 }; + const gaps: CompletionGap[] = []; + let ambiguousCalls = 0; + for (const entry of calls) { + const ambiguous = !entry.id || (byId.get(entry.id)?.length ?? 0) !== 1; + const state = ambiguous ? 'unknown' : entry.result ? relatedState(entry.result.message, entry.call) : 'no-result'; + callStates[state]++; + if (ambiguous) ambiguousCalls++; + if (state !== 'running' && state !== 'unknown' && state !== 'no-result') continue; + const result = !ambiguous ? entry.result : undefined; + gaps.push({ message: result?.message || entry.message, index: result?.index ?? entry.call.index, + toolCallId: entry.id, toolName: entry.call.name, state, + reason: ambiguous ? 'ambiguous-id' : result ? 'last-result' : 'no-result', + evidence: ambiguous ? '调用标识缺失或重复，无法唯一关联结果。' : result + ? relatedEvidence(result.message, entry.call, state === 'no-result' ? 'unknown' : state) + : '截至当前已加载日志，没有记录到此调用的结果；不代表任务失败或进程仍在运行。', + }); + } + const candidates = new Map(); + for (const event of report.events) for (const operation of event.relatedOperations) candidates.set(operation.index, operation); + const candidateStates = { success: 0, failure: 0, running: 0, cancelled: 0, unknown: 0 }; + for (const operation of candidates.values()) candidateStates[operation.state]++; + const repeated = report.events.filter((event) => event.failures.length > 1); + return { + callCount: calls.length, toolResultCount, callStates, orphanResults, unassignedResults, ambiguousCalls, gaps, + failureRecords: report.failureCount, pendingRecords: report.failures.length, pendingEvents: report.events.length, + recoveredRecords: report.recoveredCount, repeatedEvents: repeated.length, + repeatedRecords: repeated.reduce((total, event) => total + event.failures.length, 0), + candidateEvents: report.events.filter((event) => event.relatedOperations.length > 0).length, + candidateResults: candidates.size, candidateStates, + }; +} + +export interface ExecutionEvidence { + callMessage: SessionMessage; + callIndex: number; + toolCallId: string | null; + toolName: string; + resultMessage: SessionMessage | null; + resultIndex: number | null; + state: RelatedOperation['state'] | 'no-result'; + evidence: string; + directory: string | null; + directoryConflict: boolean; + target?: string; + command?: string; + basis?: 'runner-command' | 'script-name'; + commandMode?: 'direct' | 'directory-prefix' | 'compound-fragment'; + fragments?: string[]; +} + +export interface ChronologyCheck { + operation: ExecutionEvidence; + scope: 'same-recorded-directory' | 'unknown'; +} + +export interface ModificationChronology { + operation: ExecutionEvidence; + priorSuccess: ChronologyCheck | null; + laterChecks: ChronologyCheck[]; + latestLater: ChronologyCheck | null; + latestOrderAmbiguous: boolean; + overlappingChecks: ChronologyCheck[]; + excludedScopeChecks: number; +} + +interface CommandToken { value: string; quoted: boolean } +interface CommandSegment { tokens: CommandToken[]; separator: string } + +function literalSegments(command: string): CommandSegment[] | null { + if (/[`$\\(){}\0]|<<|\|\||\r/.test(command) || command.length > 20000) return null; + const segments: CommandSegment[] = []; + let tokens: CommandToken[] = []; + let value = ''; + let quoted = false; + let quote = ''; + const token = () => { if (value || quoted) tokens.push({ value, quoted }); value = ''; quoted = false; }; + for (let index = 0; index < command.length; index++) { + const char = command[index]; + if (quote) { + if (char === quote) quote = ''; + else value += char; + continue; + } + if (char === '"' || char === "'") { quote = char; quoted = true; continue; } + if (char === '#') return null; + if (char === ' ' || char === '\t') { token(); continue; } + if (char === '\n' || char === ';' || char === '|' || char === '&') { + token(); + let separator = char; + if (char === '&') { if (command[index + 1] !== '&') return null; separator = '&&'; index++; } + if (tokens.length) segments.push({ tokens, separator }); + else if (separator !== '\n') return null; + tokens = []; + continue; + } + if (char === '>' || char === '<') { + token(); + let redirect = char; + if (command[index + 1] === '>') { redirect += '>'; index++; } + if (command[index + 1] === '&' && /[0-9]/.test(command[index + 2] || '')) { + redirect += `&${command[index + 2]}`; index += 2; + } + tokens.push({ value: redirect, quoted: false }); + continue; + } + value += char; + } + if (quote) return null; + token(); + if (tokens.length) segments.push({ tokens, separator: '' }); + else if (segments.length && ['&&', '|'].includes(segments[segments.length - 1].separator)) return null; + if (segments.some((segment) => ['if', 'then', 'else', 'elif', 'fi', 'for', 'while', 'do', 'done', 'case', 'esac', 'function', 'until', '!'].includes(segment.tokens[0]?.value))) return null; + return segments; +} + +function checkTokens(tokens: string[]): 'runner-command' | 'script-name' | null { + if (tokens.some((token) => /^(?:--(?:help|version|watch(?:All)?|collect-only|collectonly|co|list-tests|listTests|list|dry-run|prefix|cwd|directory)|-[hVwCc])(?:=|$)/i.test(token))) return null; + const runner = tokens[0] === 'pytest' || (['python', 'python3'].includes(tokens[0]) && tokens[1] === '-m' && tokens[2] === 'pytest') || + (tokens[0] === 'node' && tokens[1] === '--test'); + if (runner) return 'runner-command'; + const manager = tokens[0]; + if (!['npm', 'pnpm', 'yarn'].includes(manager)) return null; + const scriptPosition = tokens[1] === 'run' ? 2 : 1; + if (!['test', 'build', 'lint', 'typecheck'].includes(tokens[scriptPosition])) return null; + if (manager === 'npm' && scriptPosition === 1 && tokens[1] !== 'test') return null; + return 'script-name'; +} + +function verificationCommand(call: RecordedCall): { + command: string; basis: 'runner-command' | 'script-name'; commandMode: 'direct' | 'directory-prefix' | 'compound-fragment'; + fragments: string[]; explicitDirectory: string | null; usesInitialDirectory: boolean; +} | null { + if (!['bash', 'Bash', 'shell', 'exec', 'exec_command', 'run_shell_command', 'execute_command', 'terminal'].includes(call.name)) return null; + const args = argumentObject(call.args); + if (!args) return null; + const values = ['command', 'cmd'].filter((key) => key in args).map((key) => args[key]); + if (!values.length || values.some((value) => typeof value !== 'string' || value !== values[0])) return null; + const command = values[0] as string; + const segments = literalSegments(command); + if (!segments?.length) return null; + const recognized = segments.flatMap((segment, index) => { + if (segment.tokens[0]?.quoted) return []; + const tokens = segment.tokens.map((item) => item.value); + const basis = checkTokens(tokens); + return basis ? [{ index, basis, fragment: tokens.join(' ') }] : []; + }); + if (!recognized.length) return null; + const simple = (segment: CommandSegment) => segment.tokens.every((item) => !/[<>]/.test(item.value)); + const cd = segments[0]; + const direct = segments.length === 1 && simple(segments[0]); + const hasDirectoryPrefix = cd.separator === '&&' && cd.tokens.length === 2 && + cd.tokens[0].value === 'cd' && !cd.tokens[0].quoted && /^\//.test(cd.tokens[1].value) && + recognized.length === 1 && recognized[0].index === 1; + const directoryPrefix = hasDirectoryPrefix && segments.length === 2 && simple(segments[1]); + return { command, basis: recognized[0].basis, fragments: recognized.map((entry) => entry.fragment), + commandMode: direct ? 'direct' : directoryPrefix ? 'directory-prefix' : 'compound-fragment', + explicitDirectory: hasDirectoryPrefix ? cd.tokens[1].value : null, + usesInitialDirectory: recognized.length === 1 && recognized[0].index === 0 }; +} + +function executionEvidence(entry: CallEvidence, ambiguous: boolean): ExecutionEvidence { + const args = argumentObject(entry.call.args); + const directories = args ? ['cwd', 'workdir', 'working_directory'].filter((key) => key in args).map((key) => args[key]) : []; + const directoryConflict = directories.some((value) => typeof value !== 'string' || !value) || new Set(directories).size > 1; + const directory = !directoryConflict && typeof directories[0] === 'string' && /^(?:\/|[A-Za-z]:[\\/]|\\\\)/.test(directories[0]) ? directories[0] : null; + const result = ambiguous ? undefined : entry.result; + const state = ambiguous ? 'unknown' : result ? relatedState(result.message, entry.call) : 'no-result'; + return { + callMessage: entry.message, callIndex: entry.call.index, toolCallId: entry.id, toolName: entry.call.name, + resultMessage: result?.message ?? null, resultIndex: result?.index ?? null, state, directory, directoryConflict, + evidence: ambiguous ? '调用标识缺失或重复，无法唯一关联结果。' : result + ? relatedEvidence(result.message, entry.call, state === 'no-result' ? 'unknown' : state) + : '未记录到此调用的结果；不推断实际进程状态。', + }; +} + +export function analyzeVerificationChronology(messages: SessionMessage[]) { + const { calls, byId } = collectCallEvidence(messages); + const modificationResults: ExecutionEvidence[] = []; + const checks: ExecutionEvidence[] = []; + let unclassifiedShellCalls = 0; + let unclassifiedModificationCalls = 0; + for (const entry of calls) { + const args = argumentObject(entry.call.args); + const ambiguous = !entry.id || (byId.get(entry.id)?.length ?? 0) !== 1; + if (['edit', 'Edit', 'write', 'Write', 'MultiEdit'].includes(entry.call.name)) { + const paths = args ? ['path', 'file_path'].filter((key) => key in args).map((key) => args[key]) : []; + if (!paths.length || paths.some((value) => typeof value !== 'string' || !value.trim() || value.includes('\0') || value !== paths[0])) { + unclassifiedModificationCalls++; + } else { + modificationResults.push({ ...executionEvidence(entry, ambiguous), target: paths[0] as string }); + } + } else if (['apply_patch', 'apply_diff', 'NotebookEdit'].includes(entry.call.name)) { + unclassifiedModificationCalls++; + } + const recognized = verificationCommand(entry.call); + if (recognized) { + const evidence = executionEvidence(entry, ambiguous); + const { explicitDirectory, usesInitialDirectory, ...command } = recognized; + if (recognized.commandMode === 'compound-fragment') { + checks.push({ ...evidence, ...command, state: evidence.state === 'no-result' ? 'no-result' : 'unknown', + directory: explicitDirectory ?? (usesInitialDirectory ? evidence.directory : null), + directoryConflict: explicitDirectory ? false : evidence.directoryConflict, + evidence: `命令文本包含检查片段；无法确认片段是否执行或通过，不能使用整个命令的退出码判定。\n${evidence.evidence}` }); + } else if (recognized.commandMode === 'directory-prefix') { + checks.push({ ...evidence, ...command, directory: explicitDirectory, directoryConflict: false, + state: evidence.state === 'failure' ? 'unknown' : evidence.state, + evidence: `记录的目录前缀：cd 后通过 && 连接检查。${evidence.state === 'failure' ? '整体失败可能发生在 cd，检查状态未知。' : ''}\n${evidence.evidence}` }); + } else checks.push({ ...evidence, ...command }); + } + else if (isExecution(entry.call, entry.message)) unclassifiedShellCalls++; + } + const modifications: ModificationChronology[] = []; + for (const operation of modificationResults) { + if (operation.state !== 'success' || operation.resultIndex === null) continue; + const prior: ChronologyCheck[] = [], laterChecks: ChronologyCheck[] = [], overlappingChecks: ChronologyCheck[] = []; + let excludedScopeChecks = 0; + for (const check of checks) { + if (operation.directoryConflict || check.directoryConflict || (operation.directory && check.directory && operation.directory !== check.directory)) { + excludedScopeChecks++; + continue; + } + const relation: ChronologyCheck = { operation: check, + scope: operation.directory && check.directory ? 'same-recorded-directory' : 'unknown' }; + if (check.callIndex > operation.resultIndex) laterChecks.push(relation); + else if (check.resultIndex !== null && check.resultIndex < operation.callIndex) { + if (check.state === 'success') prior.push(relation); + } else if (check.resultIndex === null || check.resultIndex >= operation.callIndex) overlappingChecks.push(relation); + } + prior.sort((left, right) => (left.operation.resultIndex ?? -1) - (right.operation.resultIndex ?? -1)); + const latest = laterChecks[laterChecks.length - 1]; + const latestOrderAmbiguous = !!latest && laterChecks.filter((entry) => entry.operation.callIndex === latest.operation.callIndex).length > 1; + modifications.push({ operation, priorSuccess: prior[prior.length - 1] ?? null, laterChecks, + latestLater: latestOrderAmbiguous ? null : latest ?? null, latestOrderAmbiguous, overlappingChecks, excludedScopeChecks }); + } + return { + modificationCalls: modificationResults.length, successfulModifications: modifications.length, + unconfirmedModifications: modificationResults.length - modifications.length, + unclassifiedModificationCalls, unclassifiedShellCalls, checks, modifications, + withLaterCheck: modifications.filter((row) => row.laterChecks.length).length, + withoutLaterCheck: modifications.filter((row) => !row.laterChecks.length).length, + changedAfterLastPassedCheck: modifications.filter((row) => row.priorSuccess && !row.laterChecks.length).length, + }; +} diff --git a/intent.md b/intent.md index affec28..c972fec 100644 --- a/intent.md +++ b/intent.md @@ -72,7 +72,34 @@ Portability acceptance: 17 new transfer tests and 103 focused tests pass. Browse Acceptance results: 255 tests and 125 focused checks pass. The frozen 15-session/2,024-result regression retains every original failure/event member and all counts. Strict matching finds three only-i candidates (failure/running/success); no same-file candidate qualifies in that real set because paths lack explicit directory context or turns differ. Synthetic same-file and five-state browser checks, load-more, result jumps and candidate-triggered review staleness pass. Prepare v1.20.0 via the authorized protected PR/release workflow, verify registry installation and public Pages before claiming release completion; do not publish private logs or raw evaluation artifacts. -## Non-goals +## Automatic session health (approved 2026-09-24) + +- Default to an automatic factual report, not a human-review queue. Show every diagnostic event regardless of saved manual labels. No user tagging, model invocation, account or network call is required. +- Summarize existing failure records, exact-argument recovery evidence, remaining events, repeated unresolved operations and unique follow-up result states. Candidate success is not automatic recovery; totals use documented units rather than a health score. +- Independently summarize each recorded tool call's last result in this transcript: success, failure, running, cancelled, unknown, or no recorded result. Calls/results link only by recorded ID and order. Orphan results and ambiguous/reused or missing call IDs are reported separately, never guessed. These are last-recorded states, not live process status or proof that a session ended. +- Surface running/unknown/no-result call evidence with original call/result navigation, including sessions without detected failures. Never call absence of detected failures task success. +- Keep manual notes and transfer as an explicit optional mode. Do not read/hash review storage before opt-in, delete saved notes, or allow its failures/labels to affect the automatic view. Switching back restores all automatic events. +- Preserve diagnostic rules, platform adapters, candidate matching and existing review fingerprints. Only add a derived summary and presentation changes. +- Add deterministic offline transformation tests with the same base data/rubric: append/remove success, alter command/cwd/i, insert pre-started parallel calls, pending/cancelled/unknown results, and verify exact expected outcomes plus complete evidence membership. Test shared candidate deduplication, ambiguous IDs, no-result/orphan records and input immutability. +- Validate against synthetic UI plus unchanged frozen-log failure/event outputs. Test metrics are known-case coverage and invariants, not human time saved or real-world accuracy. Publish no raw logs or automatic human labels. + +## Modification / verification chronology (2026-09-24) + +Continue the automatic-health worktree toward an execution-evidence product rather than adding another generic dashboard. User authorized autonomous improvement and roadmap changes; manual tagging is not a prerequisite. + +- Recognize explicit `edit`, `Edit`, `write`, `Write`, `MultiEdit` calls with one unambiguous recorded `path`/`file_path`. Only a uniquely paired successful result is a completed modification-tool record; it does not prove bytes changed. Failed/running/unknown/no-result mutation calls remain visible in coverage counts. Patch/shell-embedded modifications are not parsed or guessed. +- Recognize only simple direct shell invocations of common test runners (`pytest`, `python[3] -m pytest`, `node --test`) and package-script conventions (`npm test`, `npm run test/build/lint/typecheck`, equivalent direct pnpm/yarn forms). Label runner vs script-name evidence separately; script bodies and actual coverage are not known. Reject compound commands, substitutions, quotes/redirection, help/watch/list-only modes and directory-changing flags. Count unrecognized shell commands explicitly. +- Compare call/result record order, not wall-clock guesses. A verification call must begin after a modification result to count as later. A prior result must finish before the modification call to count as prior; overlapping/in-flight checks are separate, never post-change validation. Keep the latest recorded state, including failures/cancelled/unknown/missing results; do not cherry-pick a passed check. +- Timeline is within the viewed transcript, not a proof of file coverage. Matching explicit absolute working directories is stronger scope evidence; missing directory context is labelled unknown. Known-different/conflicting directory contexts are excluded from the relation and counted. Do not resolve files, infer implicit `cd`, or assume the initial session cwd remains current. +- Surface the actionable fact “a check succeeded, then a modification tool succeeded, and no later recognized check is recorded” with both source jumps; do not say “bug”, “task failed”, “tests cover the file” or “ready to ship”. Every check and modification has access to its original call/result evidence. No verification is executed. +- Preserve all automatic diagnostics/candidate/review-fingerprint semantics. Reuse conservative unique-ID pairing for health and chronology; ambiguous/orphan IDs do not supply success evidence. Keep manual storage opt-in. +- Add deterministic temporal transformations (check→edit vs edit→check, overlap, failure after earlier success, delete/missing results, cwd mismatch) and frozen real-log coverage checks. Record recognized/excluded categories and keep production claims bounded by actual data. Do not upload private logs or add model calls/dependencies. + +Real-log coverage refinement: the initial direct-only classifier recognized zero checks because logged shell commands are predominantly compound. Support a bounded, non-executing lexer for literal command segments with quotes, `&&`, `;`, newline and simple pipelines/redirections. Direct checks and a sole literal absolute `cd ... && check` may use the whole-call outcome; other recognized fragments are explicitly labelled "command contains a check; whether this fragment ran/passed is unknown", regardless of wrapper exit code. Reject substitutions, heredocs, shell control syntax and quoted executable names. Quoted strings in echo/python arguments must never become executable segments. Do not treat compound snippets as proven verification or infer their working directory. This replaces blanket compound exclusion, not the no-guessing/coverage boundary. + +Validation: 290 tests pass, including 200 fixed health transformations and 120 temporal transformations without human labels. The frozen 15-session/2,024-result set retains exact v1.20 diagnostic output. It yields 59 check-containing calls, all with unknown fragment outcome; no check-coverage or task-success claim is made. Explicit edit/write tools return successfully 240 times; 112 have later check-containing calls and 128 do not. Browser checks cover default zero review-storage reads, optional-note preservation, live missing-result resolution, before/overlap/after chronology, source navigation and 390px rendering. Prepare the combined increment as v1.21.0 through the authorized protected PR/release path after final gates; keep the continuing project goal active, and do not claim real productivity or adoption from this release. + +## Continuing non-goals - No new platform, dependency, model call, account, telemetry, cloud log storage or automatic command execution. - No automatic human judgments of real sessions, relaxed argument matching or fabricated accuracy/productivity/adoption claims. diff --git a/package-lock.json b/package-lock.json index 14cd0b8..4f596fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@alloevil/agent-xray", - "version": "1.20.0", + "version": "1.21.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@alloevil/agent-xray", - "version": "1.20.0", + "version": "1.21.0", "license": "MIT", "dependencies": { "express": "^4.21.2" diff --git a/package.json b/package.json index 5c3c83e..e8df681 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alloevil/agent-xray", - "version": "1.20.0", + "version": "1.21.0", "description": "Web dashboard for viewing AI agent session logs — supports OpenClaw, Codex, Claude Code, Hermes, OMP, DeepSeek Harness, and Gemini CLI", "main": "server.js", "bin": { diff --git a/screenshots/automatic-health.png b/screenshots/automatic-health.png new file mode 100644 index 0000000..b4e2fc9 Binary files /dev/null and b/screenshots/automatic-health.png differ diff --git a/screenshots/verification-chronology.png b/screenshots/verification-chronology.png new file mode 100644 index 0000000..cbb8b75 Binary files /dev/null and b/screenshots/verification-chronology.png differ diff --git a/scripts/demo-follow-up.cjs b/scripts/demo-follow-up.cjs index 74641a6..215698d 100644 --- a/scripts/demo-follow-up.cjs +++ b/scripts/demo-follow-up.cjs @@ -54,6 +54,12 @@ async function main() { }) ); records.push(...operation('other-turn', 'edit', { ...editArgs, newText: 'next turn' }, 'success')); + records.push( + message('no-result-call', { + role: 'assistant', + content: [{ type: 'toolCall', id: 'no-result', name: 'bash', arguments: { command: 'synthetic pending check' } }], + }) + ); try { await fs.mkdir(directory, { recursive: true }); await fs.writeFile(file, `${records.map((record) => JSON.stringify(record)).join('\n')}\n`); @@ -65,8 +71,10 @@ async function main() { console.log(`Session API: ${server.base}/api/omp/sessions/${id}`); console.log('OMP 合成会话：3 个自动事件；初始 bash 事件有 7 条不同状态候选，edit 事件有 1 条同文件候选。'); console.log('输入 n：追加仅 i 不同的成功候选，使旧复核过期；q 或 Ctrl-C 清理退出。'); + console.log('自动体检无需笔记：另有 1 条无结果调用。输入 c 为它追加成功结果，结果缺口应减少，原失败事件不变。'); const input = readline.createInterface({ input: process.stdin }); let appended = 0; + let completed = false; const stop = async () => { input.close(); await server.stop(); @@ -74,6 +82,27 @@ async function main() { }; input.on('line', async (line) => { if (line.trim() === 'q') return stop(); + if (line.trim() === 'c' && !completed) { + completed = true; + try { + const finished = message('no-result-finished', { + role: 'toolResult', + toolCallId: 'no-result', + toolName: 'bash', + isError: false, + details: { exitCode: 0 }, + content: [{ type: 'text', text: 'Synthetic pending check completed; not task acceptance.' }], + }); + await fs.appendFile(file, `${JSON.stringify(finished)}\n`); + console.log( + 'PASS: appended result for pending call; no-result count should be zero, automatic failures unchanged.' + ); + } catch (error) { + console.error(error.message); + await stop(); + } + return; + } if (line.trim() !== 'n') return; appended++; try { diff --git a/scripts/demo-verification.cjs b/scripts/demo-verification.cjs new file mode 100644 index 0000000..36aae3a --- /dev/null +++ b/scripts/demo-verification.cjs @@ -0,0 +1,95 @@ +const fs = require('node:fs/promises'); +const path = require('node:path'); +const readline = require('node:readline'); +const { startServer } = require('../test/helpers'); + +async function main() { + const server = await startServer(); + const id = '01990000-0000-7000-8000-000000000188'; + const directory = path.join(server.home, '.omp/agent/sessions/synthetic-verification'); + const file = path.join(directory, `2026-09-24T08-00-00_${id}.jsonl`); + let tick = 0; + const stamp = () => new Date(Date.UTC(2026, 8, 24, 8) + tick++ * 1000).toISOString(); + const message = (recordId, value) => ({ type: 'message', id: recordId, timestamp: stamp(), message: value }); + const call = (callId, name, args) => + message(`${callId}-call`, { + role: 'assistant', + content: [{ type: 'toolCall', id: callId, name, arguments: args }], + }); + const result = (callId, toolName, state) => + message(`${callId}-result`, { + role: 'toolResult', + toolCallId: callId, + toolName, + isError: state === 'failure', + details: state === 'running' ? { async: { state: 'running' } } : { exitCode: state === 'failure' ? 1 : 0 }, + content: [{ type: 'text', text: `Synthetic ${state} result. No tool command was executed by this demo.` }], + }); + const records = [ + { type: 'session', id, timestamp: stamp(), cwd: '/synthetic/project' }, + message('user', { + role: 'user', + content: [{ type: 'text', text: '[Synthetic] 检查成功之后又修改：先后顺序不是测试覆盖' }], + }), + call('passed-check', 'bash', { command: 'npm test', cwd: '/synthetic/project' }), + result('passed-check', 'bash', 'success'), + call('overlapping-check', 'bash', { command: 'pytest', cwd: '/synthetic/project' }), + call('changed-file', 'edit', { + path: '/synthetic/project/config.ts', + oldText: 'before', + newText: 'after', + cwd: '/synthetic/project', + }), + result('changed-file', 'edit', 'success'), + result('overlapping-check', 'bash', 'success'), + call('different-directory', 'bash', { command: 'npm run build', cwd: '/synthetic/other' }), + result('different-directory', 'bash', 'success'), + call('unclassified', 'bash', { command: 'echo synthetic-check-placeholder', cwd: '/synthetic/project' }), + result('unclassified', 'bash', 'success'), + ]; + try { + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(file, `${records.map((record) => JSON.stringify(record)).join('\n')}\n`); + } catch (error) { + await server.stop(); + throw error; + } + console.log(`Synthetic chronology demo: ${server.base}`); + console.log(`Session API: ${server.base}/api/omp/sessions/${id}`); + console.log('初始：1 次修改成功返回；先前检查通过、另一个检查与修改重叠、不同目录检查不关联。无人工标注。'); + console.log('输入 f 追加修改后的失败检查；输入 s 追加成功检查；输入 q 或 Ctrl-C 清理退出。'); + const input = readline.createInterface({ input: process.stdin }); + const stop = async () => { + input.close(); + await server.stop(); + process.exit(0); + }; + let appended = 0; + input.on('line', async (line) => { + const action = line.trim(); + if (action === 'q') return stop(); + if (!['f', 's'].includes(action)) return; + appended++; + const callId = `later-check-${appended}`; + try { + const extra = [ + call(callId, 'bash', { command: 'npm test', cwd: '/synthetic/project' }), + result(callId, 'bash', action === 'f' ? 'failure' : 'success'), + ]; + await fs.appendFile(file, `${extra.map((record) => JSON.stringify(record)).join('\n')}\n`); + console.log( + `PASS: appended ${action === 'f' ? 'failure' : 'success'} check result; chronology should update without human labels.` + ); + } catch (error) { + console.error(error.message); + await stop(); + } + }); + process.on('SIGINT', stop); + process.on('SIGTERM', stop); +} + +main().catch((error) => { + console.error(error.message); + process.exitCode = 1; +}); diff --git a/test/session-health.test.js b/test/session-health.test.js new file mode 100644 index 0000000..2b55d81 --- /dev/null +++ b/test/session-health.test.js @@ -0,0 +1,191 @@ +const { test, before } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const { stripTypeScriptTypes } = require('node:module'); + +let diagnoseSession; +let summarizeSessionHealth; +before(async () => { + const source = readFileSync(path.join(__dirname, '../frontend/src/views/sessions/diagnostics.ts'), 'utf8'); + ({ diagnoseSession, summarizeSessionHealth } = await import( + `data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(source)).toString('base64')}` + )); +}); +const call = (id, args = { command: 'synthetic test' }, name = 'bash') => ({ + role: 'toolCall', + id, + toolCallId: id, + toolName: name, + details: args, +}); +const result = (id, state, extra = {}) => ({ + role: 'toolResult', + id: `${id}-result`, + toolCallId: id, + isError: state === 'failure', + details: null, + content: [{ type: 'text', text: `Synthetic ${state}` }], + ompOutcome: { state, evidence: [`synthetic.state=${state}`], warnings: [] }, + ...extra, +}); +const summary = (messages) => summarizeSessionHealth(messages, diagnoseSession(messages)); + +test('empty report has no invented success or live status', () => { + const health = summary([]); + assert.equal(health.callCount, 0); + assert.equal(health.failureRecords, 0); + assert.equal(health.gaps.length, 0); + assert.equal(health.orphanResults, 0); +}); + +test('last result per call distinguishes five states and missing results', () => { + const states = ['success', 'failure', 'running', 'cancelled', 'unknown']; + const messages = states.flatMap((state) => [call(state), result(state, state)]); + messages.push(call('missing')); + const health = summary(messages); + assert.equal(health.callCount, 6); + assert.deepEqual(health.callStates, { success: 1, failure: 1, running: 1, cancelled: 1, unknown: 1, 'no-result': 1 }); + assert.deepEqual( + health.gaps.map((gap) => gap.state), + ['running', 'unknown', 'no-result'] + ); + assert.equal(health.gaps[0].message, messages[5]); + assert.equal(health.gaps[2].message, messages[10]); +}); + +test('running then completed updates last state rather than double-counting calls', () => { + const messages = [call('job'), result('job', 'running')]; + assert.equal(summary(messages).callStates.running, 1); + const health = summary([...messages, result('job', 'success')]); + assert.equal(health.callCount, 1); + assert.equal(health.toolResultCount, 2); + assert.equal(health.callStates.success, 1); + assert.equal(health.gaps.length, 0); +}); + +test('orphan or pre-call results do not certify later calls', () => { + const health = summary([result('later', 'success'), call('later'), result('orphan', 'success')]); + assert.equal(health.orphanResults, 2); + assert.equal(health.callStates['no-result'], 1); +}); + +test('missing and reused IDs remain ambiguous instead of assigning outcomes', () => { + const health = summary([ + call(null), + call('duplicate'), + result('duplicate', 'success'), + call('duplicate'), + result('duplicate', 'success'), + ]); + assert.equal(health.ambiguousCalls, 3); + assert.equal(health.callStates.unknown, 3); + assert.equal(health.unassignedResults, 2); + assert.ok(health.gaps.every((gap) => gap.reason === 'ambiguous-id')); +}); + +test('embedded tool calls use their source message for missing-result evidence', () => { + const embedded = { + id: 'assistant', + role: 'assistant', + content: [ + { type: 'toolCall', id: 'one', name: 'Read', arguments: { path: 'a' } }, + { type: 'toolCall', id: 'two', name: 'Read', arguments: { path: 'b' } }, + ], + }; + const health = summary([embedded, result('one', 'success')]); + assert.equal(health.callCount, 2); + assert.equal(health.gaps[0].message, embedded); + assert.equal(health.gaps[0].toolCallId, 'two'); +}); + +test('repeated unresolved event and recovered record units stay explicit', () => { + const messages = [call('one'), result('one', 'failure'), call('two'), result('two', 'failure')]; + const health = summary(messages); + assert.equal(health.failureRecords, 2); + assert.equal(health.pendingRecords, 2); + assert.equal(health.pendingEvents, 1); + assert.equal(health.repeatedEvents, 1); + assert.equal(health.repeatedRecords, 2); + const recovered = summary([...messages, call('retry'), result('retry', 'success')]); + assert.equal(recovered.recoveredRecords, 2); + assert.equal(recovered.repeatedEvents, 0); +}); + +test('one follow-up result referenced by two events is counted once by result position', () => { + const messages = [ + call('first', { command: 'same', i: 1 }), + result('first', 'failure'), + call('second', { command: 'same', i: 2 }), + result('second', 'failure'), + call('third', { command: 'same', i: 3 }), + result('third', 'success'), + ]; + const health = summary(messages); + assert.equal(health.candidateEvents, 2); + assert.equal(health.candidateStates.success, 1); + assert.equal(health.candidateStates.failure, 1); + assert.equal(health.candidateResults, 2); + assert.equal(health.recoveredRecords, 0); +}); + +test('unknown completion and non-OMP running status remain gaps', () => { + const messages = [ + call('unknown'), + result('unknown', 'success', { ompOutcome: undefined }), + call('running'), + result('running', 'success', { ompOutcome: undefined, details: { status: 'running', exitCode: 0 } }), + ]; + assert.deepEqual( + summary(messages).gaps.map((gap) => gap.state), + ['unknown', 'running'] + ); +}); + +test('health calculation does not change source or original diagnostics', () => { + const messages = [call('one'), result('one', 'failure'), call('pending')]; + const raw = JSON.stringify(messages); + const report = diagnoseSession(messages); + const before = JSON.stringify(report); + summarizeSessionHealth(messages, report); + assert.equal(JSON.stringify(report), before); + assert.equal(JSON.stringify(messages), raw); +}); + +test('fixed offline transformations preserve exact outcomes and all evidence across 20 seeds', (context) => { + let checks = 0; + for (let seed = 0; seed < 20; seed++) { + const args = { command: `synthetic test ${seed}`, cwd: `/synthetic/${seed}`, i: 'first' }; + const base = [call('failed', args), result('failed', 'failure')]; + const variants = [ + [base, 1, 0], + [[...base, call('retry', args), result('retry', 'success')], 0, 1], + [[...base, call('retry', args)], 1, 0], + [[...base, call('retry', args), result('retry', 'running')], 1, 0], + [[...base, call('retry', args), result('retry', 'cancelled')], 1, 0], + [[...base, call('retry', args), result('retry', 'unknown')], 1, 0], + [[...base, call('retry', { ...args, command: 'other' }), result('retry', 'success')], 1, 0], + [[...base, call('retry', { ...args, cwd: '/other' }), result('retry', 'success')], 1, 0], + [[...base, call('retry', { ...args, i: 'next' }), result('retry', 'success')], 1, 0], + [[base[0], call('parallel', args), base[1], result('parallel', 'success')], 1, 0], + ]; + for (const [messages, pending, recovered] of variants) { + const report = diagnoseSession(messages), + health = summarizeSessionHealth(messages, report); + assert.equal(health.pendingRecords, pending); + assert.equal(health.recoveredRecords, recovered); + const members = report.events.flatMap((event) => event.failures); + assert.deepEqual(new Set(members), new Set(report.failures)); + assert.equal(members.length, report.failures.length); + assert.equal( + Object.values(health.callStates).reduce((sum, value) => sum + value, 0), + health.callCount + ); + checks++; + } + } + assert.equal(checks, 200); + context.diagnostic( + `Known-case transformations: ${checks}/200 passed; wrong recovery decisions: 0; event evidence partitions: ${checks}/200 complete. Not a real-world accuracy estimate.` + ); +}); diff --git a/test/verification-chronology.test.js b/test/verification-chronology.test.js new file mode 100644 index 0000000..0e4ce66 --- /dev/null +++ b/test/verification-chronology.test.js @@ -0,0 +1,361 @@ +const { test, before } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const { stripTypeScriptTypes } = require('node:module'); + +let api; +before(async () => { + const source = readFileSync(path.join(__dirname, '../frontend/src/views/sessions/diagnostics.ts'), 'utf8'); + api = await import(`data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(source)).toString('base64')}`); +}); +const call = (id, toolName, details) => ({ id, role: 'toolCall', toolCallId: id, toolName, details }); +const edit = (id, extra = {}) => call(id, 'edit', { path: '/synthetic/file.ts', oldText: 'a', newText: 'b', ...extra }); +const check = (id, command = 'npm test', extra = {}) => call(id, 'bash', { command, ...extra }); +const result = (id, state = 'success', extra = {}) => ({ + id: `${id}-result`, + role: 'toolResult', + toolCallId: id, + isError: state === 'failure', + content: [{ type: 'text', text: `Synthetic ${state}` }], + details: null, + timestamp: null, + ompOutcome: { state, evidence: [`synthetic.state=${state}`], warnings: [] }, + ...extra, +}); +const inspect = (messages) => api.analyzeVerificationChronology(messages); + +test('prior passed check followed by edit is not post-change verification', () => { + const messages = [check('test'), result('test'), edit('edit'), result('edit')]; + const report = inspect(messages); + assert.equal(report.successfulModifications, 1); + assert.equal(report.withoutLaterCheck, 1); + assert.equal(report.changedAfterLastPassedCheck, 1); + const row = report.modifications[0]; + assert.equal(row.priorSuccess.operation.resultMessage, messages[1]); + assert.equal(row.operation.resultMessage, messages[3]); + assert.equal(row.laterChecks.length, 0); +}); + +test('check after completed edit is a temporal relation, never file coverage', () => { + const messages = [edit('edit'), result('edit'), check('test'), result('test')]; + const report = inspect(messages); + assert.equal(report.withoutLaterCheck, 0); + assert.equal(report.withLaterCheck, 1); + const later = report.modifications[0].laterChecks[0]; + assert.equal(later.scope, 'unknown'); + assert.equal(later.operation.state, 'success'); + assert.equal(later.operation.resultMessage, messages[3]); + assert.equal(report.modifications[0].priorSuccess, null); +}); + +test('in-flight and overlapping checks are not counted as post-edit checks', () => { + for (const messages of [ + [check('test'), edit('edit'), result('edit'), result('test')], + [edit('edit'), check('test'), result('edit'), result('test')], + [check('test'), edit('edit'), result('edit')], + ]) { + const row = inspect(messages).modifications[0]; + assert.equal(row.laterChecks.length, 0); + assert.equal(row.overlappingChecks.length, 1); + assert.equal(row.priorSuccess, null); + } +}); + +test('latest later check remains failed even when an earlier later check passed', () => { + const report = inspect([ + edit('edit'), + result('edit'), + check('pass'), + result('pass'), + check('fail'), + result('fail', 'failure'), + ]); + assert.deepEqual( + report.modifications[0].laterChecks.map((entry) => entry.operation.state), + ['success', 'failure'] + ); + assert.equal(report.modifications[0].latestLater.operation.state, 'failure'); +}); + +test('running, cancelled, unknown and missing check outcomes retain their distinction', () => { + for (const state of ['running', 'cancelled', 'unknown', 'no-result']) { + const messages = [edit('edit'), result('edit'), check('test')]; + if (state !== 'no-result') messages.push(result('test', state)); + const later = inspect(messages).modifications[0].latestLater; + assert.equal(later.operation.state, state); + assert.equal(later.operation.callMessage, messages[2]); + } +}); + +test('failed or incomplete edit calls are counted, not asserted as successful modifications', () => { + for (const state of ['failure', 'running', 'cancelled', 'unknown', 'no-result']) { + const messages = [edit('edit')]; + if (state !== 'no-result') messages.push(result('edit', state)); + const report = inspect(messages); + assert.equal(report.modificationCalls, 1); + assert.equal(report.successfulModifications, 0); + assert.equal(report.unconfirmedModifications, 1); + } +}); + +test('known different or conflicting working directories are excluded', () => { + for (const extra of [{ cwd: '/different' }, { cwd: '/synthetic', workdir: '/different' }]) { + const row = inspect([ + edit('edit', { cwd: '/synthetic' }), + result('edit'), + check('test', 'npm test', extra), + result('test'), + ]).modifications[0]; + assert.equal(row.laterChecks.length, 0); + assert.equal(row.excludedScopeChecks, 1); + } + const same = inspect([ + edit('edit', { cwd: '/synthetic' }), + result('edit'), + check('test', 'pytest', { workdir: '/synthetic' }), + result('test'), + ]); + assert.equal(same.modifications[0].latestLater.scope, 'same-recorded-directory'); +}); + +test('unknown directory stays unknown and does not imply path coverage', () => { + const row = inspect([ + edit('edit', { path: 'relative.ts' }), + result('edit'), + check('test', 'pytest', { cwd: '/synthetic' }), + result('test'), + ]).modifications[0]; + assert.equal(row.operation.directory, null); + assert.equal(row.latestLater.scope, 'unknown'); + assert.equal(row.operation.target, 'relative.ts'); +}); + +test('direct runner and conventional script recognition is bounded', () => { + const commands = [ + 'npm test', + 'npm run build', + 'npm run lint -- --quiet', + 'npm run typecheck', + 'pnpm test', + 'pnpm run lint', + 'yarn test', + 'pytest tests/test_sample.py -q', + 'python -m pytest', + 'python3 -m pytest tests', + 'node --test test/sample.test.js', + ]; + const report = inspect( + commands.flatMap((command, index) => [check(`check-${index}`, command), result(`check-${index}`)]) + ); + assert.equal(report.checks.length, commands.length); + assert.equal(report.checks.filter((entry) => entry.basis === 'runner-command').length, 4); + assert.equal(report.checks.filter((entry) => entry.basis === 'script-name').length, commands.length - 4); +}); + +test('substitutions, quoted mentions, control flow and non-verifying commands are not guessed', () => { + const commands = [ + 'echo npm test', + 'FOO=bar npm test', + 'npm run deploy', + 'npm run check', + 'npm test -- --watch', + 'pytest --collect-only', + 'pytest --help', + 'node --test --version', + 'npm --prefix /other test', + 'pnpm -C /other test', + 'npm test -- --cwd=/other', + 'python -c "print(1)"', + 'npm test $(echo x)', + 'echo "pytest file.py; npm test"', + 'if true; then npm test; fi', + 'npm test --dry-run', + 'npm test &&', + 'npm test |', + 'npm test `echo x`', + ]; + const report = inspect(commands.map((command, index) => check(`check-${index}`, command))); + assert.equal(report.checks.length, 0); + assert.equal(report.unclassifiedShellCalls, commands.length); +}); + +test('bounded compound fragments are visible but whole-command success is never test success', () => { + for (const command of [ + 'npm test && git status', + 'npm test; echo ok', + 'npm test | tee out', + 'npm test > out', + 'npm test\nnode other.js', + 'cd /synthetic && npm install && npm run lint 2>&1 | tail -30', + ]) { + const report = inspect([check('compound', command), result('compound'), edit('edit'), result('edit')]); + assert.equal(report.checks.length, 1); + assert.equal(report.checks[0].commandMode, 'compound-fragment'); + assert.equal(report.checks[0].state, 'unknown'); + assert.equal(report.modifications[0].priorSuccess, null); + } +}); + +test('single absolute cd prefix supplies explicit directory and zero exit, but failed cd is unknown', () => { + const command = 'cd /synthetic/project && npm test'; + const passed = inspect([check('test', command), result('test')]); + assert.equal(passed.checks[0].state, 'success'); + assert.equal(passed.checks[0].directory, '/synthetic/project'); + assert.equal(passed.checks[0].commandMode, 'directory-prefix'); + assert.equal(inspect([check('test', command), result('test', 'failure')]).checks[0].state, 'unknown'); + assert.equal( + inspect([check('test', 'cd relative && npm test'), result('test')]).checks[0].commandMode, + 'compound-fragment' + ); +}); + +test('literal quoted arguments do not split into executable checks', () => { + const report = inspect([ + check('echo', 'echo "npm test; pytest"'), + result('echo'), + check('python', 'python -c \'print("npm test")\''), + result('python'), + check('quoted', "pytest 'file with spaces.py'"), + result('quoted'), + ]); + assert.equal(report.checks.length, 1); + assert.equal(report.checks[0].command, "pytest 'file with spaces.py'"); + assert.equal(report.checks[0].commandMode, 'direct'); +}); + +test('a compound fragment after an edit records an unknown attempt, never a successful verification', () => { + const row = inspect([edit('edit'), result('edit'), check('test', 'npm test 2>&1 | tail -20'), result('test')]) + .modifications[0]; + assert.equal(row.latestLater.operation.state, 'unknown'); + assert.equal(row.latestLater.operation.commandMode, 'compound-fragment'); + assert.match(row.latestLater.operation.evidence, /无法确认片段是否执行/); +}); + +test('compound checks retain provable initial or cd directory evidence without certifying exit status', () => { + for (const [command, args] of [ + ['npm test | tail -20', { cwd: '/other' }], + ['cd /other && npm test | tail -20', { cwd: '/initial' }], + ]) { + const report = inspect([ + edit('edit', { cwd: '/synthetic' }), + result('edit'), + check('test', command, args), + result('test'), + ]); + assert.equal(report.checks[0].directory, '/other'); + assert.equal(report.checks[0].state, 'unknown'); + assert.equal(report.modifications[0].excludedScopeChecks, 1); + assert.equal(report.modifications[0].laterChecks.length, 0); + } +}); + +test('checks started in one assistant message have no uniquely latest call', () => { + const simultaneous = { + role: 'assistant', + content: [ + { type: 'toolCall', id: 'pass', name: 'bash', arguments: { command: 'npm test' } }, + { type: 'toolCall', id: 'fail', name: 'bash', arguments: { command: 'npm run lint' } }, + ], + }; + const row = inspect([edit('edit'), result('edit'), simultaneous, result('pass'), result('fail', 'failure')]) + .modifications[0]; + assert.equal(row.latestOrderAmbiguous, true); + assert.equal(row.latestLater, null); + assert.equal(row.laterChecks.length, 2); +}); + +test('conflicting command fields and non-shell tools do not masquerade as checks', () => { + const report = inspect([ + check('bad', 'npm test', { cmd: 'echo done' }), + call('read', 'Read', { command: 'npm test' }), + ]); + assert.equal(report.checks.length, 0); +}); + +test('ambiguous identities cannot manufacture modification or successful check evidence', () => { + const report = inspect([edit('same'), result('same'), edit('same'), result('same'), check(null), result(null)]); + assert.equal(report.successfulModifications, 0); + assert.equal(report.modificationCalls, 2); + assert.equal(report.checks[0].state, 'unknown'); +}); + +test('embedded simultaneous call positions remain overlapping', () => { + const message = { + role: 'assistant', + id: 'embedded', + content: [ + { type: 'toolCall', id: 'edit', name: 'Edit', arguments: { file_path: '/synthetic/file.ts' } }, + { type: 'toolCall', id: 'test', name: 'Bash', arguments: { command: 'npm test' } }, + ], + }; + const row = inspect([message, result('edit'), result('test')]).modifications[0]; + assert.equal(row.overlappingChecks.length, 1); + assert.equal(row.operation.callMessage, message); +}); + +test('later completion appended to a check updates its state without new execution', () => { + const messages = [edit('edit'), result('edit'), check('test'), result('test', 'running')]; + assert.equal(inspect(messages).modifications[0].latestLater.operation.state, 'running'); + const finished = inspect([...messages, result('test')]); + assert.equal(finished.checks.length, 1); + assert.equal(finished.modifications[0].latestLater.operation.state, 'success'); +}); + +test('unsupported modification encodings are counted, not silently parsed', () => { + const report = inspect([ + call('patch', 'apply_patch', '*** Begin Patch\n*** End Patch'), + result('patch'), + edit('ambiguous', { path: '/one', file_path: '/two' }), + result('ambiguous'), + edit('missing', { path: null }), + result('missing'), + ]); + assert.equal(report.modifications.length, 0); + assert.equal(report.unclassifiedModificationCalls, 3); +}); + +test('same turn is not required, but latest message order is not a timestamp guess', () => { + const messages = [ + check('test'), + result('test', 'success', { timestamp: '2099-01-01T00:00:00Z' }), + { role: 'user', content: [{ type: 'text', text: 'Synthetic later task' }] }, + edit('edit'), + result('edit', 'success', { timestamp: '2000-01-01T00:00:00Z' }), + ]; + assert.equal(inspect(messages).changedAfterLastPassedCheck, 1); +}); + +test('chronology leaves original diagnostic output and input unchanged', () => { + const messages = [check('test'), result('test', 'failure'), edit('edit'), result('edit')]; + const serialized = JSON.stringify(messages), + before = api.diagnoseSession(messages); + inspect(messages); + assert.equal(JSON.stringify(messages), serialized); + assert.deepEqual(api.diagnoseSession(messages), before); +}); + +test('known temporal permutations preserve before/after/overlap facts across 20 fixed bases', (context) => { + let checks = 0; + for (let index = 0; index < 20; index++) { + const change = edit('edit', { path: `/synthetic/file-${index}.ts` }); + const verification = check('test', index % 2 ? 'pytest' : 'npm test'); + const cases = [ + [[verification, result('test'), change, result('edit')], 1, 0, 0], + [[change, result('edit'), verification, result('test')], 0, 1, 0], + [[verification, change, result('edit'), result('test')], 1, 0, 1], + [[change, verification, result('edit'), result('test', 'failure')], 1, 0, 1], + [[change, result('edit'), verification], 0, 1, 0], + [[change, result('edit')], 1, 0, 0], + ]; + for (const [messages, missing, later, overlap] of cases) { + const report = inspect(messages); + assert.equal(report.withoutLaterCheck, missing); + assert.equal(report.withLaterCheck, later); + assert.equal(report.modifications[0].overlappingChecks.length, overlap); + assert.equal(report.withoutLaterCheck + report.withLaterCheck, report.successfulModifications); + checks++; + } + } + context.diagnostic(`Temporal transformations: ${checks}/120 PASS; no model judge or human labels.`); +});