From 7b277df62be83d4880fcea1f592b3daf864b9e74 Mon Sep 17 00:00:00 2001 From: gaoruilin Date: Thu, 24 Sep 2026 09:51:27 +0800 Subject: [PATCH] feat: expose shared execution evidence through offline inspect CLI --- .github/workflows/test.yml | 2 +- README.md | 4 +- README.zh-CN.md | 2 + bin/agentxray.js | 51 ++--- bin/inspect.js | 77 ++++++++ biome.json | 11 +- claims.json | 14 +- docs/ROADMAP.md | 6 +- docs/diagnostics-verification.md | 14 ++ docs/offline-inspect.md | 127 ++++++++++++ docs/releases/v1.23.0.md | 27 +++ intent.md | 12 ++ lib/generated/diagnostics.cjs | 13 ++ lib/inspect.js | 328 +++++++++++++++++++++++++++++++ package-lock.json | 4 +- package.json | 5 +- scripts/build-diagnostics.mjs | 33 ++++ test/inspect.test.js | 317 +++++++++++++++++++++++++++++ 18 files changed, 1009 insertions(+), 38 deletions(-) create mode 100644 bin/inspect.js create mode 100644 docs/offline-inspect.md create mode 100644 docs/releases/v1.23.0.md create mode 100644 lib/generated/diagnostics.cjs create mode 100644 lib/inspect.js create mode 100644 scripts/build-diagnostics.mjs create mode 100644 test/inspect.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f35d050..ae810a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,6 @@ jobs: - run: npm ci # prepare script builds the web UI (and regenerates public/js/pure.js) # public/js/pure.js is generated from frontend/src/lib/{pure,markdown}.ts; # fail when the committed copy has drifted from the TS sources. - - run: git diff --exit-code public/js/pure.js + - run: git diff --exit-code public/js/pure.js lib/generated/diagnostics.cjs - run: npx biome check . - run: npm test diff --git a/README.md b/README.md index 74951e4..56cae9f 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ Rule of thumb: if you build and operate your own agent in production, use a trac ## Features +- **Offline evidence CLI** — `agentxray inspect --platform codex session.jsonl --json` reads one explicitly selected log without a server or model. Versioned, minimized reports expose source lines and shared UI-rule hashes; opt-in pending-failure gates never claim task correctness. Supports Codex, OMP and Claude Code JSONL. [Automation contract](docs/offline-inspect.md). + - **Automatic session health** — Opens with factual failure, repetition, follow-up and last-recorded call-state summaries. Missing/running/unknown results have evidence links; no human labels or model calls required. Manual notes and transfers are opt-in and never hide automatic facts. [Scope and offline checks](docs/diagnostics.md#automatic-session-health). - **Codex background-process evidence** — Connect explicit `exec_command` process IDs to later `write_stdin` results, with launch/poll/exit source links. Ambiguous IDs or polling sequences stay unknown; process completion never rewrites historical tool-call states or proves a task passed. [Association limits](docs/diagnostics.md#codex-background-process-evidence). - **Modification/check chronology** — Distinguish checks before an edit, checks overlapping it and later outcomes. A passed earlier check or a successful output pipeline is not post-change validation; ambiguous command fragments remain unknown. [Recognition and coverage limits](docs/diagnostics.md#modification-and-verification-chronology). @@ -322,7 +324,7 @@ Archived sessions (`.jsonl.reset.*`, `.jsonl.deleted.*`) are shown for OpenClaw ## Development -Tests live in `test/` and use Node's built-in test runner — no extra dependencies. Run `npm ci` once, then `npm test` (`node --test test/*.test.js`). The tests start their own server on a random port with `HOME` and every platform directory pointed at a throwaway copy of `test/fixtures/home`, so your real session logs are never read or modified. CI (`.github/workflows/test.yml`) runs on Node 22 for every push and pull request to `master`, in four steps: `npm ci` (whose `prepare` script builds the web UI and regenerates `public/js/pure.js`), a drift check (`git diff --exit-code public/js/pure.js`), `npx biome check .`, and `npm test`. +Tests live in `test/` and use Node's built-in test runner — no extra dependencies. Run `npm ci` once, then `npm test` (`node --test test/*.test.js`). The tests start their own server on a random port with `HOME` and every platform directory pointed at a throwaway copy of `test/fixtures/home`, so your real session logs are never read or modified. CI (`.github/workflows/test.yml`) runs on Node 22 for every push and pull request to `master`, in four steps: `npm ci` (whose `prepare` script builds the web UI and regenerates `public/js/pure.js`), a drift check (`git diff --exit-code public/js/pure.js lib/generated/diagnostics.cjs`), `npx biome check .`, and `npm test`. **Adding a platform** takes two files: write one adapter in `lib/platforms/.js` (list / find / parse / normalize for that log format — `lib/platforms/shared.js` provides the metadata cache, the normalized-message factory and the session sort), then register it in the `PLATFORMS` table in `lib/platforms/index.js`. The generic session routes, search, watch (SSE tail), insights, prompts, tool audit, OTLP and Markdown/HTML export all resolve platforms through that registry — no other file needs to change. diff --git a/README.zh-CN.md b/README.zh-CN.md index f7e32fe..3320f64 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -54,6 +54,8 @@ LangSmith、Langfuse 这类观测平台面向的是*你自己写的* agent：接 ## 功能特性 +- **离线证据 CLI** — `agentxray inspect --platform codex session.jsonl --json` 无需启动服务或调用模型，输出带输入/规则哈希和原始行号的结构化报告，供 Agent、脚本和 CI 使用。只读明确指定的 Codex、OMP、Claude Code JSONL；默认不输出日志正文和参数，门禁须显式开启。[自动化契约](docs/offline-inspect.md)。 + - **自动会话体检** — 默认自动整理失败、重复操作、后续候选和调用最后记录状态；执行中、未知及未记录结果可追溯证据。不依赖人工标注或模型调用，笔记与迁移改为可选，不影响自动事实展示。[口径与离线验证](docs/diagnostics.md#自动体检无需人工标注)。 - **Codex 后台进程证据** — 用明确进程 ID 关联启动、`write_stdin` 轮询和退出结果，可逐步跳转原始证据。ID 重用、轮询交叠和冲突保持未知；不改写历史工具状态，不把进程退出当作任务通过。[关联边界](docs/diagnostics.md#codex-后台进程证据)。 - **修改—检查时序** — 区分修改前成功的检查、与修改重叠的检查及后续最新结果；不把先前通过或管道整体成功当作修改后的验证。复杂命令片段执行状态保持未知。[识别边界](docs/diagnostics.md#修改与验证的先后顺序)。 diff --git a/bin/agentxray.js b/bin/agentxray.js index fd46086..766faee 100755 --- a/bin/agentxray.js +++ b/bin/agentxray.js @@ -1,30 +1,35 @@ #!/usr/bin/env node // CLI entry: parse --port/--host, export them, then boot the server. const argv = process.argv.slice(2); -let port = process.env.PORT; -let host = process.env.HOST; +if (argv[0] === 'inspect') { + void require('./inspect').main(argv.slice(1)); +} else { + let port = process.env.PORT; + let host = process.env.HOST; -for (let i = 0; i < argv.length; i++) { - const arg = argv[i]; - const eq = arg.indexOf('='); - const flag = eq === -1 ? arg : arg.slice(0, eq); - const inline = eq === -1 ? null : arg.slice(eq + 1); - const next = () => (inline !== null ? inline : argv[++i]); - if (flag === '--port' || flag === '-p') port = next(); - else if (flag === '--host' || flag === '-H') host = next(); - else if (flag === '--version' || flag === '-v') { - console.log(require('../package.json').version); - process.exit(0); - } else if (flag === '--help' || flag === '-h') { - console.log('Usage: agentxray [--port ] [--host ] [--version]'); - process.exit(0); - } else { - console.error(`agentxray: unknown option '${arg}'`); - process.exit(1); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const eq = arg.indexOf('='); + const flag = eq === -1 ? arg : arg.slice(0, eq); + const inline = eq === -1 ? null : arg.slice(eq + 1); + const next = () => (inline !== null ? inline : argv[++i]); + if (flag === '--port' || flag === '-p') port = next(); + else if (flag === '--host' || flag === '-H') host = next(); + else if (flag === '--version' || flag === '-v') { + console.log(require('../package.json').version); + process.exit(0); + } else if (flag === '--help' || flag === '-h') { + console.log('Usage: agentxray [--port ] [--host ] [--version]'); + console.log('Offline evidence: agentxray inspect --help'); + process.exit(0); + } else { + console.error(`agentxray: unknown option '${arg}'`); + process.exit(1); + } } -} -if (port) process.env.PORT = String(port); -if (host) process.env.HOST = String(host); + if (port) process.env.PORT = String(port); + if (host) process.env.HOST = String(host); -require('../server.js'); + require('../server.js'); +} diff --git a/bin/inspect.js b/bin/inspect.js new file mode 100644 index 0000000..eb078b6 --- /dev/null +++ b/bin/inspect.js @@ -0,0 +1,77 @@ +const HELP = `Usage: agentxray inspect --platform [--json] [--fail-on pending-failures] + +Read one stable regular UTF-8 JSONL file (maximum 64 MiB), without starting a server. +Reports omit raw logs, paths, commands and IDs; source references are one-based lines/positions. +Exit 0: complete report, NOT task success. Exit 1: input/runtime/coverage error. +Exit 2: pending failure records found, only when --fail-on pending-failures is requested. +`; + +async function main(args) { + let filename, + platform, + json = false, + policy; + const seen = new Set(); + let positionalOnly = false; + try { + if (args.length === 1 && ['--help', '-h'].includes(args[0])) { + process.stdout.write(HELP); + return; + } + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (!positionalOnly && arg === '--') { + positionalOnly = true; + continue; + } + if (!positionalOnly && arg.startsWith('-')) { + const [flag, ...inline] = arg.split('='); + if (!['--platform', '--json', '--fail-on'].includes(flag) || seen.has(flag)) + throw new Error('Invalid or duplicate option.'); + seen.add(flag); + if (flag === '--json') { + if (inline.length) throw new Error('--json does not take a value.'); + json = true; + continue; + } + const value = inline.length ? inline.join('=') : args[++index]; + if (!value || value.startsWith('-')) throw new Error('Missing option value.'); + if (flag === '--platform') platform = value; + else policy = value; + } else { + if (filename !== undefined) throw new Error('Provide exactly one input file.'); + filename = arg; + } + } + if (!filename || !platform) throw new Error('Explicit --platform and one input file are required.'); + if (policy !== undefined && policy !== 'pending-failures') + throw new Error('Supported --fail-on policy: pending-failures.'); + } catch (error) { + process.stderr.write(`agentxray inspect: ${error.message}\n${HELP}`); + process.exitCode = 1; + return; + } + let implementation; + try { + implementation = require('../lib/inspect'); + } catch { + process.stderr.write('agentxray inspect: bundled rules unavailable; rebuild or reinstall the package.\n'); + process.exitCode = 1; + return; + } + const { inspectFile, renderText, InspectError } = implementation; + try { + const report = await inspectFile(filename, platform); + process.stdout.write(json ? `${JSON.stringify(report, null, 2)}\n` : renderText(report)); + process.exitCode = !report.complete ? 1 : policy && report.summary.pendingRecords ? 2 : 0; + if (!report.complete) + process.stderr.write('agentxray inspect: adapter coverage is incomplete; see report.coverage.issues.\n'); + } catch (error) { + process.stderr.write( + `agentxray inspect: ${error instanceof InspectError ? error.message : 'Inspection failed; no report generated.'}\n` + ); + process.exitCode = 1; + } +} + +module.exports = { main }; diff --git a/biome.json b/biome.json index ebd80d0..579522c 100644 --- a/biome.json +++ b/biome.json @@ -1,7 +1,16 @@ { "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", "files": { - "includes": ["server.js", "bin/**", "lib/**", "scripts/**", "public/js/**", "test/**/*.js", "!public/js/pure.js"] + "includes": [ + "server.js", + "bin/**", + "lib/**", + "scripts/**", + "public/js/**", + "test/**/*.js", + "!public/js/pure.js", + "!lib/generated" + ] }, "formatter": { "enabled": true, diff --git a/claims.json b/claims.json index f2716fd..cba5b0f 100644 --- a/claims.json +++ b/claims.json @@ -102,16 +102,16 @@ }, { "id": "test-count", - "claim": "313 tests pass on Node's built-in test runner, the count docs/ROADMAP.md records for `npm test`.", - "value": "313", - "metric": "passing node:test cases (# tests 313 / # pass 313 / # fail 0)", - "method": "npm test → node --test test/*.test.js, run in the claims job after npm ci, and the TAP summary is asserted. The roadmap sentence ('313 tests on Node's built-in runner (`npm test`, 2026-09-23)') is verified by the run, not read back from the prose.", + "claim": "332 tests pass on Node's built-in test runner, the count docs/ROADMAP.md records for `npm test`.", + "value": "332", + "metric": "passing node:test cases (# tests 332 / # pass 332 / # fail 0)", + "method": "npm test → node --test test/*.test.js, run in the claims job after npm ci, and the TAP summary is asserted. The roadmap sentence ('332 tests on Node's built-in runner (`npm test`, 2026-09-23)') is verified by the run, not read back from the prose.", "repro": "npm test 2>&1 | grep -E '^# (tests|pass|fail)'", "evidence": "docs/ROADMAP.md", "as_of": "2026-09-13", "check": { "cmd": "npm test 2>&1 | grep -E '^# (tests|pass|fail)'", - "expect": { "contains": ["# tests 313", "# pass 313", "# fail 0"] }, + "expect": { "contains": ["# tests 332", "# pass 332", "# fail 0"] }, "timeout": 120 } }, @@ -127,7 +127,7 @@ "check": { "cmd": "node scripts/claims-receipts.mjs ci-test-workflow", "expect": { - "equals": "1 job (test) on Node 22 · 4 steps: npm ci | git diff --exit-code public/js/pure.js | npx biome check . | npm test · on push, pull_request of master" + "equals": "1 job (test) on Node 22 · 4 steps: npm ci | git diff --exit-code public/js/pure.js lib/generated/diagnostics.cjs | npx biome check . | npm test · on push, pull_request of master" }, "timeout": 60 } @@ -238,7 +238,7 @@ "check": { "cmd": "node scripts/claims-receipts.mjs tests-node-only", "expect": { - "equals": "21 files in test/ · 15 distinct requires: 11 node builtins, 4 relative, 0 third-party" + "equals": "22 files in test/ · 16 distinct requires: 11 node builtins, 5 relative, 0 third-party" }, "timeout": 60 } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 92abafa..0e3adfd 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -8,7 +8,7 @@ - **Session browser** with tool-call inspection, trace/waterfall view, spawn tracking and message timeline - **Prompt tooling** — extraction (noise filtered), template clustering with outcome attribution, Claude-powered rewrites, and a prompt library that installs entries as native slash commands - **Global search** across all platforms, insights dashboard, incremental session backup -- **React + Vite frontend** served by an Express backend; 313 tests on Node's built-in runner (`npm test`, 2026-09-23), CI on Node 22 +- **React + Vite frontend** served by an Express backend; 332 tests on Node's built-in runner (`npm test`, 2026-09-23), CI on Node 22 - **Evidence-backed failure events and local review** with full-result invalidation, evidence navigation and narrow-screen session layout ## Current priorities @@ -24,6 +24,10 @@ External grounding: official guidance emphasizes [executable verification](https | P2 | Execution/verification evidence, not another statistics dashboard | Automatic health and modification/check chronology distinguish before/overlap/after and unknown outcomes. Codex process IDs now connect launch/poll/exit evidence without rewriting history or original check start. Next investigate explicit task/child linkage and per-step outcomes; reject ambiguous associations rather than relaxing shell assumptions. Deterministic tests and frozen-log checks are not file coverage, human time saved or universal accuracy. | | P3 | Make releases reproducible for contributors | Keep clean-install tests, generated fixtures, documentation claims and release/package verification aligned. Add browser regression automation when it can run deterministically without personal logs. | +### Machine-consumable evidence + +The next integration surface is a read-only offline CLI, not a new agent runtime or hosted service. `inspect` emits versioned facts and source references using the same generated rules as the UI. This lets an agent or CI step consume evidence without manual tagging, a browser or model scoring. Explicit exit policies and coverage failures must never become a generic "task passed" claim. Current scope is one stable Codex/OMP/Claude Code JSONL file; logs remain on the machine. + No launch dates or star-count targets are promised. Progress is gated on these observable outcomes. Physical-device/keyboard coverage and complex Trace/analytics layouts remain separate work, not implied by the session-screen checks. ## Existing backlog diff --git a/docs/diagnostics-verification.md b/docs/diagnostics-verification.md index 16469de..77c6e28 100644 --- a/docs/diagnostics-verification.md +++ b/docs/diagnostics-verification.md @@ -130,3 +130,17 @@ The frozen real corpus was re-read under content hashes: all previous `diagnoseS Synthetic browser checks validate the process summary, unlinked poll count, source-result navigation, a test launched before an edit remaining overlapping, a later failed check remaining visible and live completion changing terminal process count from 2 to 3 without changing historical failure events. At 390px the process panel has no horizontal overflow. `node scripts/demo-process-evidence.cjs` reproduces the example locally without executing transcript commands. Unknown/conflicting chains are not silently certified; callers who need live job control or cross-session task association still need stronger runtime evidence. Input is represented as a boolean in the process summary, though original call evidence remains accessible. Existing large-bundle and lint findings remain unchanged. + +## Offline inspect acceptance + +`agentxray inspect --platform --json` adds a machine-consumable surface without a web server. The CommonJS rules are generated from the UI's TypeScript source, packaged under `lib/generated/`, compared against source in tests and checked for committed drift in CI. Historical UI diagnostic rules are unchanged. + +The full suite now has **332 passing tests**, including **19 inspect tests**. Tests verify deterministic JSON, exact source references, minimized outputs, policy exits, malformed/truncated input, invalid UTF-8, directories/missing/oversized files, wrong platform, changed-read metadata, missing flags, blank/CRLF physical lines and generated-rule parity. A runtime guard blocks Express/http/https/net/server imports while the CLI runs; input bytes and temporary HOME entries remain unchanged. + +Known Claude multi-result and text/result mixed records are tested as incomplete adapter coverage: JSON contains `complete:false`, raw/normalized counts and issue lines, and exit status is 1 even when a pending-failure gate was requested. This exposes an existing parser limitation instead of claiming it is fixed. Parsing failures output no partial report or sensitive line text. + +An actual npm tarball was unpacked to an isolated directory with no `node_modules`, frontend source or TypeScript compiler. Inspect produced the expected OMP synthetic report (7 pending records, 2 events) directly from that artifact. This tests standalone inspection, not the dashboard dependency requirements. + +The previously frozen 15 real sessions / 2,024 tool results were read only in memory. CLI report summaries matched UI rules exactly: 142 failures, 141 pending records, 76 events and 60 recorded Codex launches. **2,493 source references** were verified against the frozen original line/message mapping; repeated reports were byte-identical. No real raw text, commands, paths, process identifiers or human notes were emitted into public artifacts. This is rule/report parity, not real-world accuracy or evidence of time saved. + +Reports omit raw content by construction, but hashes, counts and associations may still be sensitive. The output is not anonymized, signed task proof or an autonomous safety decision. Exit 0 means valid report production; only the explicitly requested `pending-failures` policy returns 2. See [the offline contract](offline-inspect.md). diff --git a/docs/offline-inspect.md b/docs/offline-inspect.md new file mode 100644 index 0000000..1cd2772 --- /dev/null +++ b/docs/offline-inspect.md @@ -0,0 +1,127 @@ +# Offline evidence for agents and automation + +AgentXRay's UI is useful for people, but a coding agent, local script or CI step should not need to launch a web server to inspect recorded execution evidence. `inspect` uses the same diagnostic source as the React view and produces a versioned report with original line references. + +It reads exactly one file you specify. It does not discover your sessions, connect to model APIs, execute logged commands, create an archive, read review notes or start the dashboard. + +## Quick start + +After installing AgentXRay, or through `npx`: + +```sh +agentxray inspect --platform codex /path/to/session.jsonl +agentxray inspect --platform omp /path/to/session.jsonl --json > evidence.json +npx @alloevil/agent-xray inspect --platform claude-code /path/to/session.jsonl --json +``` + +`npx`/package installation may access the npm registry. The **inspect command itself** does not use network services. Prefer an installed, pinned package in repeatable or network-isolated jobs. + +From a source checkout with dependencies installed: + +```sh +npm run build:diagnostics +node bin/agentxray.js inspect --platform omp \ + frontend/demo/sample-logs/omp/-demo-diagnostics/2026-09-23T08-00-00-000Z_0199demo-diagnostics.jsonl --json +``` + +This synthetic sample reports 8 historical failures, 7 pending records in 2 events, and 1 matching recovery. **Default exit status is zero even when there are findings**: it means the report was produced, not that the agent's task passed. + +### Input contract + +- Required `--platform`: `codex`, `omp` or `claude-code`. These are the formats validated for the offline path; the dashboard supports additional platforms, but this command does not claim parity for their containers/databases. +- Exactly one regular, stable UTF-8 JSONL file, at most **64 MiB**. No directories, stdin, compressed streams or database files. Use `--` before a filename beginning with `-`. +- Reads the initial file length and checks metadata again after reading. If the source changed, retry when it is stable. The output hash identifies the bytes actually analyzed; this is not a filesystem transaction or tamper-proof archive. +- Invalid UTF-8, malformed/truncated JSON, non-object records, incompatible shape, empty/metadata-only input and wrong selected format fail rather than being silently treated as clean sessions. Valid last lines need not end with a newline; blank lines are counted in physical line positions. +- Known metadata/unhandled records are counted as ignored. Recognized raw tool-call/result IDs are checked against normalization per line. Known loss (for example a Claude multi-result or mixed text/result record not fully represented by the adapter) produces `complete:false` and a coverage issue. This is honest reporting of adapter limits, not a new parser that fixes those formats. + +## Report contract: schemaVersion 1 + +`--json` writes one JSON document to stdout. Errors go to stderr, without original paths, input text or stack traces. Parsing/read failures produce no report; known adapter coverage gaps produce a report with `complete:false` and exit 1. + +| Field | Meaning | +| --- | --- | +| `schemaVersion` | Report contract version, currently 1. Check it before consuming fields. | +| `complete` | The input parsed and known raw tool identity/error-marker coverage checks passed. **Not full semantic coverage, task success or absence of unknown formats.** | +| `engine` | Package version, SHA-256 of the UI diagnostic source, and adapter/shared-normalization source hash. | +| `source` | Selected platform, input byte length and SHA-256; no filename/path or raw session ID. | +| `coverage` | Physical lines, JSON records, normalized messages, ignored records, raw/normalized tool counts and per-line coverage issues. | +| `summary` | Same aggregate health fields as the UI: failures, exact-match recovery, pending events, repeated records, call states and distinct related results. | +| `events` | Failure source references, grouping facts and subsequent related-call/result references. | +| `gaps` | Running/unknown/no-result evidence references and reason; not live-process state. | +| `processes` | Codex launch/poll/final-result references, state, exit code, input-observed boolean and ambiguity codes. Raw process IDs are omitted. | +| `chronology` | Recognized check and modification source references, before/overlap/after relationships, latest outcome and scope limitations. | +| `limits` | Human-readable reminders that association and timing do not prove coverage or correctness. | + +A source reference is `{ "line": 12, "messageIndex": 9 }`, with **one-based** physical JSONL line and normalized message position. One raw record can fan out to multiple normalized messages; two different references may have the same physical line. References are valid only for the input bytes matching `source.sha256`. Preserve the original file locally if you need the actual evidence. The report cannot reconstruct omitted content. + +Reports omit raw prompts, outputs, command arguments, paths, call/process IDs and human notes. Tool names outside a fixed common-tool vocabulary become `other`. Aggregate counts, relations and source hashes can still disclose activity or equality of inputs: **minimized is not anonymized**, and you should still review sharing/retention decisions. There is no raw-content opt-in flag in this version. + +The same bytes, platform, package and rule/adapter versions yield the same report bytes. No wall-clock generation timestamp, random identifier or measured runtime is mixed into the report. + +## Explicit exit policy + +| Exit | Meaning | +| --- | --- | +| 0 | Report generated with known adapter coverage checks satisfied. Findings may exist; no task-success claim. | +| 1 | Argument/input/read/runtime error or incomplete adapter coverage. Coverage failure takes precedence over findings. | +| 2 | `--fail-on pending-failures` was explicitly requested and at least one failure lacks the existing rule's exact matching later success. | + +```sh +agentxray inspect --platform codex session.jsonl --json \ + --fail-on pending-failures > evidence.json +``` + +Do not make this policy a universal "agent passed" gate. A negative probe may intentionally return nonzero; alternate verification may not match the original arguments. The command does not infer equivalent tests, implicit cwd or test coverage. + +A CI shell can keep both findings and parse errors distinguishable: + +```sh +status=0 +agentxray inspect --platform codex session.jsonl --json \ + --fail-on pending-failures > evidence.json || status=$? +case "$status" in + 0) echo "Evidence report generated; task correctness not asserted" ;; + 2) echo "Pending failure records: inspect evidence.json and source lines" ;; + *) echo "Inspection incomplete or invalid; do not accept as clean" >&2 ;; +esac +exit "$status" +``` + +## A small machine consumer + +Use stdout as data, never as executable instructions. For example, after generating a complete `evidence.json`: + +```js +const fs = require('node:fs'); +const report = JSON.parse(fs.readFileSync('evidence.json', 'utf8')); +if (report.schemaVersion !== 1 || !report.complete) { + throw new Error('Unsupported or incomplete evidence report'); +} +for (const event of report.events) { + console.log({ tool: event.tool, reason: event.reason, sourceLines: event.failures.map(ref => ref.line) }); +} +``` + +This can feed an agent's review step or a build artifact without a browser or model judge. It should inform an explicit next check, not automatically rerun a possibly destructive original command. + +## Keeping UI and CLI aligned + +The source of truth remains `frontend/src/views/sessions/diagnostics.ts`. `npm run build:diagnostics` uses the existing frontend esbuild dependency to generate `lib/generated/diagnostics.cjs`; `build:ui` regenerates it too. Installed packages contain the prebuilt CommonJS artifact and do not need TypeScript, esbuild or a frontend checkout to run `inspect`. + +Tests compare actual CLI output with the TypeScript UI functions and validate every reference. `node scripts/build-diagnostics.mjs --check` verifies generated contents, and CI checks the committed bundle diff after the normal build. Do not hand-edit the generated file. + +## 中文使用与边界 + +离线核验不要求开网页或人工标注，供本机 Agent、脚本与 CI 消费同一套诊断事实： + +```sh +agentxray inspect --platform omp session.jsonl --json > evidence.json +``` + +当前只支持明确指定的 Codex、OMP、Claude Code JSONL，单文件上限 64 MiB。读取期间变化、截断坏行、错误格式都会失败；已知标准化丢失会输出 `complete:false` 与问题行号，不会假绿。 + +报告默认不含日志正文、路径、命令参数、原始 ID 或笔记，只保留计数、状态、关系、输入/规则哈希和原始行号。哈希与活动计数仍可能敏感，不是匿名化保证。原始文件必须自行保留，报告不能重建日志。 + +退出码 **0 表示报告生成成功，不是任务通过**；1 表示输入/覆盖/运行错误；只有显式 `--fail-on pending-failures` 才因待闭合失败返回 2。调用方应检查 `schemaVersion`、`complete` 和退出码，不把“没有识别到失败”当作完成证明。 + +安装包运行无需编译器、UI 源码或服务。`npx` 首次下载可能联网，安装后的 `inspect` 本身不发网络请求、不执行日志命令、不扫描 HOME、不创建归档或人工笔记。 diff --git a/docs/releases/v1.23.0.md b/docs/releases/v1.23.0.md new file mode 100644 index 0000000..16ffd77 --- /dev/null +++ b/docs/releases/v1.23.0.md @@ -0,0 +1,27 @@ +# v1.23.0 — Offline execution evidence for agents and CI + +AgentXRay's evidence rules can now be used without a dashboard. Inspect one selected session log and give an agent, script or CI step a structured report with source references and explicit uncertainty. + +```sh +npx @alloevil/agent-xray@1.23.0 inspect --platform codex session.jsonl --json +``` + +## New + +- **No-server inspect command:** supports explicitly selected Codex, OMP and Claude Code UTF-8 JSONL files, up to 64 MiB. No model call, command execution, session discovery or background service. +- **Shared rules:** CLI uses a generated CommonJS build of the same TypeScript diagnostic source as the UI. Installed packages do not require frontend source or a compiler. CI verifies generated drift. +- **Minimized versioned JSON:** input/rule/adapter hashes, automatic summary, failure events, related calls, result gaps, Codex process chains and modification/check chronology, with one-based original line and message references. No automatically copied raw logs, paths, arguments, IDs or human notes. +- **Honest exits:** 0 means a report was generated, not task success; 1 means input/runtime/adapter-coverage failure; 2 means an explicitly requested `--fail-on pending-failures` policy found pending records. +- **Incomplete input is not clean:** reject bad/truncated JSON, invalid UTF-8, changed reads and incompatible formats. Known dropped multi-result/mixed Claude records produce coverage issues and exit 1, not a false green result. + +## Verification + +332 Node tests passed, including 19 inspect cases covering rule parity, source references, deterministic output, sensitive-content omission, no-server/network operation, read-only behavior, error codes and snapshot changes. A packed tarball ran inspect with neither node_modules nor frontend source. A frozen 15-session/2,024-result regression matched UI summaries and verified 2,493 source references. + +UI build, lint and 16 automatic claims checks pass with existing lint findings unchanged. No change to failure/recovery rules or the browser workflow. + +## Boundaries + +The CLI supports three JSONL formats, not every dashboard adapter or storage container. Reports do not reconstruct raw logs, prove test coverage or evaluate task correctness. Hashes and aggregate activity are not guaranteed anonymous. No-results/unknown state and unclassified commands stay uncertain. `npx` installation can contact the registry; the installed inspect operation itself is offline. + +[Usage, schema and CI examples](https://github.com/alloevil/AgentXRay/blob/master/docs/offline-inspect.md) · [Verification receipt](https://github.com/alloevil/AgentXRay/blob/master/docs/diagnostics-verification.md) diff --git a/intent.md b/intent.md index d8fb2c9..ca25b16 100644 --- a/intent.md +++ b/intent.md @@ -110,6 +110,18 @@ Validation: 290 tests pass, including 200 fixed health transformations and 120 t Validation results: the frozen corpus has 60 Codex starts and 53 uniquely linked polls: 48 successful process exits, 1 failed exit and 11 last-recorded running starts. All prior per-call health and failure/candidate/recovery outputs remain identical. The implementation adds 23 tests including 40 fixed terminal transformations and generated-demo/raw-parser parity. Synthetic browser checks cover terminal-result jumps, original-start overlap semantics, unlinked poll counts, live completion and 390px layout. Prepare v1.22.0 through the previously authorized protected PR/release workflow after final checks; no private evidence or logs are published. +## Offline evidence CLI (2026-09-24) + +- The next product increment makes existing evidence usable by agents, local automation and CI without starting a dashboard. Prefer a local `inspect` command over a new MCP service/hook/runtime dependency; no execution, network, recursive discovery or log upload. +- `agentxray inspect --platform omp|codex|claude-code [--json] [--fail-on pending-failures]` inspects exactly one explicitly supplied regular UTF-8 JSONL file (maximum 64 MiB). No auto-detection or directory scans. Read a fixed-sized snapshot and reject changed file metadata, malformed lines, invalid UTF-8 and wrong/empty formats. Preserve original physical line positions. +- Use the same normalizers and diagnostic source as the UI. Generate a CommonJS build from the TypeScript rule source using the existing build toolchain, package it and enforce generated drift in tests. Include rule hash and adapter hash in versioned deterministic reports; no copied rule implementation or TypeScript runtime requirement in installed packages. +- Report only whitelisted aggregate facts and source references (physical line plus normalized message index, one-based). Default reports omit file names, paths, prompts, outputs, command arguments, raw call/process IDs, private tool names and user notes. Input/content hashes still reveal equality, so this is minimized evidence, not guaranteed anonymization. +- Report failure events, source membership, follow-up relations, call-result gaps, process launch/poll/terminal links and modification/check chronology. Preserve unknown and scope limits. Known adapter loss of tool-call/result identity or error markers must be counted as coverage issues and prevent a successful exit, not quietly certify a partial report. +- Exit 0 means a complete report was produced, not task success. Exit 1 means invalid input, runtime/read error or incomplete adapter coverage. Exit 2 is reserved for an explicitly requested pending-failures policy. Stdout JSON stays parseable; human-safe errors go to stderr without dumping original input or paths. Never treat absence of recognized failures/checks as task correctness. +- Verify UI/CLI rule parity, deterministic bytes, every source reference, sensitive-content omission, error policy, no server/network startup, no HOME/fixture mutation, packaged-artifact operation and frozen real-log counts. Prior browser data and original sessions stay untouched. Add documented automation examples and an explicit contract, not claims of autonomous truth. + +Acceptance: 332 tests pass, including 19 CLI/contract cases. Packed tarball inspection runs without node_modules/frontend source. The 15 frozen sessions reproduce 2,024 result records, 142 failures, 141 pending records, 76 events and 60 process launches with UI-rule parity; all 2,493 report references are valid. Known Claude adapter loss is explicitly incomplete rather than green. Stage only code, shared generated rules, synthetic tests and public documentation, not private artifacts. Publish v1.23.0 through the authorized protected PR workflow after final gates; validate actual registry installation before claiming release success. The ongoing product goal remains active. + ## Continuing boundaries - No new platform, dependency, model call, account, telemetry, cloud log storage or automatic command execution. diff --git a/lib/generated/diagnostics.cjs b/lib/generated/diagnostics.cjs new file mode 100644 index 0000000..72ee093 --- /dev/null +++ b/lib/generated/diagnostics.cjs @@ -0,0 +1,13 @@ +var O=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var L=Object.getOwnPropertyNames;var W=Object.prototype.hasOwnProperty;var B=(e,t)=>{for(var n in t)O(e,n,{get:t[n],enumerable:!0})},G=(e,t,n,o)=>{if(t&&typeof t=="object"||typeof t=="function")for(let f of L(t))!W.call(e,f)&&f!==n&&O(e,f,{get:()=>t[f],enumerable:!(o=J(t,f))||o.enumerable});return e};var V=e=>G(O({},"__esModule",{value:!0}),e);var te={};B(te,{analyzeCodexProcesses:()=>q,analyzeVerificationChronology:()=>ne,diagnoseSession:()=>A,summarizeSessionHealth:()=>U});module.exports=V(te);function Z(e){let t=e.map(n=>n.message.timestamp?Date.parse(n.message.timestamp):NaN);return t.some((n,o)=>!Number.isFinite(n)||o>0&&nt.text||"").join(` +`)}function K(e){if(typeof e!="string")return e;try{return JSON.parse(e)}catch{return e}}function S(e){return Array.isArray(e)?e.map(S):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>[t,S(n)])):e}function _(e,t){let n=e?.name||t.toolName||"";if(/(?:^|[./_])(bash|shell|exec|exec_command|run_shell_command|execute_command|terminal|write_stdin)$/i.test(n))return!0;let o=e?.args;return!!o&&typeof o=="object"&&("command"in o||"cmd"in o)}function E(e,t){let n=e.details?.exitCode??e.details?.exit_code;if(typeof n=="number"&&Number.isInteger(n))return n;if(!_(t,e))return null;let o=v(e);if(!/^(?:Chunk ID:|Wall time:)/.test(o))return null;let f=o.search(/^(?:Final output|Output):\s*$/m);if(f<0)return null;let p=o.slice(0,f).match(/^(?:Process exited with code |Exit code: )(-?\d+)\s*$/m);return p?Number(p[1]):null}function z(e,t){if(e.ompOutcome){let o=e.ompOutcome.state;return o==="success"||o==="failure"?o:"unknown"}let n=E(e,t);return e.isError===!0||n!==null&&n!==0?"failure":n===0?"success":["running","pending","in_progress"].includes(String(e.details?.status))||_(t,e)?"unknown":e.isError===!1&&e.content!==null?"success":"unknown"}function w(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)?e:null}function H(e,t){let n=w(t);if(!n)return null;let o=Object.entries(n).filter(([f])=>f!=="i");return o.length?JSON.stringify([e,S(Object.fromEntries(o))]):null}function Q(e,t){if(!["edit","Edit","write","Write","MultiEdit"].includes(e))return null;let n=w(t);if(!n)return null;let o=["path","file_path"].filter(s=>s in n).map(s=>n[s]);if(!o.length||o.some(s=>typeof s!="string"||!s||s!==o[0]))return null;let f=o[0],p=["cwd","workdir","working_directory"].filter(s=>s in n).map(s=>[s,n[s]]);if(p.some(([,s])=>typeof s!="string"||!s)||new Set(p.map(([,s])=>s)).size>1)return null;let h=s=>/^(?:\/|[A-Za-z]:[\\/]|\\\\)/.test(s);return!h(f)&&(!p.length||!h(p[0][1]))?null:JSON.stringify([f,p])}function P(e,t){if(e.ompOutcome)return e.ompOutcome.state;let n=E(e,t);return e.isError===!0||n!==null&&n!==0?"failure":["running","pending","in_progress"].includes(String(e.details?.status))?"running":["cancelled","canceled"].includes(String(e.details?.status))?"cancelled":e.details?.status!=null&&!["ok","success","complete","completed"].includes(String(e.details.status))?"unknown":z(e,t)}function $(e,t,n){let o=E(e,t);return[...e.ompOutcome?.evidence||[...e.isError?["日志标记 isError=true"]:[],...o!==null?[`退出码 ${o}`]:[],...n==="running"?[`details.status=${e.details?.status}`]:[],...n==="cancelled"?[`details.status=${e.details?.status}`]:[],...o===null&&n==="success"?["isError=false；工具结果未报错（不等于任务通过）"]:[],...n==="unknown"?["缺少明确完成状态"]:[]],v(e)].join(` +`).slice(0,500)}function A(e){let t=new Map,n=new Map,o=[],f=new Set,p=new Map,h=new Map,s=new Map,g=new Map,a=new Map,l=0;function m(i,d,c,b){if(!i)return;let C=K(c),y=d&&C!==null&&C!==void 0?JSON.stringify([d,S(C)]):null;if(t.set(i,{name:d||"未知工具",args:C,key:y,index:b,userTurn:l,onlyIKey:d?H(d,C):null,fileKey:d?Q(d,C):null}),y)for(let k of n.get(y)||[])b>k.index&&(k.reason="unconfirmed-retry")}e.forEach((i,d)=>{if(i.role==="user"&&l++,i.role==="toolCall"&&m(i.toolCallId||i.id,i.toolName,i.details,d),i.role==="assistant")for(let C of i.content||[])C.type==="toolCall"&&m(C.id,C.name,C.arguments??C.input,d);if(i.role!=="toolResult")return;let c=i.toolCallId?t.get(i.toolCallId):void 0;if(c?.key){let C={message:i,index:d,call:c};for(let[y,k]of[[c.onlyIKey,g],[c.fileKey,a]]){if(!y)continue;let M=k.get(y)||[];M.push(C),k.set(y,M)}}let b=z(i,c);if(b==="failure"){let C=E(i,c),y={message:i,index:d,toolName:c?.name||i.toolName||"未知工具",argumentsText:c?.key?JSON.stringify(c.args):null,evidence:[...i.ompOutcome?.evidence||[],i.isError?"日志标记 isError=true":null,C!==null?`退出码 ${C}`:null,v(i)].filter(k=>k!==null).join(` +`).slice(0,500),reason:c?.key?"no-success":"missing-call"};if(o.push(y),c?.key&&s.set(y,c),p.set(y,{key:c?.key?JSON.stringify([c.userTurn,c.key,h.get(c.key)||0]):`orphan-${d}`,userTurn:c?.userTurn??l}),c?.key){let k=n.get(c.key)||[];k.push(y),n.set(c.key,k)}}else if(b==="success"&&c?.key){h.set(c.key,(h.get(c.key)||0)+1);let C=n.get(c.key)||[];for(let y of C)c.index>y.index&&f.add(y);n.set(c.key,C.filter(y=>!f.has(y)))}});let x=o.filter(i=>!f.has(i)),r=new Map;for(let i of x){let d=p.get(i),c=r.get(d.key);c?c.failures.push(i):r.set(d.key,{id:`failure-event-${i.index}`,toolName:i.toolName,argumentsText:i.argumentsText,userTurn:d.userTurn,failures:[i],spanMs:null,relatedOperations:[]})}let u=[...r.values()];for(let i of u){i.spanMs=Z(i.failures);let d=i.failures[i.failures.length-1],c=s.get(d);if(!c)continue;let b=new Map,C=[["only-i",c.onlyIKey?g.get(c.onlyIKey):void 0],["same-file",c.fileKey?a.get(c.fileKey):void 0]];for(let[y,k]of C)for(let{message:M,index:R,call:I}of k||[]){if(I.index<=d.index||I.key===c.key||b.has(R)||y==="same-file"&&I.userTurn!==c.userTurn)continue;let F=P(M,I);b.set(R,{message:M,index:R,callIndex:I.index,userTurn:I.userTurn,toolName:I.name,argumentsText:JSON.stringify(I.args),relation:y,state:F,evidence:$(M,I,F)})}i.relatedOperations=[...b.values()].sort((y,k)=>y.index-k.index)}return u.sort((i,d)=>d.failures.length-i.failures.length||i.failures[0].index-d.failures[0].index),{failures:x,failureCount:o.length,recoveredCount:f.size,events:u}}function D(e){let t=[],n=new Map,o=0;e.forEach((s,g)=>{s.role==="user"&&o++;function a(l,m,x){let r={id:l||null,message:s,call:{name:m||"未知工具",args:K(x),key:null,onlyIKey:null,fileKey:null,index:g,userTurn:o}};if(t.push(r),l){let u=n.get(l)||[];u.push(r),n.set(l,u)}}if(s.role==="toolCall"&&a(s.toolCallId||s.id,s.toolName,s.details),s.role==="assistant")for(let l of s.content||[])l.type==="toolCall"&&a(l.id,l.name,l.arguments??l.input)});let f=0,p=0,h=0;return e.forEach((s,g)=>{if(s.role!=="toolResult")return;h++;let a=s.toolCallId?n.get(s.toolCallId):void 0;if(!a?.length||a.every(l=>l.call.index>=g)){f++;return}if(a.length!==1){p++;return}a[0].result={message:s,index:g}}),{calls:t,byId:n,orphanResults:f,unassignedResults:p,toolResultCount:h}}function U(e,t=A(e)){let{calls:n,byId:o,orphanResults:f,unassignedResults:p,toolResultCount:h}=D(e),s={success:0,failure:0,running:0,cancelled:0,unknown:0,"no-result":0},g=[],a=0;for(let r of n){let u=!r.id||(o.get(r.id)?.length??0)!==1,i=u?"unknown":r.result?P(r.result.message,r.call):"no-result";if(s[i]++,u&&a++,i!=="running"&&i!=="unknown"&&i!=="no-result")continue;let d=u?void 0:r.result;g.push({message:d?.message||r.message,index:d?.index??r.call.index,toolCallId:r.id,toolName:r.call.name,state:i,reason:u?"ambiguous-id":d?"last-result":"no-result",evidence:u?"调用标识缺失或重复，无法唯一关联结果。":d?$(d.message,r.call,i==="no-result"?"unknown":i):"截至当前已加载日志，没有记录到此调用的结果；不代表任务失败或进程仍在运行。"})}let l=new Map;for(let r of t.events)for(let u of r.relatedOperations)l.set(u.index,u);let m={success:0,failure:0,running:0,cancelled:0,unknown:0};for(let r of l.values())m[r.state]++;let x=t.events.filter(r=>r.failures.length>1);return{callCount:n.length,toolResultCount:h,callStates:s,orphanResults:f,unassignedResults:p,ambiguousCalls:a,gaps:g,failureRecords:t.failureCount,pendingRecords:t.failures.length,pendingEvents:t.events.length,recoveredRecords:t.recoveredCount,repeatedEvents:x.length,repeatedRecords:x.reduce((r,u)=>r+u.failures.length,0),candidateEvents:t.events.filter(r=>r.relatedOperations.length>0).length,candidateResults:l.size,candidateStates:m}}function T(e){return e.startsWith("functions.")?e.slice(10):e}function N(e){if(e.ompOutcome||e.isError)return null;let t=v(e);if(!/^(?:Chunk ID: [^\n]+\n)?Wall time: [\d.]+ seconds\n/.test(t))return null;let n=t.search(/^(?:Final output|Output):\s*$/m);if(n<0)return null;let o=t.slice(0,n).split(` +`).filter(h=>/^(?:Process (?:running|exited)|Exit code:)/.test(h));if(o.length!==1)return null;let f=o[0].match(/^Process running with session ID (0|[1-9]\d*)$/),p=o[0].match(/^(?:Process exited with code |Exit code: )(-?\d+)$/);if(f&&Number.isSafeInteger(Number(f[1])))return e.details?.exitCode!=null||e.details?.exit_code!=null?null:{processId:Number(f[1]),exitCode:null};if(p&&Number.isSafeInteger(Number(p[1]))){let h=Number(p[1]),s=e.details?.exitCode??e.details?.exit_code;return s!=null&&s!==h?null:{processId:null,exitCode:h}}return null}function q(e){let{calls:t,byId:n}=D(e),o=new Map;e.forEach((l,m)=>{if(l.role!=="toolResult"||!l.toolCallId)return;let x=o.get(l.toolCallId)||[];x.push({message:l,index:m}),o.set(l.toolCallId,x)});let f=[],p=new Map,h=l=>!!l.id&&n.get(l.id)?.length===1;for(let l of t){if(T(l.call.name)!=="exec_command")continue;let m=l.id?o.get(l.id)||[]:[],x=m.find(c=>c.index>l.call.index&&N(c.message)?.processId!=null);if(!x)continue;let r=N(x.message).processId,u=!h(l)||m.length!==1?["ambiguous-launch"]:[],i={processId:r,launchMessage:l.message,launchIndex:l.call.index,launchCallId:l.id,launchResult:x.message,launchResultIndex:x.index,polls:[],state:"running",exitCode:null,finalMessage:null,finalIndex:null,inputObserved:!1,issues:u};f.push(i);let d=p.get(r)||[];d.push(i),p.set(r,d)}for(let l of p.values())if(l.length>1)for(let m of l)m.issues.push("reused-process-id");let s=0,g=0,a=0;for(let l of t){if(T(l.call.name)!=="write_stdin")continue;s++;let m=w(l.call.args),x=m?.session_id,r=typeof x=="number"&&Number.isSafeInteger(x)&&x>=0?p.get(x):void 0;if(!h(l)){a++,r?.length===1&&r[0].issues.push("ambiguous-poll");continue}if(r?.length!==1||r[0].issues.includes("ambiguous-launch")||l.call.index<=r[0].launchResultIndex)continue;let u=r[0],i=o.get(l.id)||[],d=i.length===1&&i[0].index>l.call.index?i[0]:void 0,c=d?N(d.message):null,b=m?.chars!=null&&m.chars!=="",C={callMessage:l.message,callIndex:l.call.index,toolCallId:l.id,resultMessage:d?.message??null,resultIndex:d?.index??null,hasInput:b,state:i.length?c?c.exitCode!==null?c.exitCode===0?"success":"failure":"running":"unknown":"no-result",exitCode:c?.exitCode??null},y=u.polls[u.polls.length-1];y&&(y.resultIndex===null||y.resultIndex>=l.call.index)&&u.issues.push("overlapping-polls"),u.finalMessage&&u.issues.push("poll-after-terminal"),(!d||!c)&&u.issues.push("unconfirmed-poll-result"),c?.processId!=null&&c.processId!==u.processId&&u.issues.push("mismatched-process-id"),u.polls.push(C),g++,u.inputObserved||=b,c?.exitCode!=null&&(u.finalMessage=d.message,u.finalIndex=d.index,u.exitCode=c.exitCode,u.state=c.exitCode===0?"success":"failure")}for(let l of f)l.issues=[...new Set(l.issues)],l.issues.length&&(l.state="unknown",l.exitCode=null,l.finalMessage=null,l.finalIndex=null);return{processes:f,pollCalls:s,linkedPolls:g,unlinkedPolls:s-g,ambiguousCalls:a}}function X(e){if(/[`$\\(){}\0]|<<|\|\||\r/.test(e)||e.length>2e4)return null;let t=[],n=[],o="",f=!1,p="",h=()=>{(o||f)&&n.push({value:o,quoted:f}),o="",f=!1};for(let s=0;s"||g==="<"){h();let a=g;e[s+1]===">"&&(a+=">",s++),e[s+1]==="&"&&/[0-9]/.test(e[s+2]||"")&&(a+=`&${e[s+2]}`,s+=2),n.push({value:a,quoted:!1});continue}o+=g}if(p)return null;if(h(),n.length)t.push({tokens:n,separator:""});else if(t.length&&["&&","|"].includes(t[t.length-1].separator))return null;return t.some(s=>["if","then","else","elif","fi","for","while","do","done","case","esac","function","until","!"].includes(s.tokens[0]?.value))?null:t}function Y(e){if(e.some(f=>/^(?:--(?:help|version|watch(?:All)?|collect-only|collectonly|co|list-tests|listTests|list|dry-run|prefix|cwd|directory)|-[hVwCc])(?:=|$)/i.test(f)))return null;if(e[0]==="pytest"||["python","python3"].includes(e[0])&&e[1]==="-m"&&e[2]==="pytest"||e[0]==="node"&&e[1]==="--test")return"runner-command";let n=e[0];if(!["npm","pnpm","yarn"].includes(n))return null;let o=e[1]==="run"?2:1;return!["test","build","lint","typecheck"].includes(e[o])||n==="npm"&&o===1&&e[1]!=="test"?null:"script-name"}function ee(e){if(!["bash","Bash","shell","exec","exec_command","run_shell_command","execute_command","terminal"].includes(T(e.name)))return null;let t=w(e.args);if(!t)return null;let n=["command","cmd"].filter(m=>m in t).map(m=>t[m]);if(!n.length||n.some(m=>typeof m!="string"||m!==n[0]))return null;let o=n[0],f=X(o);if(!f?.length)return null;let p=f.flatMap((m,x)=>{if(m.tokens[0]?.quoted)return[];let r=m.tokens.map(i=>i.value),u=Y(r);return u?[{index:x,basis:u,fragment:r.join(" ")}]:[]});if(!p.length)return null;let h=m=>m.tokens.every(x=>!/[<>]/.test(x.value)),s=f[0],g=f.length===1&&h(f[0]),a=s.separator==="&&"&&s.tokens.length===2&&s.tokens[0].value==="cd"&&!s.tokens[0].quoted&&/^\//.test(s.tokens[1].value)&&p.length===1&&p[0].index===1,l=a&&f.length===2&&h(f[1]);return{command:o,basis:p[0].basis,fragments:p.map(m=>m.fragment),commandMode:g?"direct":l?"directory-prefix":"compound-fragment",explicitDirectory:a?s.tokens[1].value:null,usesInitialDirectory:p.length===1&&p[0].index===0}}function j(e,t){let n=w(e.call.args),o=n?["cwd","workdir","working_directory"].filter(g=>g in n).map(g=>n[g]):[],f=o.some(g=>typeof g!="string"||!g)||new Set(o).size>1,p=!f&&typeof o[0]=="string"&&/^(?:\/|[A-Za-z]:[\\/]|\\\\)/.test(o[0])?o[0]:null,h=t?void 0:e.result,s=t?"unknown":h?P(h.message,e.call):"no-result";return{callMessage:e.message,callIndex:e.call.index,toolCallId:e.id,toolName:e.call.name,resultMessage:h?.message??null,resultIndex:h?.index??null,state:s,directory:p,directoryConflict:f,evidence:t?"调用标识缺失或重复，无法唯一关联结果。":h?$(h.message,e.call,s==="no-result"?"unknown":s):"未记录到此调用的结果；不推断实际进程状态。"}}function ne(e){let{calls:t,byId:n}=D(e),o=new Map(q(e).processes.map(a=>[a.launchCallId,a])),f=[],p=[],h=0,s=0;for(let a of t){let l=w(a.call.args),m=!a.id||(n.get(a.id)?.length??0)!==1;if(["edit","Edit","write","Write","MultiEdit"].includes(a.call.name)){let r=l?["path","file_path"].filter(u=>u in l).map(u=>l[u]):[];!r.length||r.some(u=>typeof u!="string"||!u.trim()||u.includes("\0")||u!==r[0])?s++:f.push({...j(a,m),target:r[0]})}else["apply_patch","apply_diff","NotebookEdit"].includes(a.call.name)&&s++;let x=ee(a.call);if(x){let r=j(a,m),u=o.get(a.id);u&&u.launchIndex===a.call.index&&(r.processEvidence=u,r.state=u.inputObserved?"unknown":u.state,r.resultMessage=u.finalMessage,r.resultIndex=u.finalIndex,r.evidence=`通过 exec_command 包装头进程标识 ${u.processId} 与 write_stdin.session_id 关联。${u.inputObserved?"过程中发送过输入，不能视为未干预的检查结果。":""} +`+(u.finalMessage?`最终进程退出码 ${u.exitCode} +${v(u.finalMessage).slice(0,400)}`:`未得到唯一可信的最终退出结果；${u.issues.join(", ")||"最后记录为执行中"}。`));let{explicitDirectory:i,usesInitialDirectory:d,...c}=x;x.commandMode==="compound-fragment"?p.push({...r,...c,state:r.state==="no-result"?"no-result":"unknown",directory:i??(d?r.directory:null),directoryConflict:i?!1:r.directoryConflict,evidence:`命令文本包含检查片段；无法确认片段是否执行或通过，不能使用整个命令的退出码判定。 +${r.evidence}`}):x.commandMode==="directory-prefix"?p.push({...r,...c,directory:i,directoryConflict:!1,state:r.state==="failure"?"unknown":r.state,evidence:`记录的目录前缀：cd 后通过 && 连接检查。${r.state==="failure"?"整体失败可能发生在 cd，检查状态未知。":""} +${r.evidence}`}):p.push({...r,...c})}else _(a.call,a.message)&&h++}let g=[];for(let a of f){if(a.state!=="success"||a.resultIndex===null)continue;let l=[],m=[],x=[],r=0;for(let d of p){if(a.directoryConflict||d.directoryConflict||a.directory&&d.directory&&a.directory!==d.directory){r++;continue}let c={operation:d,scope:a.directory&&d.directory?"same-recorded-directory":"unknown"};d.callIndex>a.resultIndex?m.push(c):d.resultIndex!==null&&d.resultIndex=a.callIndex)&&x.push(c)}l.sort((d,c)=>(d.operation.resultIndex??-1)-(c.operation.resultIndex??-1));let u=m[m.length-1],i=!!u&&m.filter(d=>d.operation.callIndex===u.operation.callIndex).length>1;g.push({operation:a,priorSuccess:l[l.length-1]??null,laterChecks:m,latestLater:i?null:u??null,latestOrderAmbiguous:i,overlappingChecks:x,excludedScopeChecks:r})}return{modificationCalls:f.length,successfulModifications:g.length,unconfirmedModifications:f.length-g.length,unclassifiedModificationCalls:s,unclassifiedShellCalls:h,checks:p,modifications:g,withLaterCheck:g.filter(a=>a.laterChecks.length).length,withoutLaterCheck:g.filter(a=>!a.laterChecks.length).length,changedAfterLastPassedCheck:g.filter(a=>a.priorSuccess&&!a.laterChecks.length).length}}0&&(module.exports={analyzeCodexProcesses,analyzeVerificationChronology,diagnoseSession,summarizeSessionHealth}); + +module.exports.rulesSha256 = "d2a8402dfc1a2533c0f58fe23e8a4334a24e585477e1992b4484f7a9c00771ad"; diff --git a/lib/inspect.js b/lib/inspect.js new file mode 100644 index 0000000..d573669 --- /dev/null +++ b/lib/inspect.js @@ -0,0 +1,328 @@ +const fs = require('node:fs/promises'); +const { constants } = require('node:fs'); +const { createHash } = require('node:crypto'); +const { TextDecoder } = require('node:util'); +const path = require('node:path'); +const rules = require('./generated/diagnostics.cjs'); +const { normalizeOmpRecord } = require('./platforms/omp'); +const { normalizeCodexRecord } = require('./platforms/codex'); +const { normalizeClaudeCodeRecord } = require('./platforms/claude'); + +const MAX_BYTES = 64 * 1024 * 1024; +const PLATFORMS = ['omp', 'codex', 'claude-code']; +const hash = (bytes) => createHash('sha256').update(bytes).digest('hex'); +class InspectError extends Error {} +const normalizers = { + omp: normalizeOmpRecord, + codex: (record) => (record.type === 'response_item' ? normalizeCodexRecord(record) : null), + 'claude-code': normalizeClaudeCodeRecord, +}; + +function toolLabel(value) { + return [ + 'bash', + 'Bash', + 'shell', + 'exec', + 'exec_command', + 'write_stdin', + 'functions.exec_command', + 'functions.write_stdin', + 'edit', + 'Edit', + 'write', + 'Write', + 'MultiEdit', + 'Read', + 'read', + 'grep', + 'glob', + 'eval', + 'web_search', + 'hub', + 'ask', + 'apply_patch', + 'run_shell_command', + 'execute_command', + 'terminal', + ].includes(value) + ? value + : 'other'; +} + +function rawToolEntries(record, platform) { + const calls = [], + results = []; + if (platform === 'codex' && record.type === 'response_item') { + const payload = record.payload || {}; + if (['function_call', 'custom_tool_call'].includes(payload.type)) calls.push(payload.call_id || null); + if (['function_call_output', 'custom_tool_call_output'].includes(payload.type)) { + const exitCode = payload.output?.metadata?.exit_code; + results.push({ id: payload.call_id || null, error: Number.isInteger(exitCode) && exitCode !== 0 }); + } + } else if (platform === 'omp' && record.type === 'message') { + const message = record.message || {}; + if (message.role === 'assistant' && Array.isArray(message.content)) { + for (const part of message.content) if (part?.type === 'toolCall') calls.push(part.id || null); + } + if (message.role === 'toolResult') + results.push({ id: message.toolCallId || null, error: message.isError === true }); + } else if ( + platform === 'claude-code' && + ['user', 'assistant'].includes(record.type) && + Array.isArray(record.message?.content) + ) { + for (const part of record.message.content) { + if (part?.type === 'tool_use') calls.push(part.id || null); + if (part?.type === 'tool_result') results.push({ id: part.tool_use_id || null, error: part.is_error === true }); + } + } + return { calls, results }; +} + +function sameIds(left, right) { + return JSON.stringify(left.map(String).sort()) === JSON.stringify(right.map(String).sort()); +} + +function normalizeRecords(bytes, platform) { + if (!PLATFORMS.includes(platform)) throw new InspectError('Unsupported platform; choose omp, codex or claude-code.'); + if (bytes.byteLength > MAX_BYTES) throw new InspectError('Input exceeds 64 MiB; select a smaller stable JSONL file.'); + let text; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new InspectError('Input is not valid UTF-8.'); + } + const lines = text.split('\n'); + if (lines.at(-1) === '') lines.pop(); + const messages = [], + lineOf = new Map(); + const coverage = { + physicalLines: lines.length, + records: 0, + normalizedMessages: 0, + rawToolCalls: 0, + normalizedToolCalls: 0, + rawToolResults: 0, + normalizedToolResults: 0, + ignoredRecords: 0, + issues: [], + }; + for (const [index, line] of lines.entries()) { + if (!line.trim()) continue; + let record; + try { + record = JSON.parse(line); + } catch { + throw new InspectError(`Invalid JSON at line ${index + 1}; no report generated.`); + } + if (!record || typeof record !== 'object' || Array.isArray(record)) + throw new InspectError(`Expected a JSON object at line ${index + 1}.`); + coverage.records++; + let normalized, raw; + try { + raw = rawToolEntries(record, platform); + const value = normalizers[platform](record); + normalized = Array.isArray(value) ? value : value ? [value] : []; + } catch { + throw new InspectError(`Unsupported record shape at line ${index + 1}; no report generated.`); + } + const callIds = [], + results = []; + for (const message of normalized) { + messages.push(message); + lineOf.set(message, index + 1); + if (message.role === 'toolCall') callIds.push(message.toolCallId || message.id || null); + for (const part of message.content || []) if (part.type === 'toolCall') callIds.push(part.id || null); + if (message.role === 'toolResult') results.push(message); + } + coverage.rawToolCalls += raw.calls.length; + coverage.normalizedToolCalls += callIds.length; + coverage.rawToolResults += raw.results.length; + coverage.normalizedToolResults += results.length; + if (!normalized.length) coverage.ignoredRecords++; + if (!sameIds(raw.calls, callIds)) coverage.issues.push({ line: index + 1, kind: 'tool-call-coverage' }); + if ( + !sameIds( + raw.results.map((entry) => entry.id), + results.map((entry) => entry.toolCallId || null) + ) + ) { + coverage.issues.push({ line: index + 1, kind: 'tool-result-coverage' }); + } + for (const entry of raw.results.filter((entry) => entry.error)) { + if (!results.some((result) => result.toolCallId === entry.id && result.isError === true)) { + coverage.issues.push({ line: index + 1, kind: 'error-marker-loss' }); + } + } + } + coverage.normalizedMessages = messages.length; + if (!messages.length) throw new InspectError('No supported messages found for the selected platform.'); + return { messages, lineOf, coverage }; +} + +async function readSnapshot(filename) { + let handle; + try { + handle = await fs.open(filename, constants.O_RDONLY | (constants.O_NONBLOCK || 0)); + const before = await handle.stat(); + if (!before.isFile()) throw new InspectError('Input must be one regular JSONL file.'); + if (before.size > MAX_BYTES) throw new InspectError('Input exceeds 64 MiB; select a smaller stable JSONL file.'); + const bytes = Buffer.alloc(before.size); + let offset = 0; + while (offset < bytes.length) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (!bytesRead) throw new InspectError('Input changed during inspection; retry a stable file.'); + offset += bytesRead; + } + const after = await handle.stat(); + if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) { + throw new InspectError('Input changed during inspection; retry a stable file.'); + } + return bytes; + } catch (error) { + if (error instanceof InspectError) throw error; + throw new InspectError('Cannot read input file. Check that it exists and is readable.'); + } finally { + if (handle) await handle.close(); + } +} + +async function inspectFile(filename, platform) { + if (!PLATFORMS.includes(platform)) throw new InspectError('Unsupported platform; choose omp, codex or claude-code.'); + const bytes = await readSnapshot(filename); + return createReport(bytes, platform); +} + +async function createReport(bytes, platform) { + const { messages, lineOf, coverage } = normalizeRecords(bytes, platform); + const messageIndex = new Map(messages.map((message, index) => [message, index + 1])); + const ref = (message) => (message ? { line: lineOf.get(message), messageIndex: messageIndex.get(message) } : null); + let diagnostics, health, processes, chronology; + try { + diagnostics = rules.diagnoseSession(messages); + health = rules.summarizeSessionHealth(messages, diagnostics); + processes = rules.analyzeCodexProcesses(messages); + chronology = rules.analyzeVerificationChronology(messages); + } catch { + throw new InspectError('Cannot analyze this record structure; no report generated.'); + } + const operation = (entry) => ({ + tool: toolLabel(entry.toolName), + call: ref(entry.callMessage), + result: ref(entry.resultMessage), + state: entry.state, + directoryKnown: entry.directory !== null, + directoryConflict: entry.directoryConflict, + basis: entry.basis || null, + commandMode: entry.commandMode || null, + }); + const check = (entry) => (entry ? { scope: entry.scope, operation: operation(entry.operation) } : null); + const { gaps, ...summary } = health; + const adapters = + platform === 'omp' ? ['omp.js', 'omp-outcome.js'] : [platform === 'codex' ? 'codex.js' : 'claude.js']; + const adapterSources = await Promise.all( + [ + ...adapters.map((file) => path.join(__dirname, 'platforms', file)), + path.join(__dirname, 'platforms/shared.js'), + ].map((file) => fs.readFile(file)) + ); + return { + schemaVersion: 1, + complete: coverage.issues.length === 0, + engine: { + version: require('../package.json').version, + rulesSha256: rules.rulesSha256, + adapterSha256: hash(Buffer.concat(adapterSources)), + }, + source: { platform, bytes: bytes.length, sha256: hash(bytes) }, + coverage, + limits: [ + 'This report describes recognized records, not a task-success verdict or live process status.', + 'Only one stable input file is inspected. No commands, models, network calls or background service are started.', + 'Unknown and unclassified operations remain unknown; check timing is not test coverage.', + 'Raw content, paths, commands, IDs and notes are omitted; hashes and aggregate facts are not guaranteed anonymous.', + 'Source references use one-based physical JSONL lines and normalized message positions. Keep the original file for evidence.', + ], + summary, + events: diagnostics.events.map((event) => ({ + tool: toolLabel(event.toolName), + userTurn: event.userTurn, + spanMs: event.spanMs, + reason: event.failures.at(-1).reason, + failures: event.failures.map((failure) => ref(failure.message)), + related: event.relatedOperations.map((entry) => ({ + relation: entry.relation, + state: entry.state, + tool: toolLabel(entry.toolName), + call: ref(messages[entry.callIndex]), + result: ref(entry.message), + })), + })), + gaps: gaps.map((gap) => ({ + tool: toolLabel(gap.toolName), + state: gap.state, + reason: gap.reason, + source: ref(gap.message), + })), + processes: { + pollCalls: processes.pollCalls, + linkedPolls: processes.linkedPolls, + unlinkedPolls: processes.unlinkedPolls, + ambiguousCalls: processes.ambiguousCalls, + entries: processes.processes.map((entry) => ({ + launch: ref(entry.launchMessage), + launchResult: ref(entry.launchResult), + finalResult: ref(entry.finalMessage), + state: entry.state, + exitCode: entry.exitCode, + inputObserved: entry.inputObserved, + issues: entry.issues, + polls: entry.polls.map((poll) => ({ + call: ref(poll.callMessage), + result: ref(poll.resultMessage), + state: poll.state, + exitCode: poll.exitCode, + hasInput: poll.hasInput, + })), + })), + }, + chronology: { + modificationCalls: chronology.modificationCalls, + successfulModifications: chronology.successfulModifications, + unconfirmedModifications: chronology.unconfirmedModifications, + unclassifiedModificationCalls: chronology.unclassifiedModificationCalls, + unclassifiedShellCalls: chronology.unclassifiedShellCalls, + withLaterCheck: chronology.withLaterCheck, + withoutLaterCheck: chronology.withoutLaterCheck, + changedAfterLastPassedCheck: chronology.changedAfterLastPassedCheck, + checks: chronology.checks.map(operation), + modifications: chronology.modifications.map((row) => ({ + modification: operation(row.operation), + priorSuccess: check(row.priorSuccess), + latestLater: check(row.latestLater), + latestOrderAmbiguous: row.latestOrderAmbiguous, + laterChecks: row.laterChecks.map(check), + overlappingChecks: row.overlappingChecks.map(check), + excludedScopeChecks: row.excludedScopeChecks, + })), + }, + }; +} + +function renderText(report) { + return `${[ + `AgentXRay offline evidence report (schema ${report.schemaVersion})`, + `Platform: ${report.source.platform} | Source SHA-256: ${report.source.sha256}`, + `Known adapter identity coverage: ${report.complete ? 'complete' : 'INCOMPLETE — inspect coverage issues in --json output'}`, + `Tool calls: ${report.summary.callCount} | Tool results: ${report.summary.toolResultCount}`, + `Failure records: ${report.summary.failureRecords} | Pending failure records: ${report.summary.pendingRecords}`, + `Pending events: ${report.summary.pendingEvents} | Matching recovery records: ${report.summary.recoveredRecords}`, + `Call-result gaps: ${report.gaps.length} | Related result candidates: ${report.summary.candidateResults}`, + `Codex launches: ${report.processes.entries.length} | Linked polls: ${report.processes.linkedPolls}`, + `Explicit modification successes: ${report.chronology.successfulModifications} | Recognized check calls: ${report.chronology.checks.length}`, + 'This is not a task-success verdict. Use --json for all source references and coverage limits.', + ].join('\n')}\n`; +} + +module.exports = { inspectFile, createReport, normalizeRecords, renderText, InspectError, PLATFORMS, MAX_BYTES }; diff --git a/package-lock.json b/package-lock.json index a1e0ba6..5b7fee2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@alloevil/agent-xray", - "version": "1.22.0", + "version": "1.23.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@alloevil/agent-xray", - "version": "1.22.0", + "version": "1.23.0", "license": "MIT", "dependencies": { "express": "^4.21.2" diff --git a/package.json b/package.json index 84a9d3d..af7a0c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alloevil/agent-xray", - "version": "1.22.0", + "version": "1.23.0", "description": "Web dashboard for viewing AI agent session logs — supports OpenClaw, Codex, Claude Code, Hermes, OMP, DeepSeek Harness, and Gemini CLI", "main": "server.js", "bin": { @@ -17,7 +17,8 @@ ], "scripts": { "start": "node server.js", - "build:ui": "node scripts/build-legacy-pure.mjs && cd frontend && npm run build", + "build:ui": "node scripts/build-diagnostics.mjs && node scripts/build-legacy-pure.mjs && cd frontend && npm run build", + "build:diagnostics": "node scripts/build-diagnostics.mjs", "build:ui:full": "cd frontend && npm ci && cd .. && npm run build:ui", "prepare": "node -e \"process.exit(require('fs').existsSync('frontend/dist/index.html') ? 0 : 1)\" || npm run build:ui:full", "test": "node --test test/*.test.js", diff --git a/scripts/build-diagnostics.mjs b/scripts/build-diagnostics.mjs new file mode 100644 index 0000000..f1b2066 --- /dev/null +++ b/scripts/build-diagnostics.mjs @@ -0,0 +1,33 @@ +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const { buildSync } = createRequire(path.join(root, 'frontend/package.json'))('esbuild'); +const input = path.join(root, 'frontend/src/views/sessions/diagnostics.ts'); +const output = path.join(root, 'lib/generated/diagnostics.cjs'); +const result = buildSync({ + entryPoints: [input], + bundle: true, + write: false, + format: 'cjs', + platform: 'node', + target: 'node22', + charset: 'utf8', + legalComments: 'none', + minify: true, +}); +const digest = createHash('sha256').update(readFileSync(input)).digest('hex'); +const content = `${result.outputFiles[0].text}\nmodule.exports.rulesSha256 = ${JSON.stringify(digest)};\n`; +if (process.argv.includes('--check')) { + if (!existsSync(output) || readFileSync(output, 'utf8') !== content) { + console.error('Diagnostic bundle drift: run npm run build:diagnostics'); + process.exitCode = 1; + } else console.log('PASS: CLI diagnostic bundle matches UI rule source'); +} else { + mkdirSync(path.dirname(output), { recursive: true }); + writeFileSync(output, content); + console.log('wrote lib/generated/diagnostics.cjs'); +} diff --git a/test/inspect.test.js b/test/inspect.test.js new file mode 100644 index 0000000..0480f4b --- /dev/null +++ b/test/inspect.test.js @@ -0,0 +1,317 @@ +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { spawnSync } = require('node:child_process'); +const { stripTypeScriptTypes } = require('node:module'); + +const ROOT = path.join(__dirname, '..'); +const BIN = path.join(ROOT, 'bin/agentxray.js'); +let home; +let rules; +before(async () => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'axr-inspect-')); + const source = fs.readFileSync(path.join(ROOT, 'frontend/src/views/sessions/diagnostics.ts'), 'utf8'); + rules = await import(`data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(source)).toString('base64')}`); +}); +after(() => fs.rmSync(home, { recursive: true, force: true })); + +function fixture(name, records) { + const filename = path.join(home, name); + fs.writeFileSync( + filename, + records.map((record) => (typeof record === 'string' ? record : JSON.stringify(record))).join('\n') + ); + return filename; +} +const ompCall = (id, args = { command: 'PRIVATE_COMMAND' }, name = 'bash') => ({ + type: 'message', + id: `${id}-call`, + message: { role: 'assistant', content: [{ type: 'toolCall', id, name, arguments: args }] }, +}); +const ompResult = (id, error = true) => ({ + type: 'message', + id: `${id}-result`, + message: { + role: 'toolResult', + toolCallId: id, + toolName: 'bash', + isError: error, + details: { exitCode: error ? 1 : 0 }, + content: [{ type: 'text', text: 'PRIVATE_OUTPUT' }], + }, +}); +const records = () => [ + { type: 'session', id: 'PRIVATE_SESSION', cwd: '/PRIVATE_PATH' }, + { type: 'message', message: { role: 'user', content: [{ type: 'text', text: 'PRIVATE_PROMPT' }] } }, + ompCall('PRIVATE_CALL'), + ompResult('PRIVATE_CALL'), +]; +function run(args, options = {}) { + return spawnSync(process.execPath, [BIN, ...args], { + cwd: ROOT, + env: { ...process.env, HOME: home }, + encoding: 'utf8', + timeout: 15000, + ...options, + }); +} +function jsonReport(file, platform = 'omp', options = []) { + return run(['inspect', '--platform', platform, file, '--json', ...options]); +} + +test('offline inspect emits deterministic versioned minimized JSON and defaults to exit zero on findings', () => { + const file = fixture('private-name.jsonl', records()); + const first = jsonReport(file), + second = jsonReport(file); + assert.equal(first.status, 0, first.stderr); + assert.equal(first.stderr, ''); + assert.equal(first.stdout, second.stdout); + const report = JSON.parse(first.stdout); + assert.equal(report.schemaVersion, 1); + assert.equal(report.complete, true); + assert.equal(report.summary.pendingRecords, 1); + assert.match(report.source.sha256, /^[a-f0-9]{64}$/); + assert.match(report.engine.rulesSha256, /^[a-f0-9]{64}$/); + assert.match(report.engine.adapterSha256, /^[a-f0-9]{64}$/); + assert.doesNotMatch(first.stdout, /PRIVATE_|private-name|agentxray-test|toolCallId|timestamp/); + assert.deepEqual(report.events[0].failures, [{ line: 4, messageIndex: 4 }]); +}); + +test('pending failure policy exits two but does not alter report contents', () => { + const file = fixture('policy.jsonl', records()); + const plain = jsonReport(file), + gated = jsonReport(file, 'omp', ['--fail-on', 'pending-failures']); + assert.equal(gated.status, 2); + assert.equal(gated.stdout, plain.stdout); + const recovered = fixture('recovered.jsonl', [...records(), ompCall('retry'), ompResult('retry', false)]); + const result = jsonReport(recovered, 'omp', ['--fail-on', 'pending-failures']); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).summary.recoveredRecords, 1); +}); + +test('plain text is a factual summary with limits, not a task success assertion', () => { + const result = run(['inspect', '--platform=omp', fixture('text.jsonl', records())]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Pending failure records: 1/); + assert.match(result.stdout, /not a task-success verdict/i); + assert.doesNotMatch(result.stdout, /PRIVATE_/); +}); + +test('malformed or truncated line fails with safe line number instead of partial green report', () => { + const file = fixture('broken.jsonl', [...records(), '{"PRIVATE_SECRET":']); + const result = jsonReport(file); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.match(result.stderr, /line 5/i); + assert.doesNotMatch(result.stderr, /PRIVATE_SECRET|broken.jsonl|SyntaxError|\n\s+at /); +}); + +test('invalid UTF-8 and non-object records fail safely', () => { + const file = path.join(home, 'utf8.jsonl'); + fs.writeFileSync(file, Buffer.from([0xff, 0xfe])); + assert.equal(jsonReport(file).status, 1); + for (const value of ['null', '[]', '42', '"secret"']) { + const result = jsonReport(fixture('invalid-object.jsonl', [value])); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + } +}); + +test('wrong platform, unsupported format, empty files and metadata-only files cannot pass', () => { + const file = fixture('wrong.jsonl', records()); + assert.equal(jsonReport(file, 'codex').status, 1); + assert.equal(jsonReport(file, 'hermes').status, 1); + assert.equal(jsonReport(fixture('empty.jsonl', [])).status, 1); + assert.equal(jsonReport(fixture('metadata.jsonl', [{ type: 'session', id: 'only-meta' }])).status, 1); +}); + +test('directories, nonexistent files and oversized regular files fail without disclosing paths', () => { + for (const file of [home, path.join(home, 'PRIVATE_NONEXISTENT')]) { + const result = jsonReport(file); + assert.equal(result.status, 1); + assert.doesNotMatch(result.stderr, /PRIVATE_NONEXISTENT|axr-inspect-/); + } + const file = path.join(home, 'large.jsonl'); + fs.closeSync(fs.openSync(file, 'w')); + fs.truncateSync(file, 64 * 1024 * 1024 + 1); + const result = jsonReport(file); + assert.equal(result.status, 1); + assert.match(result.stderr, /64 MiB/); +}); + +test('inspect requires explicit single path/platform and validates flags', () => { + const file = fixture('flags.jsonl', records()); + for (const args of [ + [], + [file], + ['--platform'], + ['--platform', 'omp'], + ['--platform', 'omp', file, file], + ['--platform', 'omp', file, '--bad'], + ['--platform', 'omp', file, '--fail-on', 'success'], + ['--platform', 'omp', '--platform', 'codex', file], + ['--platform', 'omp', file, '--fail-on'], + ]) { + assert.equal(run(['inspect', ...args]).status, 1); + } + const help = run(['inspect', '--help']); + assert.equal(help.status, 0); + assert.match(help.stdout, /--platform/); + assert.match(run(['--help']).stdout, /inspect/); +}); + +test('blank lines and CRLF retain original physical source positions', () => { + const file = fixture('blank.jsonl', ['', JSON.stringify(ompCall('call')), '', JSON.stringify(ompResult('call')), '']); + fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replaceAll('\n', '\r\n')); + const result = jsonReport(file); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout).events[0].failures, [{ line: 4, messageIndex: 3 }]); +}); + +test('Claude multi-result adapter loss is reported, never silently accepted as complete', () => { + const file = fixture('claude-multi.jsonl', [ + { + type: 'assistant', + uuid: 'call', + message: { + content: [ + { type: 'tool_use', id: 'one', name: 'Read', input: {} }, + { type: 'tool_use', id: 'two', name: 'Read', input: {} }, + ], + }, + }, + { + type: 'user', + uuid: 'results', + message: { + content: [ + { type: 'tool_result', tool_use_id: 'one', content: 'ok' }, + { type: 'tool_result', tool_use_id: 'two', content: 'PRIVATE_ERROR', is_error: true }, + ], + }, + }, + ]); + const result = jsonReport(file, 'claude-code', ['--fail-on', 'pending-failures']); + assert.equal(result.status, 1); + const report = JSON.parse(result.stdout); + assert.equal(report.complete, false); + assert.equal(report.coverage.rawToolResults, 2); + assert.equal(report.coverage.normalizedToolResults, 1); + assert.ok(report.coverage.issues.some((issue) => issue.line === 2)); + assert.doesNotMatch(result.stdout, /PRIVATE_ERROR/); +}); + +test('mixed Claude text/result record remains visible as a coverage gap', () => { + const file = fixture('claude-mixed.jsonl', [ + { + type: 'user', + uuid: 'mixed', + message: { + content: [ + { type: 'text', text: 'PRIVATE_PROMPT' }, + { type: 'tool_result', tool_use_id: 'one', content: 'PRIVATE_OUTPUT', is_error: true }, + ], + }, + }, + ]); + const result = jsonReport(file, 'claude-code'); + assert.equal(result.status, 1); + assert.equal(JSON.parse(result.stdout).complete, false); +}); + +for (const [platform, relative] of [ + ['omp', 'frontend/demo/sample-logs/omp/-demo-diagnostics/2026-09-23T08-00-00-000Z_0199demo-diagnostics.jsonl'], + [ + 'codex', + 'frontend/demo/sample-logs/codex/2026/09/24/rollout-2026-09-24T08-00-00-01990000-0000-7000-8000-000000000199.jsonl', + ], + ['claude-code', 'frontend/demo/sample-logs/claude/-demo-webapp/synthetic-feature-dark-mode.jsonl'], +]) { + test(`${platform} CLI summary matches UI rules and every evidence reference points into the input`, () => { + const { normalizeRecords } = require('../lib/inspect'); + const bytes = fs.readFileSync(path.join(ROOT, relative)); + const normalized = normalizeRecords(bytes, platform); + const expected = rules.diagnoseSession(normalized.messages); + const result = jsonReport(path.join(ROOT, relative), platform); + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.summary.failureRecords, expected.failureCount); + assert.equal(report.summary.pendingRecords, expected.failures.length); + assert.equal(report.summary.recoveredRecords, expected.recoveredCount); + const checkRefs = (value) => { + if (!value || typeof value !== 'object') return; + if ('messageIndex' in value) { + const message = normalized.messages[value.messageIndex - 1]; + assert.ok(message); + assert.equal(normalized.lineOf.get(message), value.line); + } + for (const item of Object.values(value)) checkRefs(item); + }; + checkRefs(report); + assert.equal(report.events.flatMap((event) => event.failures).length, expected.failures.length); + }); +} + +test('no service/network or writes to input/HOME are needed to inspect', () => { + const file = fixture('readonly.jsonl', records()); + const guard = path.join(home, 'guard.cjs'); + fs.writeFileSync( + guard, + `const Module=require('node:module');const load=Module._load;Module._load=function(name,...rest){if(['express','http','https','net','node:http','node:https','node:net'].includes(name)||name.endsWith('/server.js'))throw Error('Forbidden runtime dependency');return load.call(this,name,...rest)};` + ); + const before = fs.readFileSync(file); + const entries = fs.readdirSync(home).sort(); + const result = jsonReport(file, 'omp', []); + const isolated = run(['inspect', '--platform', 'omp', file, '--json'], { + env: { ...process.env, HOME: home, NODE_OPTIONS: `--require=${guard}` }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(isolated.status, 0, isolated.stderr); + assert.equal(isolated.stdout, result.stdout); + assert.deepEqual(fs.readFileSync(file), before); + assert.deepEqual(fs.readdirSync(home).sort(), entries); +}); + +test('packaged rules are generated from the UI source and drift check passes', () => { + const result = spawnSync(process.execPath, ['scripts/build-diagnostics.mjs', '--check'], { + cwd: ROOT, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr || result.stdout); +}); + +test('changed input metadata rejects the read instead of claiming a stable snapshot', () => { + const file = fixture('changing.jsonl', records()); + const guard = path.join(home, 'changed-stat.cjs'); + fs.writeFileSync( + guard, + `const fs=require('node:fs/promises');const open=fs.open;fs.open=async function(...args){const handle=await open.apply(this,args);const stat=handle.stat.bind(handle);let reads=0;handle.stat=async()=>{const value=await stat();if(++reads>1)value.mtimeMs+=1;return value;};return handle;};` + ); + const result = run(['inspect', '--platform', 'omp', file, '--json'], { + env: { ...process.env, HOME: home, NODE_OPTIONS: `--require=${guard}` }, + }); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.match(result.stderr, /changed during inspection/); +}); + +test('unrecognized private tool names are minimized and source content is never returned', () => { + const file = fixture('private-tool.jsonl', [ + ompCall('PRIVATE_ID', { secret: 'PRIVATE_ARGUMENT' }, 'PRIVATE_TOOL'), + ompResult('PRIVATE_ID'), + ]); + const result = jsonReport(file); + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.events[0].tool, 'other'); + assert.doesNotMatch(result.stdout, /PRIVATE/); +}); + +test('input option terminator allows a dash-prefixed literal filename', () => { + fixture('-literal.jsonl', records()); + const result = run(['inspect', '--json', '--platform', 'omp', '--', '-literal.jsonl'], { cwd: home }); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).summary.pendingRecords, 1); +});