From 0616906e86a83f03db78b4f1da952c3defda643c Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 14:06:39 +0200 Subject: [PATCH 1/7] fix(verify): stop feature verification reporting false green MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verify-features run with four failing Tier 2 checks reported "COMPLETED — 11 passed, 0 failed". Four independent defects produced that, and each is closed here. 1. Every step set failOnError: false, so a non-zero exit scored as a pass. Tiers still use it (all tiers should run even after one fails), but they are no longer the verdict: each check now appends pass/fail/skip to checks.jsonl and a `verdict` step aggregates it into verdict.json. 2. Nothing aggregated tier results into a run verdict. `verdict` plus a terminal `enforce-verdict` now do, and main() exits non-zero. 3. main() discarded the runner result. It now reads verdict.json and fails closed when that file is missing. 4. applyReliabilityDefaults force-enables strategy "retry" with a repair agent for any workflow declaring agents, so a failing verification gate was handed to an agent that edited the tree until it passed. onError is now "continue", which opts out of that path. The run also verified agent-relay 10.0.0 against an 11.3.0 checkout — it logged "Update available: 10.0.0 -> 11.3.0" and then reported `relay node deadletters` as unknown, a real command in 11.x. A `provenance` step now fails on CLI/repo version drift. A `capabilities` probe turns unrunnable checks (no cloud login, no provider CLI) into SKIPs with a stated cause rather than product failures, and `verdict` fails the run when a tier produced no records at all, so a crashed tier cannot look clean. Coverage now spans tiers 1-6 and all six critical paths, including a real MCP JSON-RPC handshake asserting the required tool set (previously just `relay mcp --help`). On failure the run posts to Slack, files a GitHub issue, attempts a fix on a branch, and opens a draft PR. An integrity gate refuses a branch that reduced the number of verification call sites, left the repo on main, or has no root cause on record. Harness-level breakage escalates to NightCTO via the CloudEvidenceSummary v1 contract; run and per-failure events go to PostHog. Adds scripts/audit-feature-manifest.mjs and workflows/audit-feature-manifest.ts to answer whether the feature map is current. manifest-contract.test.ts asserts a hand-maintained list, so it cannot notice a newly added command; the audit derives the surface from recursive `--help` and `tools/list` instead. Its first run found `fleet spawn` and `fleet release` shipping undocumented, and therefore unverified — both are now in the manifest. Co-Authored-By: Claude Opus 5 --- .../manifest-contract.test.ts | 2 + .agentworkforce/features/manifest.yaml | 16 +- .claude/skills/verify-features.md | 38 + .gitignore | 1 + scripts/audit-feature-manifest.mjs | 394 +++ workflows/audit-feature-manifest.ts | 814 ++++++ workflows/verify-features.ts | 2271 +++++++++++++---- 7 files changed, 3019 insertions(+), 517 deletions(-) create mode 100644 scripts/audit-feature-manifest.mjs create mode 100644 workflows/audit-feature-manifest.ts diff --git a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts index 1375741b9..7b313b6ca 100644 --- a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts +++ b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts @@ -137,6 +137,8 @@ describe('feature manifest contract', () => { 'relay skills add', 'relay mcp', 'relay fleet nodes', + 'relay fleet spawn', + 'relay fleet release', 'relay fleet config', 'relay fleet enable', 'relay fleet disable', diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 3781b9a56..dcfe895fb 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1,5 +1,5 @@ version: '1.1' -updated: '2026-07-20' +updated: '2026-07-30' # Every user-facing feature in this repo, categorized and scored. # @@ -599,6 +599,20 @@ categories: location: packages/cli/src/cli/commands/fleet.ts verify_tier: 5 + - id: fleet-spawn + name: Spawn Fleet Agent + cli: relay fleet spawn [--name ] [--task ] [--node ] [--persona ] [--model ] + description: Spawn a worker agent onto a live fleet node, by explicit node or by persona placement + location: packages/cli/src/cli/commands/fleet.ts + verify_tier: 5 + + - id: fleet-release + name: Release Fleet Agent + cli: relay fleet release [--reason ] [--delete-agent] + description: Release a spawned fleet agent, optionally deleting the agent identity + location: packages/cli/src/cli/commands/fleet.ts + verify_tier: 5 + - id: fleet-config name: Fleet Config cli: relay fleet config diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index f93c12d71..0be764e06 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -17,6 +17,37 @@ Use when you need to verify that a specific feature or set of features works cor .agentworkforce/features/verify/procedures.md # step-by-step verification by tier ``` +## Automation + +```text +workflows/verify-features.ts # runs tiers 1-6 + critical paths 1-6 +workflows/audit-feature-manifest.ts # checks the manifest still matches the CLI +scripts/audit-feature-manifest.mjs # the audit itself; run it directly +``` + +Before trusting any verification result, check two things: + +1. **Which CLI ran.** `verify-features.ts` has a `provenance` step that fails + when `relay version` disagrees with the repo's `package.json`. A run against + a stale globally-installed CLI describes that CLI, not your checkout — this + has already produced a full green run plus one bogus "unknown command" + failure against a real command. +2. **What was skipped.** Every check records `pass`, `fail`, or `skip` with a + reason into `.workflow-artifacts/verify-features/checks.jsonl`, and + `verdict.json` is the authoritative result. A SKIP means *not verified*. + Never read a skip as a pass. + +Run the manifest audit before adding checks, so you are not writing coverage +against a stale map: + +```bash +node scripts/audit-feature-manifest.mjs # human-readable +node scripts/audit-feature-manifest.mjs --json # for tooling +``` + +Exit 0 = clean, 1 = drift, 2 = the audit itself could not run. Exit 2 is +deliberately distinct: a broken audit must never be read as a clean manifest. + ## How to use it ### 1. Read the manifest to find the feature @@ -101,6 +132,13 @@ Add or update entries in `manifest.yaml` when: - A new harness is supported - A feature is removed or deprecated +Then run `node scripts/audit-feature-manifest.mjs` and confirm `MANIFEST_CLEAN`. +Note that `manifest-contract.test.ts` cannot catch a missing entry for a *new* +command — a new command is absent from both the manifest and that test's +hardcoded expectation list, so it passes. The audit script derives the surface +from `--help` and `tools/list` instead, which is why it is the check that +matters here. Add new commands to both. + Update `critical-paths.md` when: - A new path becomes foundational to the product diff --git a/.gitignore b/.gitignore index daeac1975..60dbfe742 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ __pycache__/ !/workflows/relayauth-integration/ !/workflows/cloud-connect/ !/workflows/verify-features.ts +!/workflows/audit-feature-manifest.ts # Eval harness JSON reports (generated per run) tests/integration/broker/evals-reports/ diff --git a/scripts/audit-feature-manifest.mjs b/scripts/audit-feature-manifest.mjs new file mode 100644 index 000000000..264053e1b --- /dev/null +++ b/scripts/audit-feature-manifest.mjs @@ -0,0 +1,394 @@ +#!/usr/bin/env node +/** + * Audit .agentworkforce/features/manifest.yaml against the CLI's real surface. + * + * The manifest is the input to every feature-verification run, so drift there + * silently shrinks coverage: a command nobody documented is a command nobody + * verifies. `manifest-contract.test.ts` only checks a hand-maintained list, so + * it can never notice a *newly added* command or MCP tool. This script derives + * the surface instead of asserting a snapshot of it. + * + * Two surfaces are derived: + * 1. The Commander leaf-command tree, by walking `--help` recursively. + * 2. The MCP tool list, by a `tools/list` JSON-RPC call over stdio. + * + * Usage: + * node scripts/audit-feature-manifest.mjs + * node scripts/audit-feature-manifest.mjs --json + * node scripts/audit-feature-manifest.mjs --cli "node packages/cli/dist/cli/index.js" + * + * Exit codes: + * 0 manifest matches the derived surface + * 1 drift found (undocumented or stale entries) + * 2 the audit itself could not run (CLI missing, help unparseable) + * + * Exit 2 is deliberately distinct: "the audit broke" must never be reported as + * "the manifest is clean". + */ + +import { readFileSync, readdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync, spawn } from 'node:child_process'; +import { parse } from 'yaml'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(__dirname, '..'); +const manifestPath = join(repoRoot, '.agentworkforce/features/manifest.yaml'); + +const asJson = process.argv.includes('--json'); +const cliArgIndex = process.argv.indexOf('--cli'); +const cliCommand = + cliArgIndex !== -1 && process.argv[cliArgIndex + 1] + ? process.argv[cliArgIndex + 1] + : process.env.RELAY_CLI || `node ${join(repoRoot, 'packages/cli/dist/cli/index.js')}`; + +/** + * Commands that exist but are intentionally absent from the manifest. + * + * Keep this list short and justified — every entry is coverage we are choosing + * not to have. `help` is Commander's own builtin, not a relay feature. + */ +const UNDOCUMENTED_ALLOWLIST = new Set(['help']); + +/** Subcommand trees too costly or destructive to walk during an audit. */ +const NO_WALK = new Set(['help', 'uninstall']); + +// ── surface derivation: CLI ─────────────────────────────────────────────────── + +function runHelp(argPath) { + const [bin, ...prefix] = cliCommand.split(/\s+/); + try { + return execFileSync(bin, [...prefix, ...argPath, '--help'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + env: { ...process.env, NO_COLOR: '1', AGENT_RELAY_DISABLE_UPDATE_CHECK: '1' }, + }); + } catch (err) { + // Commander exits non-zero for some help paths but still writes usage. + const out = `${err.stdout ?? ''}${err.stderr ?? ''}`; + if (out.includes('Usage:')) return out; + throw new Error(`\`${cliCommand} ${argPath.join(' ')} --help\` failed: ${err.message}`); + } +} + +/** + * Parse the "Commands:" block of a Commander help screen. + * + * Commander wraps long descriptions onto continuation lines that are indented + * further than the command names, so a naive per-line split invents commands + * out of description text. Anchor on the indentation of the first entry. + */ +function parseSubcommands(helpText) { + const lines = helpText.split('\n'); + const start = lines.findIndex((line) => /^Commands:/.test(line.trim())); + if (start === -1) return []; + + const names = []; + let entryIndent = null; + + for (const line of lines.slice(start + 1)) { + if (!line.trim()) continue; + const indent = line.length - line.trimStart().length; + if (indent === 0) break; // left the Commands block + if (entryIndent === null) entryIndent = indent; + if (indent > entryIndent) continue; // wrapped description line + + const name = line.trim().split(/\s+/)[0]; + if (!name || name.startsWith('-')) continue; + // Commander renders aliases as "status|st"; the first token is canonical. + names.push(name.split('|')[0]); + } + + return names; +} + +/** Walk the command tree and return every leaf command path. */ +function deriveCliLeaves() { + const leaves = []; + const visit = (path, depth) => { + if (depth > 4) return; // relay's tree is 3 deep; guard against a help loop + const children = parseSubcommands(runHelp(path)); + const walkable = children.filter((child) => !NO_WALK.has(child)); + + // A no-walk child is still part of the surface — record it without + // recursing, or the audit reports it as stale when the manifest has it. + for (const child of children) { + if (!NO_WALK.has(child)) continue; + if (!UNDOCUMENTED_ALLOWLIST.has(child)) leaves.push([...path, child].join(' ')); + } + + if (walkable.length === 0) { + if (path.length > 0) leaves.push(path.join(' ')); + return; + } + + for (const child of walkable) visit([...path, child], depth + 1); + + // A group can also be a runnable leaf (e.g. `relay mcp` with subcommands). + if (path.length > 0) leaves.push(path.join(' ')); + }; + + visit([], 0); + return [...new Set(leaves)].sort(); +} + +// ── surface derivation: MCP ─────────────────────────────────────────────────── + +/** + * List MCP tools via a stdio JSON-RPC handshake against `relay mcp`. + * + * Resolves to null (rather than throwing) when the server cannot be reached, so + * a broken MCP path degrades to an explicit "could not derive" in the report + * instead of an empty tool list that would read as "every tool is stale". + */ +function deriveMcpTools() { + return new Promise((resolve) => { + const [bin, ...prefix] = cliCommand.split(/\s+/); + const child = spawn(bin, [...prefix, 'mcp'], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, NO_COLOR: '1', AGENT_RELAY_DISABLE_UPDATE_CHECK: '1' }, + }); + + let buffer = ''; + let settled = false; + const finish = (value) => { + if (settled) return; + settled = true; + child.kill('SIGTERM'); + resolve(value); + }; + + const timer = setTimeout(() => finish(null), 20_000); + timer.unref?.(); + + child.stdout.on('data', (chunk) => { + buffer += chunk.toString(); + for (const line of buffer.split('\n')) { + const trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + const msg = JSON.parse(trimmed); + if (msg.id === 2 && msg.result?.tools) { + clearTimeout(timer); + finish(msg.result.tools.map((tool) => tool.name).sort()); + } + } catch { + // Partial frame — wait for the rest. + } + } + }); + + child.on('error', () => { + clearTimeout(timer); + finish(null); + }); + child.on('exit', () => { + clearTimeout(timer); + finish(null); + }); + + const send = (msg) => child.stdin.write(`${JSON.stringify(msg)}\n`); + send({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'feature-manifest-audit', version: '1.0.0' }, + }, + }); + send({ jsonrpc: '2.0', method: 'notifications/initialized' }); + send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }); + }); +} + +/** + * MCP tool names the server source still registers, whether or not a default + * stdio probe exposes them. Used to tell "removed" apart from "conditional". + */ +function mcpToolsRegisteredInSource() { + const names = new Set(); + + // Registrations are spread across the whole CLI tree, not just `mcp/`: + // `submit_result` lives in cli/agent-relay-mcp.ts. Scan recursively so a tool + // is never called stale merely because it is declared in a sibling file. + const walk = (dir) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + // No source tree (running against a published CLI) — callers then see + // every probe-missing tool as stale, the safe direction for an audit. + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full); + continue; + } + if (!entry.name.endsWith('.ts') || entry.name.includes('.test.')) continue; + // `registerTool(` and its name argument sit on separate lines here, so + // this must be a whole-file match rather than a line-based grep. + const source = readFileSync(full, 'utf8'); + for (const match of source.matchAll(/registerTool\(\s*'([a-z0-9_]+)'/g)) { + names.add(match[1]); + } + } + }; + + walk(join(repoRoot, 'packages/cli/src/cli')); + return names; +} + +// ── manifest ───────────────────────────────────────────────────────────────── + +function loadManifest() { + const manifest = parse(readFileSync(manifestPath, 'utf8')); + const features = []; + for (const [categoryKey, category] of Object.entries(manifest.categories ?? {})) { + for (const feature of category.features ?? []) { + features.push({ ...feature, category: categoryKey }); + } + } + return { manifest, features }; +} + +/** + * Reduce a documented `cli:` string to its command path. + * + * Manifest entries carry usage decoration — `relay agent add [options]` + * — so strip anything that is not a bare command word. + */ +function documentedCommandPath(cliString) { + const tokens = cliString.trim().split(/\s+/); + if (tokens[0] === 'relay' || tokens[0] === 'agent-relay') tokens.shift(); + const path = []; + for (const token of tokens) { + // Underscores are load-bearing: `set_topic`, `mark_read`, `send_group`. + if (/^[a-z][a-z0-9_-]*$/.test(token)) path.push(token); + else break; // hit , [opt], or --flag + } + return path.join(' '); +} + +// ── report ─────────────────────────────────────────────────────────────────── + +async function main() { + const { manifest, features } = loadManifest(); + + let cliLeaves; + try { + cliLeaves = deriveCliLeaves(); + } catch (err) { + const failure = { ok: false, auditable: false, error: err.message }; + console.error(asJson ? JSON.stringify(failure, null, 2) : `AUDIT_ERROR: ${err.message}`); + process.exit(2); + } + + const mcpTools = await deriveMcpTools(); + + const documentedCommands = new Set( + features.flatMap((f) => (f.cli ? [documentedCommandPath(f.cli)] : [])).filter(Boolean) + ); + const documentedMcp = new Set(features.flatMap((f) => (f.mcp ? [f.mcp] : []))); + + // A parent command counts as covered when a child of it is documented: + // `relay message` needs no entry of its own if `relay message post` has one. + const coveredPrefixes = new Set(); + for (const cmd of documentedCommands) { + const parts = cmd.split(' '); + for (let i = 1; i <= parts.length; i++) coveredPrefixes.add(parts.slice(0, i).join(' ')); + } + + const undocumentedCommands = cliLeaves.filter( + (leaf) => !coveredPrefixes.has(leaf) && !UNDOCUMENTED_ALLOWLIST.has(leaf) + ); + + const cliLeafSet = new Set(cliLeaves); + + // Some commands dispatch on a positional argument rather than a subcommand + // (`relay telemetry enable` is `telemetry [action]`). `relay telemetry enable` + // is a real, documented usage, so a documented command is stale only when no + // prefix of it exists in the CLI at all. + const staleCommands = [...documentedCommands].filter((cmd) => { + const parts = cmd.split(' '); + for (let i = parts.length; i >= 1; i--) { + if (cliLeafSet.has(parts.slice(0, i).join(' '))) return false; + } + return true; + }); + + const undocumentedMcp = mcpTools ? mcpTools.filter((tool) => !documentedMcp.has(tool)) : []; + + // A documented tool missing from a default stdio probe is not automatically + // stale: `list_actions`/`invoke_action`/`submit_result` only register when an + // actions surface is wired in (see mcp/action-tools.ts). Split the two cases + // on whether the server source still registers the tool at all — absent from + // both the probe and the source is genuine drift; absent from only the probe + // is coverage we did not exercise, and is reported as such. + const registeredInSource = mcpToolsRegisteredInSource(); + const documentedMcpMissing = mcpTools ? [...documentedMcp].filter((tool) => !mcpTools.includes(tool)) : []; + const staleMcp = documentedMcpMissing.filter((tool) => !registeredInSource.has(tool)); + const conditionalMcp = documentedMcpMissing.filter((tool) => registeredInSource.has(tool)); + + const drift = undocumentedCommands.length + staleCommands.length + undocumentedMcp.length + staleMcp.length; + + const report = { + ok: drift === 0, + auditable: true, + manifestVersion: manifest.version, + manifestUpdated: manifest.updated, + cliCommand, + mcpDerivable: mcpTools !== null, + counts: { + cliLeaves: cliLeaves.length, + documentedCommands: documentedCommands.size, + mcpTools: mcpTools?.length ?? null, + documentedMcp: documentedMcp.size, + features: features.length, + }, + undocumentedCommands, + staleCommands, + undocumentedMcp, + staleMcp, + conditionalMcp, + }; + + if (asJson) { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log(`=== Feature Manifest Audit ===`); + console.log(`manifest ${manifest.version} (updated ${manifest.updated})`); + console.log(`CLI: ${cliCommand}`); + console.log( + `derived ${cliLeaves.length} CLI leaves, ${mcpTools?.length ?? 'n/a'} MCP tools; ` + + `manifest documents ${documentedCommands.size} commands, ${documentedMcp.size} MCP tools ` + + `across ${features.length} features` + ); + if (mcpTools === null) { + console.log('\nWARN MCP tool list could not be derived — MCP drift not checked.'); + } + const section = (title, items) => { + if (items.length === 0) return; + console.log(`\n${title} (${items.length}):`); + for (const item of items) console.log(` - ${item}`); + }; + section('UNDOCUMENTED commands (exist in CLI, missing from manifest)', undocumentedCommands); + section('STALE commands (in manifest, not in CLI)', staleCommands); + section('UNDOCUMENTED MCP tools', undocumentedMcp); + section('STALE MCP tools (in manifest, not registered anywhere)', staleMcp); + section('CONDITIONAL MCP tools (registered in source, not in this probe)', conditionalMcp); + console.log(drift === 0 ? '\nMANIFEST_CLEAN' : `\nMANIFEST_DRIFT: ${drift} item(s)`); + } + + process.exit(drift === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error(`AUDIT_ERROR: ${err.message}`); + process.exit(2); +}); diff --git a/workflows/audit-feature-manifest.ts b/workflows/audit-feature-manifest.ts new file mode 100644 index 000000000..c5cca6f9b --- /dev/null +++ b/workflows/audit-feature-manifest.ts @@ -0,0 +1,814 @@ +/** + * audit-feature-manifest.ts + * + * Keeps .agentworkforce/features/manifest.yaml honest about the CLI's real + * surface, and files the work when it drifts. + * + * ## Why this exists + * + * The manifest is the input to every feature-verification run, so a command + * nobody documented is a command nobody verifies. Coverage does not fail loudly + * when it shrinks — it just quietly stops covering things. + * + * The only pre-existing guard was `manifest-contract.test.ts`, which asserts + * the manifest contains each entry of a hand-maintained list of ~110 commands. + * That catches a *deletion* from the manifest but is structurally incapable of + * noticing a newly *added* CLI command or MCP tool, because a new command is + * absent from both the manifest and the expected list. On the first run of + * scripts/audit-feature-manifest.mjs, `fleet spawn` and `fleet release` had + * been shipping undocumented and therefore unverified. + * + * This workflow derives the surface instead of asserting a snapshot of it: + * it walks `--help` recursively for the Commander tree and performs a + * `tools/list` JSON-RPC handshake for MCP tools, then diffs against the + * manifest. + * + * Schedule: + * relay cloud schedule workflows/audit-feature-manifest.ts --cron "0 2 * * 1" + * + * Manually: + * relay node workflow run workflows/audit-feature-manifest.ts + * + * ## Exit semantics + * + * The audit script distinguishes three outcomes, and so does this workflow: + * 0 manifest matches the derived surface + * 1 drift — file an issue, update the manifest on a branch, open a draft PR + * 2 the audit could not run — escalate to NightCTO, because a broken audit + * reporting "clean" is worse than no audit at all + * + * ## Environment + * + * AUDIT_SLACK_CHANNEL Slack channel for drift. Default #relay-health. + * AUDIT_AUTOFIX "0" disables the issue/manifest-update/PR path. + * RELAY_CLI CLI under audit. Defaults to the repo build. + * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. + * NIGHTCTO_EVIDENCE_URL Enables escalation when the audit cannot run. + * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. + * SLACK_BOT_TOKEN Slack local runtime; falls back to CLOUD_API_*. + */ + +import { existsSync, readFileSync } from 'node:fs'; + +import { workflow } from '@relayflows/core'; + +const ARTIFACTS = '.workflow-artifacts/audit-feature-manifest'; +const TIMESTAMP = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 16); +const RUN_ID = `audit-${TIMESTAMP}`; +const REPORT_FILE = `${ARTIFACTS}/audit.json`; +const FIX_BRANCH = `chore/${RUN_ID}-manifest`; + +const SLACK_CHANNEL = process.env.AUDIT_SLACK_CHANNEL ?? '#relay-health'; +const AUTOFIX = process.env.AUDIT_AUTOFIX !== '0'; + +/** A literal backtick; a raw one would close the template literals below. */ +const BT = '`'; + +/** + * Optional-environment seeding. `printenv` rather than `${VAR:-default}` + * because these step bodies are TypeScript template literals — a `${` in the + * shell source would be parsed as TypeScript interpolation. Also keeps `set -u` + * from aborting on an unconfigured optional knob. + */ +const ENV_DEFAULTS = String.raw` +POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" +POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" +NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" +NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" +SLACK_BOT_TOKEN="$(printenv SLACK_BOT_TOKEN || true)" +CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" +CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" +RELAY_CLI="$(printenv RELAY_CLI || true)" +VERIFY_ENVIRONMENT="$(printenv VERIFY_ENVIRONMENT || true)" +if [ -z "$POSTHOG_HOST" ]; then POSTHOG_HOST="https://i.agentrelay.com"; fi +if [ -z "$VERIFY_ENVIRONMENT" ]; then VERIFY_ENVIRONMENT="sandbox"; fi +export AUDIT_ARTIFACTS="${ARTIFACTS}" +export AUDIT_RUN_ID="${RUN_ID}" +`; + +async function main() { + const wf = workflow('relay-audit-feature-manifest') + .description( + 'Derive the CLI and MCP surface, diff it against the feature manifest, and open a ' + + 'draft PR updating the manifest when it has drifted.' + ) + .pattern('pipeline') + .channel('relay-health') + .maxConcurrency(3) + // "continue" also opts out of applyReliabilityDefaults attaching a repair + // agent to a failing gate — an audit an agent can edit until it passes is + // not an audit. + .onError('continue') + .timeout(900_000); + + wf.agent('manifest-editor', { + cli: 'claude', + role: 'Update the feature manifest to match the CLI surface the audit derived', + retries: 0, + }); + + // ── Phase 1: run the audit ─────────────────────────────────────────────── + // + // failOnError stays false so the reporting steps below can run; the exit + // code is captured into audit-exit.txt and drives everything downstream. + + wf.step('audit', { + type: 'deterministic', + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +mkdir -p "${ARTIFACTS}" +${ENV_DEFAULTS} + +CLI_ARG="" +if [ -n "$RELAY_CLI" ]; then + CLI_ARG="--cli $RELAY_CLI" +fi + +# Human-readable for the log, JSON for the machinery. Two invocations rather +# than one because the script's report goes to stdout in one format at a time. +node scripts/audit-feature-manifest.mjs $CLI_ARG 2>&1 | tee "${ARTIFACTS}/audit.txt" +EXIT_CODE=$? + +node scripts/audit-feature-manifest.mjs $CLI_ARG --json > "${ARTIFACTS}/audit.json" 2>"${ARTIFACTS}/audit.err" || true + +echo "$EXIT_CODE" > "${ARTIFACTS}/audit-exit.txt" +echo "audit exit code: $EXIT_CODE" +exit 0 +`, + }); + + // ── Phase 2: escalate an unrunnable audit ──────────────────────────────── + // + // Exit 2 means the audit itself broke (CLI missing, help unparseable). That + // is an operator problem, not a manifest problem: a silent audit means + // coverage can drift indefinitely with nobody noticing. + + wf.step('escalate-infra', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +if [ "$EXIT_CODE" != "2" ]; then + echo "Audit ran (exit $EXIT_CODE) — nothing to escalate." + exit 0 +fi + +DETAIL=$(head -c 300 "$ARTIFACTS/audit.txt" 2>/dev/null || true) +if [ -z "$DETAIL" ]; then DETAIL="no audit output"; fi +DETAIL_CLEAN=$(printf '%s' "$DETAIL" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g') + +if [ -z "$NIGHTCTO_EVIDENCE_URL" ]; then + echo "NIGHTCTO_EVIDENCE_URL unset — cannot escalate. Audit failure detail:" + printf '%s\n' "$DETAIL" + exit 0 +fi + +BODY=$(printf '{"schemaVersion":"cloud-runtime-evidence/1","service":"relay-audit-feature-manifest","environment":"%s","version":"unknown","path":"relayflow.audit.manifest","kind":"request_error","outcome":"error","severity":6,"occurredAt":"%s","requestId":"%s","correlationIds":{"ingress":"relayflow"},"summary":"feature manifest audit could not run: %s","errorCode":"audit_unrunnable","inspect":{"logQuery":"%s"}}' \ + "$VERIFY_ENVIRONMENT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$DETAIL_CLEAN" "$ARTIFACTS") + +if curl -sS -m 15 -X POST "$NIGHTCTO_EVIDENCE_URL" \ + -H 'content-type: application/json' \ + -H "authorization: Bearer $NIGHTCTO_EVIDENCE_TOKEN" \ + -H "x-nightcto-evidence-token: $NIGHTCTO_EVIDENCE_TOKEN" \ + -d "$BODY" >/dev/null 2>&1; then + echo "escalated audit_unrunnable to NightCTO" +else + echo "NightCTO escalation POST failed (non-fatal)" +fi +exit 0 +`, + }); + + // ── Phase 3: PostHog ───────────────────────────────────────────────────── + + wf.step('emit-posthog', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +${ENV_DEFAULTS} + +if [ -z "$POSTHOG_API_KEY" ]; then + echo "POSTHOG_API_KEY unset — not emitting" + exit 0 +fi + +PROPS=$(node <<'PHEOF' +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +let report = null; +try { + report = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); +} catch { + report = null; +} +const exitCode = (() => { + try { + return Number(fs.readFileSync(artifacts + '/audit-exit.txt', 'utf8').trim()); + } catch { + return 2; + } +})(); +process.stdout.write( + JSON.stringify({ + run_id: process.env.AUDIT_RUN_ID, + exit_code: exitCode, + auditable: report?.auditable ?? false, + ok: report?.ok ?? false, + manifest_version: report?.manifestVersion ?? null, + manifest_updated: report?.manifestUpdated ?? null, + cli_leaves: report?.counts?.cliLeaves ?? null, + documented_commands: report?.counts?.documentedCommands ?? null, + mcp_tools: report?.counts?.mcpTools ?? null, + features: report?.counts?.features ?? null, + undocumented_commands: (report?.undocumentedCommands ?? []).join(',') || null, + stale_commands: (report?.staleCommands ?? []).join(',') || null, + undocumented_mcp: (report?.undocumentedMcp ?? []).join(',') || null, + stale_mcp: (report?.staleMcp ?? []).join(',') || null, + drift_count: + (report?.undocumentedCommands ?? []).length + + (report?.staleCommands ?? []).length + + (report?.undocumentedMcp ?? []).length + + (report?.staleMcp ?? []).length, + }) +); +PHEOF +) + +BODY=$(printf '{"api_key":"%s","event":"relay_manifest_audit","distinct_id":"relay-audit-feature-manifest","properties":%s}' \ + "$POSTHOG_API_KEY" "$PROPS") + +if curl -sS -m 15 -X POST "$POSTHOG_HOST/capture/" \ + -H 'content-type: application/json' -d "$BODY" >/dev/null 2>&1; then + echo "emitted relay_manifest_audit" +else + echo "PostHog emit failed (non-fatal)" +fi +exit 0 +`, + }); + + // ── Phase 4: Slack ─────────────────────────────────────────────────────── + + wf.step('slack-alert', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +if [ "$EXIT_CODE" = "0" ]; then + echo "SLACK_SKIPPED: manifest is clean" + exit 0 +fi + +node <<'SLACKEOF' > "$ARTIFACTS/slack-message.txt" +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +const exitCode = Number(fs.readFileSync(artifacts + '/audit-exit.txt', 'utf8').trim()); + +const lines = []; +if (exitCode === 2) { + lines.push(':warning: *Feature manifest audit could not run* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push('This is harness breakage. Coverage drift is currently undetectable.'); + let detail = ''; + try { + detail = fs.readFileSync(artifacts + '/audit.txt', 'utf8').slice(0, 500); + } catch { + detail = 'no audit output'; + } + lines.push('${BT}${BT}${BT}'); + lines.push(detail); + lines.push('${BT}${BT}${BT}'); +} else { + const r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); + lines.push(':clipboard: *Feature manifest has drifted* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push( + 'manifest ' + r.manifestVersion + ' (updated ' + r.manifestUpdated + ') — ' + + r.counts.cliLeaves + ' CLI leaves derived, ' + r.counts.documentedCommands + ' documented' + ); + lines.push(''); + const section = (title, items) => { + if (!items || items.length === 0) return; + lines.push('*' + title + '* (' + items.length + ')'); + for (const item of items.slice(0, 20)) lines.push(' • ${BT}' + item + '${BT}'); + }; + section('Undocumented commands — shipping unverified', r.undocumentedCommands); + section('Stale commands — in manifest, not in CLI', r.staleCommands); + section('Undocumented MCP tools', r.undocumentedMcp); + section('Stale MCP tools', r.staleMcp); + if (r.conditionalMcp && r.conditionalMcp.length > 0) { + lines.push(''); + lines.push( + '_Not drift, but unexercised: ' + + r.conditionalMcp.join(', ') + + ' are registered conditionally and did not appear in the probe._' + ); + } + if (!r.mcpDerivable) { + lines.push(''); + lines.push(':warning: MCP tool list could not be derived — MCP drift was NOT checked.'); + } +} +process.stdout.write(lines.join('\n')); +SLACKEOF + +if [ -n "$SLACK_BOT_TOKEN" ]; then + RESP=$(curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ + -H "authorization: Bearer $SLACK_BOT_TOKEN" \ + -H 'content-type: application/json; charset=utf-8' \ + --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8"),unfurl_links:false}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo '{"ok":false}') + if printf '%s' "$RESP" | grep -q '"ok":true'; then + echo "SLACK_POSTED: $CHANNEL" + exit 0 + fi + echo "SLACK_FAILED: $(printf '%s' "$RESP" | head -c 300)" +fi + +if [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; then + RESP=$(curl -sS -m 20 -X POST "$CLOUD_API_URL/api/v1/integrations/slack/messages" \ + -H "authorization: Bearer $CLOUD_API_TOKEN" \ + -H 'content-type: application/json' \ + --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8")}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || true) + if [ -n "$RESP" ] && ! printf '%s' "$RESP" | grep -qi 'error\|unauthorized\|not found'; then + echo "SLACK_POSTED: $CHANNEL (cloud-relay runtime)" + exit 0 + fi +fi + +echo "SLACK_UNDELIVERED: no SLACK_BOT_TOKEN and no CLOUD_API_URL/CLOUD_API_TOKEN." +echo "---- undelivered Slack payload ----" +cat "$ARTIFACTS/slack-message.txt" +exit 0 +`, + }); + + // ── Phase 5: file an issue ─────────────────────────────────────────────── + + wf.step('file-issue', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +AUTOFIX="${AUTOFIX ? '1' : '0'}" +${ENV_DEFAULTS} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ "$EXIT_CODE" != "1" ]; then + echo "ISSUE_SKIPPED: audit exit is '$EXIT_CODE' — only drift (1) files an issue" + exit 0 +fi +if [ "$AUTOFIX" != "1" ]; then + echo "ISSUE_SKIPPED: AUDIT_AUTOFIX=0" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +node <<'ISSUEEOF' > "$ARTIFACTS/issue-body.md" +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +const r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); + +const out = []; +out.push('The feature manifest no longer matches the CLI surface.'); +out.push(''); +out.push('An undocumented command is an unverified command: the manifest is the'); +out.push('input to every feature-verification run, so drift here silently shrinks'); +out.push('coverage.'); +out.push(''); +out.push('- Manifest: ${BT}' + r.manifestVersion + '${BT} (updated ' + r.manifestUpdated + ')'); +out.push('- Derived: ' + r.counts.cliLeaves + ' CLI leaves, ' + (r.counts.mcpTools ?? 'n/a') + ' MCP tools'); +out.push('- Documented: ' + r.counts.documentedCommands + ' commands, ' + r.counts.documentedMcp + ' MCP tools across ' + r.counts.features + ' features'); +out.push(''); +const section = (title, items, note) => { + if (!items || items.length === 0) return; + out.push('## ' + title); + out.push(''); + if (note) { + out.push(note); + out.push(''); + } + for (const item of items) out.push('- ${BT}' + item + '${BT}'); + out.push(''); +}; +section( + 'Undocumented commands', + r.undocumentedCommands, + 'These exist in the CLI but have no manifest entry, so nothing verifies them.' +); +section( + 'Stale commands', + r.staleCommands, + 'These are documented but no prefix of them exists in the CLI. Either the command was removed and the entry should go, or it was renamed.' +); +section('Undocumented MCP tools', r.undocumentedMcp); +section( + 'Stale MCP tools', + r.staleMcp, + 'Documented, and not registered anywhere in the CLI source.' +); +if (r.conditionalMcp && r.conditionalMcp.length > 0) { + out.push('## Conditionally registered (not drift)'); + out.push(''); + out.push('Registered in source but absent from a default stdio probe, so this run did not exercise them:'); + out.push(''); + for (const item of r.conditionalMcp) out.push('- ${BT}' + item + '${BT}'); + out.push(''); +} +out.push('---'); +out.push('Filed by ${BT}workflows/audit-feature-manifest.ts${BT}. Reproduce with:'); +out.push(''); +out.push('${BT}${BT}${BT}bash'); +out.push('node scripts/audit-feature-manifest.mjs'); +out.push('${BT}${BT}${BT}'); +process.stdout.write(out.join('\n')); +ISSUEEOF + +ISSUE_URL=$(gh issue create --title "Feature manifest drift: ${RUN_ID}" \ + --body-file "$ARTIFACTS/issue-body.md" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || true) + +if [ -n "$ISSUE_URL" ]; then + echo "$ISSUE_URL" > "$ARTIFACTS/issue-url.txt" + echo "ISSUE_CREATED: $ISSUE_URL" +else + echo "ISSUE_FAILED: gh issue create produced no URL" +fi +exit 0 +`, + }); + + // ── Phase 6: update the manifest ───────────────────────────────────────── + + wf.step('update-manifest', { + agent: 'manifest-editor', + dependsOn: ['file-issue'], + task: `Bring ${'`.agentworkforce/features/manifest.yaml`'} back in line with the CLI surface. + +## FIRST: check whether there is anything to do + +Read ${ARTIFACTS}/audit-exit.txt. + +- If it does not contain exactly \`1\`, there is NOTHING to do. Write "No drift — + nothing to update." to ${ARTIFACTS}/update-summary.md and STOP IMMEDIATELY: + do not create a branch, edit files, or run commands. +- Only if it contains \`1\` do you continue. + +(The engine has no conditional steps, so this step is scheduled on every run. +The early exit is what keeps an agent away from a clean tree.) + +## The task + +Read ${REPORT_FILE}. It lists, as JSON: + +- \`undocumentedCommands\` — real CLI commands with no manifest entry. +- \`staleCommands\` — manifest entries whose command no longer exists. +- \`undocumentedMcp\` / \`staleMcp\` — the same for MCP tools. +- \`conditionalMcp\` — NOT drift. Registered in source but not exposed by a + default stdio probe. Leave these alone. + +## Hard rules + +1. NEVER commit or push to \`main\`. Work on a branch named exactly + \`${FIX_BRANCH}\`, created from the current HEAD. +2. For each undocumented command, add a manifest entry in the correct existing + category, matching the surrounding style exactly: \`id\`, \`name\`, \`cli\` + (with its real argument and option signature, taken from + \`relay --help\`), \`description\`, \`location\` (the actual source + file), and \`verify_tier\`. +3. Choose \`verify_tier\` honestly, using the scale documented at the top of the + manifest. Do not mark something tier 1 to make it look cheap to verify. +4. For stale entries, confirm with \`relay --help\` that the command + really is gone before removing anything. If it still exists in any form, the + audit's derivation is wrong — say so and change nothing. +5. Do NOT delete or weaken unrelated manifest entries. +6. Bump the manifest's \`updated:\` field to today's date. + +## Verify before you finish + +Run both of these and paste the real output: + +\`\`\`bash +node scripts/audit-feature-manifest.mjs +npx vitest run .agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts +\`\`\` + +The audit must print MANIFEST_CLEAN and the contract test must pass. If the +contract test's hardcoded expectation list now disagrees with the manifest, add +the new commands to that list too — it is a snapshot that must be maintained +alongside the manifest. + +## Deliverable + +Write ${ARTIFACTS}/update-summary.md with: + +- \`Added:\` each new entry and the tier you assigned it, with your reasoning. +- \`Removed:\` each removed entry and the evidence the command is gone. +- \`Evidence:\` the output of both commands above. + +Commit on the branch. Do NOT open a pull request — the next step does that.`, + }); + + // ── Phase 7: integrity gate ────────────────────────────────────────────── + // + // The audit's own credibility depends on the manifest only ever growing to + // match reality. An "update" that shrinks documented coverage, or that edits + // the audit script instead of the manifest, is the failure mode to catch. + + wf.step('update-integrity', { + type: 'deterministic', + dependsOn: ['update-manifest'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +MANIFEST=".agentworkforce/features/manifest.yaml" + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ "$EXIT_CODE" != "1" ]; then + echo "INTEGRITY_NOT_APPLICABLE: no drift to fix" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +if ! git rev-parse --git-dir >/dev/null 2>&1; then + echo "INTEGRITY_SKIPPED: not a git repository" + echo "INTEGRITY=skipped" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +CURRENT=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown) +if [ "$CURRENT" = "main" ]; then + echo "INTEGRITY_FAIL: editor left the repo on main" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +BASE_REF=$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || true) +if [ -z "$BASE_REF" ]; then + echo "INTEGRITY_SKIPPED: no base ref to diff against" + echo "INTEGRITY=skipped" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# 'grep -c' exits 1 on zero matches while still printing 0, so the fallback is +# '|| true'; '|| echo 0' would emit a second line and break the comparison. +BEFORE=$(git show "$BASE_REF:$MANIFEST" 2>/dev/null | grep -cE '^ - id: ' || true) +AFTER=$(grep -cE '^ - id: ' "$MANIFEST" 2>/dev/null || true) +if [ -z "$BEFORE" ]; then BEFORE=0; fi +if [ -z "$AFTER" ]; then AFTER=0; fi + +echo "Manifest feature entries: $BEFORE -> $AFTER" + +# A drift fix may legitimately remove genuinely stale entries, so a small +# decrease is allowed; a large one means the editor deleted real coverage. +STALE_COUNT=$(node -e 'try{const r=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));process.stdout.write(String((r.staleCommands||[]).length+(r.staleMcp||[]).length))}catch(e){process.stdout.write("0")}' "$ARTIFACTS/audit.json" 2>/dev/null || echo 0) +MIN_ALLOWED=$((BEFORE - STALE_COUNT)) + +if [ "$AFTER" -lt "$MIN_ALLOWED" ]; then + echo "INTEGRITY_FAIL: manifest dropped to $AFTER entries; only $STALE_COUNT stale entr(ies) justified removal (floor $MIN_ALLOWED)." + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# The audit script is the measuring instrument. Editing it to resolve drift is +# moving the goalposts, so require an explicit human decision for that. +if ! git diff --quiet "$BASE_REF" -- scripts/audit-feature-manifest.mjs 2>/dev/null; then + echo "INTEGRITY_FAIL: the branch modified scripts/audit-feature-manifest.mjs." + echo "The audit script is the measuring instrument — changing it to resolve drift needs human review." + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +if [ ! -f "$ARTIFACTS/update-summary.md" ]; then + echo "INTEGRITY_FAIL: no update-summary.md, so there is no reasoning on record" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# The whole point: after the edit the audit must actually be clean. +if node scripts/audit-feature-manifest.mjs >"$ARTIFACTS/audit-after.txt" 2>&1; then + echo "INTEGRITY_OK: audit is clean after the update" + echo "INTEGRITY=ok" > "$ARTIFACTS/update-integrity.env" +else + echo "INTEGRITY_FAIL: audit still reports drift after the update:" + tail -20 "$ARTIFACTS/audit-after.txt" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" +fi + +exit 0 +`, + }); + + // ── Phase 8: draft PR ──────────────────────────────────────────────────── + + wf.step('open-pr', { + type: 'deterministic', + dependsOn: ['update-integrity'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/update-integrity.env" ]; then + echo "PR_SKIPPED: no integrity result" + exit 0 +fi + +. "$ARTIFACTS/update-integrity.env" + +if [ "$INTEGRITY" != "ok" ]; then + echo "PR_SKIPPED: integrity gate returned '$INTEGRITY'" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "PR_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$BRANCH" = "main" ]; then + echo "PR_SKIPPED: refusing to open a PR from main" + exit 0 +fi + +git push -u origin "$BRANCH" 2>&1 | tail -3 || { + echo "PR_FAILED: could not push $BRANCH" + exit 0 +} + +{ + echo "## Feature manifest drift correction" + echo "" + echo "Run: ${RUN_ID}" + echo "" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + echo "Fixes $(cat "$ARTIFACTS/issue-url.txt")" + echo "" + fi + echo "### What drifted" + echo "" + echo '${BT}${BT}${BT}' + cat "$ARTIFACTS/audit.txt" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Editor summary" + echo "" + cat "$ARTIFACTS/update-summary.md" 2>/dev/null || echo "_No summary produced._" + echo "" + echo "### Audit after the change" + echo "" + echo '${BT}${BT}${BT}' + cat "$ARTIFACTS/audit-after.txt" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Review notes" + echo "" + echo "- Opened as a **draft** by ${BT}workflows/audit-feature-manifest.ts${BT}. Not auto-merged." + echo "- The integrity gate confirmed the audit is clean after the change, that the" + echo " manifest did not lose unjustified entries, and that the audit script itself" + echo " was not modified." + echo "- Still needs a human on the ${BT}verify_tier${BT} assignments: the gate cannot tell a" + echo " correct tier from a convenient one." +} > "$ARTIFACTS/pr-body.md" + +PR_URL=$(gh pr create --draft --title "chore: sync feature manifest with CLI surface (${RUN_ID})" \ + --body-file "$ARTIFACTS/pr-body.md" --base main --head "$BRANCH" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || true) + +if [ -n "$PR_URL" ]; then + echo "$PR_URL" > "$ARTIFACTS/pr-url.txt" + echo "PR_CREATED: $PR_URL" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + gh issue comment "$(cat "$ARTIFACTS/issue-url.txt")" \ + --body "Draft manifest sync PR opened: $PR_URL" >/dev/null 2>&1 || true + fi +else + echo "PR_FAILED: gh pr create produced no URL" +fi +exit 0 +`, + }); + + // ── Phase 9: enforce ───────────────────────────────────────────────────── + // + // Terminal, and the only failing step. Does not depend on the fix chain, so + // a hiccup while opening a PR cannot swallow the drift signal. + + wf.step('enforce', { + type: 'deterministic', + dependsOn: ['slack-alert', 'emit-posthog', 'escalate-infra'], + captureOutput: true, + failOnError: true, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +case "$EXIT_CODE" in + 0) + echo "ENFORCE: manifest is clean" + exit 0 + ;; + 1) + echo "ENFORCE: manifest drift — see the issue and draft PR" + exit 1 + ;; + *) + echo "ENFORCE: audit could not run (exit $EXIT_CODE) — escalated to NightCTO" + exit 2 + ;; +esac +`, + }); + + const result = await wf.run(); + + if (process.env.DRY_RUN || !('status' in result)) { + return; + } + + // Make the process exit code tell the truth, the same way verify-features + // does: a scheduler must be able to see drift without reading logs. + const exitFile = `${ARTIFACTS}/audit-exit.txt`; + if (!existsSync(exitFile)) { + console.error( + `[audit-feature-manifest] no ${exitFile} — the audit step did not complete. ` + + `Treating as harness breakage.` + ); + process.exitCode = 2; + return; + } + + const auditExit = Number(readFileSync(exitFile, 'utf8').trim()); + let report: { undocumentedCommands?: string[]; staleCommands?: string[] } | null = null; + if (existsSync(REPORT_FILE)) { + try { + report = JSON.parse(readFileSync(REPORT_FILE, 'utf8')); + } catch { + report = null; + } + } + + if (auditExit === 0) { + console.log('[audit-feature-manifest] manifest is clean'); + return; + } + + if (auditExit === 1) { + const undocumented = report?.undocumentedCommands ?? []; + const stale = report?.staleCommands ?? []; + console.error( + `[audit-feature-manifest] DRIFT: ${undocumented.length} undocumented, ${stale.length} stale. ` + + `Undocumented commands are unverified commands.` + ); + process.exitCode = 1; + return; + } + + console.error(`[audit-feature-manifest] the audit could not run (exit ${auditExit})`); + process.exitCode = 2; +} + +main().catch((err) => { + console.error( + `[audit-feature-manifest] harness failure: ${err instanceof Error ? err.stack : String(err)}` + ); + process.exitCode = 2; +}); diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index c7bef777f..0755dc9f2 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -1,10 +1,10 @@ /** * verify-features.ts * - * Automated feature health check for agent-relay. - * Runs through every user-facing feature tier by tier, captures - * pass/fail per check, and has a cheap opencode agent write the - * PASS/FAIL report into #relay-health. + * Automated feature health check for agent-relay. Walks every user-facing + * feature tier by tier, records one structured result per check, and turns a + * failure into action: a Slack alert, a GitHub issue, an attempted fix on a + * branch, and a draft PR carrying re-verified evidence. * * Designed to run on a schedule (nightly or post-merge): * relay cloud schedule workflows/verify-features.ts --cron "0 3 * * *" @@ -12,796 +12,2035 @@ * Or manually: * relay node workflow run workflows/verify-features.ts * - * Pattern: pipeline (sequential phases — each tier depends on the prior passing) + * Pattern: pipeline (sequential phases — later tiers assume earlier ones ran) * - * Acceptance contract (V1–V5): - * V1 Tier 1: CLI health — version, status, doctor, telemetry, workspace list - * V2 Tier 2: Broker lifecycle — up/down/status/metrics + agent register/list/remove - * V3 Tier 3: Channel messaging round-trip — create/join/post/list/reply/thread/search - * V4 Tier 4: Cross-agent — DM send/list, channel invite, read receipts - * V5 Critical paths — the 4 sequences that must work for the product to function + * ## Why this file is shaped the way it is + * + * An earlier version of this workflow reported "COMPLETED — 11 passed, 0 + * failed" on a run where Tier 2 had four failing checks. Three independent + * defects produced that false green, and each one is now closed here: + * + * 1. Every step set `failOnError: false`, so a non-zero exit still scored as + * a pass. Tiers still use it (we want all tiers to run even after one + * fails), but they are no longer the verdict — `verdict` is, and it reads + * a machine-readable ledger rather than grepping prose logs. + * 2. Nothing aggregated the per-tier results into a run verdict. `verdict` + * plus `enforce-verdict` now do, and `main()` sets a non-zero process exit + * so a scheduler sees red. + * 3. `main()` discarded the runner result entirely. It now reads verdict.json + * and fails closed when that file is missing. + * + * A fourth defect was subtler. `applyReliabilityDefaults` in @relayflows/core + * force-enables `strategy: "retry"` with a repair agent for any workflow that + * declares agents, so a failing verification gate got handed to an agent that + * edited the working tree until the gate passed. A verification workflow whose + * assertions can be rewritten to make them pass measures nothing. `onError` + * is set to "continue" below specifically to opt out of that path. + * + * ## Honest accounting + * + * Every check records `pass`, `fail`, or `skip` with a reason. A check that + * cannot run in this environment (no cloud login, no provider CLI) is a SKIP + * with a stated cause — never a pass, and never a fail. `verdict` also fails + * the run when a tier produced no records at all, so a tier that crashes + * before its first check cannot masquerade as a clean tier. * * Feature manifest: .agentworkforce/features/manifest.yaml * Critical paths: .agentworkforce/features/critical-paths.md * Procedures ref: .agentworkforce/features/verify/procedures.md + * Manifest audit: workflows/audit-feature-manifest.ts + * + * ## Environment + * + * VERIFY_SLACK_CHANNEL Slack channel for failures. Default #relay-health. + * VERIFY_AUTOFIX "0" disables the issue/fix/PR path. Default on. + * VERIFY_ALLOW_CLI_DRIFT "1" downgrades a CLI-vs-repo version mismatch from + * a failure to a warning. + * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. + * NIGHTCTO_EVIDENCE_URL Enables infra escalation to NightCTO. + * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. + * SLACK_BOT_TOKEN Slack local runtime. Falls back to CLOUD_API_URL + + * CLOUD_API_TOKEN (cloud-relay runtime). */ +import { existsSync, readFileSync } from 'node:fs'; + import { workflow } from '@relayflows/core'; const ARTIFACTS = '.workflow-artifacts/verify-features'; const TIMESTAMP = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 16); const RUN_ID = `verify-${TIMESTAMP}`; +const VERDICT_FILE = `${ARTIFACTS}/verdict.json`; -// Unique suffixes to avoid collisions with existing workspace state +/** Unique suffix so a run never collides with existing workspace state. */ const SUFFIX = `vf-${Date.now()}`; -const SUFFIX2 = `cp-${Date.now()}`; + +/** Canonical fix branch. RUN_ID already carries the "verify-" prefix. */ +const FIX_BRANCH = `fix/${RUN_ID}`; + +const SLACK_CHANNEL = process.env.VERIFY_SLACK_CHANNEL ?? '#relay-health'; +const AUTOFIX = process.env.VERIFY_AUTOFIX !== '0'; + +/** + * A literal backtick. + * + * Step bodies below are template literals, so a raw backtick would close the + * template. The Slack and GitHub payloads want Markdown code spans, so those + * are written as `${BT}` and interpolated in. + */ +const BT = '`'; + +/** + * Optional-environment seeding, interpolated ahead of every step body. + * + * `printenv` is used instead of the natural `${VAR:-default}` for one hard + * reason: these step bodies are TypeScript template literals, so a `${` in the + * shell source is parsed as TypeScript interpolation and the file would not + * compile. Every shell variable below is therefore written `$VAR`, and defaults + * are applied with explicit `if` blocks. Seeding also keeps `set -u` from + * aborting a step because an operator did not configure an optional knob. + */ +const ENV_DEFAULTS = String.raw` +VERIFY_ALLOW_CLI_DRIFT="$(printenv VERIFY_ALLOW_CLI_DRIFT || true)" +POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" +POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" +NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" +NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" +SLACK_BOT_TOKEN="$(printenv SLACK_BOT_TOKEN || true)" +CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" +CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" +VERIFY_ENVIRONMENT="$(printenv VERIFY_ENVIRONMENT || true)" +if [ -z "$POSTHOG_HOST" ]; then POSTHOG_HOST="https://i.agentrelay.com"; fi +if [ -z "$VERIFY_ENVIRONMENT" ]; then VERIFY_ENVIRONMENT="sandbox"; fi +export VERIFY_ARTIFACTS="${ARTIFACTS}" +export VERIFY_RUN_ID="${RUN_ID}" +`; + +/** + * Shell harness interpolated into every check-running step. + * + * Every check appends one JSON line to checks.jsonl. Downstream steps read + * that ledger instead of grepping the human-readable logs — the previous + * version's analyzer reported "1 issue(s)" for a tier with four failures + * because it counted matching *tiers*, not matching checks. + * + * Written with String.raw so the sed escapes survive into the shell. Shell + * variables must be written `$VAR`, never `${VAR}` — braces would be read as + * TypeScript interpolation. + */ +const PRELUDE = + ENV_DEFAULTS + + String.raw` +PASS=0 +FAIL=0 +SKIP=0 +ARTIFACTS="${ARTIFACTS}" +CHECKS="$ARTIFACTS/checks.jsonl" +CAPS="$ARTIFACTS/caps.env" +RUN_ID="${RUN_ID}" + +# Append one structured result AND move the tier counters. +# +# The counters live here, not in the callers, so that the per-tier summary line +# can never disagree with the ledger. Several checks below record directly +# (multi-step sequences that do not fit run_check); when the callers owned the +# counters those sites reported "0 passed, 0 failed" while the ledger held real +# results. +# +# Reason text is flattened and bounded so a multi-line stack trace cannot +# corrupt the ledger. +record() { + _reason=$(printf '%s' "$4" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-400) + printf '{"run":"%s","tier":"%s","check":"%s","status":"%s","reason":"%s"}\n' \ + "$RUN_ID" "$1" "$2" "$3" "$_reason" >> "$CHECKS" + case "$3" in + pass) PASS=$((PASS + 1)) ;; + fail) FAIL=$((FAIL + 1)) ;; + skip) SKIP=$((SKIP + 1)) ;; + esac +} + +# Run a command and match its output. Records pass/fail either way. +run_check() { + _name="$1" + _cmd="$2" + _expect="$3" + if _out=$(eval "$_cmd" 2>&1) && printf '%s' "$_out" | grep -qi -- "$_expect"; then + echo " PASS $_name" | tee -a "$LOG" + record "$TIER" "$_name" pass "" + else + echo " FAIL $_name (expected: $_expect)" | tee -a "$LOG" + echo " output: $(printf '%s' "$_out" | head -3)" | tee -a "$LOG" + record "$TIER" "$_name" fail "$_out" + fi +} + +# A check we could not run here. Recorded with a cause so that reading the +# report can never confuse "not exercised" with "verified working". +skip_check() { + echo " SKIP $1 ($2)" | tee -a "$LOG" + record "$TIER" "$1" skip "$2" +} + +# Capability flags are written by the "capabilities" step. +have_cap() { grep -q "^$1=1$" "$CAPS" 2>/dev/null; } + +# Run a check only when a capability is present, else skip with the reason. +gated_check() { + _cap="$1" + shift + if have_cap "$_cap"; then + run_check "$@" + else + skip_check "$1" "requires capability: $_cap" + fi +} + +finish_tier() { + echo "" | tee -a "$LOG" + echo "$TIER result: $PASS passed, $FAIL failed, $SKIP skipped" | tee -a "$LOG" + if [ "$FAIL" -gt 0 ]; then + echo "TIER_FAIL" >> "$LOG" + else + echo "TIER_PASS" >> "$LOG" + fi +} +`; + +/** + * Bounded evidence POST to NightCTO, matching the CloudEvidenceSummary v1 + * contract in ../cloud/docs/dogfood-telemetry-brief.md. Byte compatibility + * matters — NightCTO's cloud-runtime adapter consumes exactly this shape. + * + * This is for failures of the verification *machinery* (the relayflow, the + * sandbox, the executor), not for product features being broken. A broken + * feature is a PR; a broken harness is an operator alert, because nobody is + * watching a check that never ran. + */ +const NIGHTCTO_FN = String.raw` +escalate_infra() { + _path="$1" + _code="$2" + _summary="$3" + if [ -z "$NIGHTCTO_EVIDENCE_URL" ]; then + echo " [nightcto] NIGHTCTO_EVIDENCE_URL unset — infra escalation skipped: $_summary" + return 0 + fi + _summary_clean=$(printf '%s' "$_summary" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-300) + _body=$(printf '{"schemaVersion":"cloud-runtime-evidence/1","service":"relay-verify-features","environment":"%s","version":"%s","path":"%s","kind":"request_error","outcome":"error","severity":6,"occurredAt":"%s","requestId":"%s","correlationIds":{"ingress":"relayflow"},"summary":"%s","errorCode":"%s","inspect":{"logQuery":"%s"}}' \ + "$VERIFY_ENVIRONMENT" "$VERIFY_CLI_VERSION" "$_path" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$_summary_clean" "$_code" "$ARTIFACTS") + # Fire-and-forget: an unreachable NightCTO must never mask the product result. + if curl -sS -m 15 -X POST "$NIGHTCTO_EVIDENCE_URL" \ + -H 'content-type: application/json' \ + -H "authorization: Bearer $NIGHTCTO_EVIDENCE_TOKEN" \ + -H "x-nightcto-evidence-token: $NIGHTCTO_EVIDENCE_TOKEN" \ + -d "$_body" >/dev/null 2>&1; then + echo " [nightcto] escalated: $_code" + else + echo " [nightcto] escalation POST failed (non-fatal): $_code" + fi + return 0 +} +`; + +/** + * PostHog emission via the public capture endpoint. + * + * Deliberately not routed through the CLI's telemetry client: that client is + * opt-in *product* telemetry keyed to a real user's distinct_id, and CI health + * events would pollute it. This posts CI-owned events under a stable + * synthetic distinct_id instead. + */ +const POSTHOG_FN = String.raw` +posthog_capture() { + _event="$1" + _props="$2" + if [ -z "$POSTHOG_API_KEY" ]; then + echo " [posthog] POSTHOG_API_KEY unset — not emitting $_event" + return 0 + fi + _body=$(printf '{"api_key":"%s","event":"%s","distinct_id":"relay-verify-features","properties":%s}' \ + "$POSTHOG_API_KEY" "$_event" "$_props") + if curl -sS -m 15 -X POST "$POSTHOG_HOST/capture/" \ + -H 'content-type: application/json' -d "$_body" >/dev/null 2>&1; then + echo " [posthog] emitted $_event" + else + echo " [posthog] emit failed (non-fatal): $_event" + fi + return 0 +} +`; async function main() { const wf = workflow('relay-verify-features') .description( - 'Automated feature health check. Runs CLI verification tiers 1–4 and the 5 critical paths. ' + - 'Posts a structured PASS/FAIL report to #relay-health.' + 'Automated feature health check. Runs verification tiers 1-6 and the 6 critical paths, ' + + 'posts PASS/FAIL to Slack, and opens an issue plus a draft fix PR when something breaks.' ) .pattern('pipeline') .channel('relay-health') .maxConcurrency(1) - .timeout(600_000); // 10 minutes + // Not cosmetic: "continue" is the only strategy that keeps + // applyReliabilityDefaults from attaching a repair agent to failing + // verification gates. See the file header. + .onError('continue') + .timeout(3_600_000); // 1 hour — tiers 5/6 drive real cloud and provider calls // ── Agents ──────────────────────────────────────────────────────────────── - // opencode is cheap for routine summarization — no complex reasoning needed + // opencode is cheap and this is routine summarization of files already written. wf.agent('reporter', { cli: 'opencode', - role: 'Read verification artifact files and write a structured PASS/FAIL report for the relay-health channel', + role: 'Read verification artifacts and write a structured PASS/FAIL/SKIP report', retries: 1, }); - // ── Phase 0: Emit acceptance contract ──────────────────────────────────── + // The fixer edits source, so it gets the stronger model. + wf.agent('fixer', { + cli: 'claude', + role: 'Root-cause a failing feature verification check and fix the underlying defect', + retries: 0, + }); + + // ── Phase 0: acceptance contract ───────────────────────────────────────── + // + // Unquoted heredoc terminator so $(date) actually expands. The previous + // version used <<'EOF' and printed the literal text "$(date -u ...)". wf.step('acceptance-contract', { type: 'deterministic', captureOutput: true, failOnError: false, - command: `cat <<'EOF' + command: String.raw`cat < run FAIL. + - Any tier with no records => run FAIL (a crashed tier is not a clean tier). + - A check that cannot run => SKIP with a stated cause. Never a pass. + +A FAIL posts to Slack, opens a GitHub issue, and attempts a fix on a branch +with a draft PR. A failure of the harness itself escalates to NightCTO. EOF `, }); - // ── Phase 1: Setup ─────────────────────────────────────────────────────── + // ── Phase 1: setup ─────────────────────────────────────────────────────── wf.step('setup', { type: 'deterministic', dependsOn: ['acceptance-contract'], captureOutput: true, failOnError: false, - command: ` -set -euo pipefail + command: String.raw` +set -uo pipefail mkdir -p "${ARTIFACTS}" +LOG="${ARTIFACTS}/setup.log" +: > "${ARTIFACTS}/checks.jsonl" -echo "=== Setup: ${RUN_ID} ===" | tee "${ARTIFACTS}/setup.log" -echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "${ARTIFACTS}/setup.log" +echo "=== Setup: ${RUN_ID} ===" | tee "$LOG" +echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "$LOG" -# Ensure broker is up — --background daemonizes and returns immediately -relay node up --background 2>&1 | tee -a "${ARTIFACTS}/setup.log" || true +relay node up --background 2>&1 | tee -a "$LOG" || true -# Wait for broker to be ready (up to 15 seconds) BROKER_READY=0 -for i in $(seq 1 15); do - if relay status 2>&1 | grep -qi "running"; then +for i in $(seq 1 30); do + if relay node status 2>&1 | grep -qi "running"; then BROKER_READY=1 break fi sleep 1 done -relay status 2>&1 | tee -a "${ARTIFACTS}/setup.log" +relay status 2>&1 | tee -a "$LOG" if [ "$BROKER_READY" -eq 1 ]; then - echo "SETUP_OK" >> "${ARTIFACTS}/setup.log" + echo "SETUP_OK" >> "$LOG" + exit 0 +fi + +echo "SETUP_FAIL: broker did not become ready within 30 seconds" >> "$LOG" +exit 1 +`, + }); + + // ── Phase 2: provenance ────────────────────────────────────────────────── + // + // The run this workflow was rebuilt from verified agent-relay 10.0.0 while + // the repo was at 11.3.0 — it even logged "Update available: 10.0.0 → + // 11.3.0" and then reported `relay node deadletters` as an unknown command, + // which was a real command in 11.x. A whole run's worth of green (and one + // loud red herring) measured a stale published CLI instead of the code under + // test. Version drift is now an explicit, failing check. + + wf.step('provenance', { + type: 'deterministic', + dependsOn: ['setup'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="provenance" +LOG="${ARTIFACTS}/provenance.log" +${PRELUDE} + +echo "=== Provenance ===" | tee "$LOG" + +CLI_PATH=$(command -v relay 2>/dev/null || echo "not-found") +CLI_VERSION=$(relay version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || echo "unknown") +REPO_VERSION=$(node -e 'try{process.stdout.write(require("./package.json").version)}catch(e){process.stdout.write("unknown")}' 2>/dev/null || echo "unknown") +GIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +echo " relay path: $CLI_PATH" | tee -a "$LOG" +echo " relay version: $CLI_VERSION" | tee -a "$LOG" +echo " repo version: $REPO_VERSION" | tee -a "$LOG" +echo " git sha: $GIT_SHA" | tee -a "$LOG" + +# Recorded for the report, Slack, PostHog, and NightCTO payloads. +{ + echo "VERIFY_CLI_VERSION=$CLI_VERSION" + echo "VERIFY_REPO_VERSION=$REPO_VERSION" + echo "VERIFY_GIT_SHA=$GIT_SHA" + echo "VERIFY_CLI_PATH=$CLI_PATH" +} > "${ARTIFACTS}/provenance.env" + +if [ "$CLI_PATH" = "not-found" ]; then + record "$TIER" "cli-present" fail "relay is not on PATH" + echo " FAIL relay is not on PATH" | tee -a "$LOG" + exit 0 +fi +record "$TIER" "cli-present" pass "" + +if [ "$REPO_VERSION" = "unknown" ]; then + skip_check "cli-matches-repo" "no package.json in cwd — cannot compare versions" +elif [ "$CLI_VERSION" = "$REPO_VERSION" ]; then + echo " PASS cli-matches-repo ($CLI_VERSION)" | tee -a "$LOG" + record "$TIER" "cli-matches-repo" pass "" +elif [ "$VERIFY_ALLOW_CLI_DRIFT" = "1" ]; then + skip_check "cli-matches-repo" "CLI $CLI_VERSION != repo $REPO_VERSION (drift allowed by env)" else - echo "SETUP_FAIL: broker did not become ready within 15 seconds" >> "${ARTIFACTS}/setup.log" - exit 1 + echo " FAIL cli-matches-repo: verifying $CLI_VERSION but repo is $REPO_VERSION" | tee -a "$LOG" + record "$TIER" "cli-matches-repo" fail "CLI under test is $CLI_VERSION but the repo is $REPO_VERSION; results do not describe this checkout" fi + +finish_tier +exit 0 `, }); - // ── Phase 2: Tier 1 — CLI health (no broker deps) ─────────────────────── + // ── Phase 3: capability probe ──────────────────────────────────────────── + // + // Tier 5/6 checks need cloud auth and provider CLIs. Probing once up front + // lets every later check state a cause for skipping instead of emitting a + // failure that only means "this sandbox is not logged in" — the exact reason + // `cloud whoami` and `cloud session` were scored as product failures before. - wf.step('tier1-cli-health', { + wf.step('capabilities', { type: 'deterministic', - dependsOn: ['setup'], + dependsOn: ['provenance'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/tier1.log" -echo "=== Tier 1: CLI Health ===" | tee "$LOG" +LOG="${ARTIFACTS}/capabilities.log" +CAPS="${ARTIFACTS}/caps.env" +: > "$CAPS" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) +echo "=== Capability probe ===" | tee "$LOG" + +probe() { + _name="$1" + _cmd="$2" + if eval "$_cmd" >/dev/null 2>&1; then + echo "$_name=1" >> "$CAPS" + echo " YES $_name" | tee -a "$LOG" else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + echo "$_name=0" >> "$CAPS" + echo " no $_name" | tee -a "$LOG" fi } -run_check "relay version" "relay version" "[0-9]\\+\\.[0-9]" -run_check "relay --help" "relay --help" "Usage" -run_check "relay status (any)" "relay status" "." -run_check "relay node --help" "relay node --help" "up" -run_check "relay telemetry status" "relay telemetry status" "." -run_check "relay workspace list" "relay workspace list" "." - -echo "" | tee -a "$LOG" -echo "Tier 1 result: $PASS passed, $FAIL failed" | tee -a "$LOG" - -if [ "$FAIL" -gt 0 ]; then - echo "TIER1_FAIL" >> "$LOG" - exit 1 +probe broker 'relay node status 2>&1 | grep -qi running' +probe workspace 'relay workspace active' +probe cloud 'relay cloud whoami' +probe gh 'gh auth status' +probe git_repo 'git rev-parse --git-dir' +probe jq 'command -v jq' +probe slack '[ -n "$SLACK_BOT_TOKEN" ] || { [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; }' +probe provider_claude 'command -v claude' +probe provider_codex 'command -v codex' +probe provider_opencode 'command -v opencode' +probe provider_gemini 'command -v gemini' + +# Any provider at all is enough for the managed-agent paths. +if grep -q '^provider_.*=1$' "$CAPS"; then + echo "provider_any=1" >> "$CAPS" else - echo "TIER1_PASS" >> "$LOG" - exit 0 + echo "provider_any=0" >> "$CAPS" fi + +cat "$CAPS" +exit 0 `, }); - // ── Phase 3: Tier 2 — Broker + agent management ───────────────────────── + // ── Phase 4: Tier 1 — CLI health and command discovery ─────────────────── - wf.step('tier2-broker-agents', { + wf.step('tier1-cli-health', { type: 'deterministic', - dependsOn: ['tier1-cli-health'], + dependsOn: ['capabilities'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/tier2.log" -AGENT_NAME="vf-agent-${SUFFIX}" -echo "=== Tier 2: Broker + Agent Management ===" | tee "$LOG" +TIER="tier1" +LOG="${ARTIFACTS}/tier1.log" +${PRELUDE} + +echo "=== Tier 1: CLI Health and Command Discovery ===" | tee "$LOG" + +run_check "relay version" "relay version" "[0-9]\+\.[0-9]" +run_check "relay --help" "relay --help" "Usage" +run_check "relay status" "relay status" "." +run_check "relay telemetry status" "relay telemetry status" "." +run_check "relay workspace list" "relay workspace list" "." +run_check "relay update --help" "relay update --help" "Usage" +run_check "relay uninstall --help" "relay uninstall --help" "Usage" + +# Command-group discovery: a missing group means a whole feature category is +# unreachable, which no individual check below would attribute correctly. +for group in node cloud fleet workspace agent channel message integration capabilities skills reflex telemetry mcp; do + run_check "group discovery: $group" "relay $group --help" "Usage" +done -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +run_check "node subcommands" "relay node --help" "up" +run_check "node agent subcommands" "relay node agent --help" "spawn" +run_check "node workflow subcommands" "relay node workflow --help" "run" +run_check "message dm subcommands" "relay message dm --help" "send" +run_check "message inbox subcommands" "relay message inbox --help" "check" +run_check "integration webhook subs" "relay integration webhook --help" "create" -# Broker checks -run_check "broker status shows running" "relay status" "running" -run_check "broker metrics" "relay node metrics" "." -run_check "broker deadletters" "relay node deadletters" "." +finish_tier +exit 0 +`, + }); -# Agent management -run_check "agent register" "relay agent register '$AGENT_NAME'" "." -run_check "agent list shows registered agent" \ - "relay agent list" "$AGENT_NAME" + // ── Phase 5: Tier 2 — broker lifecycle and node agents ─────────────────── -# Workspace (active may 404 if workspace key is stale — just check command runs) -run_check "workspace active" "relay workspace active 2>&1 || true" "." -run_check "workspace list" "relay workspace list" "." + wf.step('tier2-broker', { + type: 'deterministic', + dependsOn: ['tier1-cli-health'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail -# Node agent list (requires broker running) -run_check "node agent list" "relay node agent list 2>&1 || true" "." +TIER="tier2" +LOG="${ARTIFACTS}/tier2.log" +${PRELUDE} -# Fleet (broker only — no cloud state modified) -run_check "fleet status" "relay fleet status" "." -run_check "fleet nodes" "relay fleet nodes" "." +echo "=== Tier 2: Broker Lifecycle and Node Agents ===" | tee "$LOG" -# Cloud auth reads (already logged in — no browser needed) -run_check "cloud whoami" "relay cloud whoami" "." -run_check "cloud session" "relay cloud session" "." +run_check "node status shows running" "relay node status" "running" +run_check "node metrics" "relay node metrics" "." +run_check "node deadletters" "relay node deadletters" "." +run_check "node deadletters --json" "relay node deadletters --json" "." +run_check "node agent list" "relay node agent list" "." +run_check "fleet status" "relay fleet status" "." -# Cleanup -relay agent remove "$AGENT_NAME" 2>/dev/null || true +# 'relay node tail' streams until interrupted; bound it and accept a timeout +# kill as success. Without the bound this step hangs until the run times out. +run_check "node tail (bounded)" \ + "timeout 5 relay node tail >/dev/null 2>&1; [ \$? -le 124 ] && echo tail-ok" "tail-ok" -echo "" | tee -a "$LOG" -echo "Tier 2 result: $PASS passed, $FAIL failed" | tee -a "$LOG" +# Redeliver against an empty queue is a no-op, so this exercises the command +# path without mutating real delivery state. +run_check "node redeliver --all (empty queue)" \ + "relay node redeliver --all 2>&1 || true" "." -if [ "$FAIL" -gt 0 ]; then - echo "TIER2_FAIL" >> "$LOG" - exit 1 -else - echo "TIER2_PASS" >> "$LOG" - exit 0 -fi +finish_tier +exit 0 `, }); - // ── Phase 4: Tier 3 — Channel messaging round-trip ────────────────────── + // ── Phase 6: Tier 3 — channels, messages, threads, reactions, inbox ────── - wf.step('tier3-channel-messaging', { + wf.step('tier3-messaging', { type: 'deterministic', - dependsOn: ['tier2-broker-agents'], + dependsOn: ['tier2-broker'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 +TIER="tier3" LOG="${ARTIFACTS}/tier3.log" +${PRELUDE} + AGENT="vf-msg-${SUFFIX}" CHANNEL="vf-channel-${SUFFIX}" -MSG_TEXT="verify-$(date +%s)" -echo "=== Tier 3: Channel Messaging Round-Trip ===" | tee "$LOG" +MSG_TEXT="verify-msg-${SUFFIX}" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +echo "=== Tier 3: Channels, Messages, Threads, Reactions, Inbox ===" | tee "$LOG" -# Register a test agent and get its token TOKEN=$(relay agent register "$AGENT" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") if [ -z "$TOKEN" ]; then + record "$TIER" "agent register + token" fail "no token returned from relay agent register" echo " FAIL agent register (no token returned)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -else - echo " PASS agent register + token" | tee -a "$LOG" - PASS=$((PASS + 1)) + echo " Remaining tier 3 checks cannot run without an agent token." | tee -a "$LOG" + for remaining in "channel create" "channel list" "channel join" "channel set_topic" \ + "message post" "message list" "message reply" "message get_thread" \ + "reaction add" "reaction remove" "inbox check" "inbox mark_read" \ + "message search" "channel leave" "channel archive"; do + skip_check "$remaining" "agent registration failed — no token" + done + finish_tier + exit 0 fi + +echo " PASS agent register + token" | tee -a "$LOG" +record "$TIER" "agent register + token" pass "" export RELAY_AGENT_TOKEN="$TOKEN" -# Channel operations (token passed via RELAY_AGENT_TOKEN env var) -run_check "channel create" "relay channel create '$CHANNEL'" "." -run_check "channel list" "relay channel list" "$CHANNEL" -run_check "channel join" "relay channel join '$CHANNEL'" "." -run_check "channel set_topic" "relay channel set_topic '$CHANNEL' 'verify topic'" "." +run_check "agent list shows agent" "relay agent list" "$AGENT" + +run_check "channel create" "relay channel create '$CHANNEL'" "." +run_check "channel list" "relay channel list" "$CHANNEL" +run_check "channel join" "relay channel join '$CHANNEL'" "." +run_check "channel set_topic" "relay channel set_topic '$CHANNEL' 'verify topic'" "." -# Message operations (positional args: ) -run_check "message post" "relay message post '$CHANNEL' '$MSG_TEXT'" "." -run_check "message list" "relay message list '$CHANNEL' --limit 5" "$MSG_TEXT" +run_check "message post" "relay message post '$CHANNEL' '$MSG_TEXT'" "." +# Asserting the posted text reads back is the actual round-trip proof; a bare +# "command exited 0" would pass even if the message were dropped. +run_check "message list round-trip" "relay message list '$CHANNEL' --limit 5" "$MSG_TEXT" -# Get the message id for thread operations MSG_ID=$(relay message list "$CHANNEL" 2>/dev/null | grep -o '"messageId": *"[^"]*"' | head -1 | sed 's/.*"messageId": *"//;s/".*//' || echo "") if [ -n "$MSG_ID" ]; then run_check "message reply" "relay message reply '$MSG_ID' 'thread-reply-${SUFFIX}'" "." - run_check "message get_thread" "relay message get_thread '$MSG_ID'" "." - run_check "reaction add" "relay message reaction add '$MSG_ID' thumbsup" "." - run_check "reaction remove" "relay message reaction remove '$MSG_ID' thumbsup" "." - run_check "inbox check" "relay message inbox check" "." - run_check "inbox mark_read" "relay message inbox mark_read '$MSG_ID'" "." + run_check "message get_thread" "relay message get_thread '$MSG_ID'" "thread-reply-${SUFFIX}" + run_check "reaction add" "relay message reaction add '$MSG_ID' thumbsup" "." + run_check "reaction remove" "relay message reaction remove '$MSG_ID' thumbsup" "." + run_check "inbox check" "relay message inbox check" "." + run_check "inbox mark_read" "relay message inbox mark_read '$MSG_ID'" "." else - echo " SKIP thread/reaction/inbox checks (no message id)" | tee -a "$LOG" + for remaining in "message reply" "message get_thread" "reaction add" "reaction remove" \ + "inbox check" "inbox mark_read"; do + skip_check "$remaining" "could not extract a messageId from message list output" + done fi -run_check "message search" "relay message search '$MSG_TEXT'" "$MSG_TEXT" - -# Channel leave + archive -run_check "channel leave" "relay channel leave '$CHANNEL'" "." -run_check "channel archive" "relay channel archive '$CHANNEL'" "." +run_check "message search" "relay message search '$MSG_TEXT'" "$MSG_TEXT" +run_check "channel leave" "relay channel leave '$CHANNEL'" "." +run_check "channel archive" "relay channel archive '$CHANNEL'" "." -# Cleanup relay agent remove "$AGENT" 2>/dev/null || true -echo "" | tee -a "$LOG" -echo "Tier 3 result: $PASS passed, $FAIL failed" | tee -a "$LOG" - -if [ "$FAIL" -gt 0 ]; then - echo "TIER3_FAIL" >> "$LOG" - exit 1 -else - echo "TIER3_PASS" >> "$LOG" - exit 0 -fi +finish_tier +exit 0 `, }); - // ── Phase 5: Tier 4 — Cross-agent DMs and invite ──────────────────────── + // ── Phase 7: Tier 4 — cross-agent coordination ─────────────────────────── wf.step('tier4-cross-agent', { type: 'deterministic', - dependsOn: ['tier3-channel-messaging'], + dependsOn: ['tier3-messaging'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 +TIER="tier4" LOG="${ARTIFACTS}/tier4.log" +${PRELUDE} + AGENT_A="vf-alice-${SUFFIX}" AGENT_B="vf-bob-${SUFFIX}" CHANNEL="vf-private-${SUFFIX}" -echo "=== Tier 4: Cross-Agent DMs + Channel Invite ===" | tee "$LOG" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +echo "=== Tier 4: Cross-Agent DMs, Invites, Read Receipts, Files ===" | tee "$LOG" -# Register two agents TOKEN_A=$(relay agent register "$AGENT_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") TOKEN_B=$(relay agent register "$AGENT_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") if [ -z "$TOKEN_A" ] || [ -z "$TOKEN_B" ]; then - echo " FAIL register two agents (tokens missing)" | tee -a "$LOG" - FAIL=$((FAIL + 2)) -else - echo " PASS register agent A ($AGENT_A)" | tee -a "$LOG" - echo " PASS register agent B ($AGENT_B)" | tee -a "$LOG" - PASS=$((PASS + 2)) + record "$TIER" "register two agents" fail "token A empty=$([ -z "$TOKEN_A" ] && echo yes || echo no), token B empty=$([ -z "$TOKEN_B" ] && echo yes || echo no)" + echo " FAIL register two agents" | tee -a "$LOG" + for remaining in "dm send A to B" "dm list" "channel invite" "dm send_group" \ + "inbox get_readers" "message file upload"; do + skip_check "$remaining" "two-agent registration failed" + done + finish_tier + exit 0 fi -# DM: A sends to B (capture conversationId for dm list) -DM_TEXT="dm-verify-$(date +%s)" -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CHANNEL" 2>/dev/null || true -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CHANNEL" 2>/dev/null || true +echo " PASS register agent A ($AGENT_A)" | tee -a "$LOG" +echo " PASS register agent B ($AGENT_B)" | tee -a "$LOG" +record "$TIER" "register agent A" pass "" +record "$TIER" "register agent B" pass "" -if RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "$AGENT_B" "$DM_TEXT" > /tmp/dm-send-out.txt 2>&1; then - DM_SEND_OUT=$(cat /tmp/dm-send-out.txt) - echo " PASS dm send A→B" | tee -a "$LOG" - PASS=$((PASS + 1)) +DM_TEXT="dm-verify-${SUFFIX}" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CHANNEL" >/dev/null 2>&1 || true +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CHANNEL" >/dev/null 2>&1 || true + +DM_OUT_FILE="${ARTIFACTS}/dm-send.out" +if RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "$AGENT_B" "$DM_TEXT" > "$DM_OUT_FILE" 2>&1; then + echo " PASS dm send A to B" | tee -a "$LOG" + record "$TIER" "dm send A to B" pass "" else - DM_SEND_OUT=$(cat /tmp/dm-send-out.txt) - echo " FAIL dm send A→B (exit code non-zero)" | tee -a "$LOG" - echo " output: $DM_SEND_OUT" >> "$LOG" - FAIL=$((FAIL + 1)) + echo " FAIL dm send A to B" | tee -a "$LOG" + record "$TIER" "dm send A to B" fail "$(cat "$DM_OUT_FILE")" fi -CONV_ID=$(echo "$DM_SEND_OUT" | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") -# B lists DMs using conversationId from send response +DM_SEND_OUT=$(cat "$DM_OUT_FILE" 2>/dev/null || echo "") +CONV_ID=$(printf '%s' "$DM_SEND_OUT" | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") +DM_ID=$(printf '%s' "$DM_SEND_OUT" | grep -oE '"(messageId|id)": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + if [ -n "$CONV_ID" ]; then - if RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV_ID" 2>&1 | grep -qi "."; then - echo " PASS dm list (B sees DM from A)" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL dm list (B could not list conversation $CONV_ID)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi + # B must see A's exact text, not merely a listable conversation. + run_check "dm list (B sees A's text)" \ + "RELAY_AGENT_TOKEN='$TOKEN_B' relay message dm list '$CONV_ID'" "$DM_TEXT" else - echo " FAIL dm list (no conversationId in send response — cannot verify receipt)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "dm list (B sees A's text)" "no conversationId in dm send response" fi -# Channel invite B into A's channel -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite "$CHANNEL" "$AGENT_B" 2>&1 | tee -a "$LOG" > /tmp/invite-out.txt -if grep -qi "." /tmp/invite-out.txt; then - echo " PASS channel invite B into private channel" | tee -a "$LOG" - PASS=$((PASS + 1)) +run_check "channel invite B into private channel" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay channel invite '$CHANNEL' '$AGENT_B'" "." +run_check "dm send_group" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message dm send_group 'group-dm-${SUFFIX}' --to '$AGENT_B'" "." + +if [ -n "$DM_ID" ]; then + run_check "inbox mark_read (B)" \ + "RELAY_AGENT_TOKEN='$TOKEN_B' relay message inbox mark_read '$DM_ID'" "." + run_check "inbox get_readers (A sees B)" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message inbox get_readers '$DM_ID'" "$AGENT_B" else - echo " FAIL channel invite B into private channel" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "inbox mark_read (B)" "no message id in dm send response" + skip_check "inbox get_readers (A sees B)" "no message id in dm send response" fi -# Group DM (positional , then --to ) -RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group-dm-$(date +%s)" --to "$AGENT_B" 2>&1 | grep -qi "." && { - echo " PASS dm send_group" | tee -a "$LOG" - PASS=$((PASS + 1)) -} || { - echo " FAIL dm send_group" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -} +UPLOAD_FILE="${ARTIFACTS}/upload-fixture.txt" +echo "verify-upload-${SUFFIX}" > "$UPLOAD_FILE" +run_check "message file upload" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message file upload '$CHANNEL' '$UPLOAD_FILE'" "." -# Cleanup -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CHANNEL" 2>/dev/null || true +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CHANNEL" >/dev/null 2>&1 || true relay agent remove "$AGENT_A" 2>/dev/null || true relay agent remove "$AGENT_B" 2>/dev/null || true -echo "" | tee -a "$LOG" -echo "Tier 4 result: $PASS passed, $FAIL failed" | tee -a "$LOG" - -if [ "$FAIL" -gt 0 ]; then - echo "TIER4_FAIL" >> "$LOG" - exit 1 -else - echo "TIER4_PASS" >> "$LOG" - exit 0 -fi +finish_tier +exit 0 `, }); - // ── Phase 6: Critical paths ────────────────────────────────────────────── + // ── Phase 8: Tier 5 — cloud, fleet, integrations, skills, reflex ───────── // - // Runs the 4 non-cloud critical paths from critical-paths.md in sequence. - // If any critical path fails, the whole check is FAIL regardless of tiers. + // Everything here needs real cloud auth. Each check is gated so an + // unauthenticated sandbox produces SKIPs with a cause rather than a wall of + // failures that say nothing about the product. - wf.step('critical-paths', { + wf.step('tier5-cloud', { type: 'deterministic', dependsOn: ['tier4-cross-agent'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/critical-paths.log" -SUFFIX2="cp-${SUFFIX}" -echo "=== Critical Paths ===" | tee "$LOG" -echo "See .agentworkforce/features/critical-paths.md for definitions." | tee -a "$LOG" -echo "" | tee -a "$LOG" +TIER="tier5" +LOG="${ARTIFACTS}/tier5.log" +${PRELUDE} -# ── Critical Path 1: Broker + Agent Registration ────────────────────────── -echo "-- CP1: Broker + Agent Registration --" | tee -a "$LOG" -CP1=0 +echo "=== Tier 5: Cloud, Fleet, Integrations, Skills, Reflex ===" | tee "$LOG" -relay status 2>&1 | grep -qi "running" && { - echo " PASS broker is running" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || { - echo " FAIL broker is not running" | tee -a "$LOG" -} +if ! have_cap cloud; then + echo " Cloud capability absent — every tier 5 check is a stated SKIP." | tee -a "$LOG" +fi -CP1_AGENT="cp1-${SUFFIX2}" -relay agent register "$CP1_AGENT" 2>&1 | grep -qi "." && { - echo " PASS agent register" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || echo " FAIL agent register" | tee -a "$LOG" +gated_check cloud "cloud whoami" "relay cloud whoami" "." +gated_check cloud "cloud session" "relay cloud session" "." +gated_check cloud "cloud status" "relay cloud status" "." +gated_check cloud "cloud schedules" "relay cloud schedules" "." +gated_check cloud "cloud logs" "relay cloud logs 2>&1 || true" "." +gated_check cloud "cloud sync --help" "relay cloud sync --help" "Usage" + +# Deliberately not exercised: 'cloud run' bills real sandbox time, and +# cloud login/connect/enroll need an interactive browser. Recorded as +# skips so the report shows the coverage hole instead of hiding it. +skip_check "cloud run" "spends real cloud budget — excluded from scheduled verification" +skip_check "cloud login" "requires interactive browser auth" +skip_check "cloud connect" "requires interactive browser auth" +skip_check "cloud enroll" "requires interactive browser auth" + +gated_check cloud "fleet nodes" "relay fleet nodes" "." +gated_check cloud "fleet config" "relay fleet config" "." +gated_check cloud "fleet inherit" "relay fleet inherit --help" "Usage" +skip_check "fleet enable" "mutates workspace cloud state" +skip_check "fleet disable" "mutates workspace cloud state" + +# fleet spawn/release were undocumented in the manifest until this change and +# are still unverified: spawning burns provider credits on a remote node. +skip_check "fleet spawn" "spawns a billed remote agent — needs a dedicated fixture node" +skip_check "fleet release" "depends on fleet spawn" + +gated_check cloud "reflex status" "relay reflex status" "." +skip_check "reflex on" "mutates history sync state for the workspace" +skip_check "reflex off" "mutates history sync state for the workspace" + +WEBHOOK_NAME="vf-hook-${SUFFIX}" +if have_cap cloud; then + run_check "integration webhook list" "relay integration webhook list" "." + run_check "integration subscription list" "relay integration subscription list" "." + if relay integration webhook create "$WEBHOOK_NAME" >/dev/null 2>&1; then + echo " PASS integration webhook create" | tee -a "$LOG" + record "$TIER" "integration webhook create" pass "" + run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_NAME' 2>&1 || true" "." + run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_NAME'" "." + else + record "$TIER" "integration webhook create" fail "relay integration webhook create exited non-zero" + echo " FAIL integration webhook create" | tee -a "$LOG" + skip_check "integration webhook trigger" "webhook create failed" + skip_check "integration webhook delete" "webhook create failed" + fi +else + for remaining in "integration webhook list" "integration subscription list" \ + "integration webhook create" "integration webhook trigger" "integration webhook delete"; do + skip_check "$remaining" "requires capability: cloud" + done +fi -relay agent list 2>&1 | grep -qi "$CP1_AGENT" && { - echo " PASS agent list shows registered agent" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || echo " FAIL agent list did not show agent" | tee -a "$LOG" +gated_check cloud "capabilities list" "relay capabilities list" "." +gated_check cloud "skills add --help" "relay skills add --help" "Usage" -relay agent remove "$CP1_AGENT" 2>/dev/null || true +finish_tier +exit 0 +`, + }); -if [ "$CP1" -eq 3 ]; then - echo " PASS CP1 complete" | tee -a "$LOG" - PASS=$((PASS + 1)) -else - echo " FAIL CP1 incomplete ($CP1/3 checks passed)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -fi - -echo "" | tee -a "$LOG" - -# ── Critical Path 2: Channel Messaging ──────────────────────────────────── -echo "-- CP2: Channel Messaging Round-Trip --" | tee -a "$LOG" -CP2=0 -CP2_AGENT="cp2-${SUFFIX2}" -CP2_CH="cp2-ch-${SUFFIX2}" -CP2_MSG="cp2-msg-$(date +%s)" - -TOKEN_CP2=$(relay agent register "$CP2_AGENT" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -if [ -n "$TOKEN_CP2" ]; then - CP2=$((CP2 + 1)) - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel create "$CP2_CH" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel join "$CP2_CH" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay message post "$CP2_CH" "$CP2_MSG" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay message list "$CP2_CH" --limit 3 2>&1 | grep -qi "$CP2_MSG" && { - echo " PASS posted message appears in list" | tee -a "$LOG" - CP2=$((CP2 + 1)) - } || echo " FAIL posted message not in list" | tee -a "$LOG" - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel archive "$CP2_CH" 2>/dev/null || true - relay agent remove "$CP2_AGENT" 2>/dev/null || true -fi - -if [ "$CP2" -ge 4 ]; then - echo " PASS CP2 complete ($CP2/5 checks)" | tee -a "$LOG" - PASS=$((PASS + 1)) + // ── Phase 9: Tier 6 — managed agents, harnesses, PTY, SDK ──────────────── + + wf.step('tier6-harnesses', { + type: 'deterministic', + dependsOn: ['tier5-cloud'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="tier6" +LOG="${ARTIFACTS}/tier6.log" +${PRELUDE} + +echo "=== Tier 6: Managed Agents, Harnesses, PTY, SDK ===" | tee "$LOG" + +# Discovery works without credentials and catches a removed subcommand. +run_check "node agent spawn --help" "relay node agent spawn --help" "Usage" +run_check "node agent release --help" "relay node agent release --help" "Usage" +run_check "node agent attach --help" "relay node agent attach --help" "Usage" +run_check "node agent set-model --help" "relay node agent set-model --help" "Usage" +run_check "node agent new --help" "relay node agent new --help" "Usage" +run_check "node agent message hold --help" "relay node agent message hold --help" "Usage" + +PROVIDER="" +for candidate in claude codex opencode gemini; do + if command -v "$candidate" >/dev/null 2>&1; then + PROVIDER="$candidate" + break + fi +done + +if [ -z "$PROVIDER" ]; then + echo " No provider CLI on PATH — managed-agent lifecycle is a stated SKIP." | tee -a "$LOG" + for remaining in "node agent spawn" "node agent list shows worker" \ + "node agent message hold" "node agent message auto" "node agent release"; do + skip_check "$remaining" "no provider CLI (claude/codex/opencode/gemini) on PATH" + done else - echo " FAIL CP2 incomplete ($CP2/5 checks)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -fi - -echo "" | tee -a "$LOG" - -# ── Critical Path 3: MCP Server Starts ──────────────────────────────────── -echo "-- CP3: MCP Server --" | tee -a "$LOG" -# We can only verify the MCP server starts without crashing on startup -# Full tool invocation requires a connected harness (tier 6) -# Capture output first to avoid pipefail interaction with grep -MCP_HELP=$(relay mcp --help 2>&1 || true) -echo " mcp help output: $(echo "$MCP_HELP" | head -1)" | tee -a "$LOG" -if echo "$MCP_HELP" | grep -qiE "mcp|server|stdio|relay|Usage"; then - echo " PASS relay mcp --help responds" | tee -a "$LOG" - PASS=$((PASS + 1)) + echo " Using provider: $PROVIDER" | tee -a "$LOG" + WORKER="vf-worker-${SUFFIX}" + if relay node agent spawn "$PROVIDER" --name "$WORKER" \ + --task 'Reply verify-ok and exit' --spawn-mode task-exit --exit-after-task >/dev/null 2>&1; then + echo " PASS node agent spawn" | tee -a "$LOG" + record "$TIER" "node agent spawn" pass "" + run_check "node agent list shows worker" "relay node agent list" "$WORKER" + run_check "node agent message hold" "relay node agent message hold '$WORKER'" "." + run_check "node agent message auto" "relay node agent message auto '$WORKER'" "." + run_check "node agent release" "relay node agent release '$WORKER'" "." + else + record "$TIER" "node agent spawn" fail "spawn of provider $PROVIDER exited non-zero" + echo " FAIL node agent spawn ($PROVIDER)" | tee -a "$LOG" + for remaining in "node agent list shows worker" "node agent message hold" \ + "node agent message auto" "node agent release"; do + skip_check "$remaining" "spawn failed" + done + fi +fi + +# SDK: presence of the package manifest only. A full SDK round trip belongs in +# the SDK package's own tests, not in a CLI feature sweep. +if [ -d packages/sdk ]; then + run_check "sdk package resolves" "test -f packages/sdk/package.json && echo sdk-ok" "sdk-ok" else - echo " FAIL relay mcp --help returned no recognizable output" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "sdk package resolves" "packages/sdk not present in this checkout" fi -echo "" | tee -a "$LOG" +skip_check "browser primitive" "requires a provisioned browser session (tier 6 interactive)" +skip_check "python sdk" "verified in the python-sdk repo test suite" +skip_check "swift sdk" "verified in the swift-sdk repo test suite" + +finish_tier +exit 0 +`, + }); + + // ── Phase 10: critical paths 1-6 ───────────────────────────────────────── -# ── Critical Path 4: DM Cross-Agent ─────────────────────────────────────── -echo "-- CP4: DM Between Two Agents --" | tee -a "$LOG" -CP4=0 -CP4_A="cp4a-${SUFFIX2}" -CP4_B="cp4b-${SUFFIX2}" + wf.step('critical-paths', { + type: 'deterministic', + dependsOn: ['tier6-harnesses'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="critical-paths" +LOG="${ARTIFACTS}/critical-paths.log" +${PRELUDE} -TOKEN_CP4A=$(relay agent register "$CP4_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_CP4B=$(relay agent register "$CP4_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +echo "=== Critical Paths (see .agentworkforce/features/critical-paths.md) ===" | tee "$LOG" -if [ -n "$TOKEN_CP4A" ] && [ -n "$TOKEN_CP4B" ]; then - CP4=$((CP4 + 1)) - DM_MSG="dm-cp4-$(date +%s)" - if RELAY_AGENT_TOKEN="$TOKEN_CP4A" relay message dm send "$CP4_B" "$DM_MSG" > /tmp/cp4-dm-out.txt 2>&1; then - CP4=$((CP4 + 1)) +# ── CP1: local broker lifecycle ────────────────────────────────────────── +# Uses an isolated state dir so bringing the broker down cannot disturb the +# broker the remaining paths depend on. +echo "-- CP1: Local Broker Lifecycle --" | tee -a "$LOG" +CP1_STATE="${ARTIFACTS}/cp1-state" +mkdir -p "$CP1_STATE" +CP1=0 +relay node up --background --no-spawn --state-dir "$CP1_STATE" >/dev/null 2>&1 || true +sleep 2 +relay node status --state-dir "$CP1_STATE" 2>&1 | grep -qi running && CP1=$((CP1 + 1)) +relay node metrics >/dev/null 2>&1 && CP1=$((CP1 + 1)) +relay node deadletters --json >/dev/null 2>&1 && CP1=$((CP1 + 1)) +relay node down --state-dir "$CP1_STATE" >/dev/null 2>&1 || true +sleep 2 +relay node status --state-dir "$CP1_STATE" 2>&1 | grep -qiv running && CP1=$((CP1 + 1)) +if [ "$CP1" -ge 4 ]; then + echo " PASS CP1 ($CP1/4)" | tee -a "$LOG" + record "$TIER" "CP1 broker lifecycle" pass "" +else + echo " FAIL CP1 ($CP1/4)" | tee -a "$LOG" + record "$TIER" "CP1 broker lifecycle" fail "only $CP1 of 4 lifecycle assertions held" +fi + +# ── CP2: cross-agent channel message ──────────────────────────────────── +echo "-- CP2: Cross-Agent Channel Message --" | tee -a "$LOG" +CP2_A="cp2a-${SUFFIX}" +CP2_B="cp2b-${SUFFIX}" +CP2_CH="cp2-ch-${SUFFIX}" +CP2_MSG="cp2-msg-${SUFFIX}" +TOKEN_2A=$(relay agent register "$CP2_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_2B=$(relay agent register "$CP2_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +if [ -n "$TOKEN_2A" ] && [ -n "$TOKEN_2B" ]; then + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel create "$CP2_CH" >/dev/null 2>&1 || true + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel invite "$CP2_CH" "$CP2_B" >/dev/null 2>&1 || true + RELAY_AGENT_TOKEN="$TOKEN_2A" relay message post "$CP2_CH" "$CP2_MSG" >/dev/null 2>&1 || true + if RELAY_AGENT_TOKEN="$TOKEN_2B" relay message list "$CP2_CH" --limit 10 2>&1 | grep -q "$CP2_MSG"; then + echo " PASS CP2 (B read A's exact text)" | tee -a "$LOG" + record "$TIER" "CP2 cross-agent channel message" pass "" else - echo " dm send output: $(cat /tmp/cp4-dm-out.txt)" >> "$LOG" + echo " FAIL CP2 (B did not see A's text)" | tee -a "$LOG" + record "$TIER" "CP2 cross-agent channel message" fail "agent B could not read the message agent A posted to $CP2_CH" fi - CONV4_ID=$(cat /tmp/cp4-dm-out.txt | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") - if [ -n "$CONV4_ID" ]; then - RELAY_AGENT_TOKEN="$TOKEN_CP4B" relay message dm list "$CONV4_ID" 2>&1 | grep -qi "." && { - echo " PASS B can list DMs" | tee -a "$LOG" - CP4=$((CP4 + 1)) - } || echo " FAIL B cannot list DMs" | tee -a "$LOG" + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel archive "$CP2_CH" >/dev/null 2>&1 || true +else + skip_check "CP2 cross-agent channel message" "could not register two agents" +fi +relay agent remove "$CP2_A" 2>/dev/null || true +relay agent remove "$CP2_B" 2>/dev/null || true + +# ── CP3: managed local agent ──────────────────────────────────────────── +echo "-- CP3: Managed Local Agent --" | tee -a "$LOG" +CP3_PROVIDER="" +for candidate in claude codex opencode gemini; do + command -v "$candidate" >/dev/null 2>&1 && CP3_PROVIDER="$candidate" && break +done +if [ -z "$CP3_PROVIDER" ]; then + skip_check "CP3 managed local agent" "no provider CLI on PATH; provider cost makes this a pre-provisioned check" +else + CP3_WORKER="cp3-${SUFFIX}" + if relay node agent spawn "$CP3_PROVIDER" --name "$CP3_WORKER" \ + --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task >/dev/null 2>&1 \ + && relay node agent list 2>&1 | grep -q "$CP3_WORKER"; then + relay node agent message hold "$CP3_WORKER" >/dev/null 2>&1 || true + relay node agent message auto "$CP3_WORKER" >/dev/null 2>&1 || true + relay node agent release "$CP3_WORKER" >/dev/null 2>&1 || true + echo " PASS CP3" | tee -a "$LOG" + record "$TIER" "CP3 managed local agent" pass "" else - echo " FAIL dm list (no conversationId in send response)" | tee -a "$LOG" + echo " FAIL CP3" | tee -a "$LOG" + record "$TIER" "CP3 managed local agent" fail "spawn or list did not show worker $CP3_WORKER with provider $CP3_PROVIDER" fi fi -relay agent remove "$CP4_A" 2>/dev/null || true -relay agent remove "$CP4_B" 2>/dev/null || true - -if [ "$CP4" -ge 2 ]; then - echo " PASS CP4 complete" | tee -a "$LOG" - PASS=$((PASS + 1)) +# ── CP4: MCP stdio round trip ─────────────────────────────────────────── +# A real JSON-RPC handshake. The old version only ran 'relay mcp --help', +# which proves argument parsing and nothing about the server. +echo "-- CP4: MCP Stdio Round Trip --" | tee -a "$LOG" +MCP_OUT="${ARTIFACTS}/cp4-mcp.json" +cat > "${ARTIFACTS}/cp4-mcp.mjs" <<'MCPEOF' +import { spawn } from 'node:child_process'; + +const child = spawn('relay', ['mcp'], { stdio: ['pipe', 'pipe', 'pipe'] }); +let buffer = ''; +let done = false; + +const finish = (payload) => { + if (done) return; + done = true; + child.kill('SIGTERM'); + console.log(JSON.stringify(payload)); + process.exit(0); +}; + +const timer = setTimeout(() => finish({ ok: false, error: 'timeout waiting for tools/list' }), 25000); +timer.unref(); + +child.stdout.on('data', (chunk) => { + buffer += chunk.toString(); + for (const line of buffer.split('\n')) { + const trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + const msg = JSON.parse(trimmed); + if (msg.id === 2 && msg.result?.tools) { + clearTimeout(timer); + finish({ ok: true, tools: msg.result.tools.map((t) => t.name).sort() }); + } + } catch { + // partial frame + } + } +}); +child.on('error', (err) => finish({ ok: false, error: err.message })); +child.on('exit', (code) => finish({ ok: false, error: 'mcp server exited with code ' + code })); + +const send = (msg) => child.stdin.write(JSON.stringify(msg) + '\n'); +send({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'verify-features', version: '1.0.0' }, + }, +}); +send({ jsonrpc: '2.0', method: 'notifications/initialized' }); +send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }); +MCPEOF + +if node "${ARTIFACTS}/cp4-mcp.mjs" > "$MCP_OUT" 2>/dev/null && grep -q '"ok":true' "$MCP_OUT"; then + TOOL_COUNT=$(grep -o '"' "$MCP_OUT" | wc -l | tr -d ' ') + # Assert the tools we depend on are actually listed, not just that some + # list came back. + MCP_MISSING="" + for tool in register_agent create_channel post_message list_messages send_dm check_inbox; do + grep -q "\"$tool\"" "$MCP_OUT" || MCP_MISSING="$MCP_MISSING $tool" + done + if [ -z "$MCP_MISSING" ]; then + echo " PASS CP4 (tools/list returned the required tool set)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" pass "" + else + echo " FAIL CP4 (missing tools:$MCP_MISSING)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" fail "tools/list is missing:$MCP_MISSING" + fi else - echo " FAIL CP4 incomplete ($CP4/3 checks)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + echo " FAIL CP4 (no tools/list response)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" fail "$(head -c 300 "$MCP_OUT" 2>/dev/null || echo 'no output from mcp probe')" fi -echo "" | tee -a "$LOG" -echo "Critical paths: $PASS/4 passed" | tee -a "$LOG" +# ── CP5: local workflow run, logs, sync ───────────────────────────────── +echo "-- CP5: Local Workflow Run, Logs, Sync --" | tee -a "$LOG" +CP5_DIR="${ARTIFACTS}/cp5" +mkdir -p "$CP5_DIR" +echo 'console.log("critical-workflow-ok")' > "$CP5_DIR/workflow.js" +CP5_JSON=$(relay node workflow run "$CP5_DIR/workflow.js" --json 2>&1 || echo "") +CP5_RUN_ID=$(printf '%s' "$CP5_JSON" | grep -o '"runId": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +if [ -n "$CP5_RUN_ID" ] && printf '%s' "$CP5_JSON" | grep -q "critical-workflow-ok\|completed"; then + relay node workflow logs "$CP5_RUN_ID" --json >/dev/null 2>&1 || true + relay node workflow sync "$CP5_RUN_ID" --dry-run --json >/dev/null 2>&1 || true + echo " PASS CP5 (run $CP5_RUN_ID)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" pass "" +else + echo " FAIL CP5" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "$(printf '%s' "$CP5_JSON" | head -c 300)" +fi -if [ "$FAIL" -gt 0 ]; then - echo "CRITICAL_PATHS_FAIL" >> "$LOG" - exit 1 +# ── CP6: direct message and read receipt ──────────────────────────────── +echo "-- CP6: Direct Message and Read Receipt --" | tee -a "$LOG" +CP6_A="cp6a-${SUFFIX}" +CP6_B="cp6b-${SUFFIX}" +TOKEN_6A=$(relay agent register "$CP6_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_6B=$(relay agent register "$CP6_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +if [ -n "$TOKEN_6A" ] && [ -n "$TOKEN_6B" ]; then + CP6_MSG="cp6-dm-${SUFFIX}" + CP6_OUT="${ARTIFACTS}/cp6-dm.out" + RELAY_AGENT_TOKEN="$TOKEN_6A" relay message dm send "$CP6_B" "$CP6_MSG" > "$CP6_OUT" 2>&1 || true + CP6_CONV=$(grep -o '"conversationId": *"[^"]*"' "$CP6_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + CP6_ID=$(grep -oE '"(messageId|id)": *"[^"]*"' "$CP6_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + CP6=0 + if [ -n "$CP6_CONV" ]; then + RELAY_AGENT_TOKEN="$TOKEN_6B" relay message dm list "$CP6_CONV" 2>&1 | grep -q "$CP6_MSG" && CP6=$((CP6 + 1)) + fi + if [ -n "$CP6_ID" ]; then + RELAY_AGENT_TOKEN="$TOKEN_6B" relay message inbox mark_read "$CP6_ID" >/dev/null 2>&1 && CP6=$((CP6 + 1)) + RELAY_AGENT_TOKEN="$TOKEN_6A" relay message inbox get_readers "$CP6_ID" 2>&1 | grep -q "$CP6_B" && CP6=$((CP6 + 1)) + fi + if [ "$CP6" -ge 3 ]; then + echo " PASS CP6 ($CP6/3)" | tee -a "$LOG" + record "$TIER" "CP6 dm and read receipt" pass "" + else + echo " FAIL CP6 ($CP6/3)" | tee -a "$LOG" + record "$TIER" "CP6 dm and read receipt" fail "only $CP6 of 3 assertions held (conv=$CP6_CONV id=$CP6_ID)" + fi else - echo "CRITICAL_PATHS_PASS" >> "$LOG" - exit 0 + skip_check "CP6 dm and read receipt" "could not register two agents" fi +relay agent remove "$CP6_A" 2>/dev/null || true +relay agent remove "$CP6_B" 2>/dev/null || true + +finish_tier +exit 0 `, }); - // ── Phase 7: Cleanup (always runs) ────────────────────────────────────── + // ── Phase 11: cleanup ──────────────────────────────────────────────────── wf.step('cleanup', { type: 'deterministic', dependsOn: ['critical-paths'], - captureOutput: false, + captureOutput: true, failOnError: false, - command: ` -# Remove any test agents that may have leaked (best-effort) -relay agent list 2>/dev/null | grep "^vf-\\|^cp[0-9]" | awk '{print $1}' | while read name; do - relay agent remove "$name" 2>/dev/null || true + command: String.raw` +relay agent list 2>/dev/null | grep -oE '(vf|cp[0-9])[A-Za-z0-9_-]*' | while read -r name; do + relay agent remove "$name" >/dev/null 2>&1 || true done -# Archive any test channels that may have leaked -relay channel list 2>/dev/null | grep "^vf-\\|^cp[0-9]" | awk '{print $1}' | while read name; do - relay channel archive "$name" 2>/dev/null || true +relay channel list 2>/dev/null | grep -oE '(vf|cp[0-9])[A-Za-z0-9_-]*' | while read -r name; do + relay channel archive "$name" >/dev/null 2>&1 || true done echo "Cleanup complete" +exit 0 `, }); - // ── Phase 8: Collect results ───────────────────────────────────────────── - - wf.step('collect-results', { + // ── Phase 12: verdict ──────────────────────────────────────────────────── + // + // The authoritative aggregation. It exits 0 even on FAIL, on purpose: + // findReadySteps() in @relayflows/core only schedules a step whose + // dependencies are "completed" or "skipped", so a failing gate here would + // permanently block the Slack, issue, and PR steps below — the escalation + // path would silently never run. Enforcement is `enforce-verdict` plus the + // process exit code in main(). + + wf.step('verdict', { type: 'deterministic', dependsOn: ['cleanup'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail ARTIFACTS="${ARTIFACTS}" -SUMMARY="$ARTIFACTS/summary.txt" - -echo "=== Verification Run: ${RUN_ID} ===" > "$SUMMARY" -echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$SUMMARY" -echo "" >> "$SUMMARY" - -for tier in tier1 tier2 tier3 tier4; do - log="$ARTIFACTS/$tier.log" - if [ -f "$log" ]; then - result=$(grep -E "^(TIER|FAIL|PASS)" "$log" | tail -1 || echo "UNKNOWN") - counts=$(grep -E "result:" "$log" | tail -1 || echo "") - echo "[$tier] $result $counts" >> "$SUMMARY" - else - echo "[$tier] NOT_RUN" >> "$SUMMARY" +${ENV_DEFAULTS} + +node <<'VERDICTEOF' +const fs = require('node:fs'); + +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const ledgerPath = artifacts + '/checks.jsonl'; + +// Every tier that must have produced records. A tier missing from the ledger +// crashed before its first check; reporting that as "clean" is the exact +// failure mode this gate exists to prevent. +const EXPECTED_TIERS = [ + 'provenance', + 'tier1', + 'tier2', + 'tier3', + 'tier4', + 'tier5', + 'tier6', + 'critical-paths', +]; + +let checks = []; +let ledgerError = null; +try { + checks = fs + .readFileSync(ledgerPath, 'utf8') + .split('\n') + .filter((line) => line.trim().startsWith('{')) + .map((line) => { + try { + return JSON.parse(line); + } catch { + return null; + } + }) + .filter(Boolean); +} catch (err) { + ledgerError = err.message; +} + +const byTier = {}; +for (const tier of EXPECTED_TIERS) { + byTier[tier] = { pass: 0, fail: 0, skip: 0, failures: [] }; +} +for (const check of checks) { + const bucket = (byTier[check.tier] ??= { pass: 0, fail: 0, skip: 0, failures: [] }); + if (check.status === 'pass') bucket.pass++; + else if (check.status === 'fail') { + bucket.fail++; + bucket.failures.push({ check: check.check, reason: check.reason }); + } else if (check.status === 'skip') bucket.skip++; +} + +const notRun = EXPECTED_TIERS.filter( + (tier) => byTier[tier].pass + byTier[tier].fail + byTier[tier].skip === 0 +); + +const totals = { pass: 0, fail: 0, skip: 0 }; +for (const bucket of Object.values(byTier)) { + totals.pass += bucket.pass; + totals.fail += bucket.fail; + totals.skip += bucket.skip; +} + +const provenance = {}; +try { + for (const line of fs.readFileSync(artifacts + '/provenance.env', 'utf8').split('\n')) { + const [key, ...rest] = line.split('='); + if (key) provenance[key] = rest.join('='); + } +} catch { + // provenance step did not run; reported via notRun +} + +const reasons = []; +if (ledgerError) reasons.push('check ledger unreadable: ' + ledgerError); +if (notRun.length > 0) reasons.push('tiers produced no records: ' + notRun.join(', ')); +if (totals.fail > 0) reasons.push(totals.fail + ' check(s) failed'); + +const verdict = { + runId: process.env.VERIFY_RUN_ID, + verdict: reasons.length === 0 ? 'PASS' : 'FAIL', + reasons, + totals, + tiers: byTier, + tiersNotRun: notRun, + provenance, + generatedAt: new Date().toISOString(), +}; + +fs.writeFileSync(artifacts + '/verdict.json', JSON.stringify(verdict, null, 2)); + +console.log('=== Verdict: ' + verdict.verdict + ' ==='); +console.log( + 'checks: ' + totals.pass + ' passed, ' + totals.fail + ' failed, ' + totals.skip + ' skipped' +); +for (const [tier, bucket] of Object.entries(byTier)) { + const state = bucket.pass + bucket.fail + bucket.skip === 0 ? 'NOT_RUN' : bucket.fail > 0 ? 'FAIL' : 'PASS'; + console.log( + ' [' + tier + '] ' + state + ' ' + bucket.pass + 'p/' + bucket.fail + 'f/' + bucket.skip + 's' + ); + for (const failure of bucket.failures) { + console.log(' FAIL: ' + failure.check + ' — ' + String(failure.reason).slice(0, 160)); + } +} +if (reasons.length > 0) console.log('reasons: ' + reasons.join('; ')); +VERDICTEOF + +exit 0 +`, + }); + + // ── Phase 13: observability — PostHog ──────────────────────────────────── + + wf.step('emit-posthog', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} +${POSTHOG_FN} + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo " [posthog] no verdict.json — nothing to emit" + exit 0 +fi + +# Build the property payloads with node so the JSON is always well-formed. +node <<'PHEOF' > "$ARTIFACTS/posthog-payloads.txt" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const verdict = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const runProps = { + run_id: verdict.runId, + verdict: verdict.verdict, + checks_passed: verdict.totals.pass, + checks_failed: verdict.totals.fail, + checks_skipped: verdict.totals.skip, + tiers_not_run: verdict.tiersNotRun.join(',') || null, + cli_version: verdict.provenance.VERIFY_CLI_VERSION || null, + repo_version: verdict.provenance.VERIFY_REPO_VERSION || null, + git_sha: verdict.provenance.VERIFY_GIT_SHA || null, + reasons: verdict.reasons.join('; ') || null, +}; +console.log('relay_verify_run\t' + JSON.stringify(runProps)); + +// One event per failing check, bounded so a catastrophic run cannot emit +// thousands of events. +const failures = Object.entries(verdict.tiers).flatMap(([tier, bucket]) => + bucket.failures.map((f) => ({ tier, ...f })) +); +for (const failure of failures.slice(0, 50)) { + console.log( + 'relay_verify_check_failed\t' + + JSON.stringify({ + run_id: verdict.runId, + tier: failure.tier, + check: failure.check, + reason: String(failure.reason).slice(0, 300), + cli_version: verdict.provenance.VERIFY_CLI_VERSION || null, + }) + ); +} +PHEOF + +while IFS="$(printf '\t')" read -r event props; do + [ -n "$event" ] || continue + posthog_capture "$event" "$props" +done < "$ARTIFACTS/posthog-payloads.txt" + +exit 0 +`, + }); + + // ── Phase 14: infra escalation to NightCTO ─────────────────────────────── + // + // A broken *harness* is a different failure class from a broken feature. A + // product regression gets an issue and a PR; a harness that cannot run gets + // an operator alert, because otherwise the run just goes quiet and nobody + // notices that verification stopped happening. The pasted run showed exactly + // this class: an expired openai refresh token during bootstrap, logged as + // "non-fatal" and never surfaced to anyone. + + wf.step('escalate-infra', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} +VERIFY_CLI_VERSION=$(grep '^VERIFY_CLI_VERSION=' "$ARTIFACTS/provenance.env" 2>/dev/null | cut -d= -f2 || true) +if [ -z "$VERIFY_CLI_VERSION" ]; then VERIFY_CLI_VERSION="unknown"; fi +${NIGHTCTO_FN} + +ESCALATED=0 + +# 1. The verdict gate itself did not produce a verdict. +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + escalate_infra "relayflow.verify.verdict" "verdict_missing" \ + "verify-features produced no verdict.json — the verification pipeline did not complete" + ESCALATED=1 +fi + +# 2. Tiers that never recorded a single check: harness breakage, not a +# product signal. +if [ -f "$ARTIFACTS/verdict.json" ]; then + NOT_RUN=$(node -e 'const v=require("node:fs").readFileSync(process.argv[1],"utf8");process.stdout.write((JSON.parse(v).tiersNotRun||[]).join(","))' "$ARTIFACTS/verdict.json" 2>/dev/null || echo "") + if [ -n "$NOT_RUN" ]; then + escalate_infra "relayflow.verify.tier_not_run" "tier_not_run" \ + "verify-features tiers produced no records: $NOT_RUN" + ESCALATED=1 fi -done +fi -cp_log="$ARTIFACTS/critical-paths.log" -if [ -f "$cp_log" ]; then - cp_result=$(grep -E "^CRITICAL" "$cp_log" | tail -1 || echo "UNKNOWN") - echo "[critical-paths] $cp_result" >> "$SUMMARY" -else - echo "[critical-paths] NOT_RUN" >> "$SUMMARY" +# 3. The broker never came up, so nothing below tier 1 could be meaningful. +if grep -q "SETUP_FAIL" "$ARTIFACTS/setup.log" 2>/dev/null; then + escalate_infra "relayflow.verify.setup" "broker_start_failed" \ + "verify-features could not start the local broker within 30s" + ESCALATED=1 +fi + +# 4. No provider CLI at all. Bootstrap logs an expired provider credential as +# "non-fatal" and continues, which silently converts tier 6 and CP3 into +# permanent SKIPs — verification quietly stops covering managed agents. +if grep -q "^provider_any=0$" "$ARTIFACTS/caps.env" 2>/dev/null; then + escalate_infra "relayflow.verify.providers" "no_provider_cli" \ + "verify-features found no provider CLI on PATH — tier 6 and CP3 cannot be verified in this environment" + ESCALATED=1 fi -echo "" >> "$SUMMARY" -echo "Artifact files:" >> "$SUMMARY" -ls -la "$ARTIFACTS/" >> "$SUMMARY" +if [ "$ESCALATED" -eq 0 ]; then + echo " No harness-level failures to escalate." +fi -cat "$SUMMARY" +exit 0 `, }); - // ── Phase 9: Post verification report ─────────────────────────────────── + // ── Phase 15: Slack escalation ─────────────────────────────────────────── // - // Posts a structured PASS/FAIL report to #relay-health. - // Must be fast — just read logs and post. No code exploration, no waiting. + // Posts on FAIL. Delivery is attempted through the Slack local runtime + // (SLACK_BOT_TOKEN) and falls back to the cloud-relay runtime + // (CLOUD_API_URL + CLOUD_API_TOKEN). When neither is configured the step + // says so loudly and records it — a silent no-op must never be mistakable + // for a delivered alert. + + wf.step('slack-alert', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "SLACK_SKIPPED: no verdict.json to report" + exit 0 +fi + +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") + +if [ "$VERDICT" = "PASS" ]; then + echo "SLACK_SKIPPED: verdict is PASS — not alerting" + exit 0 +fi + +node <<'SLACKEOF' > "$ARTIFACTS/slack-message.txt" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const v = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const lines = []; +lines.push(':rotating_light: *Feature verification FAILED* — ${BT}' + v.runId + '${BT}'); +lines.push( + 'CLI ' + + (v.provenance.VERIFY_CLI_VERSION || '?') + + ' / repo ' + + (v.provenance.VERIFY_REPO_VERSION || '?') + + ' @ ' + + (v.provenance.VERIFY_GIT_SHA || '?') +); +lines.push( + v.totals.pass + ' passed, *' + v.totals.fail + ' failed*, ' + v.totals.skip + ' skipped' +); +lines.push(''); +for (const [tier, bucket] of Object.entries(v.tiers)) { + const total = bucket.pass + bucket.fail + bucket.skip; + const state = total === 0 ? 'NOT_RUN' : bucket.fail > 0 ? 'FAIL' : 'PASS'; + lines.push( + '• ${BT}' + tier + '${BT} ' + state + ' — ' + bucket.pass + 'p/' + bucket.fail + 'f/' + bucket.skip + 's' + ); + for (const f of bucket.failures.slice(0, 5)) { + lines.push(' ↳ ' + f.check + ': ' + String(f.reason).slice(0, 180)); + } +} +if (v.tiersNotRun.length > 0) { + lines.push(''); + lines.push('*Tiers that produced no records:* ' + v.tiersNotRun.join(', ')); +} +lines.push(''); +lines.push('Artifacts: ${BT}' + artifacts + '${BT}'); +process.stdout.write(lines.join('\n')); +SLACKEOF + +MESSAGE=$(cat "$ARTIFACTS/slack-message.txt") + +if [ -n "$SLACK_BOT_TOKEN" ]; then + RESP=$(curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ + -H "authorization: Bearer $SLACK_BOT_TOKEN" \ + -H 'content-type: application/json; charset=utf-8' \ + --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8"),unfurl_links:false}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo '{"ok":false}') + if printf '%s' "$RESP" | grep -q '"ok":true'; then + echo "SLACK_POSTED: $CHANNEL (local runtime)" + exit 0 + fi + echo "SLACK_FAILED via SLACK_BOT_TOKEN: $(printf '%s' "$RESP" | head -c 300)" +fi + +if [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; then + RESP=$(curl -sS -m 20 -X POST "$CLOUD_API_URL/api/v1/integrations/slack/messages" \ + -H "authorization: Bearer $CLOUD_API_TOKEN" \ + -H 'content-type: application/json' \ + --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8")}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo "") + if [ -n "$RESP" ] && ! printf '%s' "$RESP" | grep -qi 'error\|unauthorized\|not found'; then + echo "SLACK_POSTED: $CHANNEL (cloud-relay runtime)" + exit 0 + fi + echo "SLACK_FAILED via CLOUD_API: $(printf '%s' "$RESP" | head -c 300)" +fi + +# Neither runtime available. Print the payload so the alert is at least +# recoverable from the run log, and make the gap explicit. +echo "SLACK_UNDELIVERED: no SLACK_BOT_TOKEN and no CLOUD_API_URL/CLOUD_API_TOKEN." +echo "---- undelivered Slack payload ----" +printf '%s\n' "$MESSAGE" +exit 0 +`, + }); + + // ── Phase 16: relay-native report ──────────────────────────────────────── wf.step('report', { agent: 'reporter', - dependsOn: ['collect-results'], + dependsOn: ['verdict'], task: `You are an automated feature health agent for agent-relay. -First, check whether the verification artifact directory exists. Look for the directory "${ARTIFACTS}". +Read ${VERDICT_FILE}. It is the authoritative result — do NOT re-derive the +verdict from the individual tier logs, and do NOT run any relay commands to +re-test anything. -If the directory does NOT exist or NONE of the expected log files are present: - Post this message to relay-health: +If ${VERDICT_FILE} does not exist, post exactly this to the relay-health channel +and then finish: ## Feature Verification: ${RUN_ID} -**Overall: NOT_RUN** +**Overall: NOT_RUN** — the verification pipeline produced no verdict. +This is a harness failure, not a product result. Check the workflow logs. -| Check | Result | Notes | -|-------|--------|-------| -| V1: CLI Health | NOT_RUN | verification pipeline did not produce artifacts | -| V2: Broker + Agents | NOT_RUN | | -| V3: Channel Messaging | NOT_RUN | | -| V4: Cross-Agent DMs | NOT_RUN | | -| V5: Critical Paths | NOT_RUN | | +Otherwise post a message to the relay-health channel in this format, filling it +in from verdict.json: -**Failures:** verification pipeline did not run — check broker startup and workflow logs -**Next:** Run \`relay node workflow run workflows/verify-features.ts\` or debug broker startup +## Feature Verification: ${RUN_ID} -Then finish immediately — do not attempt to read missing files. +**Overall: ** ( passed, failed, skipped) +CLI / repo -If the directory DOES exist and has log files, read these artifact files directly: -- ${ARTIFACTS}/summary.txt -- ${ARTIFACTS}/tier1.log -- ${ARTIFACTS}/tier2.log -- ${ARTIFACTS}/tier3.log -- ${ARTIFACTS}/tier4.log -- ${ARTIFACTS}/critical-paths.log +| Tier | Result | p/f/s | Notes | +|------|--------|-------|-------| +| one row per tier in verdict.json "tiers" | -Then post a message to the relay-health channel with this format: +**Failures:** every entry in each tier's "failures" array, as +\`: \`. Write "None" only if there are genuinely zero. +**Skipped:** the skip count per tier — state plainly that skipped checks were +NOT verified. +**Tiers not run:** verdict.json "tiersNotRun", or "None". -## Feature Verification: ${RUN_ID} +Be accurate over brief. Never describe a skipped check as passing.`, + }); + + // ── Phase 17: GitHub issue ─────────────────────────────────────────────── + + wf.step('file-issue', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +AUTOFIX="${AUTOFIX ? '1' : '0'}" +${ENV_DEFAULTS} + +if [ "$AUTOFIX" != "1" ]; then + echo "ISSUE_SKIPPED: VERIFY_AUTOFIX=0" + exit 0 +fi +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "ISSUE_SKIPPED: no verdict.json" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh CLI not available" + exit 0 +fi +if ! gh auth status >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh is not authenticated" + exit 0 +fi + +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") +if [ "$VERDICT" = "PASS" ]; then + echo "ISSUE_SKIPPED: verdict is PASS" + exit 0 +fi + +node <<'ISSUEEOF' > "$ARTIFACTS/issue-body.md" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const v = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const out = []; +out.push('Automated feature verification failed.'); +out.push(''); +out.push('- Run: ${BT}' + v.runId + '${BT}'); +out.push('- CLI under test: ${BT}' + (v.provenance.VERIFY_CLI_VERSION || '?') + '${BT}'); +out.push('- Repo version: ${BT}' + (v.provenance.VERIFY_REPO_VERSION || '?') + '${BT}'); +out.push('- Git sha: ${BT}' + (v.provenance.VERIFY_GIT_SHA || '?') + '${BT}'); +out.push('- Totals: ' + v.totals.pass + ' passed, ' + v.totals.fail + ' failed, ' + v.totals.skip + ' skipped'); +out.push(''); +out.push('## Failing checks'); +out.push(''); +let any = false; +for (const [tier, bucket] of Object.entries(v.tiers)) { + if (bucket.failures.length === 0) continue; + any = true; + out.push('### ' + tier); + for (const f of bucket.failures) { + out.push('- **' + f.check + '** — ' + String(f.reason).slice(0, 500)); + } + out.push(''); +} +if (!any) out.push('_No individual check failures; see the reasons below._'); +out.push('## Verdict reasons'); +out.push(''); +for (const reason of v.reasons) out.push('- ' + reason); +if (v.tiersNotRun.length > 0) { + out.push(''); + out.push('## Tiers that produced no records'); + out.push(''); + out.push('These crashed before their first check — treat as harness breakage, not a clean pass.'); + out.push(''); + for (const tier of v.tiersNotRun) out.push('- ${BT}' + tier + '${BT}'); +} +out.push(''); +out.push('---'); +out.push('Filed by ${BT}workflows/verify-features.ts${BT}. Artifacts: ${BT}' + artifacts + '${BT}'); +process.stdout.write(out.join('\n')); +ISSUEEOF + +TITLE="Feature verification failed: ${RUN_ID}" +ISSUE_URL=$(gh issue create --title "$TITLE" --body-file "$ARTIFACTS/issue-body.md" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || echo "") + +if [ -n "$ISSUE_URL" ]; then + echo "$ISSUE_URL" > "$ARTIFACTS/issue-url.txt" + echo "ISSUE_CREATED: $ISSUE_URL" +else + echo "ISSUE_FAILED: gh issue create produced no URL" +fi -**Overall: PASS** or **Overall: FAIL** +exit 0 +`, + }); + + // ── Phase 18: attempt a fix ────────────────────────────────────────────── + + wf.step('attempt-fix', { + agent: 'fixer', + dependsOn: ['file-issue'], + task: `Fix the underlying defect behind a feature verification failure. + +## FIRST: check whether there is anything to do + +Read ${VERDICT_FILE}. + +- If the file does not exist, or its "verdict" field is "PASS", there is NOTHING + to fix. Write "No failures — nothing to fix." to + ${ARTIFACTS}/fix-summary.md and STOP IMMEDIATELY. Do not read other files, do + not create a branch, do not edit anything, do not run any commands. +- Only if "verdict" is "FAIL" do you continue with the rest of this task. -| Check | Result | Notes | -|-------|--------|-------| -| V1: CLI Health | PASS/FAIL | brief note if failed | -| V2: Broker + Agents | PASS/FAIL | | -| V3: Channel Messaging | PASS/FAIL | | -| V4: Cross-Agent DMs | PASS/FAIL | | -| V5: Critical Paths | PASS/FAIL | | +(The workflow engine has no conditional steps, so this step is scheduled on +every run. The early exit above is what keeps a green run from spending model +time and, more importantly, from letting an agent loose on a clean tree.) -**Failures:** list specific failed commands, or "None" -**Next:** "System healthy" if all pass, or what to debug if not +## The task -Do NOT explore the codebase, run builds, or test CLI commands. Just read the files and post.`, +Read ${VERDICT_FILE} for the authoritative list of failing checks, and the +per-tier logs in ${ARTIFACTS}/ for detail. + +## Hard rules + +1. NEVER commit or push to \`main\`. Work on a branch named exactly + \`${FIX_BRANCH}\`. Create it from the current HEAD. +2. Fix the ROOT CAUSE in product code. Do NOT make a check pass by weakening, + deleting, or loosening the check itself. If a check is genuinely wrong (it + tests a command that was intentionally removed, or asserts something the + product never promised), you may correct the check — but you must say so + explicitly and justify it in your summary. +3. Do NOT reduce the number of checks in \`workflows/verify-features.ts\`. A + later gate counts them and will reject the branch if the count drops. +4. Re-run the specific failing tier after your fix and paste the real output as + evidence. A fix with no re-run is not a fix. +5. If a failure is environmental (no cloud login, no provider CLI, expired + credentials), do NOT patch code. Report it as environmental and change + nothing — those cases are already recorded as SKIPs or escalated to NightCTO. + +## Deliverables + +Write a file \`${ARTIFACTS}/fix-summary.md\` containing: + +- \`Root cause:\` one paragraph per distinct failure, naming the file and symbol. +- \`Change:\` what you edited and why it addresses the root cause. +- \`Evidence:\` the exact command you re-ran and its output. +- \`Environmental:\` any failures you deliberately did not patch, and why. +- \`Assertions changed:\` any verification check you modified, with justification, + or "none". + +Commit your work on the branch. Do NOT open a pull request — a later step does +that. Do NOT merge anything.`, }); - // ── Phase 10: Analyze for improvements and drift ───────────────────────── + // ── Phase 19: fix integrity gate ───────────────────────────────────────── // - // Reads logs to identify feature drift, broken surfaces, or improvement - // opportunities. Produces an analysis artifact. Runs as deterministic - // to avoid opencode agent spawning failures. + // The one guard that makes the auto-fix path trustworthy. A fixer agent's + // cheapest route to green is always to weaken the assertion, so this + // compares the check count before and after and refuses a branch that + // shrank coverage. - wf.step('analyze-improvements', { + wf.step('fix-integrity', { type: 'deterministic', - dependsOn: ['collect-results'], + dependsOn: ['attempt-fix'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail ARTIFACTS="${ARTIFACTS}" -SUMMARY="$ARTIFACTS/summary.txt" -REPORT="$ARTIFACTS/improvements-report.txt" -MANIFEST=".agentworkforce/features/manifest.yaml" - -echo "=== Improvement Analysis: ${RUN_ID} ===" > "$REPORT" -echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$REPORT" -echo "" >> "$REPORT" - -# Check if artifacts exist -if [ ! -d "$ARTIFACTS" ] || [ -z "$(ls -A "$ARTIFACTS"/*.log 2>/dev/null)" ]; then - echo "Status: NO_ARTIFACTS" >> "$REPORT" - echo "No verification artifacts found — pipeline did not produce results." >> "$REPORT" - cat "$REPORT" +WORKFLOW_FILE="workflows/verify-features.ts" + +# This step is scheduled unconditionally (no conditional steps in the engine), +# so a green run reaches it with nothing to check. Exit before the git +# assertions below, which would otherwise report a misleading INTEGRITY_FAIL +# just because HEAD is still main on a clean run. +if [ -f "$ARTIFACTS/verdict.json" ]; then + VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json" 2>/dev/null || true) + if [ "$VERDICT" = "PASS" ]; then + echo "INTEGRITY_NOT_APPLICABLE: verdict is PASS, no fix was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/fix-integrity.env" + exit 0 + fi +fi + +if ! git rev-parse --git-dir >/dev/null 2>&1; then + echo "INTEGRITY_SKIPPED: not a git repository" + echo "INTEGRITY=skipped" > "$ARTIFACTS/fix-integrity.env" exit 0 fi -FOUND=0 +CURRENT=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +echo "Current branch: $CURRENT" -# ── Check each tier and critical paths for failures ──────────────────────── -for tier in tier1 tier2 tier3 tier4; do - log="$ARTIFACTS/$tier.log" - if [ -f "$log" ]; then - if grep -q "FAIL" "$log" 2>/dev/null; then - FOUND=$((FOUND + 1)) - failed=$(grep "FAIL" "$log" | head -5) - echo "[$tier] FAILURES found:" >> "$REPORT" - echo "$failed" | sed 's/^/ /' >> "$REPORT" - echo "" >> "$REPORT" - fi - fi -done +if [ "$CURRENT" = "main" ]; then + echo "INTEGRITY_FAIL: the fixer left the repo on main — refusing to proceed" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi -if [ -f "$ARTIFACTS/critical-paths.log" ]; then - if grep -q "FAIL" "$ARTIFACTS/critical-paths.log" 2>/dev/null; then - FOUND=$((FOUND + 1)) - failed=$(grep "FAIL" "$ARTIFACTS/critical-paths.log" | head -10) - echo "[critical-paths] FAILURES found:" >> "$REPORT" - echo "$failed" | sed 's/^/ /' >> "$REPORT" - echo "" >> "$REPORT" - - if grep -q "relay mcp.*no recognizable output" "$ARTIFACTS/critical-paths.log" 2>/dev/null; then - echo " [mcp-server-start] CP3 MCP help check failed — verify relay mcp --help returns expected output" >> "$REPORT" - echo "" >> "$REPORT" - fi - fi +# 'grep -c' prints 0 and exits 1 when there are no matches, so the fallback +# must be '|| true' — '|| echo 0' would emit a second line and break the +# numeric comparison below. POSIX bracket classes, not \s, for portability. +CHECK_PATTERN='^[[:space:]]*(run_check|gated_check|skip_check|record) ' + +count_checks_at() { + git show "$1:$WORKFLOW_FILE" 2>/dev/null | grep -cE "$CHECK_PATTERN" || true +} + +BASE_REF=$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || true) +if [ -z "$BASE_REF" ]; then + echo "INTEGRITY_SKIPPED: cannot resolve a base ref to diff against" + echo "INTEGRITY=skipped" > "$ARTIFACTS/fix-integrity.env" + exit 0 fi -if [ "$FOUND" -eq 0 ]; then - echo "Result: NO_IMPROVEMENTS_NEEDED" >> "$REPORT" - echo "" >> "$REPORT" - echo "All tiers and critical paths passed. No drift or improvements identified." >> "$REPORT" +BEFORE=$(count_checks_at "$BASE_REF") +AFTER=$(grep -cE "$CHECK_PATTERN" "$WORKFLOW_FILE" 2>/dev/null || true) +if [ -z "$BEFORE" ]; then BEFORE=0; fi +if [ -z "$AFTER" ]; then AFTER=0; fi + +echo "Verification call sites: $BEFORE (base $BASE_REF) -> $AFTER (working tree)" + +if [ "$AFTER" -lt "$BEFORE" ]; then + echo "INTEGRITY_FAIL: the fix removed $((BEFORE - AFTER)) verification call site(s)." + echo "A fix that deletes checks has not fixed anything." + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +if [ ! -f "$ARTIFACTS/fix-summary.md" ]; then + echo "INTEGRITY_FAIL: fixer produced no fix-summary.md, so there is no root cause on record" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +if ! grep -qi "Root cause:" "$ARTIFACTS/fix-summary.md"; then + echo "INTEGRITY_FAIL: fix-summary.md has no 'Root cause:' section" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +echo "INTEGRITY_OK" +echo "INTEGRITY=ok" > "$ARTIFACTS/fix-integrity.env" +exit 0 +`, + }); + + // ── Phase 20: draft PR ─────────────────────────────────────────────────── + + wf.step('open-pr', { + type: 'deterministic', + dependsOn: ['fix-integrity'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/fix-integrity.env" ]; then + echo "PR_SKIPPED: no integrity result — the fix path did not complete" + exit 0 +fi + +. "$ARTIFACTS/fix-integrity.env" + +if [ "$INTEGRITY" != "ok" ]; then + echo "PR_SKIPPED: integrity gate returned '$INTEGRITY' — not opening a PR" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "PR_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$BRANCH" = "main" ]; then + echo "PR_SKIPPED: refusing to open a PR from main" + exit 0 +fi + +if git diff --quiet HEAD 2>/dev/null && [ -z "$(git log origin/main..HEAD --oneline 2>/dev/null)" ]; then + echo "PR_SKIPPED: no commits on $BRANCH relative to origin/main" + exit 0 +fi + +git push -u origin "$BRANCH" 2>&1 | tail -3 || { + echo "PR_FAILED: could not push $BRANCH" + exit 0 +} + +ISSUE_REF="" +if [ -f "$ARTIFACTS/issue-url.txt" ]; then + ISSUE_REF="Fixes $(cat "$ARTIFACTS/issue-url.txt")" +fi + +{ + echo "## Automated fix for feature verification failure" + echo "" + echo "Run: \`${RUN_ID}\`" + echo "" + [ -n "$ISSUE_REF" ] && echo "$ISSUE_REF" && echo "" + echo "### Fixer summary" + echo "" + cat "$ARTIFACTS/fix-summary.md" 2>/dev/null || echo "_No fix summary produced._" + echo "" + echo "### Verdict at time of failure" + echo "" + echo '${BT}${BT}${BT}' + node -e 'const v=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));console.log(v.totals.pass+" passed, "+v.totals.fail+" failed, "+v.totals.skip+" skipped");for(const[t,b]of Object.entries(v.tiers)){if(b.failures.length)for(const f of b.failures)console.log("["+t+"] "+f.check+": "+String(f.reason).slice(0,200))}' "$ARTIFACTS/verdict.json" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Review notes" + echo "" + echo "- Opened as a **draft** by \`workflows/verify-features.ts\`. Not auto-merged." + echo "- The integrity gate confirmed this branch does not reduce the number of" + echo " verification call sites, but a human still needs to confirm the fix" + echo " addresses the root cause rather than the symptom." +} > "$ARTIFACTS/pr-body.md" + +PR_URL=$(gh pr create --draft --title "fix: feature verification failures (${RUN_ID})" \ + --body-file "$ARTIFACTS/pr-body.md" --base main --head "$BRANCH" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || echo "") + +if [ -n "$PR_URL" ]; then + echo "$PR_URL" > "$ARTIFACTS/pr-url.txt" + echo "PR_CREATED: $PR_URL" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + gh issue comment "$(cat "$ARTIFACTS/issue-url.txt")" \ + --body "Draft fix PR opened: $PR_URL" >/dev/null 2>&1 || true + fi else - echo "Result: IMPROVEMENTS_FOUND" >> "$REPORT" - echo "" >> "$REPORT" - echo "Summary: $FOUND issue(s) above. Review the verification logs for details." >> "$REPORT" - echo "To fix: review each FAIL and correct the command/check or the test expectation." >> "$REPORT" + echo "PR_FAILED: gh pr create produced no URL" +fi + +exit 0 +`, + }); + + // ── Phase 21: Slack follow-up with issue and PR links ──────────────────── + + wf.step('slack-followup', { + type: 'deterministic', + dependsOn: ['open-pr'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} + +ISSUE=$(cat "$ARTIFACTS/issue-url.txt" 2>/dev/null || true) +PR=$(cat "$ARTIFACTS/pr-url.txt" 2>/dev/null || true) + +if [ -z "$ISSUE" ] && [ -z "$PR" ]; then + echo "FOLLOWUP_SKIPPED: no issue or PR to report" + exit 0 +fi + +TEXT="Follow-up for \`${RUN_ID}\`:" +[ -n "$ISSUE" ] && TEXT="$TEXT +• Issue: $ISSUE" +[ -n "$PR" ] && TEXT="$TEXT +• Draft fix PR: $PR" +[ -z "$PR" ] && TEXT="$TEXT +• No fix PR — the fix attempt did not pass the integrity gate. Needs a human." + +if [ -n "$SLACK_BOT_TOKEN" ]; then + curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ + -H "authorization: Bearer $SLACK_BOT_TOKEN" \ + -H 'content-type: application/json; charset=utf-8' \ + --data "$(TEXT="$TEXT" CH="$CHANNEL" node -e 'process.stdout.write(JSON.stringify({channel:process.env.CH,text:process.env.TEXT}))')" >/dev/null 2>&1 \ + && echo "FOLLOWUP_POSTED" && exit 0 +fi + +echo "FOLLOWUP_UNDELIVERED (no SLACK_BOT_TOKEN):" +printf '%s\n' "$TEXT" +exit 0 +`, + }); + + // ── Phase 22: enforce the verdict ──────────────────────────────────────── + // + // Terminal step, and the only one that fails on a bad result. Nothing + // depends on it, so its failure cannot block the escalation steps above — + // it exists to colour the run red after all reporting has happened. + // + // Deliberately does NOT depend on the fix chain (attempt-fix → fix-integrity + // → open-pr → slack-followup). A step failure marks its whole downstream + // subtree skipped (markDownstreamSkipped is transitive), so hanging + // enforcement off the fix chain would mean any hiccup while opening a PR + // silently swallowed the red verdict. Reporting and alerting are the only + // things enforcement waits on. + + wf.step('enforce-verdict', { + type: 'deterministic', + dependsOn: ['report', 'emit-posthog', 'escalate-infra', 'slack-alert'], + captureOutput: true, + failOnError: true, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "ENFORCE: no verdict.json — treating as FAIL (harness did not complete)" + exit 2 fi -cat "$REPORT" +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") +node -e 'const v=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));console.log("verdict="+v.verdict);console.log("reasons="+(v.reasons.join("; ")||"none"))' "$ARTIFACTS/verdict.json" + +if [ "$VERDICT" = "PASS" ]; then + echo "ENFORCE: PASS" + exit 0 +fi + +echo "ENFORCE: FAIL — see verdict.json" +exit 1 `, }); - await wf.run(); + const result = await wf.run(); + + // A dry run plans the graph without executing it, so there is no verdict to + // read and nothing to enforce. Bail before the checks below, which would + // otherwise report a validated plan as harness breakage. + if (process.env.DRY_RUN || !('status' in result)) { + return; + } + + // ── Post-run: make the process exit code tell the truth ────────────────── + // + // The previous version awaited run() and discarded the result, so a run with + // four failing checks still exited 0 and every scheduler saw success. Read + // the verdict directly rather than trusting the runner's row status, and + // fail closed when the verdict is missing. + + let verdict: { verdict?: string; reasons?: string[]; totals?: Record } | null = null; + if (existsSync(VERDICT_FILE)) { + try { + verdict = JSON.parse(readFileSync(VERDICT_FILE, 'utf8')); + } catch (err) { + console.error(`[verify-features] verdict.json is unreadable: ${(err as Error).message}`); + } + } + + if (!verdict) { + console.error( + `[verify-features] no usable verdict at ${VERDICT_FILE} — treating this run as FAILED. ` + + `The verification pipeline did not complete; this is harness breakage, not a clean run.` + ); + process.exitCode = 2; + return; + } + + const totals = verdict.totals ?? {}; + console.log( + `[verify-features] verdict=${verdict.verdict} ` + + `pass=${totals.pass ?? '?'} fail=${totals.fail ?? '?'} skip=${totals.skip ?? '?'} ` + + `(workflow row status: ${'status' in result ? String(result.status) : 'unknown'})` + ); + + if (verdict.verdict !== 'PASS') { + console.error(`[verify-features] FAILED: ${(verdict.reasons ?? []).join('; ')}`); + process.exitCode = 1; + } } -main(); +main().catch((err) => { + // A throw here means the harness itself broke — the runner could not even + // produce a result. That is the NightCTO escalation case, and it must not + // exit 0. + console.error(`[verify-features] harness failure: ${err instanceof Error ? err.stack : String(err)}`); + process.exitCode = 2; +}); From 72539f7b88d4ab800fda9218d6e197eb811b8036 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 15:07:59 +0200 Subject: [PATCH 2/7] fix(verify): post Slack via relayflows integration, not a bot token Both workflows now post through @relayflows/slack-primitive pinned to its cloud-relay runtime, which proxies /api/v1/slack/post-message and delivers via the workspace's configured Slack integration (the Nango connection). Pinning the runtime is load-bearing. selectRuntime() prefers cloud-relay but falls back to a local SLACK_BOT_TOKEN runtime that talks to slack.com directly, so leaving it on auto meant a stray bot token in the environment silently changed the delivery route. Pinned, a missing CLOUD_API_* pair raises auth_token_missing instead. Verified: with SLACK_BOT_TOKEN set and CLOUD_API_* absent, delivery fails loudly rather than falling back. Also fixes the endpoint. The previous cloud fallback POSTed to /api/v1/integrations/slack/messages, which does not exist; the primitive's actual path is /api/v1/slack/post-message. Confirmed against a stub server: POST /api/v1/slack/post-message, bearer CLOUD_API_TOKEN, body {channel, text, unfurlLinks, unfurlMedia}. Default channel is now the ID C0AEKNLDNKW rather than a #name. The cloud-relay runtime does not implement channel resolution, so an ID is the only form guaranteed to resolve. Override with VERIFY_SLACK_CHANNEL / AUDIT_SLACK_CHANNEL. An undelivered alert still echoes its full payload to the run log, so a delivery failure can never read as a delivered alert. Co-Authored-By: Claude Opus 5 --- workflows/audit-feature-manifest.ts | 91 +++++++++++------ workflows/verify-features.ts | 149 +++++++++++++++++----------- 2 files changed, 152 insertions(+), 88 deletions(-) diff --git a/workflows/audit-feature-manifest.ts b/workflows/audit-feature-manifest.ts index c5cca6f9b..c36b18dfb 100644 --- a/workflows/audit-feature-manifest.ts +++ b/workflows/audit-feature-manifest.ts @@ -39,13 +39,16 @@ * * ## Environment * - * AUDIT_SLACK_CHANNEL Slack channel for drift. Default #relay-health. + * AUDIT_SLACK_CHANNEL Slack channel ID for drift. Default C0AEKNLDNKW. * AUDIT_AUTOFIX "0" disables the issue/manifest-update/PR path. * RELAY_CLI CLI under audit. Defaults to the repo build. * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. * NIGHTCTO_EVIDENCE_URL Enables escalation when the audit cannot run. * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. - * SLACK_BOT_TOKEN Slack local runtime; falls back to CLOUD_API_*. + * CLOUD_API_URL Required for Slack delivery, which goes through the + * CLOUD_API_TOKEN relayflows Slack primitive's cloud-relay runtime + * (/api/v1/slack/post-message) using the workspace's + * configured Slack integration. No bot token is read. */ import { existsSync, readFileSync } from 'node:fs'; @@ -58,7 +61,11 @@ const RUN_ID = `audit-${TIMESTAMP}`; const REPORT_FILE = `${ARTIFACTS}/audit.json`; const FIX_BRANCH = `chore/${RUN_ID}-manifest`; -const SLACK_CHANNEL = process.env.AUDIT_SLACK_CHANNEL ?? '#relay-health'; +/** + * Slack destination, as a channel ID. The cloud-relay runtime does not + * implement channel resolution, so an ID is the only form guaranteed to work. + */ +const SLACK_CHANNEL = process.env.AUDIT_SLACK_CHANNEL ?? 'C0AEKNLDNKW'; const AUTOFIX = process.env.AUDIT_AUTOFIX !== '0'; /** A literal backtick; a raw one would close the template literals below. */ @@ -75,7 +82,6 @@ POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" -SLACK_BOT_TOKEN="$(printenv SLACK_BOT_TOKEN || true)" CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" RELAY_CLI="$(printenv RELAY_CLI || true)" @@ -86,6 +92,55 @@ export AUDIT_ARTIFACTS="${ARTIFACTS}" export AUDIT_RUN_ID="${RUN_ID}" `; +/** + * Post to Slack through the relayflows Slack primitive. + * + * Pins `runtime: 'cloud-relay'` on purpose: the adapter's auto-detection falls + * back to a local `SLACK_BOT_TOKEN` runtime that talks to slack.com directly, + * and drift reports must go through the workspace's configured Slack + * integration. Pinning turns a missing CLOUD_API_* pair into an explicit + * `auth_token_missing` rather than a silent change of route. + */ +const SLACK_POST_FN = String.raw` +slack_post() { + _sp_channel="$1" + _sp_textfile="$2" + + cat > "$ARTIFACTS/slack-post.mjs" <<'SLACKPOSTEOF' +import { readFileSync } from 'node:fs'; +import { SlackClient } from '@relayflows/slack-primitive'; + +const [channel, textFile] = process.argv.slice(2); +const text = readFileSync(textFile, 'utf8'); + +const client = new SlackClient({ + runtime: 'cloud-relay', + cloudApiUrl: process.env.CLOUD_API_URL, + cloudApiToken: process.env.CLOUD_API_TOKEN, +}); + +try { + const out = await client.postMessage({ channel, text, unfurl: false }); + console.log('SLACK_POSTED channel=' + out.channel + ' ts=' + out.ts); +} catch (err) { + const code = err && err.code ? err.code : 'unknown'; + console.log('SLACK_ERROR ' + code + ': ' + (err && err.message ? err.message : String(err))); + process.exitCode = 1; +} +SLACKPOSTEOF + + if node "$ARTIFACTS/slack-post.mjs" "$_sp_channel" "$_sp_textfile"; then + return 0 + fi + + echo "SLACK_UNDELIVERED to $_sp_channel — payload follows:" + echo "---- undelivered Slack payload ----" + cat "$_sp_textfile" + echo "---- end payload ----" + return 1 +} +`; + async function main() { const wf = workflow('relay-audit-feature-manifest') .description( @@ -278,6 +333,7 @@ set -uo pipefail ARTIFACTS="${ARTIFACTS}" CHANNEL="${SLACK_CHANNEL}" ${ENV_DEFAULTS} +${SLACK_POST_FN} EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi @@ -338,32 +394,7 @@ if (exitCode === 2) { process.stdout.write(lines.join('\n')); SLACKEOF -if [ -n "$SLACK_BOT_TOKEN" ]; then - RESP=$(curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ - -H "authorization: Bearer $SLACK_BOT_TOKEN" \ - -H 'content-type: application/json; charset=utf-8' \ - --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8"),unfurl_links:false}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo '{"ok":false}') - if printf '%s' "$RESP" | grep -q '"ok":true'; then - echo "SLACK_POSTED: $CHANNEL" - exit 0 - fi - echo "SLACK_FAILED: $(printf '%s' "$RESP" | head -c 300)" -fi - -if [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; then - RESP=$(curl -sS -m 20 -X POST "$CLOUD_API_URL/api/v1/integrations/slack/messages" \ - -H "authorization: Bearer $CLOUD_API_TOKEN" \ - -H 'content-type: application/json' \ - --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8")}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || true) - if [ -n "$RESP" ] && ! printf '%s' "$RESP" | grep -qi 'error\|unauthorized\|not found'; then - echo "SLACK_POSTED: $CHANNEL (cloud-relay runtime)" - exit 0 - fi -fi - -echo "SLACK_UNDELIVERED: no SLACK_BOT_TOKEN and no CLOUD_API_URL/CLOUD_API_TOKEN." -echo "---- undelivered Slack payload ----" -cat "$ARTIFACTS/slack-message.txt" +slack_post "$CHANNEL" "$ARTIFACTS/slack-message.txt" || true exit 0 `, }); diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index 0755dc9f2..73521d644 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -59,8 +59,11 @@ * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. * NIGHTCTO_EVIDENCE_URL Enables infra escalation to NightCTO. * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. - * SLACK_BOT_TOKEN Slack local runtime. Falls back to CLOUD_API_URL + - * CLOUD_API_TOKEN (cloud-relay runtime). + * CLOUD_API_URL Required for Slack delivery. Slack posts go through + * CLOUD_API_TOKEN the relayflows Slack primitive's cloud-relay runtime + * (/api/v1/slack/post-message), which uses the + * workspace's configured Slack integration. No bot + * token is read. */ import { existsSync, readFileSync } from 'node:fs'; @@ -78,7 +81,14 @@ const SUFFIX = `vf-${Date.now()}`; /** Canonical fix branch. RUN_ID already carries the "verify-" prefix. */ const FIX_BRANCH = `fix/${RUN_ID}`; -const SLACK_CHANNEL = process.env.VERIFY_SLACK_CHANNEL ?? '#relay-health'; +/** + * Slack destination, as a channel ID. + * + * An ID rather than a #name because the cloud-relay runtime does not implement + * channel resolution — `resolveChannel` is unsupported there, so a "#name" only + * works if Slack itself accepts it for that workspace. An ID always resolves. + */ +const SLACK_CHANNEL = process.env.VERIFY_SLACK_CHANNEL ?? 'C0AEKNLDNKW'; const AUTOFIX = process.env.VERIFY_AUTOFIX !== '0'; /** @@ -106,7 +116,6 @@ POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" -SLACK_BOT_TOKEN="$(printenv SLACK_BOT_TOKEN || true)" CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" VERIFY_ENVIRONMENT="$(printenv VERIFY_ENVIRONMENT || true)" @@ -207,6 +216,63 @@ finish_tier() { } `; +/** + * Post to Slack through the relayflows Slack primitive. + * + * Deliberately pins `runtime: 'cloud-relay'` rather than letting the adapter + * auto-detect. `selectRuntime` in @relayflows/slack-primitive prefers + * cloud-relay but falls back to a local `SLACK_BOT_TOKEN` runtime that talks to + * slack.com directly — and this workflow must post through the workspace's + * configured Slack integration (the Nango connection behind + * `/api/v1/slack/post-message`), not whatever bot token happens to be in the + * environment. Pinning makes a missing CLOUD_API_* pair throw + * `auth_token_missing` instead of silently taking a different path. + * + * The primitive is a hard dependency of @relayflows/core, which this workflow + * already imports, so it resolves wherever the workflow itself does. + */ +const SLACK_POST_FN = String.raw` +slack_post() { + _sp_channel="$1" + _sp_textfile="$2" + + cat > "$ARTIFACTS/slack-post.mjs" <<'SLACKPOSTEOF' +import { readFileSync } from 'node:fs'; +import { SlackClient } from '@relayflows/slack-primitive'; + +const [channel, textFile] = process.argv.slice(2); +const text = readFileSync(textFile, 'utf8'); + +const client = new SlackClient({ + runtime: 'cloud-relay', + cloudApiUrl: process.env.CLOUD_API_URL, + cloudApiToken: process.env.CLOUD_API_TOKEN, +}); + +try { + const out = await client.postMessage({ channel, text, unfurl: false }); + console.log('SLACK_POSTED channel=' + out.channel + ' ts=' + out.ts); +} catch (err) { + const code = err && err.code ? err.code : 'unknown'; + console.log('SLACK_ERROR ' + code + ': ' + (err && err.message ? err.message : String(err))); + process.exitCode = 1; +} +SLACKPOSTEOF + + if node "$ARTIFACTS/slack-post.mjs" "$_sp_channel" "$_sp_textfile"; then + return 0 + fi + + # Undelivered alerts must be loud. Echo the payload so it is at least + # recoverable from the run log rather than lost with the process. + echo "SLACK_UNDELIVERED to $_sp_channel — payload follows:" + echo "---- undelivered Slack payload ----" + cat "$_sp_textfile" + echo "---- end payload ----" + return 1 +} +`; + /** * Bounded evidence POST to NightCTO, matching the CloudEvidenceSummary v1 * contract in ../cloud/docs/dogfood-telemetry-brief.md. Byte compatibility @@ -485,7 +551,7 @@ probe cloud 'relay cloud whoami' probe gh 'gh auth status' probe git_repo 'git rev-parse --git-dir' probe jq 'command -v jq' -probe slack '[ -n "$SLACK_BOT_TOKEN" ] || { [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; }' +probe slack '[ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]' probe provider_claude 'command -v claude' probe provider_codex 'command -v codex' probe provider_opencode 'command -v opencode' @@ -1424,11 +1490,12 @@ exit 0 // ── Phase 15: Slack escalation ─────────────────────────────────────────── // - // Posts on FAIL. Delivery is attempted through the Slack local runtime - // (SLACK_BOT_TOKEN) and falls back to the cloud-relay runtime - // (CLOUD_API_URL + CLOUD_API_TOKEN). When neither is configured the step - // says so loudly and records it — a silent no-op must never be mistakable - // for a delivered alert. + // Posts on FAIL through the relayflows Slack primitive, pinned to its + // cloud-relay runtime so delivery always goes via the workspace's Slack + // integration rather than a bot token. When CLOUD_API_URL/CLOUD_API_TOKEN are + // absent the primitive throws auth_token_missing and the step echoes the + // undelivered payload — a silent no-op must never be mistakable for a + // delivered alert. wf.step('slack-alert', { type: 'deterministic', @@ -1441,6 +1508,7 @@ set -uo pipefail ARTIFACTS="${ARTIFACTS}" CHANNEL="${SLACK_CHANNEL}" ${ENV_DEFAULTS} +${SLACK_POST_FN} if [ ! -f "$ARTIFACTS/verdict.json" ]; then echo "SLACK_SKIPPED: no verdict.json to report" @@ -1492,37 +1560,7 @@ lines.push('Artifacts: ${BT}' + artifacts + '${BT}'); process.stdout.write(lines.join('\n')); SLACKEOF -MESSAGE=$(cat "$ARTIFACTS/slack-message.txt") - -if [ -n "$SLACK_BOT_TOKEN" ]; then - RESP=$(curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ - -H "authorization: Bearer $SLACK_BOT_TOKEN" \ - -H 'content-type: application/json; charset=utf-8' \ - --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8"),unfurl_links:false}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo '{"ok":false}') - if printf '%s' "$RESP" | grep -q '"ok":true'; then - echo "SLACK_POSTED: $CHANNEL (local runtime)" - exit 0 - fi - echo "SLACK_FAILED via SLACK_BOT_TOKEN: $(printf '%s' "$RESP" | head -c 300)" -fi - -if [ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]; then - RESP=$(curl -sS -m 20 -X POST "$CLOUD_API_URL/api/v1/integrations/slack/messages" \ - -H "authorization: Bearer $CLOUD_API_TOKEN" \ - -H 'content-type: application/json' \ - --data "$(node -e 'const fs=require("node:fs");process.stdout.write(JSON.stringify({channel:process.argv[1],text:fs.readFileSync(process.argv[2],"utf8")}))' "$CHANNEL" "$ARTIFACTS/slack-message.txt")" 2>&1 || echo "") - if [ -n "$RESP" ] && ! printf '%s' "$RESP" | grep -qi 'error\|unauthorized\|not found'; then - echo "SLACK_POSTED: $CHANNEL (cloud-relay runtime)" - exit 0 - fi - echo "SLACK_FAILED via CLOUD_API: $(printf '%s' "$RESP" | head -c 300)" -fi - -# Neither runtime available. Print the payload so the alert is at least -# recoverable from the run log, and make the gap explicit. -echo "SLACK_UNDELIVERED: no SLACK_BOT_TOKEN and no CLOUD_API_URL/CLOUD_API_TOKEN." -echo "---- undelivered Slack payload ----" -printf '%s\n' "$MESSAGE" +slack_post "$CHANNEL" "$ARTIFACTS/slack-message.txt" || true exit 0 `, }); @@ -1918,6 +1956,7 @@ set -uo pipefail ARTIFACTS="${ARTIFACTS}" CHANNEL="${SLACK_CHANNEL}" ${ENV_DEFAULTS} +${SLACK_POST_FN} ISSUE=$(cat "$ARTIFACTS/issue-url.txt" 2>/dev/null || true) PR=$(cat "$ARTIFACTS/pr-url.txt" 2>/dev/null || true) @@ -1927,24 +1966,18 @@ if [ -z "$ISSUE" ] && [ -z "$PR" ]; then exit 0 fi -TEXT="Follow-up for \`${RUN_ID}\`:" -[ -n "$ISSUE" ] && TEXT="$TEXT -• Issue: $ISSUE" -[ -n "$PR" ] && TEXT="$TEXT -• Draft fix PR: $PR" -[ -z "$PR" ] && TEXT="$TEXT -• No fix PR — the fix attempt did not pass the integrity gate. Needs a human." - -if [ -n "$SLACK_BOT_TOKEN" ]; then - curl -sS -m 20 -X POST https://slack.com/api/chat.postMessage \ - -H "authorization: Bearer $SLACK_BOT_TOKEN" \ - -H 'content-type: application/json; charset=utf-8' \ - --data "$(TEXT="$TEXT" CH="$CHANNEL" node -e 'process.stdout.write(JSON.stringify({channel:process.env.CH,text:process.env.TEXT}))')" >/dev/null 2>&1 \ - && echo "FOLLOWUP_POSTED" && exit 0 -fi +FOLLOWUP="$ARTIFACTS/slack-followup.txt" +{ + echo "Follow-up for \`${RUN_ID}\`:" + [ -n "$ISSUE" ] && echo "• Issue: $ISSUE" + if [ -n "$PR" ]; then + echo "• Draft fix PR: $PR" + else + echo "• No fix PR — the fix attempt did not pass the integrity gate. Needs a human." + fi +} > "$FOLLOWUP" -echo "FOLLOWUP_UNDELIVERED (no SLACK_BOT_TOKEN):" -printf '%s\n' "$TEXT" +slack_post "$CHANNEL" "$FOLLOWUP" || true exit 0 `, }); From 0f010e616187e83ef2dedafc73969db6bd5a0dc9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 30 Jul 2026 13:09:49 +0000 Subject: [PATCH 3/7] style: auto-format with Prettier --- .claude/skills/verify-features.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index 0be764e06..584653ad0 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -34,7 +34,7 @@ Before trusting any verification result, check two things: failure against a real command. 2. **What was skipped.** Every check records `pass`, `fail`, or `skip` with a reason into `.workflow-artifacts/verify-features/checks.jsonl`, and - `verdict.json` is the authoritative result. A SKIP means *not verified*. + `verdict.json` is the authoritative result. A SKIP means _not verified_. Never read a skip as a pass. Run the manifest audit before adding checks, so you are not writing coverage @@ -133,7 +133,7 @@ Add or update entries in `manifest.yaml` when: - A feature is removed or deprecated Then run `node scripts/audit-feature-manifest.mjs` and confirm `MANIFEST_CLEAN`. -Note that `manifest-contract.test.ts` cannot catch a missing entry for a *new* +Note that `manifest-contract.test.ts` cannot catch a missing entry for a _new_ command — a new command is absent from both the manifest and that test's hardcoded expectation list, so it passes. The audit script derives the surface from `--help` and `tools/list` instead, which is why it is the check that From ec17d7ecf83516036e2c54fcc853396356ba89ec Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 16:07:56 +0200 Subject: [PATCH 4/7] fix(verify): correct defects found by a live end-to-end run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the pipeline for real against the 11.3.0 build with a live broker, cloud login, and all four provider CLIs: 90 passed, 3 failed, 17 skipped, exit 1. Fixes below are the workflow's own bugs that the run exposed. - grep -q under `set -o pipefail`: -q exits at the first match, the upstream writer takes SIGPIPE, and the pipeline reports failure. A demonstrably RUNNING broker probed as absent. Fixed at 10 piped call sites, including run_check itself, where it was a latent false-FAIL generator for any check whose command produced long output. - The capabilities step read $CLOUD_API_URL without seeding it, so `set -u` killed it mid-probe and every provider_* flag went unwritten. Tier 5/6 gating and the NightCTO provider escalation both read a truncated file. - Token extraction matched any 20+ char string, so a failed registration ("agent already exists") yielded the agent's own NAME as its token and every downstream call failed for unrelated reasons. Now parses the JSON token field. Fixed at 7 sites. - VERIFY_AUTOFIX=0 gated file-issue but not attempt-fix, so the documented off switch did not stop an agent from editing the tree. The flag is now written to autofix.env and honoured by attempt-fix and fix-integrity. - Three checks contradicted signatures the manifest already documented correctly: `message file upload --channel`, `cloud status ` (now a stated skip — no disposable run is made), and `integration webhook create `. - CP5 asserted the sentinel against the launch response, but `workflow run` is asynchronous and returns status "running". It now polls the log for the sentinel and asserts a completed sync, as critical-path 5 documents. - Added a post-removal assertion in tier 3: `agent remove` reporting success was assumed to mean the agent was gone. It is not. Co-Authored-By: Claude Opus 5 --- workflows/verify-features.ts | 140 ++++++++++++++++++++++++++--------- 1 file changed, 104 insertions(+), 36 deletions(-) diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index 73521d644..205757bd3 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -170,11 +170,16 @@ record() { } # Run a command and match its output. Records pass/fail either way. +# +# Note the plain "grep -i" rather than "grep -qi": under "set -o pipefail", a +# -q grep exits at the first match, the writer upstream takes SIGPIPE, and the +# pipeline reports failure even though the match succeeded. That turned a +# demonstrably running broker into a "capability absent" result in a live run. run_check() { _name="$1" _cmd="$2" _expect="$3" - if _out=$(eval "$_cmd" 2>&1) && printf '%s' "$_out" | grep -qi -- "$_expect"; then + if _out=$(eval "$_cmd" 2>&1) && printf '%s' "$_out" | grep -i -- "$_expect" >/dev/null; then echo " PASS $_name" | tee -a "$LOG" record "$TIER" "$_name" pass "" else @@ -428,7 +433,7 @@ relay node up --background 2>&1 | tee -a "$LOG" || true BROKER_READY=0 for i in $(seq 1 30); do - if relay node status 2>&1 | grep -qi "running"; then + if relay node status 2>&1 | grep -i "running" >/dev/null; then BROKER_READY=1 break fi @@ -529,6 +534,7 @@ set -uo pipefail LOG="${ARTIFACTS}/capabilities.log" CAPS="${ARTIFACTS}/caps.env" +${ENV_DEFAULTS} : > "$CAPS" echo "=== Capability probe ===" | tee "$LOG" @@ -545,7 +551,7 @@ probe() { fi } -probe broker 'relay node status 2>&1 | grep -qi running' +probe broker 'relay node status 2>&1 | grep -i running >/dev/null' probe workspace 'relay workspace active' probe cloud 'relay cloud whoami' probe gh 'gh auth status' @@ -669,7 +675,12 @@ MSG_TEXT="verify-msg-${SUFFIX}" echo "=== Tier 3: Channels, Messages, Threads, Reactions, Inbox ===" | tee "$LOG" -TOKEN=$(relay agent register "$AGENT" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +# Parse the JSON token field rather than matching any long-ish string: +# "agent register" on an existing name prints that the agent already exists, +# and a 20+ char agent name matched the old heuristic, so a failed registration +# silently yielded the agent own name as its token. Every downstream call then +# failed for reasons that had nothing to do with the feature under test. +TOKEN=$(relay agent register "$AGENT" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -z "$TOKEN" ]; then record "$TIER" "agent register + token" fail "no token returned from relay agent register" echo " FAIL agent register (no token returned)" | tee -a "$LOG" @@ -719,7 +730,18 @@ run_check "message search" "relay message search '$MSG_TEXT'" "$MSG_TEXT" run_check "channel leave" "relay channel leave '$CHANNEL'" "." run_check "channel archive" "relay channel archive '$CHANNEL'" "." -relay agent remove "$AGENT" 2>/dev/null || true +# Assert the removal actually took effect rather than assuming it did. A live +# run found "agent remove" failing for any agent that had sent a message, +# leaking a raw SQL error, which left test identities accumulating in the +# workspace while every tier still reported clean. +run_check "agent remove" "relay agent remove '$AGENT'" "." +if relay agent list 2>&1 | grep "\"name\": \"$AGENT\"" >/dev/null; then + echo " FAIL agent gone after remove" | tee -a "$LOG" + record "$TIER" "agent gone after remove" fail "$AGENT is still listed after a remove that reported success" +else + echo " PASS agent gone after remove" | tee -a "$LOG" + record "$TIER" "agent gone after remove" pass "" +fi finish_tier exit 0 @@ -746,8 +768,8 @@ CHANNEL="vf-private-${SUFFIX}" echo "=== Tier 4: Cross-Agent DMs, Invites, Read Receipts, Files ===" | tee "$LOG" -TOKEN_A=$(relay agent register "$AGENT_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_B=$(relay agent register "$AGENT_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_A=$(relay agent register "$AGENT_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_B=$(relay agent register "$AGENT_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -z "$TOKEN_A" ] || [ -z "$TOKEN_B" ]; then record "$TIER" "register two agents" fail "token A empty=$([ -z "$TOKEN_A" ] && echo yes || echo no), token B empty=$([ -z "$TOKEN_B" ] && echo yes || echo no)" @@ -807,8 +829,14 @@ fi UPLOAD_FILE="${ARTIFACTS}/upload-fixture.txt" echo "verify-upload-${SUFFIX}" > "$UPLOAD_FILE" +# Signature is "upload --channel " — the channel is an option, +# not a positional. A live run caught this check passing the channel first. +# --text is passed explicitly even though --help documents it as defaulting to +# "": the API rejects an empty text with "text is required", so the documented +# default cannot succeed. Tracked as a product finding, not worked around +# silently — the check exercises the path that actually works. run_check "message file upload" \ - "RELAY_AGENT_TOKEN='$TOKEN_A' relay message file upload '$CHANNEL' '$UPLOAD_FILE'" "." + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message file upload '$UPLOAD_FILE' --channel '$CHANNEL' --text 'verify upload ${SUFFIX}'" "." RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CHANNEL" >/dev/null 2>&1 || true relay agent remove "$AGENT_A" 2>/dev/null || true @@ -845,7 +873,7 @@ fi gated_check cloud "cloud whoami" "relay cloud whoami" "." gated_check cloud "cloud session" "relay cloud session" "." -gated_check cloud "cloud status" "relay cloud status" "." +skip_check "cloud status" "takes a required ; no disposable cloud run is made (see the cloud run skip)" gated_check cloud "cloud schedules" "relay cloud schedules" "." gated_check cloud "cloud logs" "relay cloud logs 2>&1 || true" "." gated_check cloud "cloud sync --help" "relay cloud sync --help" "Usage" @@ -873,15 +901,17 @@ gated_check cloud "reflex status" "relay reflex status" "." skip_check "reflex on" "mutates history sync state for the workspace" skip_check "reflex off" "mutates history sync state for the workspace" -WEBHOOK_NAME="vf-hook-${SUFFIX}" +# The command registers a URL, not a name — example.com is reserved by RFC 2606 +# precisely so test registrations cannot reach anyone's real endpoint. +WEBHOOK_URL="https://example.com/relay-verify/${SUFFIX}" if have_cap cloud; then run_check "integration webhook list" "relay integration webhook list" "." run_check "integration subscription list" "relay integration subscription list" "." - if relay integration webhook create "$WEBHOOK_NAME" >/dev/null 2>&1; then + if relay integration webhook create "$WEBHOOK_URL" >/dev/null 2>&1; then echo " PASS integration webhook create" | tee -a "$LOG" record "$TIER" "integration webhook create" pass "" - run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_NAME' 2>&1 || true" "." - run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_NAME'" "." + run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_URL' 2>&1 || true" "." + run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_URL'" "." else record "$TIER" "integration webhook create" fail "relay integration webhook create exited non-zero" echo " FAIL integration webhook create" | tee -a "$LOG" @@ -1004,12 +1034,12 @@ mkdir -p "$CP1_STATE" CP1=0 relay node up --background --no-spawn --state-dir "$CP1_STATE" >/dev/null 2>&1 || true sleep 2 -relay node status --state-dir "$CP1_STATE" 2>&1 | grep -qi running && CP1=$((CP1 + 1)) +relay node status --state-dir "$CP1_STATE" 2>&1 | grep -i running >/dev/null && CP1=$((CP1 + 1)) relay node metrics >/dev/null 2>&1 && CP1=$((CP1 + 1)) relay node deadletters --json >/dev/null 2>&1 && CP1=$((CP1 + 1)) relay node down --state-dir "$CP1_STATE" >/dev/null 2>&1 || true sleep 2 -relay node status --state-dir "$CP1_STATE" 2>&1 | grep -qiv running && CP1=$((CP1 + 1)) +relay node status --state-dir "$CP1_STATE" 2>&1 | grep -iv running >/dev/null && CP1=$((CP1 + 1)) if [ "$CP1" -ge 4 ]; then echo " PASS CP1 ($CP1/4)" | tee -a "$LOG" record "$TIER" "CP1 broker lifecycle" pass "" @@ -1024,13 +1054,13 @@ CP2_A="cp2a-${SUFFIX}" CP2_B="cp2b-${SUFFIX}" CP2_CH="cp2-ch-${SUFFIX}" CP2_MSG="cp2-msg-${SUFFIX}" -TOKEN_2A=$(relay agent register "$CP2_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_2B=$(relay agent register "$CP2_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_2A=$(relay agent register "$CP2_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_2B=$(relay agent register "$CP2_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -n "$TOKEN_2A" ] && [ -n "$TOKEN_2B" ]; then RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel create "$CP2_CH" >/dev/null 2>&1 || true RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel invite "$CP2_CH" "$CP2_B" >/dev/null 2>&1 || true RELAY_AGENT_TOKEN="$TOKEN_2A" relay message post "$CP2_CH" "$CP2_MSG" >/dev/null 2>&1 || true - if RELAY_AGENT_TOKEN="$TOKEN_2B" relay message list "$CP2_CH" --limit 10 2>&1 | grep -q "$CP2_MSG"; then + if RELAY_AGENT_TOKEN="$TOKEN_2B" relay message list "$CP2_CH" --limit 10 2>&1 | grep "$CP2_MSG" >/dev/null; then echo " PASS CP2 (B read A's exact text)" | tee -a "$LOG" record "$TIER" "CP2 cross-agent channel message" pass "" else @@ -1056,7 +1086,7 @@ else CP3_WORKER="cp3-${SUFFIX}" if relay node agent spawn "$CP3_PROVIDER" --name "$CP3_WORKER" \ --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task >/dev/null 2>&1 \ - && relay node agent list 2>&1 | grep -q "$CP3_WORKER"; then + && relay node agent list 2>&1 | grep "$CP3_WORKER" >/dev/null; then relay node agent message hold "$CP3_WORKER" >/dev/null 2>&1 || true relay node agent message auto "$CP3_WORKER" >/dev/null 2>&1 || true relay node agent release "$CP3_WORKER" >/dev/null 2>&1 || true @@ -1152,22 +1182,45 @@ mkdir -p "$CP5_DIR" echo 'console.log("critical-workflow-ok")' > "$CP5_DIR/workflow.js" CP5_JSON=$(relay node workflow run "$CP5_DIR/workflow.js" --json 2>&1 || echo "") CP5_RUN_ID=$(printf '%s' "$CP5_JSON" | grep -o '"runId": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") -if [ -n "$CP5_RUN_ID" ] && printf '%s' "$CP5_JSON" | grep -q "critical-workflow-ok\|completed"; then - relay node workflow logs "$CP5_RUN_ID" --json >/dev/null 2>&1 || true - relay node workflow sync "$CP5_RUN_ID" --dry-run --json >/dev/null 2>&1 || true - echo " PASS CP5 (run $CP5_RUN_ID)" | tee -a "$LOG" - record "$TIER" "CP5 local workflow lifecycle" pass "" -else - echo " FAIL CP5" | tee -a "$LOG" + +# "workflow run" is asynchronous: it returns status "running" with a run id and +# the sentinel only reaches the log later. Poll the log the way critical-path 5 +# documents (--follow), rather than asserting against the launch response, which +# can never contain the output. +CP5_SEEN=0 +if [ -n "$CP5_RUN_ID" ]; then + for _ in $(seq 1 20); do + if relay node workflow logs "$CP5_RUN_ID" 2>&1 | grep "critical-workflow-ok" >/dev/null; then + CP5_SEEN=1 + break + fi + sleep 1 + done +fi + +if [ "$CP5_SEEN" -eq 1 ]; then + CP5_SYNC=$(relay node workflow sync "$CP5_RUN_ID" --dry-run --json 2>&1 || echo "") + if printf '%s' "$CP5_SYNC" | grep '"status": *"completed"' >/dev/null; then + echo " PASS CP5 (run $CP5_RUN_ID, sentinel + completed)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" pass "" + else + echo " FAIL CP5 (sentinel seen but run did not report completed)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "sync did not report completed: $(printf '%s' "$CP5_SYNC" | head -c 200)" + fi +elif [ -z "$CP5_RUN_ID" ]; then + echo " FAIL CP5 (no runId from workflow run)" | tee -a "$LOG" record "$TIER" "CP5 local workflow lifecycle" fail "$(printf '%s' "$CP5_JSON" | head -c 300)" +else + echo " FAIL CP5 (sentinel never appeared in the run log)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "run $CP5_RUN_ID produced no critical-workflow-ok within 20s" fi # ── CP6: direct message and read receipt ──────────────────────────────── echo "-- CP6: Direct Message and Read Receipt --" | tee -a "$LOG" CP6_A="cp6a-${SUFFIX}" CP6_B="cp6b-${SUFFIX}" -TOKEN_6A=$(relay agent register "$CP6_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_6B=$(relay agent register "$CP6_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_6A=$(relay agent register "$CP6_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_6B=$(relay agent register "$CP6_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -n "$TOKEN_6A" ] && [ -n "$TOKEN_6B" ]; then CP6_MSG="cp6-dm-${SUFFIX}" CP6_OUT="${ARTIFACTS}/cp6-dm.out" @@ -1176,11 +1229,11 @@ if [ -n "$TOKEN_6A" ] && [ -n "$TOKEN_6B" ]; then CP6_ID=$(grep -oE '"(messageId|id)": *"[^"]*"' "$CP6_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || echo "") CP6=0 if [ -n "$CP6_CONV" ]; then - RELAY_AGENT_TOKEN="$TOKEN_6B" relay message dm list "$CP6_CONV" 2>&1 | grep -q "$CP6_MSG" && CP6=$((CP6 + 1)) + RELAY_AGENT_TOKEN="$TOKEN_6B" relay message dm list "$CP6_CONV" 2>&1 | grep "$CP6_MSG" >/dev/null && CP6=$((CP6 + 1)) fi if [ -n "$CP6_ID" ]; then RELAY_AGENT_TOKEN="$TOKEN_6B" relay message inbox mark_read "$CP6_ID" >/dev/null 2>&1 && CP6=$((CP6 + 1)) - RELAY_AGENT_TOKEN="$TOKEN_6A" relay message inbox get_readers "$CP6_ID" 2>&1 | grep -q "$CP6_B" && CP6=$((CP6 + 1)) + RELAY_AGENT_TOKEN="$TOKEN_6A" relay message inbox get_readers "$CP6_ID" 2>&1 | grep "$CP6_B" >/dev/null && CP6=$((CP6 + 1)) fi if [ "$CP6" -ge 3 ]; then echo " PASS CP6 ($CP6/3)" | tee -a "$LOG" @@ -1241,6 +1294,11 @@ set -uo pipefail ARTIFACTS="${ARTIFACTS}" ${ENV_DEFAULTS} +# Published for the fix chain: attempt-fix is an agent step and cannot read the +# workflow's own constants, so the operator's autofix decision has to reach it +# through the filesystem. +echo "AUTOFIX=${AUTOFIX ? '1' : '0'}" > "$ARTIFACTS/autofix.env" + node <<'VERDICTEOF' const fs = require('node:fs'); @@ -1709,13 +1767,17 @@ exit 0 ## FIRST: check whether there is anything to do -Read ${VERDICT_FILE}. +Read ${ARTIFACTS}/autofix.env and ${VERDICT_FILE}. -- If the file does not exist, or its "verdict" field is "PASS", there is NOTHING - to fix. Write "No failures — nothing to fix." to - ${ARTIFACTS}/fix-summary.md and STOP IMMEDIATELY. Do not read other files, do - not create a branch, do not edit anything, do not run any commands. -- Only if "verdict" is "FAIL" do you continue with the rest of this task. +- If ${ARTIFACTS}/autofix.env contains \`AUTOFIX=0\`, the operator has disabled + the fix path. Write "Autofix disabled." to ${ARTIFACTS}/fix-summary.md and + STOP IMMEDIATELY. +- If ${VERDICT_FILE} does not exist, or its "verdict" field is "PASS", there is + NOTHING to fix. Write "No failures — nothing to fix." to + ${ARTIFACTS}/fix-summary.md and STOP IMMEDIATELY. +- In either case: do not read other files, do not create a branch, do not edit + anything, do not run any commands. +- Only if autofix is enabled AND "verdict" is "FAIL" do you continue. (The workflow engine has no conditional steps, so this step is scheduled on every run. The early exit above is what keeps a green run from spending model @@ -1780,6 +1842,12 @@ WORKFLOW_FILE="workflows/verify-features.ts" # so a green run reaches it with nothing to check. Exit before the git # assertions below, which would otherwise report a misleading INTEGRITY_FAIL # just because HEAD is still main on a clean run. +if grep -q '^AUTOFIX=0$' "$ARTIFACTS/autofix.env" 2>/dev/null; then + echo "INTEGRITY_NOT_APPLICABLE: autofix disabled, no fix was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + if [ -f "$ARTIFACTS/verdict.json" ]; then VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json" 2>/dev/null || true) if [ "$VERDICT" = "PASS" ]; then From fcbfd7f0addf3c4ca93a50855e6e47d621329b49 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 17:49:54 +0200 Subject: [PATCH 5/7] fix(cli): repair integration webhook create, document 16 hidden commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes what the live verification run found, and one thing that run could not see because it was measuring a stale build. ## integration webhook create never worked `POST /v1/webhooks` accepts `{ channel, name? }` and RETURNS the url. The command took a `` argument and sent `{ url, event }`, so the required channel was never sent and every invocation failed with "channel is required". `url` is not a request field at all, and `event` belongs to subscriptions. - CLI now takes `` with an optional `--name`, matching create-inbound, which posts to the same endpoint with the same schema. - `RelayCreateWebhookInput` corrected from `{ url, event? }` to `{ channel, name? }`. Callers passing url/event were already failing. - Verified against the live API: create returns a real webhookId/url/token, and the workflow's create -> trigger -> delete lifecycle now passes. ## The audit was reading a stale dist `packages/cli/dist` was older than `packages/cli/src`, so the audit derived a surface missing 16 shipped commands and reported MANIFEST_CLEAN across several runs — the same class of error as the sandbox verifying CLI 10.0.0 against an 11.3.0 checkout, and the reason it went unnoticed. Now documented, so they are covered rather than invisible: `agent me`, `agent presence`, the `cloud integration` tree (catalog, connect, connections, disconnect) and the `cloud room` tree (session, revoke-session, members, remove-member, invite, invites, accept, revoke-invite). Two new manifest categories, each mapped to a verification procedure. 180 -> 194 features. The audit now warns when dist is older than src, so a stale build cannot produce a falsely clean result again. Co-Authored-By: Claude Opus 5 --- .agentworkforce/features/manifest.yaml | 112 +++++++++++++++++- CHANGELOG.md | 7 +- .../integration-webhook-create.test.ts | 77 ++++++++++++ packages/cli/src/cli/commands/integration.ts | 12 +- .../sdk/src/__tests__/integrations.test.ts | 6 +- packages/sdk/src/messaging/types.ts | 14 ++- scripts/audit-feature-manifest.mjs | 62 +++++++++- workflows/verify-features.ts | 27 ++++- 8 files changed, 298 insertions(+), 19 deletions(-) create mode 100644 packages/cli/src/cli/commands/integration-webhook-create.test.ts diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index dcfe895fb..38bb7e457 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -33,6 +33,8 @@ verification: local-workflows: local-workflow-lifecycle cloud: cloud-workflows cloud-workers: cloud-workers + cloud-integrations: integrations-and-webhooks + cloud-rooms: cloud-workflows fleet: fleet-management workspace: workspace-management skills: skills-installation @@ -124,6 +126,20 @@ categories: location: packages/cli/src/cli/commands/agent.ts verify_tier: 3 + - id: agent-me + name: Show Current Agent Identity + cli: relay agent me + description: Show the identity the current agent token resolves to + location: packages/cli/src/cli/commands/agent.ts + verify_tier: 3 + + - id: agent-presence + name: List Agent Presence + cli: relay agent presence + description: List visible agent presence in the workspace + location: packages/cli/src/cli/commands/agent.ts + verify_tier: 3 + - id: agent-remove name: Remove Agent cli: relay agent remove @@ -554,6 +570,100 @@ categories: location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 + cloud-integrations: + name: Cloud Integrations + description: Connect third-party providers to the workspace through Cloud + criticality: standard + features: + - id: cloud-integration-catalog + name: Integration Catalog + cli: relay cloud integration catalog [--static] [--json] + description: Discover static and dynamic (Nango, Composio) Cloud integrations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-integration-connect + name: Connect Integration + cli: relay cloud integration connect [--json] + description: Create a Cloud connection session for a provider + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 6 + + - id: cloud-integration-connections + name: List Connections + cli: relay cloud integration connections [--json] + description: List connected workspace integrations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-integration-disconnect + name: Disconnect Integration + cli: relay cloud integration disconnect [--json] + description: Disconnect a provider from the workspace + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + cloud-rooms: + name: Cloud Rooms + description: Workspace-scoped multiplayer rooms and their membership + criticality: standard + features: + - id: cloud-room-session + name: Create Room Session + cli: relay cloud room session + description: Create or resume this device's full room-participant session + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-revoke-session + name: Revoke Room Session + cli: relay cloud room revoke-session + description: Revoke this member's scoped session for one device + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-members + name: List Room Members + cli: relay cloud room members + description: List workspace room members + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-remove-member + name: Remove Room Member + cli: relay cloud room remove-member + description: Remove a member and revoke their live room access + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-invite + name: Invite Room Participant + cli: relay cloud room invite + description: Invite a full participant to a workspace room + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-invites + name: List Room Invitations + cli: relay cloud room invites + description: List workspace room invitations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-accept + name: Accept Room Invitation + cli: relay cloud room accept + description: Accept an email-bound room invitation + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 6 + + - id: cloud-room-revoke-invite + name: Revoke Room Invitation + cli: relay cloud room revoke-invite + description: Revoke an unused workspace room invitation + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + cloud-workers: name: Cloud Worker Nodes description: Register and manage machines as cloud worker nodes @@ -735,7 +845,7 @@ categories: - id: webhook-create name: Create Webhook - cli: relay integration webhook create [--event ] + cli: relay integration webhook create [--name ] description: Register an outbound webhook location: packages/cli/src/cli/commands/integration.ts verify_tier: 5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 016197b4c..a9a4cc541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- `agent-relay integration webhook create` now works. It took a `` argument and sent `{ url, event }`, but `POST /v1/webhooks` accepts `{ channel, name? }` and returns the URL — so every invocation failed with `channel is required`. It now takes `` with an optional `--name`, matching `create-inbound`, which posts to the same endpoint. +- `@agent-relay/sdk` `RelayCreateWebhookInput` declared a required `url` and an `event`, neither of which the endpoint accepts. It is now `{ channel, name? }`. Code passing `url`/`event` was already failing at runtime. ## [11.3.0] - 2026-07-30 diff --git a/packages/cli/src/cli/commands/integration-webhook-create.test.ts b/packages/cli/src/cli/commands/integration-webhook-create.test.ts new file mode 100644 index 000000000..4124a1eb5 --- /dev/null +++ b/packages/cli/src/cli/commands/integration-webhook-create.test.ts @@ -0,0 +1,77 @@ +import { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; + +import { registerIntegrationCommands, type IntegrationCommandDependencies } from './integration.js'; + +/** + * `relay integration webhook create` posts to `POST /v1/webhooks`, whose request + * schema is `{ channel, name? }` — the URL is part of the *response*. + * + * The command previously took a `` argument and sent `{ url, event }`, so + * every invocation was rejected with "channel is required" and the required + * channel was never sent at all. A live verification run caught it. These tests + * pin the corrected contract so the argument order cannot silently regress. + */ +function makeProgram() { + const relay = { + integrations: { + webhooks: { + create: vi.fn(async (input: unknown) => ({ webhookId: 'wh_1', ...(input as object) })), + }, + }, + }; + const log = vi.fn(); + const error = vi.fn(); + const exit = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerIntegrationCommands(program, { + createAgentRelay: () => relay as never, + resolveLocalRelayOptions: async () => ({ workspaceKey: 'rk_live_test' }), + isInteractive: () => false, + log, + error, + exit: exit as never, + } satisfies Partial as IntegrationCommandDependencies); + return { program, relay, log, error, exit }; +} + +describe('integration webhook create', () => { + it('sends the channel the API requires', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync(['integration', 'webhook', 'create', 'deploy-status'], { + from: 'user', + }); + + expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ + channel: 'deploy-status', + name: undefined, + }); + }); + + it('passes an optional name through', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync( + ['integration', 'webhook', 'create', 'deploy-status', '--name', 'GitHub Alerts'], + { from: 'user' } + ); + + expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ + channel: 'deploy-status', + name: 'GitHub Alerts', + }); + }); + + it('never sends url or event, which the endpoint rejects', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync(['integration', 'webhook', 'create', 'ops'], { from: 'user' }); + + const payload = relay.integrations.webhooks.create.mock.calls[0]?.[0] as Record; + expect(payload).toBeDefined(); + expect(payload).not.toHaveProperty('url'); + expect(payload).not.toHaveProperty('event'); + }); +}); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index a714d35b2..d820d8b09 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -1844,17 +1844,17 @@ export function registerIntegrationCommands( addSdkOptions( webhook .command('create') - .description('Register a webhook') - .argument('', 'Webhook URL') - .option('--event ', 'Event to deliver') - ).action(async (url: string, o: Record) => { + .description('Register an inbound webhook that delivers into a channel (alias of create-inbound)') + .argument('', 'Target channel the webhook posts into') + .option('--name ', 'Human-readable webhook name (e.g. "GitHub Alerts")') + ).action(async (channel: string, o: Record) => { await runSdk(deps, async () => { printJson( deps, await runIntegrationOperation(deps, o, (relay) => relay.integrations.webhooks.create({ - url, - event: o.event as string | undefined, + channel, + name: o.name as string | undefined, }) ) ); diff --git a/packages/sdk/src/__tests__/integrations.test.ts b/packages/sdk/src/__tests__/integrations.test.ts index 16671fc57..2979c0ddf 100644 --- a/packages/sdk/src/__tests__/integrations.test.ts +++ b/packages/sdk/src/__tests__/integrations.test.ts @@ -41,9 +41,11 @@ describe('SDK integrations / capabilities / workspace passthrough', () => { const relaycast = createRelaycastMock(); const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); - const webhook = await client.integrations.webhooks.create({ url: 'https://x', event: 'message.created' }); + // POST /v1/webhooks takes { channel, name? } and returns the generated URL. + // This previously asserted { url, event }, a shape the endpoint rejects. + const webhook = await client.integrations.webhooks.create({ channel: 'deploy-status' }); expect(webhook.id).toBe('wh1'); - expect(relaycast.webhooks.create).toHaveBeenCalledWith({ url: 'https://x', event: 'message.created' }); + expect(relaycast.webhooks.create).toHaveBeenCalledWith({ channel: 'deploy-status' }); await client.integrations.webhooks.trigger('wh1', { hello: 'world' }); expect(relaycast.webhooks.trigger).toHaveBeenCalledWith('wh1', { hello: 'world' }); diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 9df42cf89..b1bf6cadf 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -437,9 +437,19 @@ export interface RelayWebhook { [key: string]: unknown; } +/** + * Input for `integrations.webhooks.create`. + * + * The endpoint (`POST /v1/webhooks`) takes the channel to deliver into and + * returns the generated URL. It previously declared a required `url` and an + * `event`, neither of which the API accepts — `url` is part of the *response*, + * and `event` belongs to subscriptions — so every call was rejected with + * "channel is required". + */ export interface RelayCreateWebhookInput { - url: string; - event?: string; + /** Channel the webhook posts into. A leading `#` is stripped. */ + channel: string; + name?: string; [key: string]: unknown; } diff --git a/scripts/audit-feature-manifest.mjs b/scripts/audit-feature-manifest.mjs index 264053e1b..12346f10e 100644 --- a/scripts/audit-feature-manifest.mjs +++ b/scripts/audit-feature-manifest.mjs @@ -26,7 +26,7 @@ * "the manifest is clean". */ -import { readFileSync, readdirSync } from 'node:fs'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync, spawn } from 'node:child_process'; @@ -244,6 +244,61 @@ function mcpToolsRegisteredInSource() { return names; } +/** + * Warn when the built CLI is older than the sources it is derived from. + * + * The audit can only see what the built CLI exposes, so a stale `dist` makes it + * under-report and print MANIFEST_CLEAN while real commands go undocumented. + * That is exactly what happened here: a stale dist hid 16 shipped commands + * (`agent me`, `agent presence`, the whole `cloud room` and `cloud integration` + * trees) across several audit runs that all reported clean. + * + * @returns A warning string, or null when the build is current or unknowable. + */ +function staleBuildWarning() { + const distEntry = join(repoRoot, 'packages/cli/dist/cli/index.js'); + const srcDir = join(repoRoot, 'packages/cli/src'); + + let distMtime; + try { + distMtime = statSync(distEntry).mtimeMs; + } catch { + return null; // auditing a published CLI, not this checkout + } + + let newest = 0; + const walk = (dir) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full); + continue; + } + if (!entry.name.endsWith('.ts') || entry.name.includes('.test.')) continue; + try { + newest = Math.max(newest, statSync(full).mtimeMs); + } catch { + // unreadable file; ignore + } + } + }; + walk(srcDir); + + if (newest === 0 || newest <= distMtime) return null; + const minutes = Math.round((newest - distMtime) / 60_000); + return ( + `packages/cli/dist is ${minutes} minute(s) older than packages/cli/src. ` + + `This audit only sees what the BUILT CLI exposes, so newer commands are invisible ` + + `and a "clean" result understates drift. Rebuild first: npm run build --workspace=agent-relay` + ); +} + // ── manifest ───────────────────────────────────────────────────────────────── function loadManifest() { @@ -290,6 +345,7 @@ async function main() { } const mcpTools = await deriveMcpTools(); + const staleBuild = staleBuildWarning(); const documentedCommands = new Set( features.flatMap((f) => (f.cli ? [documentedCommandPath(f.cli)] : [])).filter(Boolean) @@ -344,6 +400,7 @@ async function main() { manifestUpdated: manifest.updated, cliCommand, mcpDerivable: mcpTools !== null, + staleBuildWarning: staleBuild, counts: { cliLeaves: cliLeaves.length, documentedCommands: documentedCommands.size, @@ -369,6 +426,9 @@ async function main() { `manifest documents ${documentedCommands.size} commands, ${documentedMcp.size} MCP tools ` + `across ${features.length} features` ); + if (staleBuild) { + console.log(`\nWARN ${staleBuild}`); + } if (mcpTools === null) { console.log('\nWARN MCP tool list could not be derived — MCP drift not checked.'); } diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index 205757bd3..b6b33f797 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -901,17 +901,29 @@ gated_check cloud "reflex status" "relay reflex status" "." skip_check "reflex on" "mutates history sync state for the workspace" skip_check "reflex off" "mutates history sync state for the workspace" -# The command registers a URL, not a name — example.com is reserved by RFC 2606 -# precisely so test registrations cannot reach anyone's real endpoint. -WEBHOOK_URL="https://example.com/relay-verify/${SUFFIX}" +# The endpoint takes the channel to deliver into and returns a generated URL, +# and the channel must already exist, so create it before registering the hook. +WEBHOOK_CHANNEL="vf-hook-${SUFFIX}" +WEBHOOK_AGENT="vf-hookagent-${SUFFIX}" +WEBHOOK_TOKEN=$(relay agent register "$WEBHOOK_AGENT" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || true) +RELAY_AGENT_TOKEN="$WEBHOOK_TOKEN" relay channel create "$WEBHOOK_CHANNEL" >/dev/null 2>&1 || true if have_cap cloud; then run_check "integration webhook list" "relay integration webhook list" "." run_check "integration subscription list" "relay integration subscription list" "." - if relay integration webhook create "$WEBHOOK_URL" >/dev/null 2>&1; then + # create returns the generated webhook; trigger and delete take its id, so the + # id has to be carried between them rather than reusing the channel name. + WEBHOOK_OUT="${ARTIFACTS}/webhook-create.json" + if relay integration webhook create "$WEBHOOK_CHANNEL" > "$WEBHOOK_OUT" 2>&1; then echo " PASS integration webhook create" | tee -a "$LOG" record "$TIER" "integration webhook create" pass "" - run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_URL' 2>&1 || true" "." - run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_URL'" "." + WEBHOOK_ID=$(grep -o '"webhookId": *"[^"]*"' "$WEBHOOK_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || true) + if [ -n "$WEBHOOK_ID" ]; then + run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_ID' 2>&1 || true" "." + run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_ID'" "." + else + skip_check "integration webhook trigger" "no webhookId in create response" + skip_check "integration webhook delete" "no webhookId in create response" + fi else record "$TIER" "integration webhook create" fail "relay integration webhook create exited non-zero" echo " FAIL integration webhook create" | tee -a "$LOG" @@ -925,6 +937,9 @@ else done fi +RELAY_AGENT_TOKEN="$WEBHOOK_TOKEN" relay channel archive "$WEBHOOK_CHANNEL" >/dev/null 2>&1 || true +relay agent remove "$WEBHOOK_AGENT" >/dev/null 2>&1 || true + gated_check cloud "capabilities list" "relay capabilities list" "." gated_check cloud "skills add --help" "relay skills add --help" "Usage" From 1062989c5a3a6f1881cd0236ff223c435eee560d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 21:11:56 +0200 Subject: [PATCH 6/7] fix(verify): address PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex and CodeRabbit findings on #1392. Every item is a real hole in code this PR added; each is fixed and exercised. Audit script: - A version match is not provenance. Between releases a globally installed CLI reports the same version as the checkout, so the old gate passed while every check ran the global binary. It now resolves the binary and requires it to live inside this checkout; outside the checkout fails, and a non-git tree is a stated SKIP rather than a pass. - An underivable MCP surface now exits 2. Previously mcpTools === null forced both MCP drift arrays empty, so a broken probe printed MANIFEST_CLEAN and exited 0 — half an audit reported as a clean one. - Stale detection required only that SOME prefix of a documented command exist. Since deriveCliLeaves records every parent group, removing `fleet spawn` left `fleet` and hid the removal. It now requires the exact leaf, with positional-dispatch parents (`telemetry`) named explicitly. - child.stdin now has an error handler. A broken pipe raised an uncaught 'error' — not a rejection, so it escaped main().catch and exited 1, which the audit workflow reads as confirmed drift and acts on. verify-features: - The ledger stripped only \n\r\t, so ANSI/control bytes made a line unparseable and the verdict reader dropped it silently — a failing check could vanish. Control bytes are stripped, and unparseable lines are now counted and fail the run. - `[ $? -le 124 ]` accepted exit 1, so a broken `node tail` scored PASS. Only 0, 124 and 137 now count. - CP1's down-assertion used `grep -v`, which succeeds when ANY line lacks the pattern; a multi-line status still saying running satisfied it. Negated over the whole output instead. - Cleanup matched "vf"/"cp" anywhere in the listing and could remove unrelated workspace agents and channels. Now anchored to this run's SUFFIX. - `cloud logs` masked a missing with `|| true` and recorded a false pass off the error text; it is a stated skip like `cloud status`. audit workflow: - AUDIT_AUTOFIX=0 only stopped file-issue. The editor still branched and committed, and open-pr could push it. The flag is written to autofix.env and honoured by the editor and the integrity gate. - Both audit.json consumers parsed it unguarded, though it is written under `|| true` and can be truncated — drift with no alert. Both now degrade to a stated "report unreadable" message. - The integrity gate only protected the audit script. It now asserts the whole changed-file set is the manifest alone, so drift cannot be resolved by deleting a command or relaxing the contract test's expectations. manifest: the webhook entry described an outbound webhook; it is inbound. Not applied: CodeRabbit asked for a plain `[Unreleased]` changelog heading. CLAUDE.md requires the pending release level in the heading (`[Unreleased - Patch]`), so the existing form is correct for this repo. Co-Authored-By: Claude Opus 5 --- .agentworkforce/features/manifest.yaml | 2 +- scripts/audit-feature-manifest.mjs | 53 ++++++++++++-- workflows/audit-feature-manifest.ts | 89 ++++++++++++++++++----- workflows/verify-features.ts | 99 ++++++++++++++++++++------ 4 files changed, 200 insertions(+), 43 deletions(-) diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 38bb7e457..6d19cbb91 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -846,7 +846,7 @@ categories: - id: webhook-create name: Create Webhook cli: relay integration webhook create [--name ] - description: Register an outbound webhook + description: Register an inbound webhook that external services POST to, delivering into a channel location: packages/cli/src/cli/commands/integration.ts verify_tier: 5 diff --git a/scripts/audit-feature-manifest.mjs b/scripts/audit-feature-manifest.mjs index 12346f10e..50e62a277 100644 --- a/scripts/audit-feature-manifest.mjs +++ b/scripts/audit-feature-manifest.mjs @@ -54,6 +54,17 @@ const UNDOCUMENTED_ALLOWLIST = new Set(['help']); /** Subcommand trees too costly or destructive to walk during an audit. */ const NO_WALK = new Set(['help', 'uninstall']); +/** + * Commands that dispatch on a positional argument instead of a subcommand, so + * their documented usages (`relay telemetry enable`) are real but never appear + * as leaves in the Commander tree. + * + * Keep this explicit. Treating any surviving parent as proof would mean a + * removed child command could never be detected, because `deriveCliLeaves` + * records every parent group as a leaf too. + */ +const POSITIONAL_DISPATCH = new Set(['telemetry']); + // ── surface derivation: CLI ─────────────────────────────────────────────────── function runHelp(argPath) { @@ -189,6 +200,12 @@ function deriveMcpTools() { finish(null); }); + // A broken pipe here — spawn failure, or our own SIGTERM in finish() — must + // not surface as an uncaught 'error' event. That is not a promise + // rejection, so it would escape main().catch and exit 1, which the audit + // workflow reads as confirmed drift and acts on. + child.stdin.on('error', () => finish(null)); + const send = (msg) => child.stdin.write(`${JSON.stringify(msg)}\n`); send({ jsonrpc: '2.0', @@ -367,14 +384,19 @@ async function main() { const cliLeafSet = new Set(cliLeaves); // Some commands dispatch on a positional argument rather than a subcommand - // (`relay telemetry enable` is `telemetry [action]`). `relay telemetry enable` - // is a real, documented usage, so a documented command is stale only when no - // prefix of it exists in the CLI at all. + // (`relay telemetry enable` is `telemetry [action]`), so those documented + // usages have no leaf of their own. + // + // Accepting *any* surviving prefix would hide almost every real removal: + // deleting `fleet spawn` still leaves `fleet`, so the entry would validate + // against its own parent and never be reported. A documented command must + // therefore match a real leaf exactly, unless its parent is a known + // positional-dispatch command listed above. const staleCommands = [...documentedCommands].filter((cmd) => { + if (cliLeafSet.has(cmd)) return false; const parts = cmd.split(' '); - for (let i = parts.length; i >= 1; i--) { - if (cliLeafSet.has(parts.slice(0, i).join(' '))) return false; - } + const parent = parts.slice(0, -1).join(' '); + if (parts.length > 1 && POSITIONAL_DISPATCH.has(parent) && cliLeafSet.has(parent)) return false; return true; }); @@ -393,6 +415,25 @@ async function main() { const drift = undocumentedCommands.length + staleCommands.length + undocumentedMcp.length + staleMcp.length; + // Half an audit is not a clean audit. If the MCP surface could not be + // derived, MCP drift was never checked, so reporting "clean" would be a + // false negative of exactly the kind this script exists to prevent. Exit 2 + // marks it unrunnable rather than clean or drifted. + if (mcpTools === null) { + const failure = { + ok: false, + auditable: false, + error: + 'MCP tool list could not be derived, so MCP drift was not checked. ' + + 'Refusing to report a partial audit as clean.', + cliCommand, + undocumentedCommands, + staleCommands, + }; + console.error(asJson ? JSON.stringify(failure, null, 2) : `AUDIT_ERROR: ${failure.error}`); + process.exit(2); + } + const report = { ok: drift === 0, auditable: true, diff --git a/workflows/audit-feature-manifest.ts b/workflows/audit-feature-manifest.ts index c36b18dfb..be82db646 100644 --- a/workflows/audit-feature-manifest.ts +++ b/workflows/audit-feature-manifest.ts @@ -190,6 +190,12 @@ EXIT_CODE=$? node scripts/audit-feature-manifest.mjs $CLI_ARG --json > "${ARTIFACTS}/audit.json" 2>"${ARTIFACTS}/audit.err" || true echo "$EXIT_CODE" > "${ARTIFACTS}/audit-exit.txt" + +# Published for the fix chain: update-manifest is an agent step and cannot read +# this workflow's constants, so the operator's autofix decision has to reach it +# through the filesystem. +echo "AUTOFIX=${AUTOFIX ? '1' : '0'}" > "${ARTIFACTS}/autofix.env" + echo "audit exit code: $EXIT_CODE" exit 0 `, @@ -362,7 +368,20 @@ if (exitCode === 2) { lines.push(detail); lines.push('${BT}${BT}${BT}'); } else { - const r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); + // audit.json is produced by a "|| true" invocation, so it can be missing, + // empty, or truncated while audit-exit.txt still says 1. Falling over here + // would leave the message file empty and the surrounding "|| true" would + // swallow it — drift with no alert. + let r = null; + try { + r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); + } catch (err) { + lines.push(':warning: *Feature manifest drifted, but the report is unreadable* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push('audit.json could not be parsed: ' + err.message); + lines.push('Run ${BT}node scripts/audit-feature-manifest.mjs${BT} locally to see the drift.'); + process.stdout.write(lines.join('\n')); + process.exit(0); + } lines.push(':clipboard: *Feature manifest has drifted* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); lines.push( 'manifest ' + r.manifestVersion + ' (updated ' + r.manifestUpdated + ') — ' + @@ -430,7 +449,19 @@ fi node <<'ISSUEEOF' > "$ARTIFACTS/issue-body.md" const fs = require('node:fs'); const artifacts = process.env.AUDIT_ARTIFACTS; -const r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); +// Same guard as the Slack payload: a truncated audit.json must still produce +// a filed issue, not an empty body swallowed by the gh call. +let r = null; +try { + r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); +} catch (err) { + process.stdout.write( + 'The feature manifest drifted, but the machine-readable report could not be parsed: ' + + err.message + + '\n\nRun ${BT}node scripts/audit-feature-manifest.mjs${BT} locally to see the drift.\n' + ); + process.exit(0); +} const out = []; out.push('The feature manifest no longer matches the CLI surface.'); @@ -509,12 +540,16 @@ exit 0 ## FIRST: check whether there is anything to do -Read ${ARTIFACTS}/audit-exit.txt. +Read ${ARTIFACTS}/autofix.env and ${ARTIFACTS}/audit-exit.txt. -- If it does not contain exactly \`1\`, there is NOTHING to do. Write "No drift — - nothing to update." to ${ARTIFACTS}/update-summary.md and STOP IMMEDIATELY: - do not create a branch, edit files, or run commands. -- Only if it contains \`1\` do you continue. +- If ${ARTIFACTS}/autofix.env contains \`AUTOFIX=0\`, the operator has disabled + the fix path. Write "Autofix disabled." to ${ARTIFACTS}/update-summary.md and + STOP IMMEDIATELY. +- If audit-exit.txt does not contain exactly \`1\`, there is NOTHING to do. + Write "No drift — nothing to update." to ${ARTIFACTS}/update-summary.md and + STOP IMMEDIATELY. +- In either case: do not create a branch, edit files, or run commands. +- Only if autofix is enabled AND audit-exit.txt contains \`1\` do you continue. (The engine has no conditional steps, so this step is scheduled on every run. The early exit is what keeps an agent away from a clean tree.) @@ -555,10 +590,13 @@ node scripts/audit-feature-manifest.mjs npx vitest run .agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts \`\`\` -The audit must print MANIFEST_CLEAN and the contract test must pass. If the -contract test's hardcoded expectation list now disagrees with the manifest, add -the new commands to that list too — it is a snapshot that must be maintained -alongside the manifest. +The audit must print MANIFEST_CLEAN and the contract test must pass. + +Do NOT edit any file other than the manifest. The integrity gate rejects a +branch that touches CLI sources, the audit script, or +manifest-contract.test.ts — resolving drift by changing what is measured is +exactly what it exists to stop. If the contract test's hardcoded expectation +list needs new entries, say so in your summary and leave it to a human. ## Deliverable @@ -588,6 +626,12 @@ set -uo pipefail ARTIFACTS="${ARTIFACTS}" MANIFEST=".agentworkforce/features/manifest.yaml" +if grep -q '^AUTOFIX=0$' "$ARTIFACTS/autofix.env" 2>/dev/null; then + echo "INTEGRITY_NOT_APPLICABLE: autofix disabled, no update was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) if [ "$EXIT_CODE" != "1" ]; then echo "INTEGRITY_NOT_APPLICABLE: no drift to fix" @@ -635,11 +679,24 @@ if [ "$AFTER" -lt "$MIN_ALLOWED" ]; then exit 0 fi -# The audit script is the measuring instrument. Editing it to resolve drift is -# moving the goalposts, so require an explicit human decision for that. -if ! git diff --quiet "$BASE_REF" -- scripts/audit-feature-manifest.mjs 2>/dev/null; then - echo "INTEGRITY_FAIL: the branch modified scripts/audit-feature-manifest.mjs." - echo "The audit script is the measuring instrument — changing it to resolve drift needs human review." +# An editor can make the audit clean three ways: document the command (wanted), +# delete the command (not wanted), or move the goalposts by editing the audit +# script or its expectation list (not wanted). Only the manifest is allowed to +# change, so assert the whole changed-file set rather than one script. +ALLOWED_CHANGES=".agentworkforce/features/manifest.yaml" +CHANGED=$(git diff --name-only "$BASE_REF" 2>/dev/null || true) +UNEXPECTED="" +for f in $CHANGED; do + case "$f" in + "$ALLOWED_CHANGES") ;; + *) UNEXPECTED="$UNEXPECTED $f" ;; + esac +done + +if [ -n "$UNEXPECTED" ]; then + echo "INTEGRITY_FAIL: the branch changed files other than the manifest:$UNEXPECTED" + echo "A manifest sync may only edit the manifest. Changing CLI sources, the audit" + echo "script, or manifest-contract.test.ts resolves drift by moving what is measured." echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" exit 0 fi diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index b6b33f797..6a75ec541 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -159,7 +159,11 @@ RUN_ID="${RUN_ID}" # Reason text is flattened and bounded so a multi-line stack trace cannot # corrupt the ledger. record() { - _reason=$(printf '%s' "$4" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-400) + # Strip ALL control bytes, not just newlines: ANSI escapes and stray control + # characters from CLI output would otherwise land raw inside the JSON string + # and make the line unparseable. The verdict reader skips unparseable lines, + # so a corrupted reason could silently erase a failing check. + _reason=$(printf '%s' "$4" | tr -d '\000-\010\013\014\016-\037\177' | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-400) printf '{"run":"%s","tier":"%s","check":"%s","status":"%s","reason":"%s"}\n' \ "$RUN_ID" "$1" "$2" "$3" "$_reason" >> "$CHECKS" case "$3" in @@ -512,6 +516,39 @@ else record "$TIER" "cli-matches-repo" fail "CLI under test is $CLI_VERSION but the repo is $REPO_VERSION; results do not describe this checkout" fi +# Matching versions prove nothing on a feature branch: between releases the +# globally installed CLI and this checkout report the same version, so a global +# binary would pass the check above while exercising none of the branch's +# changes. Resolve the binary and require it to live inside this checkout. +REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || true) +CLI_REAL=$(node -e 'const fs=require("node:fs");try{process.stdout.write(fs.realpathSync(process.argv[1]))}catch(e){process.stdout.write(process.argv[1])}' "$CLI_PATH" 2>/dev/null || printf '%s' "$CLI_PATH") +REPO_REAL=$(node -e 'const fs=require("node:fs");try{process.stdout.write(fs.realpathSync(process.argv[1]))}catch(e){process.stdout.write(process.argv[1])}' "$REPO_ROOT" 2>/dev/null || printf '%s' "$REPO_ROOT") +echo " resolved CLI: $CLI_REAL" | tee -a "$LOG" +echo " repo root: $REPO_REAL" | tee -a "$LOG" + +if [ -z "$REPO_ROOT" ]; then + # Falling back to $PWD would only prove the binary sits under the current + # directory, which is not provenance. Say so instead of claiming a pass. + skip_check "cli-belongs-to-checkout" "not a git checkout — cannot establish which tree the CLI was built from" +else + +case "$CLI_REAL" in + "$REPO_REAL"/*) + echo " PASS cli-belongs-to-checkout" | tee -a "$LOG" + record "$TIER" "cli-belongs-to-checkout" pass "" + ;; + *) + if [ "$VERIFY_ALLOW_CLI_DRIFT" = "1" ]; then + skip_check "cli-belongs-to-checkout" "CLI at $CLI_REAL is outside $REPO_REAL (allowed by env)" + else + echo " FAIL cli-belongs-to-checkout: $CLI_REAL is outside $REPO_REAL" | tee -a "$LOG" + record "$TIER" "cli-belongs-to-checkout" fail "the CLI under test resolves to $CLI_REAL, outside this checkout ($REPO_REAL); a matching version number does not make it this branch's build" + fi + ;; +esac + +fi + finish_tier exit 0 `, @@ -642,8 +679,11 @@ run_check "fleet status" "relay fleet status" "." # 'relay node tail' streams until interrupted; bound it and accept a timeout # kill as success. Without the bound this step hangs until the run times out. +# Only a clean exit (0) or the timeout's own kill (124/137) counts. The old +# "-le 124" also accepted exit 1, so a removed or broken 'node tail' still +# scored PASS — precisely the false green this file exists to prevent. run_check "node tail (bounded)" \ - "timeout 5 relay node tail >/dev/null 2>&1; [ \$? -le 124 ] && echo tail-ok" "tail-ok" + "timeout 5 relay node tail >/dev/null 2>&1; _rc=\$?; case \$_rc in 0|124|137) echo tail-ok ;; *) echo \"tail-failed rc=\$_rc\" ;; esac" "tail-ok" # Redeliver against an empty queue is a no-op, so this exercises the command # path without mutating real delivery state. @@ -875,7 +915,7 @@ gated_check cloud "cloud whoami" "relay cloud whoami" "." gated_check cloud "cloud session" "relay cloud session" "." skip_check "cloud status" "takes a required ; no disposable cloud run is made (see the cloud run skip)" gated_check cloud "cloud schedules" "relay cloud schedules" "." -gated_check cloud "cloud logs" "relay cloud logs 2>&1 || true" "." +skip_check "cloud logs" "takes a required ; no disposable cloud run is made (see the cloud run skip)" gated_check cloud "cloud sync --help" "relay cloud sync --help" "Usage" # Deliberately not exercised: 'cloud run' bills real sandbox time, and @@ -1054,7 +1094,11 @@ relay node metrics >/dev/null 2>&1 && CP1=$((CP1 + 1)) relay node deadletters --json >/dev/null 2>&1 && CP1=$((CP1 + 1)) relay node down --state-dir "$CP1_STATE" >/dev/null 2>&1 || true sleep 2 -relay node status --state-dir "$CP1_STATE" 2>&1 | grep -iv running >/dev/null && CP1=$((CP1 + 1)) +# "grep -v" succeeds when ANY line lacks the pattern, so a multi-line status +# that still says running satisfied it. Negate the match over the whole output. +if ! relay node status --state-dir "$CP1_STATE" 2>&1 | grep -i running >/dev/null; then + CP1=$((CP1 + 1)) +fi if [ "$CP1" -ge 4 ]; then echo " PASS CP1 ($CP1/4)" | tee -a "$LOG" record "$TIER" "CP1 broker lifecycle" pass "" @@ -1276,13 +1320,21 @@ exit 0 captureOutput: true, failOnError: false, command: String.raw` -relay agent list 2>/dev/null | grep -oE '(vf|cp[0-9])[A-Za-z0-9_-]*' | while read -r name; do - relay agent remove "$name" >/dev/null 2>&1 || true -done +# Scope teardown to THIS run's suffix. The previous pattern matched the +# substrings "vf"/"cp" anywhere in the listing, so any pre-existing +# workspace agent or channel whose name happened to contain them — or an +# unrelated column of the JSON — would have been removed or archived. +SUFFIX="${SUFFIX}" + +relay agent list 2>/dev/null | grep -oE '"name": "[^"]*"' | sed 's/"name": "//;s/"$//' | + grep -E "^(vf|cp[0-9])[A-Za-z0-9_-]*-$SUFFIX\$" | while read -r name; do + relay agent remove "$name" >/dev/null 2>&1 || true + done -relay channel list 2>/dev/null | grep -oE '(vf|cp[0-9])[A-Za-z0-9_-]*' | while read -r name; do - relay channel archive "$name" >/dev/null 2>&1 || true -done +relay channel list 2>/dev/null | grep -oE '"name": "[^"]*"' | sed 's/"name": "//;s/"$//' | + grep -E "^(vf|cp[0-9])[A-Za-z0-9_-]*-$SUFFIX\$" | while read -r name; do + relay channel archive "$name" >/dev/null 2>&1 || true + done echo "Cleanup complete" exit 0 @@ -1336,19 +1388,22 @@ const EXPECTED_TIERS = [ let checks = []; let ledgerError = null; +// A line the reader cannot parse is a check whose result was lost. Counting +// them is the difference between "no failures" and "we could not tell", so +// they are tracked and fail the run rather than being quietly discarded. +let malformedLines = 0; try { - checks = fs + const raw = fs .readFileSync(ledgerPath, 'utf8') .split('\n') - .filter((line) => line.trim().startsWith('{')) - .map((line) => { - try { - return JSON.parse(line); - } catch { - return null; - } - }) - .filter(Boolean); + .filter((line) => line.trim().startsWith('{')); + for (const line of raw) { + try { + checks.push(JSON.parse(line)); + } catch { + malformedLines += 1; + } + } } catch (err) { ledgerError = err.message; } @@ -1391,12 +1446,16 @@ const reasons = []; if (ledgerError) reasons.push('check ledger unreadable: ' + ledgerError); if (notRun.length > 0) reasons.push('tiers produced no records: ' + notRun.join(', ')); if (totals.fail > 0) reasons.push(totals.fail + ' check(s) failed'); +if (malformedLines > 0) { + reasons.push(malformedLines + ' unparseable ledger line(s) — check results were lost'); +} const verdict = { runId: process.env.VERIFY_RUN_ID, verdict: reasons.length === 0 ? 'PASS' : 'FAIL', reasons, totals, + malformedLines, tiers: byTier, tiersNotRun: notRun, provenance, From 724ff58d6b7dbdd8b8a935e7b465b4341b0095ff Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Thu, 30 Jul 2026 21:23:27 +0200 Subject: [PATCH 7/7] fix(cli): update remaining callers of the old webhook create signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught two tests in relaycast-groups.test.ts still driving `webhook create --event`, which now fails with "unknown option '--event'". My mistake: I changed a public CLI signature and verified only the tests I had touched instead of the full suite. - relaycast-groups.test.ts: both cases moved to ` [--name]`. - verify/procedures.md documented the same broken invocation, and built the whole webhook procedure on an outbound model the API does not have — it registered CAPTURE_URL as a delivery destination. `POST /v1/webhooks` is inbound only: it takes the channel and returns the URL external services post to. Corrected, and the surrounding note now states the inbound flow. Full suite green: 1644 passed, 16 skipped. (integration-relayfile-contract flaked once under full-suite ordering and passed standalone and on re-run; it references nothing in this change.) Co-Authored-By: Claude Opus 5 --- .agentworkforce/features/verify/procedures.md | 7 +++++-- .../src/cli/commands/relaycast-groups.test.ts | 17 ++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 5d4cfbf70..f9137d8a4 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -382,7 +382,10 @@ Assert the downloaded skill and delete only this disposable project. Do not run ```bash : "${CAPTURE_URL:?See Externally provisioned fixtures}" : "${CAPTURE_FETCH_URL:?GET endpoint returning the current run captured requests}" -HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" +# `webhook create` is INBOUND: it takes the channel to deliver into and returns +# the URL external services POST to. It does not take a destination URL. +HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CHANNEL")"; HOOK_ID="$(jq -er '.webhookId // .id' <<<"$HOOK")" +HOOK_URL="$(jq -er '.url' <<<"$HOOK")"; HOOK_TOKEN="$(jq -er '.token' <<<"$HOOK")" RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' DELIVERED=false for _ in $(seq 1 15); do @@ -407,7 +410,7 @@ RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook list | jq -e --arg id "$H RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook delete "$HOOK_ID" ``` -The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. +`webhook create` and `create-inbound` are aliases over the same `POST /v1/webhooks`, whose request is `{ channel, name? }` — there is no outbound registration surface, so a capture receiver is only reachable through `webhook trigger`, not by registering `CAPTURE_URL` as a destination. The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For the inbound path proper, create the channel and hook, POST `$HOOK_URL` with `$HOOK_TOKEN` and the documented payload, assert the message lands in the channel, then delete the hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. ## reflex-history diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 4884da968..d0a0e1acf 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -165,12 +165,12 @@ describe('SDK-backed CLI groups', () => { it('integration webhook create routes to integrations.webhooks.create', async () => { const { program, relay } = harness(registerIntegrationCommands); await program.parseAsync( - ['integration', 'webhook', 'create', 'https://x', '--event', 'message.created'], + ['integration', 'webhook', 'create', 'deploy-status', '--name', 'GitHub Alerts'], { from: 'user' } ); expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ - url: 'https://x', - event: 'message.created', + channel: 'deploy-status', + name: 'GitHub Alerts', }); }); @@ -196,10 +196,9 @@ describe('SDK-backed CLI groups', () => { resolveLocalRelayOptions, } satisfies Partial); - await program.parseAsync( - ['integration', 'webhook', 'create', 'https://x', '--event', 'message.created'], - { from: 'user' } - ); + await program.parseAsync(['integration', 'webhook', 'create', 'deploy-status'], { + from: 'user', + }); expect(resolveLocalRelayOptions).toHaveBeenCalled(); expect(createAgentRelay).toHaveBeenNthCalledWith( @@ -207,8 +206,8 @@ describe('SDK-backed CLI groups', () => { expect.objectContaining({ workspaceKey: 'rk_live_local', baseUrl: 'https://relay.local' }) ); expect(secondRelay.integrations.webhooks.create).toHaveBeenCalledWith({ - url: 'https://x', - event: 'message.created', + channel: 'deploy-status', + name: undefined, }); expect(log).toHaveBeenCalled(); expect(error).not.toHaveBeenCalled();