diff --git a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts index 1375741b9..7b313b6ca 100644 --- a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts +++ b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts @@ -137,6 +137,8 @@ describe('feature manifest contract', () => { 'relay skills add', 'relay mcp', 'relay fleet nodes', + 'relay fleet spawn', + 'relay fleet release', 'relay fleet config', 'relay fleet enable', 'relay fleet disable', diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 3781b9a56..6d19cbb91 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1,5 +1,5 @@ version: '1.1' -updated: '2026-07-20' +updated: '2026-07-30' # Every user-facing feature in this repo, categorized and scored. # @@ -33,6 +33,8 @@ verification: local-workflows: local-workflow-lifecycle cloud: cloud-workflows cloud-workers: cloud-workers + cloud-integrations: integrations-and-webhooks + cloud-rooms: cloud-workflows fleet: fleet-management workspace: workspace-management skills: skills-installation @@ -124,6 +126,20 @@ categories: location: packages/cli/src/cli/commands/agent.ts verify_tier: 3 + - id: agent-me + name: Show Current Agent Identity + cli: relay agent me + description: Show the identity the current agent token resolves to + location: packages/cli/src/cli/commands/agent.ts + verify_tier: 3 + + - id: agent-presence + name: List Agent Presence + cli: relay agent presence + description: List visible agent presence in the workspace + location: packages/cli/src/cli/commands/agent.ts + verify_tier: 3 + - id: agent-remove name: Remove Agent cli: relay agent remove @@ -554,6 +570,100 @@ categories: location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 + cloud-integrations: + name: Cloud Integrations + description: Connect third-party providers to the workspace through Cloud + criticality: standard + features: + - id: cloud-integration-catalog + name: Integration Catalog + cli: relay cloud integration catalog [--static] [--json] + description: Discover static and dynamic (Nango, Composio) Cloud integrations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-integration-connect + name: Connect Integration + cli: relay cloud integration connect [--json] + description: Create a Cloud connection session for a provider + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 6 + + - id: cloud-integration-connections + name: List Connections + cli: relay cloud integration connections [--json] + description: List connected workspace integrations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-integration-disconnect + name: Disconnect Integration + cli: relay cloud integration disconnect [--json] + description: Disconnect a provider from the workspace + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + cloud-rooms: + name: Cloud Rooms + description: Workspace-scoped multiplayer rooms and their membership + criticality: standard + features: + - id: cloud-room-session + name: Create Room Session + cli: relay cloud room session + description: Create or resume this device's full room-participant session + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-revoke-session + name: Revoke Room Session + cli: relay cloud room revoke-session + description: Revoke this member's scoped session for one device + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-members + name: List Room Members + cli: relay cloud room members + description: List workspace room members + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-remove-member + name: Remove Room Member + cli: relay cloud room remove-member + description: Remove a member and revoke their live room access + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-invite + name: Invite Room Participant + cli: relay cloud room invite + description: Invite a full participant to a workspace room + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-invites + name: List Room Invitations + cli: relay cloud room invites + description: List workspace room invitations + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + + - id: cloud-room-accept + name: Accept Room Invitation + cli: relay cloud room accept + description: Accept an email-bound room invitation + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 6 + + - id: cloud-room-revoke-invite + name: Revoke Room Invitation + cli: relay cloud room revoke-invite + description: Revoke an unused workspace room invitation + location: packages/cli/src/cli/commands/cloud.ts + verify_tier: 5 + cloud-workers: name: Cloud Worker Nodes description: Register and manage machines as cloud worker nodes @@ -599,6 +709,20 @@ categories: location: packages/cli/src/cli/commands/fleet.ts verify_tier: 5 + - id: fleet-spawn + name: Spawn Fleet Agent + cli: relay fleet spawn [--name ] [--task ] [--node ] [--persona ] [--model ] + description: Spawn a worker agent onto a live fleet node, by explicit node or by persona placement + location: packages/cli/src/cli/commands/fleet.ts + verify_tier: 5 + + - id: fleet-release + name: Release Fleet Agent + cli: relay fleet release [--reason ] [--delete-agent] + description: Release a spawned fleet agent, optionally deleting the agent identity + location: packages/cli/src/cli/commands/fleet.ts + verify_tier: 5 + - id: fleet-config name: Fleet Config cli: relay fleet config @@ -721,8 +845,8 @@ categories: - id: webhook-create name: Create Webhook - cli: relay integration webhook create [--event ] - description: Register an outbound webhook + cli: relay integration webhook create [--name ] + description: Register an inbound webhook that external services POST to, delivering into a channel location: packages/cli/src/cli/commands/integration.ts verify_tier: 5 diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 5d4cfbf70..f9137d8a4 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -382,7 +382,10 @@ Assert the downloaded skill and delete only this disposable project. Do not run ```bash : "${CAPTURE_URL:?See Externally provisioned fixtures}" : "${CAPTURE_FETCH_URL:?GET endpoint returning the current run captured requests}" -HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" +# `webhook create` is INBOUND: it takes the channel to deliver into and returns +# the URL external services POST to. It does not take a destination URL. +HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CHANNEL")"; HOOK_ID="$(jq -er '.webhookId // .id' <<<"$HOOK")" +HOOK_URL="$(jq -er '.url' <<<"$HOOK")"; HOOK_TOKEN="$(jq -er '.token' <<<"$HOOK")" RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' DELIVERED=false for _ in $(seq 1 15); do @@ -407,7 +410,7 @@ RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook list | jq -e --arg id "$H RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook delete "$HOOK_ID" ``` -The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. +`webhook create` and `create-inbound` are aliases over the same `POST /v1/webhooks`, whose request is `{ channel, name? }` — there is no outbound registration surface, so a capture receiver is only reachable through `webhook trigger`, not by registering `CAPTURE_URL` as a destination. The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For the inbound path proper, create the channel and hook, POST `$HOOK_URL` with `$HOOK_TOKEN` and the documented payload, assert the message lands in the channel, then delete the hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. ## reflex-history diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index f93c12d71..584653ad0 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -17,6 +17,37 @@ Use when you need to verify that a specific feature or set of features works cor .agentworkforce/features/verify/procedures.md # step-by-step verification by tier ``` +## Automation + +```text +workflows/verify-features.ts # runs tiers 1-6 + critical paths 1-6 +workflows/audit-feature-manifest.ts # checks the manifest still matches the CLI +scripts/audit-feature-manifest.mjs # the audit itself; run it directly +``` + +Before trusting any verification result, check two things: + +1. **Which CLI ran.** `verify-features.ts` has a `provenance` step that fails + when `relay version` disagrees with the repo's `package.json`. A run against + a stale globally-installed CLI describes that CLI, not your checkout — this + has already produced a full green run plus one bogus "unknown command" + failure against a real command. +2. **What was skipped.** Every check records `pass`, `fail`, or `skip` with a + reason into `.workflow-artifacts/verify-features/checks.jsonl`, and + `verdict.json` is the authoritative result. A SKIP means _not verified_. + Never read a skip as a pass. + +Run the manifest audit before adding checks, so you are not writing coverage +against a stale map: + +```bash +node scripts/audit-feature-manifest.mjs # human-readable +node scripts/audit-feature-manifest.mjs --json # for tooling +``` + +Exit 0 = clean, 1 = drift, 2 = the audit itself could not run. Exit 2 is +deliberately distinct: a broken audit must never be read as a clean manifest. + ## How to use it ### 1. Read the manifest to find the feature @@ -101,6 +132,13 @@ Add or update entries in `manifest.yaml` when: - A new harness is supported - A feature is removed or deprecated +Then run `node scripts/audit-feature-manifest.mjs` and confirm `MANIFEST_CLEAN`. +Note that `manifest-contract.test.ts` cannot catch a missing entry for a _new_ +command — a new command is absent from both the manifest and that test's +hardcoded expectation list, so it passes. The audit script derives the surface +from `--help` and `tools/list` instead, which is why it is the check that +matters here. Add new commands to both. + Update `critical-paths.md` when: - A new path becomes foundational to the product diff --git a/.gitignore b/.gitignore index daeac1975..60dbfe742 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ __pycache__/ !/workflows/relayauth-integration/ !/workflows/cloud-connect/ !/workflows/verify-features.ts +!/workflows/audit-feature-manifest.ts # Eval harness JSON reports (generated per run) tests/integration/broker/evals-reports/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 016197b4c..a9a4cc541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- `agent-relay integration webhook create` now works. It took a `` argument and sent `{ url, event }`, but `POST /v1/webhooks` accepts `{ channel, name? }` and returns the URL — so every invocation failed with `channel is required`. It now takes `` with an optional `--name`, matching `create-inbound`, which posts to the same endpoint. +- `@agent-relay/sdk` `RelayCreateWebhookInput` declared a required `url` and an `event`, neither of which the endpoint accepts. It is now `{ channel, name? }`. Code passing `url`/`event` was already failing at runtime. ## [11.3.0] - 2026-07-30 diff --git a/packages/cli/src/cli/commands/integration-webhook-create.test.ts b/packages/cli/src/cli/commands/integration-webhook-create.test.ts new file mode 100644 index 000000000..4124a1eb5 --- /dev/null +++ b/packages/cli/src/cli/commands/integration-webhook-create.test.ts @@ -0,0 +1,77 @@ +import { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; + +import { registerIntegrationCommands, type IntegrationCommandDependencies } from './integration.js'; + +/** + * `relay integration webhook create` posts to `POST /v1/webhooks`, whose request + * schema is `{ channel, name? }` — the URL is part of the *response*. + * + * The command previously took a `` argument and sent `{ url, event }`, so + * every invocation was rejected with "channel is required" and the required + * channel was never sent at all. A live verification run caught it. These tests + * pin the corrected contract so the argument order cannot silently regress. + */ +function makeProgram() { + const relay = { + integrations: { + webhooks: { + create: vi.fn(async (input: unknown) => ({ webhookId: 'wh_1', ...(input as object) })), + }, + }, + }; + const log = vi.fn(); + const error = vi.fn(); + const exit = vi.fn(); + const program = new Command(); + program.exitOverride(); + registerIntegrationCommands(program, { + createAgentRelay: () => relay as never, + resolveLocalRelayOptions: async () => ({ workspaceKey: 'rk_live_test' }), + isInteractive: () => false, + log, + error, + exit: exit as never, + } satisfies Partial as IntegrationCommandDependencies); + return { program, relay, log, error, exit }; +} + +describe('integration webhook create', () => { + it('sends the channel the API requires', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync(['integration', 'webhook', 'create', 'deploy-status'], { + from: 'user', + }); + + expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ + channel: 'deploy-status', + name: undefined, + }); + }); + + it('passes an optional name through', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync( + ['integration', 'webhook', 'create', 'deploy-status', '--name', 'GitHub Alerts'], + { from: 'user' } + ); + + expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ + channel: 'deploy-status', + name: 'GitHub Alerts', + }); + }); + + it('never sends url or event, which the endpoint rejects', async () => { + const { program, relay } = makeProgram(); + + await program.parseAsync(['integration', 'webhook', 'create', 'ops'], { from: 'user' }); + + const payload = relay.integrations.webhooks.create.mock.calls[0]?.[0] as Record; + expect(payload).toBeDefined(); + expect(payload).not.toHaveProperty('url'); + expect(payload).not.toHaveProperty('event'); + }); +}); diff --git a/packages/cli/src/cli/commands/integration.ts b/packages/cli/src/cli/commands/integration.ts index a714d35b2..d820d8b09 100644 --- a/packages/cli/src/cli/commands/integration.ts +++ b/packages/cli/src/cli/commands/integration.ts @@ -1844,17 +1844,17 @@ export function registerIntegrationCommands( addSdkOptions( webhook .command('create') - .description('Register a webhook') - .argument('', 'Webhook URL') - .option('--event ', 'Event to deliver') - ).action(async (url: string, o: Record) => { + .description('Register an inbound webhook that delivers into a channel (alias of create-inbound)') + .argument('', 'Target channel the webhook posts into') + .option('--name ', 'Human-readable webhook name (e.g. "GitHub Alerts")') + ).action(async (channel: string, o: Record) => { await runSdk(deps, async () => { printJson( deps, await runIntegrationOperation(deps, o, (relay) => relay.integrations.webhooks.create({ - url, - event: o.event as string | undefined, + channel, + name: o.name as string | undefined, }) ) ); diff --git a/packages/cli/src/cli/commands/relaycast-groups.test.ts b/packages/cli/src/cli/commands/relaycast-groups.test.ts index 4884da968..d0a0e1acf 100644 --- a/packages/cli/src/cli/commands/relaycast-groups.test.ts +++ b/packages/cli/src/cli/commands/relaycast-groups.test.ts @@ -165,12 +165,12 @@ describe('SDK-backed CLI groups', () => { it('integration webhook create routes to integrations.webhooks.create', async () => { const { program, relay } = harness(registerIntegrationCommands); await program.parseAsync( - ['integration', 'webhook', 'create', 'https://x', '--event', 'message.created'], + ['integration', 'webhook', 'create', 'deploy-status', '--name', 'GitHub Alerts'], { from: 'user' } ); expect(relay.integrations.webhooks.create).toHaveBeenCalledWith({ - url: 'https://x', - event: 'message.created', + channel: 'deploy-status', + name: 'GitHub Alerts', }); }); @@ -196,10 +196,9 @@ describe('SDK-backed CLI groups', () => { resolveLocalRelayOptions, } satisfies Partial); - await program.parseAsync( - ['integration', 'webhook', 'create', 'https://x', '--event', 'message.created'], - { from: 'user' } - ); + await program.parseAsync(['integration', 'webhook', 'create', 'deploy-status'], { + from: 'user', + }); expect(resolveLocalRelayOptions).toHaveBeenCalled(); expect(createAgentRelay).toHaveBeenNthCalledWith( @@ -207,8 +206,8 @@ describe('SDK-backed CLI groups', () => { expect.objectContaining({ workspaceKey: 'rk_live_local', baseUrl: 'https://relay.local' }) ); expect(secondRelay.integrations.webhooks.create).toHaveBeenCalledWith({ - url: 'https://x', - event: 'message.created', + channel: 'deploy-status', + name: undefined, }); expect(log).toHaveBeenCalled(); expect(error).not.toHaveBeenCalled(); diff --git a/packages/sdk/src/__tests__/integrations.test.ts b/packages/sdk/src/__tests__/integrations.test.ts index 16671fc57..2979c0ddf 100644 --- a/packages/sdk/src/__tests__/integrations.test.ts +++ b/packages/sdk/src/__tests__/integrations.test.ts @@ -41,9 +41,11 @@ describe('SDK integrations / capabilities / workspace passthrough', () => { const relaycast = createRelaycastMock(); const client = new RelaycastMessagingClient({ relaycast: relaycast as never }); - const webhook = await client.integrations.webhooks.create({ url: 'https://x', event: 'message.created' }); + // POST /v1/webhooks takes { channel, name? } and returns the generated URL. + // This previously asserted { url, event }, a shape the endpoint rejects. + const webhook = await client.integrations.webhooks.create({ channel: 'deploy-status' }); expect(webhook.id).toBe('wh1'); - expect(relaycast.webhooks.create).toHaveBeenCalledWith({ url: 'https://x', event: 'message.created' }); + expect(relaycast.webhooks.create).toHaveBeenCalledWith({ channel: 'deploy-status' }); await client.integrations.webhooks.trigger('wh1', { hello: 'world' }); expect(relaycast.webhooks.trigger).toHaveBeenCalledWith('wh1', { hello: 'world' }); diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index 9df42cf89..b1bf6cadf 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -437,9 +437,19 @@ export interface RelayWebhook { [key: string]: unknown; } +/** + * Input for `integrations.webhooks.create`. + * + * The endpoint (`POST /v1/webhooks`) takes the channel to deliver into and + * returns the generated URL. It previously declared a required `url` and an + * `event`, neither of which the API accepts — `url` is part of the *response*, + * and `event` belongs to subscriptions — so every call was rejected with + * "channel is required". + */ export interface RelayCreateWebhookInput { - url: string; - event?: string; + /** Channel the webhook posts into. A leading `#` is stripped. */ + channel: string; + name?: string; [key: string]: unknown; } diff --git a/scripts/audit-feature-manifest.mjs b/scripts/audit-feature-manifest.mjs new file mode 100644 index 000000000..50e62a277 --- /dev/null +++ b/scripts/audit-feature-manifest.mjs @@ -0,0 +1,495 @@ +#!/usr/bin/env node +/** + * Audit .agentworkforce/features/manifest.yaml against the CLI's real surface. + * + * The manifest is the input to every feature-verification run, so drift there + * silently shrinks coverage: a command nobody documented is a command nobody + * verifies. `manifest-contract.test.ts` only checks a hand-maintained list, so + * it can never notice a *newly added* command or MCP tool. This script derives + * the surface instead of asserting a snapshot of it. + * + * Two surfaces are derived: + * 1. The Commander leaf-command tree, by walking `--help` recursively. + * 2. The MCP tool list, by a `tools/list` JSON-RPC call over stdio. + * + * Usage: + * node scripts/audit-feature-manifest.mjs + * node scripts/audit-feature-manifest.mjs --json + * node scripts/audit-feature-manifest.mjs --cli "node packages/cli/dist/cli/index.js" + * + * Exit codes: + * 0 manifest matches the derived surface + * 1 drift found (undocumented or stale entries) + * 2 the audit itself could not run (CLI missing, help unparseable) + * + * Exit 2 is deliberately distinct: "the audit broke" must never be reported as + * "the manifest is clean". + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync, spawn } from 'node:child_process'; +import { parse } from 'yaml'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(__dirname, '..'); +const manifestPath = join(repoRoot, '.agentworkforce/features/manifest.yaml'); + +const asJson = process.argv.includes('--json'); +const cliArgIndex = process.argv.indexOf('--cli'); +const cliCommand = + cliArgIndex !== -1 && process.argv[cliArgIndex + 1] + ? process.argv[cliArgIndex + 1] + : process.env.RELAY_CLI || `node ${join(repoRoot, 'packages/cli/dist/cli/index.js')}`; + +/** + * Commands that exist but are intentionally absent from the manifest. + * + * Keep this list short and justified — every entry is coverage we are choosing + * not to have. `help` is Commander's own builtin, not a relay feature. + */ +const UNDOCUMENTED_ALLOWLIST = new Set(['help']); + +/** Subcommand trees too costly or destructive to walk during an audit. */ +const NO_WALK = new Set(['help', 'uninstall']); + +/** + * Commands that dispatch on a positional argument instead of a subcommand, so + * their documented usages (`relay telemetry enable`) are real but never appear + * as leaves in the Commander tree. + * + * Keep this explicit. Treating any surviving parent as proof would mean a + * removed child command could never be detected, because `deriveCliLeaves` + * records every parent group as a leaf too. + */ +const POSITIONAL_DISPATCH = new Set(['telemetry']); + +// ── surface derivation: CLI ─────────────────────────────────────────────────── + +function runHelp(argPath) { + const [bin, ...prefix] = cliCommand.split(/\s+/); + try { + return execFileSync(bin, [...prefix, ...argPath, '--help'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + env: { ...process.env, NO_COLOR: '1', AGENT_RELAY_DISABLE_UPDATE_CHECK: '1' }, + }); + } catch (err) { + // Commander exits non-zero for some help paths but still writes usage. + const out = `${err.stdout ?? ''}${err.stderr ?? ''}`; + if (out.includes('Usage:')) return out; + throw new Error(`\`${cliCommand} ${argPath.join(' ')} --help\` failed: ${err.message}`); + } +} + +/** + * Parse the "Commands:" block of a Commander help screen. + * + * Commander wraps long descriptions onto continuation lines that are indented + * further than the command names, so a naive per-line split invents commands + * out of description text. Anchor on the indentation of the first entry. + */ +function parseSubcommands(helpText) { + const lines = helpText.split('\n'); + const start = lines.findIndex((line) => /^Commands:/.test(line.trim())); + if (start === -1) return []; + + const names = []; + let entryIndent = null; + + for (const line of lines.slice(start + 1)) { + if (!line.trim()) continue; + const indent = line.length - line.trimStart().length; + if (indent === 0) break; // left the Commands block + if (entryIndent === null) entryIndent = indent; + if (indent > entryIndent) continue; // wrapped description line + + const name = line.trim().split(/\s+/)[0]; + if (!name || name.startsWith('-')) continue; + // Commander renders aliases as "status|st"; the first token is canonical. + names.push(name.split('|')[0]); + } + + return names; +} + +/** Walk the command tree and return every leaf command path. */ +function deriveCliLeaves() { + const leaves = []; + const visit = (path, depth) => { + if (depth > 4) return; // relay's tree is 3 deep; guard against a help loop + const children = parseSubcommands(runHelp(path)); + const walkable = children.filter((child) => !NO_WALK.has(child)); + + // A no-walk child is still part of the surface — record it without + // recursing, or the audit reports it as stale when the manifest has it. + for (const child of children) { + if (!NO_WALK.has(child)) continue; + if (!UNDOCUMENTED_ALLOWLIST.has(child)) leaves.push([...path, child].join(' ')); + } + + if (walkable.length === 0) { + if (path.length > 0) leaves.push(path.join(' ')); + return; + } + + for (const child of walkable) visit([...path, child], depth + 1); + + // A group can also be a runnable leaf (e.g. `relay mcp` with subcommands). + if (path.length > 0) leaves.push(path.join(' ')); + }; + + visit([], 0); + return [...new Set(leaves)].sort(); +} + +// ── surface derivation: MCP ─────────────────────────────────────────────────── + +/** + * List MCP tools via a stdio JSON-RPC handshake against `relay mcp`. + * + * Resolves to null (rather than throwing) when the server cannot be reached, so + * a broken MCP path degrades to an explicit "could not derive" in the report + * instead of an empty tool list that would read as "every tool is stale". + */ +function deriveMcpTools() { + return new Promise((resolve) => { + const [bin, ...prefix] = cliCommand.split(/\s+/); + const child = spawn(bin, [...prefix, 'mcp'], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, NO_COLOR: '1', AGENT_RELAY_DISABLE_UPDATE_CHECK: '1' }, + }); + + let buffer = ''; + let settled = false; + const finish = (value) => { + if (settled) return; + settled = true; + child.kill('SIGTERM'); + resolve(value); + }; + + const timer = setTimeout(() => finish(null), 20_000); + timer.unref?.(); + + child.stdout.on('data', (chunk) => { + buffer += chunk.toString(); + for (const line of buffer.split('\n')) { + const trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + const msg = JSON.parse(trimmed); + if (msg.id === 2 && msg.result?.tools) { + clearTimeout(timer); + finish(msg.result.tools.map((tool) => tool.name).sort()); + } + } catch { + // Partial frame — wait for the rest. + } + } + }); + + child.on('error', () => { + clearTimeout(timer); + finish(null); + }); + child.on('exit', () => { + clearTimeout(timer); + finish(null); + }); + + // A broken pipe here — spawn failure, or our own SIGTERM in finish() — must + // not surface as an uncaught 'error' event. That is not a promise + // rejection, so it would escape main().catch and exit 1, which the audit + // workflow reads as confirmed drift and acts on. + child.stdin.on('error', () => finish(null)); + + const send = (msg) => child.stdin.write(`${JSON.stringify(msg)}\n`); + send({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'feature-manifest-audit', version: '1.0.0' }, + }, + }); + send({ jsonrpc: '2.0', method: 'notifications/initialized' }); + send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }); + }); +} + +/** + * MCP tool names the server source still registers, whether or not a default + * stdio probe exposes them. Used to tell "removed" apart from "conditional". + */ +function mcpToolsRegisteredInSource() { + const names = new Set(); + + // Registrations are spread across the whole CLI tree, not just `mcp/`: + // `submit_result` lives in cli/agent-relay-mcp.ts. Scan recursively so a tool + // is never called stale merely because it is declared in a sibling file. + const walk = (dir) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + // No source tree (running against a published CLI) — callers then see + // every probe-missing tool as stale, the safe direction for an audit. + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full); + continue; + } + if (!entry.name.endsWith('.ts') || entry.name.includes('.test.')) continue; + // `registerTool(` and its name argument sit on separate lines here, so + // this must be a whole-file match rather than a line-based grep. + const source = readFileSync(full, 'utf8'); + for (const match of source.matchAll(/registerTool\(\s*'([a-z0-9_]+)'/g)) { + names.add(match[1]); + } + } + }; + + walk(join(repoRoot, 'packages/cli/src/cli')); + return names; +} + +/** + * Warn when the built CLI is older than the sources it is derived from. + * + * The audit can only see what the built CLI exposes, so a stale `dist` makes it + * under-report and print MANIFEST_CLEAN while real commands go undocumented. + * That is exactly what happened here: a stale dist hid 16 shipped commands + * (`agent me`, `agent presence`, the whole `cloud room` and `cloud integration` + * trees) across several audit runs that all reported clean. + * + * @returns A warning string, or null when the build is current or unknowable. + */ +function staleBuildWarning() { + const distEntry = join(repoRoot, 'packages/cli/dist/cli/index.js'); + const srcDir = join(repoRoot, 'packages/cli/src'); + + let distMtime; + try { + distMtime = statSync(distEntry).mtimeMs; + } catch { + return null; // auditing a published CLI, not this checkout + } + + let newest = 0; + const walk = (dir) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full); + continue; + } + if (!entry.name.endsWith('.ts') || entry.name.includes('.test.')) continue; + try { + newest = Math.max(newest, statSync(full).mtimeMs); + } catch { + // unreadable file; ignore + } + } + }; + walk(srcDir); + + if (newest === 0 || newest <= distMtime) return null; + const minutes = Math.round((newest - distMtime) / 60_000); + return ( + `packages/cli/dist is ${minutes} minute(s) older than packages/cli/src. ` + + `This audit only sees what the BUILT CLI exposes, so newer commands are invisible ` + + `and a "clean" result understates drift. Rebuild first: npm run build --workspace=agent-relay` + ); +} + +// ── manifest ───────────────────────────────────────────────────────────────── + +function loadManifest() { + const manifest = parse(readFileSync(manifestPath, 'utf8')); + const features = []; + for (const [categoryKey, category] of Object.entries(manifest.categories ?? {})) { + for (const feature of category.features ?? []) { + features.push({ ...feature, category: categoryKey }); + } + } + return { manifest, features }; +} + +/** + * Reduce a documented `cli:` string to its command path. + * + * Manifest entries carry usage decoration — `relay agent add [options]` + * — so strip anything that is not a bare command word. + */ +function documentedCommandPath(cliString) { + const tokens = cliString.trim().split(/\s+/); + if (tokens[0] === 'relay' || tokens[0] === 'agent-relay') tokens.shift(); + const path = []; + for (const token of tokens) { + // Underscores are load-bearing: `set_topic`, `mark_read`, `send_group`. + if (/^[a-z][a-z0-9_-]*$/.test(token)) path.push(token); + else break; // hit , [opt], or --flag + } + return path.join(' '); +} + +// ── report ─────────────────────────────────────────────────────────────────── + +async function main() { + const { manifest, features } = loadManifest(); + + let cliLeaves; + try { + cliLeaves = deriveCliLeaves(); + } catch (err) { + const failure = { ok: false, auditable: false, error: err.message }; + console.error(asJson ? JSON.stringify(failure, null, 2) : `AUDIT_ERROR: ${err.message}`); + process.exit(2); + } + + const mcpTools = await deriveMcpTools(); + const staleBuild = staleBuildWarning(); + + const documentedCommands = new Set( + features.flatMap((f) => (f.cli ? [documentedCommandPath(f.cli)] : [])).filter(Boolean) + ); + const documentedMcp = new Set(features.flatMap((f) => (f.mcp ? [f.mcp] : []))); + + // A parent command counts as covered when a child of it is documented: + // `relay message` needs no entry of its own if `relay message post` has one. + const coveredPrefixes = new Set(); + for (const cmd of documentedCommands) { + const parts = cmd.split(' '); + for (let i = 1; i <= parts.length; i++) coveredPrefixes.add(parts.slice(0, i).join(' ')); + } + + const undocumentedCommands = cliLeaves.filter( + (leaf) => !coveredPrefixes.has(leaf) && !UNDOCUMENTED_ALLOWLIST.has(leaf) + ); + + const cliLeafSet = new Set(cliLeaves); + + // Some commands dispatch on a positional argument rather than a subcommand + // (`relay telemetry enable` is `telemetry [action]`), so those documented + // usages have no leaf of their own. + // + // Accepting *any* surviving prefix would hide almost every real removal: + // deleting `fleet spawn` still leaves `fleet`, so the entry would validate + // against its own parent and never be reported. A documented command must + // therefore match a real leaf exactly, unless its parent is a known + // positional-dispatch command listed above. + const staleCommands = [...documentedCommands].filter((cmd) => { + if (cliLeafSet.has(cmd)) return false; + const parts = cmd.split(' '); + const parent = parts.slice(0, -1).join(' '); + if (parts.length > 1 && POSITIONAL_DISPATCH.has(parent) && cliLeafSet.has(parent)) return false; + return true; + }); + + const undocumentedMcp = mcpTools ? mcpTools.filter((tool) => !documentedMcp.has(tool)) : []; + + // A documented tool missing from a default stdio probe is not automatically + // stale: `list_actions`/`invoke_action`/`submit_result` only register when an + // actions surface is wired in (see mcp/action-tools.ts). Split the two cases + // on whether the server source still registers the tool at all — absent from + // both the probe and the source is genuine drift; absent from only the probe + // is coverage we did not exercise, and is reported as such. + const registeredInSource = mcpToolsRegisteredInSource(); + const documentedMcpMissing = mcpTools ? [...documentedMcp].filter((tool) => !mcpTools.includes(tool)) : []; + const staleMcp = documentedMcpMissing.filter((tool) => !registeredInSource.has(tool)); + const conditionalMcp = documentedMcpMissing.filter((tool) => registeredInSource.has(tool)); + + const drift = undocumentedCommands.length + staleCommands.length + undocumentedMcp.length + staleMcp.length; + + // Half an audit is not a clean audit. If the MCP surface could not be + // derived, MCP drift was never checked, so reporting "clean" would be a + // false negative of exactly the kind this script exists to prevent. Exit 2 + // marks it unrunnable rather than clean or drifted. + if (mcpTools === null) { + const failure = { + ok: false, + auditable: false, + error: + 'MCP tool list could not be derived, so MCP drift was not checked. ' + + 'Refusing to report a partial audit as clean.', + cliCommand, + undocumentedCommands, + staleCommands, + }; + console.error(asJson ? JSON.stringify(failure, null, 2) : `AUDIT_ERROR: ${failure.error}`); + process.exit(2); + } + + const report = { + ok: drift === 0, + auditable: true, + manifestVersion: manifest.version, + manifestUpdated: manifest.updated, + cliCommand, + mcpDerivable: mcpTools !== null, + staleBuildWarning: staleBuild, + counts: { + cliLeaves: cliLeaves.length, + documentedCommands: documentedCommands.size, + mcpTools: mcpTools?.length ?? null, + documentedMcp: documentedMcp.size, + features: features.length, + }, + undocumentedCommands, + staleCommands, + undocumentedMcp, + staleMcp, + conditionalMcp, + }; + + if (asJson) { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log(`=== Feature Manifest Audit ===`); + console.log(`manifest ${manifest.version} (updated ${manifest.updated})`); + console.log(`CLI: ${cliCommand}`); + console.log( + `derived ${cliLeaves.length} CLI leaves, ${mcpTools?.length ?? 'n/a'} MCP tools; ` + + `manifest documents ${documentedCommands.size} commands, ${documentedMcp.size} MCP tools ` + + `across ${features.length} features` + ); + if (staleBuild) { + console.log(`\nWARN ${staleBuild}`); + } + if (mcpTools === null) { + console.log('\nWARN MCP tool list could not be derived — MCP drift not checked.'); + } + const section = (title, items) => { + if (items.length === 0) return; + console.log(`\n${title} (${items.length}):`); + for (const item of items) console.log(` - ${item}`); + }; + section('UNDOCUMENTED commands (exist in CLI, missing from manifest)', undocumentedCommands); + section('STALE commands (in manifest, not in CLI)', staleCommands); + section('UNDOCUMENTED MCP tools', undocumentedMcp); + section('STALE MCP tools (in manifest, not registered anywhere)', staleMcp); + section('CONDITIONAL MCP tools (registered in source, not in this probe)', conditionalMcp); + console.log(drift === 0 ? '\nMANIFEST_CLEAN' : `\nMANIFEST_DRIFT: ${drift} item(s)`); + } + + process.exit(drift === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error(`AUDIT_ERROR: ${err.message}`); + process.exit(2); +}); diff --git a/workflows/audit-feature-manifest.ts b/workflows/audit-feature-manifest.ts new file mode 100644 index 000000000..be82db646 --- /dev/null +++ b/workflows/audit-feature-manifest.ts @@ -0,0 +1,902 @@ +/** + * audit-feature-manifest.ts + * + * Keeps .agentworkforce/features/manifest.yaml honest about the CLI's real + * surface, and files the work when it drifts. + * + * ## Why this exists + * + * The manifest is the input to every feature-verification run, so a command + * nobody documented is a command nobody verifies. Coverage does not fail loudly + * when it shrinks — it just quietly stops covering things. + * + * The only pre-existing guard was `manifest-contract.test.ts`, which asserts + * the manifest contains each entry of a hand-maintained list of ~110 commands. + * That catches a *deletion* from the manifest but is structurally incapable of + * noticing a newly *added* CLI command or MCP tool, because a new command is + * absent from both the manifest and the expected list. On the first run of + * scripts/audit-feature-manifest.mjs, `fleet spawn` and `fleet release` had + * been shipping undocumented and therefore unverified. + * + * This workflow derives the surface instead of asserting a snapshot of it: + * it walks `--help` recursively for the Commander tree and performs a + * `tools/list` JSON-RPC handshake for MCP tools, then diffs against the + * manifest. + * + * Schedule: + * relay cloud schedule workflows/audit-feature-manifest.ts --cron "0 2 * * 1" + * + * Manually: + * relay node workflow run workflows/audit-feature-manifest.ts + * + * ## Exit semantics + * + * The audit script distinguishes three outcomes, and so does this workflow: + * 0 manifest matches the derived surface + * 1 drift — file an issue, update the manifest on a branch, open a draft PR + * 2 the audit could not run — escalate to NightCTO, because a broken audit + * reporting "clean" is worse than no audit at all + * + * ## Environment + * + * AUDIT_SLACK_CHANNEL Slack channel ID for drift. Default C0AEKNLDNKW. + * AUDIT_AUTOFIX "0" disables the issue/manifest-update/PR path. + * RELAY_CLI CLI under audit. Defaults to the repo build. + * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. + * NIGHTCTO_EVIDENCE_URL Enables escalation when the audit cannot run. + * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. + * CLOUD_API_URL Required for Slack delivery, which goes through the + * CLOUD_API_TOKEN relayflows Slack primitive's cloud-relay runtime + * (/api/v1/slack/post-message) using the workspace's + * configured Slack integration. No bot token is read. + */ + +import { existsSync, readFileSync } from 'node:fs'; + +import { workflow } from '@relayflows/core'; + +const ARTIFACTS = '.workflow-artifacts/audit-feature-manifest'; +const TIMESTAMP = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 16); +const RUN_ID = `audit-${TIMESTAMP}`; +const REPORT_FILE = `${ARTIFACTS}/audit.json`; +const FIX_BRANCH = `chore/${RUN_ID}-manifest`; + +/** + * Slack destination, as a channel ID. The cloud-relay runtime does not + * implement channel resolution, so an ID is the only form guaranteed to work. + */ +const SLACK_CHANNEL = process.env.AUDIT_SLACK_CHANNEL ?? 'C0AEKNLDNKW'; +const AUTOFIX = process.env.AUDIT_AUTOFIX !== '0'; + +/** A literal backtick; a raw one would close the template literals below. */ +const BT = '`'; + +/** + * Optional-environment seeding. `printenv` rather than `${VAR:-default}` + * because these step bodies are TypeScript template literals — a `${` in the + * shell source would be parsed as TypeScript interpolation. Also keeps `set -u` + * from aborting on an unconfigured optional knob. + */ +const ENV_DEFAULTS = String.raw` +POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" +POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" +NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" +NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" +CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" +CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" +RELAY_CLI="$(printenv RELAY_CLI || true)" +VERIFY_ENVIRONMENT="$(printenv VERIFY_ENVIRONMENT || true)" +if [ -z "$POSTHOG_HOST" ]; then POSTHOG_HOST="https://i.agentrelay.com"; fi +if [ -z "$VERIFY_ENVIRONMENT" ]; then VERIFY_ENVIRONMENT="sandbox"; fi +export AUDIT_ARTIFACTS="${ARTIFACTS}" +export AUDIT_RUN_ID="${RUN_ID}" +`; + +/** + * Post to Slack through the relayflows Slack primitive. + * + * Pins `runtime: 'cloud-relay'` on purpose: the adapter's auto-detection falls + * back to a local `SLACK_BOT_TOKEN` runtime that talks to slack.com directly, + * and drift reports must go through the workspace's configured Slack + * integration. Pinning turns a missing CLOUD_API_* pair into an explicit + * `auth_token_missing` rather than a silent change of route. + */ +const SLACK_POST_FN = String.raw` +slack_post() { + _sp_channel="$1" + _sp_textfile="$2" + + cat > "$ARTIFACTS/slack-post.mjs" <<'SLACKPOSTEOF' +import { readFileSync } from 'node:fs'; +import { SlackClient } from '@relayflows/slack-primitive'; + +const [channel, textFile] = process.argv.slice(2); +const text = readFileSync(textFile, 'utf8'); + +const client = new SlackClient({ + runtime: 'cloud-relay', + cloudApiUrl: process.env.CLOUD_API_URL, + cloudApiToken: process.env.CLOUD_API_TOKEN, +}); + +try { + const out = await client.postMessage({ channel, text, unfurl: false }); + console.log('SLACK_POSTED channel=' + out.channel + ' ts=' + out.ts); +} catch (err) { + const code = err && err.code ? err.code : 'unknown'; + console.log('SLACK_ERROR ' + code + ': ' + (err && err.message ? err.message : String(err))); + process.exitCode = 1; +} +SLACKPOSTEOF + + if node "$ARTIFACTS/slack-post.mjs" "$_sp_channel" "$_sp_textfile"; then + return 0 + fi + + echo "SLACK_UNDELIVERED to $_sp_channel — payload follows:" + echo "---- undelivered Slack payload ----" + cat "$_sp_textfile" + echo "---- end payload ----" + return 1 +} +`; + +async function main() { + const wf = workflow('relay-audit-feature-manifest') + .description( + 'Derive the CLI and MCP surface, diff it against the feature manifest, and open a ' + + 'draft PR updating the manifest when it has drifted.' + ) + .pattern('pipeline') + .channel('relay-health') + .maxConcurrency(3) + // "continue" also opts out of applyReliabilityDefaults attaching a repair + // agent to a failing gate — an audit an agent can edit until it passes is + // not an audit. + .onError('continue') + .timeout(900_000); + + wf.agent('manifest-editor', { + cli: 'claude', + role: 'Update the feature manifest to match the CLI surface the audit derived', + retries: 0, + }); + + // ── Phase 1: run the audit ─────────────────────────────────────────────── + // + // failOnError stays false so the reporting steps below can run; the exit + // code is captured into audit-exit.txt and drives everything downstream. + + wf.step('audit', { + type: 'deterministic', + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +mkdir -p "${ARTIFACTS}" +${ENV_DEFAULTS} + +CLI_ARG="" +if [ -n "$RELAY_CLI" ]; then + CLI_ARG="--cli $RELAY_CLI" +fi + +# Human-readable for the log, JSON for the machinery. Two invocations rather +# than one because the script's report goes to stdout in one format at a time. +node scripts/audit-feature-manifest.mjs $CLI_ARG 2>&1 | tee "${ARTIFACTS}/audit.txt" +EXIT_CODE=$? + +node scripts/audit-feature-manifest.mjs $CLI_ARG --json > "${ARTIFACTS}/audit.json" 2>"${ARTIFACTS}/audit.err" || true + +echo "$EXIT_CODE" > "${ARTIFACTS}/audit-exit.txt" + +# Published for the fix chain: update-manifest is an agent step and cannot read +# this workflow's constants, so the operator's autofix decision has to reach it +# through the filesystem. +echo "AUTOFIX=${AUTOFIX ? '1' : '0'}" > "${ARTIFACTS}/autofix.env" + +echo "audit exit code: $EXIT_CODE" +exit 0 +`, + }); + + // ── Phase 2: escalate an unrunnable audit ──────────────────────────────── + // + // Exit 2 means the audit itself broke (CLI missing, help unparseable). That + // is an operator problem, not a manifest problem: a silent audit means + // coverage can drift indefinitely with nobody noticing. + + wf.step('escalate-infra', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +if [ "$EXIT_CODE" != "2" ]; then + echo "Audit ran (exit $EXIT_CODE) — nothing to escalate." + exit 0 +fi + +DETAIL=$(head -c 300 "$ARTIFACTS/audit.txt" 2>/dev/null || true) +if [ -z "$DETAIL" ]; then DETAIL="no audit output"; fi +DETAIL_CLEAN=$(printf '%s' "$DETAIL" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g') + +if [ -z "$NIGHTCTO_EVIDENCE_URL" ]; then + echo "NIGHTCTO_EVIDENCE_URL unset — cannot escalate. Audit failure detail:" + printf '%s\n' "$DETAIL" + exit 0 +fi + +BODY=$(printf '{"schemaVersion":"cloud-runtime-evidence/1","service":"relay-audit-feature-manifest","environment":"%s","version":"unknown","path":"relayflow.audit.manifest","kind":"request_error","outcome":"error","severity":6,"occurredAt":"%s","requestId":"%s","correlationIds":{"ingress":"relayflow"},"summary":"feature manifest audit could not run: %s","errorCode":"audit_unrunnable","inspect":{"logQuery":"%s"}}' \ + "$VERIFY_ENVIRONMENT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$DETAIL_CLEAN" "$ARTIFACTS") + +if curl -sS -m 15 -X POST "$NIGHTCTO_EVIDENCE_URL" \ + -H 'content-type: application/json' \ + -H "authorization: Bearer $NIGHTCTO_EVIDENCE_TOKEN" \ + -H "x-nightcto-evidence-token: $NIGHTCTO_EVIDENCE_TOKEN" \ + -d "$BODY" >/dev/null 2>&1; then + echo "escalated audit_unrunnable to NightCTO" +else + echo "NightCTO escalation POST failed (non-fatal)" +fi +exit 0 +`, + }); + + // ── Phase 3: PostHog ───────────────────────────────────────────────────── + + wf.step('emit-posthog', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +${ENV_DEFAULTS} + +if [ -z "$POSTHOG_API_KEY" ]; then + echo "POSTHOG_API_KEY unset — not emitting" + exit 0 +fi + +PROPS=$(node <<'PHEOF' +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +let report = null; +try { + report = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); +} catch { + report = null; +} +const exitCode = (() => { + try { + return Number(fs.readFileSync(artifacts + '/audit-exit.txt', 'utf8').trim()); + } catch { + return 2; + } +})(); +process.stdout.write( + JSON.stringify({ + run_id: process.env.AUDIT_RUN_ID, + exit_code: exitCode, + auditable: report?.auditable ?? false, + ok: report?.ok ?? false, + manifest_version: report?.manifestVersion ?? null, + manifest_updated: report?.manifestUpdated ?? null, + cli_leaves: report?.counts?.cliLeaves ?? null, + documented_commands: report?.counts?.documentedCommands ?? null, + mcp_tools: report?.counts?.mcpTools ?? null, + features: report?.counts?.features ?? null, + undocumented_commands: (report?.undocumentedCommands ?? []).join(',') || null, + stale_commands: (report?.staleCommands ?? []).join(',') || null, + undocumented_mcp: (report?.undocumentedMcp ?? []).join(',') || null, + stale_mcp: (report?.staleMcp ?? []).join(',') || null, + drift_count: + (report?.undocumentedCommands ?? []).length + + (report?.staleCommands ?? []).length + + (report?.undocumentedMcp ?? []).length + + (report?.staleMcp ?? []).length, + }) +); +PHEOF +) + +BODY=$(printf '{"api_key":"%s","event":"relay_manifest_audit","distinct_id":"relay-audit-feature-manifest","properties":%s}' \ + "$POSTHOG_API_KEY" "$PROPS") + +if curl -sS -m 15 -X POST "$POSTHOG_HOST/capture/" \ + -H 'content-type: application/json' -d "$BODY" >/dev/null 2>&1; then + echo "emitted relay_manifest_audit" +else + echo "PostHog emit failed (non-fatal)" +fi +exit 0 +`, + }); + + // ── Phase 4: Slack ─────────────────────────────────────────────────────── + + wf.step('slack-alert', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} +${SLACK_POST_FN} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +if [ "$EXIT_CODE" = "0" ]; then + echo "SLACK_SKIPPED: manifest is clean" + exit 0 +fi + +node <<'SLACKEOF' > "$ARTIFACTS/slack-message.txt" +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +const exitCode = Number(fs.readFileSync(artifacts + '/audit-exit.txt', 'utf8').trim()); + +const lines = []; +if (exitCode === 2) { + lines.push(':warning: *Feature manifest audit could not run* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push('This is harness breakage. Coverage drift is currently undetectable.'); + let detail = ''; + try { + detail = fs.readFileSync(artifacts + '/audit.txt', 'utf8').slice(0, 500); + } catch { + detail = 'no audit output'; + } + lines.push('${BT}${BT}${BT}'); + lines.push(detail); + lines.push('${BT}${BT}${BT}'); +} else { + // audit.json is produced by a "|| true" invocation, so it can be missing, + // empty, or truncated while audit-exit.txt still says 1. Falling over here + // would leave the message file empty and the surrounding "|| true" would + // swallow it — drift with no alert. + let r = null; + try { + r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); + } catch (err) { + lines.push(':warning: *Feature manifest drifted, but the report is unreadable* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push('audit.json could not be parsed: ' + err.message); + lines.push('Run ${BT}node scripts/audit-feature-manifest.mjs${BT} locally to see the drift.'); + process.stdout.write(lines.join('\n')); + process.exit(0); + } + lines.push(':clipboard: *Feature manifest has drifted* — ${BT}' + process.env.AUDIT_RUN_ID + '${BT}'); + lines.push( + 'manifest ' + r.manifestVersion + ' (updated ' + r.manifestUpdated + ') — ' + + r.counts.cliLeaves + ' CLI leaves derived, ' + r.counts.documentedCommands + ' documented' + ); + lines.push(''); + const section = (title, items) => { + if (!items || items.length === 0) return; + lines.push('*' + title + '* (' + items.length + ')'); + for (const item of items.slice(0, 20)) lines.push(' • ${BT}' + item + '${BT}'); + }; + section('Undocumented commands — shipping unverified', r.undocumentedCommands); + section('Stale commands — in manifest, not in CLI', r.staleCommands); + section('Undocumented MCP tools', r.undocumentedMcp); + section('Stale MCP tools', r.staleMcp); + if (r.conditionalMcp && r.conditionalMcp.length > 0) { + lines.push(''); + lines.push( + '_Not drift, but unexercised: ' + + r.conditionalMcp.join(', ') + + ' are registered conditionally and did not appear in the probe._' + ); + } + if (!r.mcpDerivable) { + lines.push(''); + lines.push(':warning: MCP tool list could not be derived — MCP drift was NOT checked.'); + } +} +process.stdout.write(lines.join('\n')); +SLACKEOF + +slack_post "$CHANNEL" "$ARTIFACTS/slack-message.txt" || true +exit 0 +`, + }); + + // ── Phase 5: file an issue ─────────────────────────────────────────────── + + wf.step('file-issue', { + type: 'deterministic', + dependsOn: ['audit'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +AUTOFIX="${AUTOFIX ? '1' : '0'}" +${ENV_DEFAULTS} + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ "$EXIT_CODE" != "1" ]; then + echo "ISSUE_SKIPPED: audit exit is '$EXIT_CODE' — only drift (1) files an issue" + exit 0 +fi +if [ "$AUTOFIX" != "1" ]; then + echo "ISSUE_SKIPPED: AUDIT_AUTOFIX=0" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +node <<'ISSUEEOF' > "$ARTIFACTS/issue-body.md" +const fs = require('node:fs'); +const artifacts = process.env.AUDIT_ARTIFACTS; +// Same guard as the Slack payload: a truncated audit.json must still produce +// a filed issue, not an empty body swallowed by the gh call. +let r = null; +try { + r = JSON.parse(fs.readFileSync(artifacts + '/audit.json', 'utf8')); +} catch (err) { + process.stdout.write( + 'The feature manifest drifted, but the machine-readable report could not be parsed: ' + + err.message + + '\n\nRun ${BT}node scripts/audit-feature-manifest.mjs${BT} locally to see the drift.\n' + ); + process.exit(0); +} + +const out = []; +out.push('The feature manifest no longer matches the CLI surface.'); +out.push(''); +out.push('An undocumented command is an unverified command: the manifest is the'); +out.push('input to every feature-verification run, so drift here silently shrinks'); +out.push('coverage.'); +out.push(''); +out.push('- Manifest: ${BT}' + r.manifestVersion + '${BT} (updated ' + r.manifestUpdated + ')'); +out.push('- Derived: ' + r.counts.cliLeaves + ' CLI leaves, ' + (r.counts.mcpTools ?? 'n/a') + ' MCP tools'); +out.push('- Documented: ' + r.counts.documentedCommands + ' commands, ' + r.counts.documentedMcp + ' MCP tools across ' + r.counts.features + ' features'); +out.push(''); +const section = (title, items, note) => { + if (!items || items.length === 0) return; + out.push('## ' + title); + out.push(''); + if (note) { + out.push(note); + out.push(''); + } + for (const item of items) out.push('- ${BT}' + item + '${BT}'); + out.push(''); +}; +section( + 'Undocumented commands', + r.undocumentedCommands, + 'These exist in the CLI but have no manifest entry, so nothing verifies them.' +); +section( + 'Stale commands', + r.staleCommands, + 'These are documented but no prefix of them exists in the CLI. Either the command was removed and the entry should go, or it was renamed.' +); +section('Undocumented MCP tools', r.undocumentedMcp); +section( + 'Stale MCP tools', + r.staleMcp, + 'Documented, and not registered anywhere in the CLI source.' +); +if (r.conditionalMcp && r.conditionalMcp.length > 0) { + out.push('## Conditionally registered (not drift)'); + out.push(''); + out.push('Registered in source but absent from a default stdio probe, so this run did not exercise them:'); + out.push(''); + for (const item of r.conditionalMcp) out.push('- ${BT}' + item + '${BT}'); + out.push(''); +} +out.push('---'); +out.push('Filed by ${BT}workflows/audit-feature-manifest.ts${BT}. Reproduce with:'); +out.push(''); +out.push('${BT}${BT}${BT}bash'); +out.push('node scripts/audit-feature-manifest.mjs'); +out.push('${BT}${BT}${BT}'); +process.stdout.write(out.join('\n')); +ISSUEEOF + +ISSUE_URL=$(gh issue create --title "Feature manifest drift: ${RUN_ID}" \ + --body-file "$ARTIFACTS/issue-body.md" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || true) + +if [ -n "$ISSUE_URL" ]; then + echo "$ISSUE_URL" > "$ARTIFACTS/issue-url.txt" + echo "ISSUE_CREATED: $ISSUE_URL" +else + echo "ISSUE_FAILED: gh issue create produced no URL" +fi +exit 0 +`, + }); + + // ── Phase 6: update the manifest ───────────────────────────────────────── + + wf.step('update-manifest', { + agent: 'manifest-editor', + dependsOn: ['file-issue'], + task: `Bring ${'`.agentworkforce/features/manifest.yaml`'} back in line with the CLI surface. + +## FIRST: check whether there is anything to do + +Read ${ARTIFACTS}/autofix.env and ${ARTIFACTS}/audit-exit.txt. + +- If ${ARTIFACTS}/autofix.env contains \`AUTOFIX=0\`, the operator has disabled + the fix path. Write "Autofix disabled." to ${ARTIFACTS}/update-summary.md and + STOP IMMEDIATELY. +- If audit-exit.txt does not contain exactly \`1\`, there is NOTHING to do. + Write "No drift — nothing to update." to ${ARTIFACTS}/update-summary.md and + STOP IMMEDIATELY. +- In either case: do not create a branch, edit files, or run commands. +- Only if autofix is enabled AND audit-exit.txt contains \`1\` do you continue. + +(The engine has no conditional steps, so this step is scheduled on every run. +The early exit is what keeps an agent away from a clean tree.) + +## The task + +Read ${REPORT_FILE}. It lists, as JSON: + +- \`undocumentedCommands\` — real CLI commands with no manifest entry. +- \`staleCommands\` — manifest entries whose command no longer exists. +- \`undocumentedMcp\` / \`staleMcp\` — the same for MCP tools. +- \`conditionalMcp\` — NOT drift. Registered in source but not exposed by a + default stdio probe. Leave these alone. + +## Hard rules + +1. NEVER commit or push to \`main\`. Work on a branch named exactly + \`${FIX_BRANCH}\`, created from the current HEAD. +2. For each undocumented command, add a manifest entry in the correct existing + category, matching the surrounding style exactly: \`id\`, \`name\`, \`cli\` + (with its real argument and option signature, taken from + \`relay --help\`), \`description\`, \`location\` (the actual source + file), and \`verify_tier\`. +3. Choose \`verify_tier\` honestly, using the scale documented at the top of the + manifest. Do not mark something tier 1 to make it look cheap to verify. +4. For stale entries, confirm with \`relay --help\` that the command + really is gone before removing anything. If it still exists in any form, the + audit's derivation is wrong — say so and change nothing. +5. Do NOT delete or weaken unrelated manifest entries. +6. Bump the manifest's \`updated:\` field to today's date. + +## Verify before you finish + +Run both of these and paste the real output: + +\`\`\`bash +node scripts/audit-feature-manifest.mjs +npx vitest run .agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts +\`\`\` + +The audit must print MANIFEST_CLEAN and the contract test must pass. + +Do NOT edit any file other than the manifest. The integrity gate rejects a +branch that touches CLI sources, the audit script, or +manifest-contract.test.ts — resolving drift by changing what is measured is +exactly what it exists to stop. If the contract test's hardcoded expectation +list needs new entries, say so in your summary and leave it to a human. + +## Deliverable + +Write ${ARTIFACTS}/update-summary.md with: + +- \`Added:\` each new entry and the tier you assigned it, with your reasoning. +- \`Removed:\` each removed entry and the evidence the command is gone. +- \`Evidence:\` the output of both commands above. + +Commit on the branch. Do NOT open a pull request — the next step does that.`, + }); + + // ── Phase 7: integrity gate ────────────────────────────────────────────── + // + // The audit's own credibility depends on the manifest only ever growing to + // match reality. An "update" that shrinks documented coverage, or that edits + // the audit script instead of the manifest, is the failure mode to catch. + + wf.step('update-integrity', { + type: 'deterministic', + dependsOn: ['update-manifest'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +MANIFEST=".agentworkforce/features/manifest.yaml" + +if grep -q '^AUTOFIX=0$' "$ARTIFACTS/autofix.env" 2>/dev/null; then + echo "INTEGRITY_NOT_APPLICABLE: autofix disabled, no update was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ "$EXIT_CODE" != "1" ]; then + echo "INTEGRITY_NOT_APPLICABLE: no drift to fix" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +if ! git rev-parse --git-dir >/dev/null 2>&1; then + echo "INTEGRITY_SKIPPED: not a git repository" + echo "INTEGRITY=skipped" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +CURRENT=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown) +if [ "$CURRENT" = "main" ]; then + echo "INTEGRITY_FAIL: editor left the repo on main" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +BASE_REF=$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || true) +if [ -z "$BASE_REF" ]; then + echo "INTEGRITY_SKIPPED: no base ref to diff against" + echo "INTEGRITY=skipped" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# 'grep -c' exits 1 on zero matches while still printing 0, so the fallback is +# '|| true'; '|| echo 0' would emit a second line and break the comparison. +BEFORE=$(git show "$BASE_REF:$MANIFEST" 2>/dev/null | grep -cE '^ - id: ' || true) +AFTER=$(grep -cE '^ - id: ' "$MANIFEST" 2>/dev/null || true) +if [ -z "$BEFORE" ]; then BEFORE=0; fi +if [ -z "$AFTER" ]; then AFTER=0; fi + +echo "Manifest feature entries: $BEFORE -> $AFTER" + +# A drift fix may legitimately remove genuinely stale entries, so a small +# decrease is allowed; a large one means the editor deleted real coverage. +STALE_COUNT=$(node -e 'try{const r=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));process.stdout.write(String((r.staleCommands||[]).length+(r.staleMcp||[]).length))}catch(e){process.stdout.write("0")}' "$ARTIFACTS/audit.json" 2>/dev/null || echo 0) +MIN_ALLOWED=$((BEFORE - STALE_COUNT)) + +if [ "$AFTER" -lt "$MIN_ALLOWED" ]; then + echo "INTEGRITY_FAIL: manifest dropped to $AFTER entries; only $STALE_COUNT stale entr(ies) justified removal (floor $MIN_ALLOWED)." + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# An editor can make the audit clean three ways: document the command (wanted), +# delete the command (not wanted), or move the goalposts by editing the audit +# script or its expectation list (not wanted). Only the manifest is allowed to +# change, so assert the whole changed-file set rather than one script. +ALLOWED_CHANGES=".agentworkforce/features/manifest.yaml" +CHANGED=$(git diff --name-only "$BASE_REF" 2>/dev/null || true) +UNEXPECTED="" +for f in $CHANGED; do + case "$f" in + "$ALLOWED_CHANGES") ;; + *) UNEXPECTED="$UNEXPECTED $f" ;; + esac +done + +if [ -n "$UNEXPECTED" ]; then + echo "INTEGRITY_FAIL: the branch changed files other than the manifest:$UNEXPECTED" + echo "A manifest sync may only edit the manifest. Changing CLI sources, the audit" + echo "script, or manifest-contract.test.ts resolves drift by moving what is measured." + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +if [ ! -f "$ARTIFACTS/update-summary.md" ]; then + echo "INTEGRITY_FAIL: no update-summary.md, so there is no reasoning on record" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" + exit 0 +fi + +# The whole point: after the edit the audit must actually be clean. +if node scripts/audit-feature-manifest.mjs >"$ARTIFACTS/audit-after.txt" 2>&1; then + echo "INTEGRITY_OK: audit is clean after the update" + echo "INTEGRITY=ok" > "$ARTIFACTS/update-integrity.env" +else + echo "INTEGRITY_FAIL: audit still reports drift after the update:" + tail -20 "$ARTIFACTS/audit-after.txt" + echo "INTEGRITY=fail" > "$ARTIFACTS/update-integrity.env" +fi + +exit 0 +`, + }); + + // ── Phase 8: draft PR ──────────────────────────────────────────────────── + + wf.step('open-pr', { + type: 'deterministic', + dependsOn: ['update-integrity'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/update-integrity.env" ]; then + echo "PR_SKIPPED: no integrity result" + exit 0 +fi + +. "$ARTIFACTS/update-integrity.env" + +if [ "$INTEGRITY" != "ok" ]; then + echo "PR_SKIPPED: integrity gate returned '$INTEGRITY'" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "PR_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$BRANCH" = "main" ]; then + echo "PR_SKIPPED: refusing to open a PR from main" + exit 0 +fi + +git push -u origin "$BRANCH" 2>&1 | tail -3 || { + echo "PR_FAILED: could not push $BRANCH" + exit 0 +} + +{ + echo "## Feature manifest drift correction" + echo "" + echo "Run: ${RUN_ID}" + echo "" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + echo "Fixes $(cat "$ARTIFACTS/issue-url.txt")" + echo "" + fi + echo "### What drifted" + echo "" + echo '${BT}${BT}${BT}' + cat "$ARTIFACTS/audit.txt" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Editor summary" + echo "" + cat "$ARTIFACTS/update-summary.md" 2>/dev/null || echo "_No summary produced._" + echo "" + echo "### Audit after the change" + echo "" + echo '${BT}${BT}${BT}' + cat "$ARTIFACTS/audit-after.txt" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Review notes" + echo "" + echo "- Opened as a **draft** by ${BT}workflows/audit-feature-manifest.ts${BT}. Not auto-merged." + echo "- The integrity gate confirmed the audit is clean after the change, that the" + echo " manifest did not lose unjustified entries, and that the audit script itself" + echo " was not modified." + echo "- Still needs a human on the ${BT}verify_tier${BT} assignments: the gate cannot tell a" + echo " correct tier from a convenient one." +} > "$ARTIFACTS/pr-body.md" + +PR_URL=$(gh pr create --draft --title "chore: sync feature manifest with CLI surface (${RUN_ID})" \ + --body-file "$ARTIFACTS/pr-body.md" --base main --head "$BRANCH" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || true) + +if [ -n "$PR_URL" ]; then + echo "$PR_URL" > "$ARTIFACTS/pr-url.txt" + echo "PR_CREATED: $PR_URL" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + gh issue comment "$(cat "$ARTIFACTS/issue-url.txt")" \ + --body "Draft manifest sync PR opened: $PR_URL" >/dev/null 2>&1 || true + fi +else + echo "PR_FAILED: gh pr create produced no URL" +fi +exit 0 +`, + }); + + // ── Phase 9: enforce ───────────────────────────────────────────────────── + // + // Terminal, and the only failing step. Does not depend on the fix chain, so + // a hiccup while opening a PR cannot swallow the drift signal. + + wf.step('enforce', { + type: 'deterministic', + dependsOn: ['slack-alert', 'emit-posthog', 'escalate-infra'], + captureOutput: true, + failOnError: true, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +EXIT_CODE=$(cat "$ARTIFACTS/audit-exit.txt" 2>/dev/null || true) +if [ -z "$EXIT_CODE" ]; then EXIT_CODE="2"; fi + +case "$EXIT_CODE" in + 0) + echo "ENFORCE: manifest is clean" + exit 0 + ;; + 1) + echo "ENFORCE: manifest drift — see the issue and draft PR" + exit 1 + ;; + *) + echo "ENFORCE: audit could not run (exit $EXIT_CODE) — escalated to NightCTO" + exit 2 + ;; +esac +`, + }); + + const result = await wf.run(); + + if (process.env.DRY_RUN || !('status' in result)) { + return; + } + + // Make the process exit code tell the truth, the same way verify-features + // does: a scheduler must be able to see drift without reading logs. + const exitFile = `${ARTIFACTS}/audit-exit.txt`; + if (!existsSync(exitFile)) { + console.error( + `[audit-feature-manifest] no ${exitFile} — the audit step did not complete. ` + + `Treating as harness breakage.` + ); + process.exitCode = 2; + return; + } + + const auditExit = Number(readFileSync(exitFile, 'utf8').trim()); + let report: { undocumentedCommands?: string[]; staleCommands?: string[] } | null = null; + if (existsSync(REPORT_FILE)) { + try { + report = JSON.parse(readFileSync(REPORT_FILE, 'utf8')); + } catch { + report = null; + } + } + + if (auditExit === 0) { + console.log('[audit-feature-manifest] manifest is clean'); + return; + } + + if (auditExit === 1) { + const undocumented = report?.undocumentedCommands ?? []; + const stale = report?.staleCommands ?? []; + console.error( + `[audit-feature-manifest] DRIFT: ${undocumented.length} undocumented, ${stale.length} stale. ` + + `Undocumented commands are unverified commands.` + ); + process.exitCode = 1; + return; + } + + console.error(`[audit-feature-manifest] the audit could not run (exit ${auditExit})`); + process.exitCode = 2; +} + +main().catch((err) => { + console.error( + `[audit-feature-manifest] harness failure: ${err instanceof Error ? err.stack : String(err)}` + ); + process.exitCode = 2; +}); diff --git a/workflows/verify-features.ts b/workflows/verify-features.ts index c7bef777f..6a75ec541 100644 --- a/workflows/verify-features.ts +++ b/workflows/verify-features.ts @@ -1,10 +1,10 @@ /** * verify-features.ts * - * Automated feature health check for agent-relay. - * Runs through every user-facing feature tier by tier, captures - * pass/fail per check, and has a cheap opencode agent write the - * PASS/FAIL report into #relay-health. + * Automated feature health check for agent-relay. Walks every user-facing + * feature tier by tier, records one structured result per check, and turns a + * failure into action: a Slack alert, a GitHub issue, an attempted fix on a + * branch, and a draft PR carrying re-verified evidence. * * Designed to run on a schedule (nightly or post-merge): * relay cloud schedule workflows/verify-features.ts --cron "0 3 * * *" @@ -12,796 +12,2210 @@ * Or manually: * relay node workflow run workflows/verify-features.ts * - * Pattern: pipeline (sequential phases — each tier depends on the prior passing) + * Pattern: pipeline (sequential phases — later tiers assume earlier ones ran) * - * Acceptance contract (V1–V5): - * V1 Tier 1: CLI health — version, status, doctor, telemetry, workspace list - * V2 Tier 2: Broker lifecycle — up/down/status/metrics + agent register/list/remove - * V3 Tier 3: Channel messaging round-trip — create/join/post/list/reply/thread/search - * V4 Tier 4: Cross-agent — DM send/list, channel invite, read receipts - * V5 Critical paths — the 4 sequences that must work for the product to function + * ## Why this file is shaped the way it is + * + * An earlier version of this workflow reported "COMPLETED — 11 passed, 0 + * failed" on a run where Tier 2 had four failing checks. Three independent + * defects produced that false green, and each one is now closed here: + * + * 1. Every step set `failOnError: false`, so a non-zero exit still scored as + * a pass. Tiers still use it (we want all tiers to run even after one + * fails), but they are no longer the verdict — `verdict` is, and it reads + * a machine-readable ledger rather than grepping prose logs. + * 2. Nothing aggregated the per-tier results into a run verdict. `verdict` + * plus `enforce-verdict` now do, and `main()` sets a non-zero process exit + * so a scheduler sees red. + * 3. `main()` discarded the runner result entirely. It now reads verdict.json + * and fails closed when that file is missing. + * + * A fourth defect was subtler. `applyReliabilityDefaults` in @relayflows/core + * force-enables `strategy: "retry"` with a repair agent for any workflow that + * declares agents, so a failing verification gate got handed to an agent that + * edited the working tree until the gate passed. A verification workflow whose + * assertions can be rewritten to make them pass measures nothing. `onError` + * is set to "continue" below specifically to opt out of that path. + * + * ## Honest accounting + * + * Every check records `pass`, `fail`, or `skip` with a reason. A check that + * cannot run in this environment (no cloud login, no provider CLI) is a SKIP + * with a stated cause — never a pass, and never a fail. `verdict` also fails + * the run when a tier produced no records at all, so a tier that crashes + * before its first check cannot masquerade as a clean tier. * * Feature manifest: .agentworkforce/features/manifest.yaml * Critical paths: .agentworkforce/features/critical-paths.md * Procedures ref: .agentworkforce/features/verify/procedures.md + * Manifest audit: workflows/audit-feature-manifest.ts + * + * ## Environment + * + * VERIFY_SLACK_CHANNEL Slack channel for failures. Default #relay-health. + * VERIFY_AUTOFIX "0" disables the issue/fix/PR path. Default on. + * VERIFY_ALLOW_CLI_DRIFT "1" downgrades a CLI-vs-repo version mismatch from + * a failure to a warning. + * POSTHOG_API_KEY Enables PostHog emission. Host: POSTHOG_HOST. + * NIGHTCTO_EVIDENCE_URL Enables infra escalation to NightCTO. + * NIGHTCTO_EVIDENCE_TOKEN Bearer token for the above. + * CLOUD_API_URL Required for Slack delivery. Slack posts go through + * CLOUD_API_TOKEN the relayflows Slack primitive's cloud-relay runtime + * (/api/v1/slack/post-message), which uses the + * workspace's configured Slack integration. No bot + * token is read. */ +import { existsSync, readFileSync } from 'node:fs'; + import { workflow } from '@relayflows/core'; const ARTIFACTS = '.workflow-artifacts/verify-features'; const TIMESTAMP = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 16); const RUN_ID = `verify-${TIMESTAMP}`; +const VERDICT_FILE = `${ARTIFACTS}/verdict.json`; -// Unique suffixes to avoid collisions with existing workspace state +/** Unique suffix so a run never collides with existing workspace state. */ const SUFFIX = `vf-${Date.now()}`; -const SUFFIX2 = `cp-${Date.now()}`; + +/** Canonical fix branch. RUN_ID already carries the "verify-" prefix. */ +const FIX_BRANCH = `fix/${RUN_ID}`; + +/** + * Slack destination, as a channel ID. + * + * An ID rather than a #name because the cloud-relay runtime does not implement + * channel resolution — `resolveChannel` is unsupported there, so a "#name" only + * works if Slack itself accepts it for that workspace. An ID always resolves. + */ +const SLACK_CHANNEL = process.env.VERIFY_SLACK_CHANNEL ?? 'C0AEKNLDNKW'; +const AUTOFIX = process.env.VERIFY_AUTOFIX !== '0'; + +/** + * A literal backtick. + * + * Step bodies below are template literals, so a raw backtick would close the + * template. The Slack and GitHub payloads want Markdown code spans, so those + * are written as `${BT}` and interpolated in. + */ +const BT = '`'; + +/** + * Optional-environment seeding, interpolated ahead of every step body. + * + * `printenv` is used instead of the natural `${VAR:-default}` for one hard + * reason: these step bodies are TypeScript template literals, so a `${` in the + * shell source is parsed as TypeScript interpolation and the file would not + * compile. Every shell variable below is therefore written `$VAR`, and defaults + * are applied with explicit `if` blocks. Seeding also keeps `set -u` from + * aborting a step because an operator did not configure an optional knob. + */ +const ENV_DEFAULTS = String.raw` +VERIFY_ALLOW_CLI_DRIFT="$(printenv VERIFY_ALLOW_CLI_DRIFT || true)" +POSTHOG_API_KEY="$(printenv POSTHOG_API_KEY || true)" +POSTHOG_HOST="$(printenv POSTHOG_HOST || true)" +NIGHTCTO_EVIDENCE_URL="$(printenv NIGHTCTO_EVIDENCE_URL || true)" +NIGHTCTO_EVIDENCE_TOKEN="$(printenv NIGHTCTO_EVIDENCE_TOKEN || true)" +CLOUD_API_URL="$(printenv CLOUD_API_URL || true)" +CLOUD_API_TOKEN="$(printenv CLOUD_API_TOKEN || true)" +VERIFY_ENVIRONMENT="$(printenv VERIFY_ENVIRONMENT || true)" +if [ -z "$POSTHOG_HOST" ]; then POSTHOG_HOST="https://i.agentrelay.com"; fi +if [ -z "$VERIFY_ENVIRONMENT" ]; then VERIFY_ENVIRONMENT="sandbox"; fi +export VERIFY_ARTIFACTS="${ARTIFACTS}" +export VERIFY_RUN_ID="${RUN_ID}" +`; + +/** + * Shell harness interpolated into every check-running step. + * + * Every check appends one JSON line to checks.jsonl. Downstream steps read + * that ledger instead of grepping the human-readable logs — the previous + * version's analyzer reported "1 issue(s)" for a tier with four failures + * because it counted matching *tiers*, not matching checks. + * + * Written with String.raw so the sed escapes survive into the shell. Shell + * variables must be written `$VAR`, never `${VAR}` — braces would be read as + * TypeScript interpolation. + */ +const PRELUDE = + ENV_DEFAULTS + + String.raw` +PASS=0 +FAIL=0 +SKIP=0 +ARTIFACTS="${ARTIFACTS}" +CHECKS="$ARTIFACTS/checks.jsonl" +CAPS="$ARTIFACTS/caps.env" +RUN_ID="${RUN_ID}" + +# Append one structured result AND move the tier counters. +# +# The counters live here, not in the callers, so that the per-tier summary line +# can never disagree with the ledger. Several checks below record directly +# (multi-step sequences that do not fit run_check); when the callers owned the +# counters those sites reported "0 passed, 0 failed" while the ledger held real +# results. +# +# Reason text is flattened and bounded so a multi-line stack trace cannot +# corrupt the ledger. +record() { + # Strip ALL control bytes, not just newlines: ANSI escapes and stray control + # characters from CLI output would otherwise land raw inside the JSON string + # and make the line unparseable. The verdict reader skips unparseable lines, + # so a corrupted reason could silently erase a failing check. + _reason=$(printf '%s' "$4" | tr -d '\000-\010\013\014\016-\037\177' | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-400) + printf '{"run":"%s","tier":"%s","check":"%s","status":"%s","reason":"%s"}\n' \ + "$RUN_ID" "$1" "$2" "$3" "$_reason" >> "$CHECKS" + case "$3" in + pass) PASS=$((PASS + 1)) ;; + fail) FAIL=$((FAIL + 1)) ;; + skip) SKIP=$((SKIP + 1)) ;; + esac +} + +# Run a command and match its output. Records pass/fail either way. +# +# Note the plain "grep -i" rather than "grep -qi": under "set -o pipefail", a +# -q grep exits at the first match, the writer upstream takes SIGPIPE, and the +# pipeline reports failure even though the match succeeded. That turned a +# demonstrably running broker into a "capability absent" result in a live run. +run_check() { + _name="$1" + _cmd="$2" + _expect="$3" + if _out=$(eval "$_cmd" 2>&1) && printf '%s' "$_out" | grep -i -- "$_expect" >/dev/null; then + echo " PASS $_name" | tee -a "$LOG" + record "$TIER" "$_name" pass "" + else + echo " FAIL $_name (expected: $_expect)" | tee -a "$LOG" + echo " output: $(printf '%s' "$_out" | head -3)" | tee -a "$LOG" + record "$TIER" "$_name" fail "$_out" + fi +} + +# A check we could not run here. Recorded with a cause so that reading the +# report can never confuse "not exercised" with "verified working". +skip_check() { + echo " SKIP $1 ($2)" | tee -a "$LOG" + record "$TIER" "$1" skip "$2" +} + +# Capability flags are written by the "capabilities" step. +have_cap() { grep -q "^$1=1$" "$CAPS" 2>/dev/null; } + +# Run a check only when a capability is present, else skip with the reason. +gated_check() { + _cap="$1" + shift + if have_cap "$_cap"; then + run_check "$@" + else + skip_check "$1" "requires capability: $_cap" + fi +} + +finish_tier() { + echo "" | tee -a "$LOG" + echo "$TIER result: $PASS passed, $FAIL failed, $SKIP skipped" | tee -a "$LOG" + if [ "$FAIL" -gt 0 ]; then + echo "TIER_FAIL" >> "$LOG" + else + echo "TIER_PASS" >> "$LOG" + fi +} +`; + +/** + * Post to Slack through the relayflows Slack primitive. + * + * Deliberately pins `runtime: 'cloud-relay'` rather than letting the adapter + * auto-detect. `selectRuntime` in @relayflows/slack-primitive prefers + * cloud-relay but falls back to a local `SLACK_BOT_TOKEN` runtime that talks to + * slack.com directly — and this workflow must post through the workspace's + * configured Slack integration (the Nango connection behind + * `/api/v1/slack/post-message`), not whatever bot token happens to be in the + * environment. Pinning makes a missing CLOUD_API_* pair throw + * `auth_token_missing` instead of silently taking a different path. + * + * The primitive is a hard dependency of @relayflows/core, which this workflow + * already imports, so it resolves wherever the workflow itself does. + */ +const SLACK_POST_FN = String.raw` +slack_post() { + _sp_channel="$1" + _sp_textfile="$2" + + cat > "$ARTIFACTS/slack-post.mjs" <<'SLACKPOSTEOF' +import { readFileSync } from 'node:fs'; +import { SlackClient } from '@relayflows/slack-primitive'; + +const [channel, textFile] = process.argv.slice(2); +const text = readFileSync(textFile, 'utf8'); + +const client = new SlackClient({ + runtime: 'cloud-relay', + cloudApiUrl: process.env.CLOUD_API_URL, + cloudApiToken: process.env.CLOUD_API_TOKEN, +}); + +try { + const out = await client.postMessage({ channel, text, unfurl: false }); + console.log('SLACK_POSTED channel=' + out.channel + ' ts=' + out.ts); +} catch (err) { + const code = err && err.code ? err.code : 'unknown'; + console.log('SLACK_ERROR ' + code + ': ' + (err && err.message ? err.message : String(err))); + process.exitCode = 1; +} +SLACKPOSTEOF + + if node "$ARTIFACTS/slack-post.mjs" "$_sp_channel" "$_sp_textfile"; then + return 0 + fi + + # Undelivered alerts must be loud. Echo the payload so it is at least + # recoverable from the run log rather than lost with the process. + echo "SLACK_UNDELIVERED to $_sp_channel — payload follows:" + echo "---- undelivered Slack payload ----" + cat "$_sp_textfile" + echo "---- end payload ----" + return 1 +} +`; + +/** + * Bounded evidence POST to NightCTO, matching the CloudEvidenceSummary v1 + * contract in ../cloud/docs/dogfood-telemetry-brief.md. Byte compatibility + * matters — NightCTO's cloud-runtime adapter consumes exactly this shape. + * + * This is for failures of the verification *machinery* (the relayflow, the + * sandbox, the executor), not for product features being broken. A broken + * feature is a PR; a broken harness is an operator alert, because nobody is + * watching a check that never ran. + */ +const NIGHTCTO_FN = String.raw` +escalate_infra() { + _path="$1" + _code="$2" + _summary="$3" + if [ -z "$NIGHTCTO_EVIDENCE_URL" ]; then + echo " [nightcto] NIGHTCTO_EVIDENCE_URL unset — infra escalation skipped: $_summary" + return 0 + fi + _summary_clean=$(printf '%s' "$_summary" | tr '\n\r\t' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g' | cut -c1-300) + _body=$(printf '{"schemaVersion":"cloud-runtime-evidence/1","service":"relay-verify-features","environment":"%s","version":"%s","path":"%s","kind":"request_error","outcome":"error","severity":6,"occurredAt":"%s","requestId":"%s","correlationIds":{"ingress":"relayflow"},"summary":"%s","errorCode":"%s","inspect":{"logQuery":"%s"}}' \ + "$VERIFY_ENVIRONMENT" "$VERIFY_CLI_VERSION" "$_path" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$RUN_ID" "$_summary_clean" "$_code" "$ARTIFACTS") + # Fire-and-forget: an unreachable NightCTO must never mask the product result. + if curl -sS -m 15 -X POST "$NIGHTCTO_EVIDENCE_URL" \ + -H 'content-type: application/json' \ + -H "authorization: Bearer $NIGHTCTO_EVIDENCE_TOKEN" \ + -H "x-nightcto-evidence-token: $NIGHTCTO_EVIDENCE_TOKEN" \ + -d "$_body" >/dev/null 2>&1; then + echo " [nightcto] escalated: $_code" + else + echo " [nightcto] escalation POST failed (non-fatal): $_code" + fi + return 0 +} +`; + +/** + * PostHog emission via the public capture endpoint. + * + * Deliberately not routed through the CLI's telemetry client: that client is + * opt-in *product* telemetry keyed to a real user's distinct_id, and CI health + * events would pollute it. This posts CI-owned events under a stable + * synthetic distinct_id instead. + */ +const POSTHOG_FN = String.raw` +posthog_capture() { + _event="$1" + _props="$2" + if [ -z "$POSTHOG_API_KEY" ]; then + echo " [posthog] POSTHOG_API_KEY unset — not emitting $_event" + return 0 + fi + _body=$(printf '{"api_key":"%s","event":"%s","distinct_id":"relay-verify-features","properties":%s}' \ + "$POSTHOG_API_KEY" "$_event" "$_props") + if curl -sS -m 15 -X POST "$POSTHOG_HOST/capture/" \ + -H 'content-type: application/json' -d "$_body" >/dev/null 2>&1; then + echo " [posthog] emitted $_event" + else + echo " [posthog] emit failed (non-fatal): $_event" + fi + return 0 +} +`; async function main() { const wf = workflow('relay-verify-features') .description( - 'Automated feature health check. Runs CLI verification tiers 1–4 and the 5 critical paths. ' + - 'Posts a structured PASS/FAIL report to #relay-health.' + 'Automated feature health check. Runs verification tiers 1-6 and the 6 critical paths, ' + + 'posts PASS/FAIL to Slack, and opens an issue plus a draft fix PR when something breaks.' ) .pattern('pipeline') .channel('relay-health') .maxConcurrency(1) - .timeout(600_000); // 10 minutes + // Not cosmetic: "continue" is the only strategy that keeps + // applyReliabilityDefaults from attaching a repair agent to failing + // verification gates. See the file header. + .onError('continue') + .timeout(3_600_000); // 1 hour — tiers 5/6 drive real cloud and provider calls // ── Agents ──────────────────────────────────────────────────────────────── - // opencode is cheap for routine summarization — no complex reasoning needed + // opencode is cheap and this is routine summarization of files already written. wf.agent('reporter', { cli: 'opencode', - role: 'Read verification artifact files and write a structured PASS/FAIL report for the relay-health channel', + role: 'Read verification artifacts and write a structured PASS/FAIL/SKIP report', retries: 1, }); - // ── Phase 0: Emit acceptance contract ──────────────────────────────────── + // The fixer edits source, so it gets the stronger model. + wf.agent('fixer', { + cli: 'claude', + role: 'Root-cause a failing feature verification check and fix the underlying defect', + retries: 0, + }); + + // ── Phase 0: acceptance contract ───────────────────────────────────────── + // + // Unquoted heredoc terminator so $(date) actually expands. The previous + // version used <<'EOF' and printed the literal text "$(date -u ...)". wf.step('acceptance-contract', { type: 'deterministic', captureOutput: true, failOnError: false, - command: `cat <<'EOF' + command: String.raw`cat < run FAIL. + - Any tier with no records => run FAIL (a crashed tier is not a clean tier). + - A check that cannot run => SKIP with a stated cause. Never a pass. + +A FAIL posts to Slack, opens a GitHub issue, and attempts a fix on a branch +with a draft PR. A failure of the harness itself escalates to NightCTO. EOF `, }); - // ── Phase 1: Setup ─────────────────────────────────────────────────────── + // ── Phase 1: setup ─────────────────────────────────────────────────────── wf.step('setup', { type: 'deterministic', dependsOn: ['acceptance-contract'], captureOutput: true, failOnError: false, - command: ` -set -euo pipefail + command: String.raw` +set -uo pipefail mkdir -p "${ARTIFACTS}" +LOG="${ARTIFACTS}/setup.log" +: > "${ARTIFACTS}/checks.jsonl" -echo "=== Setup: ${RUN_ID} ===" | tee "${ARTIFACTS}/setup.log" -echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "${ARTIFACTS}/setup.log" +echo "=== Setup: ${RUN_ID} ===" | tee "$LOG" +echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "$LOG" -# Ensure broker is up — --background daemonizes and returns immediately -relay node up --background 2>&1 | tee -a "${ARTIFACTS}/setup.log" || true +relay node up --background 2>&1 | tee -a "$LOG" || true -# Wait for broker to be ready (up to 15 seconds) BROKER_READY=0 -for i in $(seq 1 15); do - if relay status 2>&1 | grep -qi "running"; then +for i in $(seq 1 30); do + if relay node status 2>&1 | grep -i "running" >/dev/null; then BROKER_READY=1 break fi sleep 1 done -relay status 2>&1 | tee -a "${ARTIFACTS}/setup.log" +relay status 2>&1 | tee -a "$LOG" if [ "$BROKER_READY" -eq 1 ]; then - echo "SETUP_OK" >> "${ARTIFACTS}/setup.log" -else - echo "SETUP_FAIL: broker did not become ready within 15 seconds" >> "${ARTIFACTS}/setup.log" - exit 1 + echo "SETUP_OK" >> "$LOG" + exit 0 fi + +echo "SETUP_FAIL: broker did not become ready within 30 seconds" >> "$LOG" +exit 1 `, }); - // ── Phase 2: Tier 1 — CLI health (no broker deps) ─────────────────────── - - wf.step('tier1-cli-health', { + // ── Phase 2: provenance ────────────────────────────────────────────────── + // + // The run this workflow was rebuilt from verified agent-relay 10.0.0 while + // the repo was at 11.3.0 — it even logged "Update available: 10.0.0 → + // 11.3.0" and then reported `relay node deadletters` as an unknown command, + // which was a real command in 11.x. A whole run's worth of green (and one + // loud red herring) measured a stale published CLI instead of the code under + // test. Version drift is now an explicit, failing check. + + wf.step('provenance', { type: 'deterministic', dependsOn: ['setup'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/tier1.log" -echo "=== Tier 1: CLI Health ===" | tee "$LOG" - -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +TIER="provenance" +LOG="${ARTIFACTS}/provenance.log" +${PRELUDE} + +echo "=== Provenance ===" | tee "$LOG" + +CLI_PATH=$(command -v relay 2>/dev/null || echo "not-found") +CLI_VERSION=$(relay version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || echo "unknown") +REPO_VERSION=$(node -e 'try{process.stdout.write(require("./package.json").version)}catch(e){process.stdout.write("unknown")}' 2>/dev/null || echo "unknown") +GIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +echo " relay path: $CLI_PATH" | tee -a "$LOG" +echo " relay version: $CLI_VERSION" | tee -a "$LOG" +echo " repo version: $REPO_VERSION" | tee -a "$LOG" +echo " git sha: $GIT_SHA" | tee -a "$LOG" + +# Recorded for the report, Slack, PostHog, and NightCTO payloads. +{ + echo "VERIFY_CLI_VERSION=$CLI_VERSION" + echo "VERIFY_REPO_VERSION=$REPO_VERSION" + echo "VERIFY_GIT_SHA=$GIT_SHA" + echo "VERIFY_CLI_PATH=$CLI_PATH" +} > "${ARTIFACTS}/provenance.env" + +if [ "$CLI_PATH" = "not-found" ]; then + record "$TIER" "cli-present" fail "relay is not on PATH" + echo " FAIL relay is not on PATH" | tee -a "$LOG" + exit 0 +fi +record "$TIER" "cli-present" pass "" + +if [ "$REPO_VERSION" = "unknown" ]; then + skip_check "cli-matches-repo" "no package.json in cwd — cannot compare versions" +elif [ "$CLI_VERSION" = "$REPO_VERSION" ]; then + echo " PASS cli-matches-repo ($CLI_VERSION)" | tee -a "$LOG" + record "$TIER" "cli-matches-repo" pass "" +elif [ "$VERIFY_ALLOW_CLI_DRIFT" = "1" ]; then + skip_check "cli-matches-repo" "CLI $CLI_VERSION != repo $REPO_VERSION (drift allowed by env)" +else + echo " FAIL cli-matches-repo: verifying $CLI_VERSION but repo is $REPO_VERSION" | tee -a "$LOG" + record "$TIER" "cli-matches-repo" fail "CLI under test is $CLI_VERSION but the repo is $REPO_VERSION; results do not describe this checkout" +fi -run_check "relay version" "relay version" "[0-9]\\+\\.[0-9]" -run_check "relay --help" "relay --help" "Usage" -run_check "relay status (any)" "relay status" "." -run_check "relay node --help" "relay node --help" "up" -run_check "relay telemetry status" "relay telemetry status" "." -run_check "relay workspace list" "relay workspace list" "." +# Matching versions prove nothing on a feature branch: between releases the +# globally installed CLI and this checkout report the same version, so a global +# binary would pass the check above while exercising none of the branch's +# changes. Resolve the binary and require it to live inside this checkout. +REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || true) +CLI_REAL=$(node -e 'const fs=require("node:fs");try{process.stdout.write(fs.realpathSync(process.argv[1]))}catch(e){process.stdout.write(process.argv[1])}' "$CLI_PATH" 2>/dev/null || printf '%s' "$CLI_PATH") +REPO_REAL=$(node -e 'const fs=require("node:fs");try{process.stdout.write(fs.realpathSync(process.argv[1]))}catch(e){process.stdout.write(process.argv[1])}' "$REPO_ROOT" 2>/dev/null || printf '%s' "$REPO_ROOT") +echo " resolved CLI: $CLI_REAL" | tee -a "$LOG" +echo " repo root: $REPO_REAL" | tee -a "$LOG" + +if [ -z "$REPO_ROOT" ]; then + # Falling back to $PWD would only prove the binary sits under the current + # directory, which is not provenance. Say so instead of claiming a pass. + skip_check "cli-belongs-to-checkout" "not a git checkout — cannot establish which tree the CLI was built from" +else -echo "" | tee -a "$LOG" -echo "Tier 1 result: $PASS passed, $FAIL failed" | tee -a "$LOG" +case "$CLI_REAL" in + "$REPO_REAL"/*) + echo " PASS cli-belongs-to-checkout" | tee -a "$LOG" + record "$TIER" "cli-belongs-to-checkout" pass "" + ;; + *) + if [ "$VERIFY_ALLOW_CLI_DRIFT" = "1" ]; then + skip_check "cli-belongs-to-checkout" "CLI at $CLI_REAL is outside $REPO_REAL (allowed by env)" + else + echo " FAIL cli-belongs-to-checkout: $CLI_REAL is outside $REPO_REAL" | tee -a "$LOG" + record "$TIER" "cli-belongs-to-checkout" fail "the CLI under test resolves to $CLI_REAL, outside this checkout ($REPO_REAL); a matching version number does not make it this branch's build" + fi + ;; +esac -if [ "$FAIL" -gt 0 ]; then - echo "TIER1_FAIL" >> "$LOG" - exit 1 -else - echo "TIER1_PASS" >> "$LOG" - exit 0 fi + +finish_tier +exit 0 `, }); - // ── Phase 3: Tier 2 — Broker + agent management ───────────────────────── + // ── Phase 3: capability probe ──────────────────────────────────────────── + // + // Tier 5/6 checks need cloud auth and provider CLIs. Probing once up front + // lets every later check state a cause for skipping instead of emitting a + // failure that only means "this sandbox is not logged in" — the exact reason + // `cloud whoami` and `cloud session` were scored as product failures before. - wf.step('tier2-broker-agents', { + wf.step('capabilities', { type: 'deterministic', - dependsOn: ['tier1-cli-health'], + dependsOn: ['provenance'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/tier2.log" -AGENT_NAME="vf-agent-${SUFFIX}" -echo "=== Tier 2: Broker + Agent Management ===" | tee "$LOG" +LOG="${ARTIFACTS}/capabilities.log" +CAPS="${ARTIFACTS}/caps.env" +${ENV_DEFAULTS} +: > "$CAPS" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) +echo "=== Capability probe ===" | tee "$LOG" + +probe() { + _name="$1" + _cmd="$2" + if eval "$_cmd" >/dev/null 2>&1; then + echo "$_name=1" >> "$CAPS" + echo " YES $_name" | tee -a "$LOG" else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + echo "$_name=0" >> "$CAPS" + echo " no $_name" | tee -a "$LOG" fi } -# Broker checks -run_check "broker status shows running" "relay status" "running" -run_check "broker metrics" "relay node metrics" "." -run_check "broker deadletters" "relay node deadletters" "." +probe broker 'relay node status 2>&1 | grep -i running >/dev/null' +probe workspace 'relay workspace active' +probe cloud 'relay cloud whoami' +probe gh 'gh auth status' +probe git_repo 'git rev-parse --git-dir' +probe jq 'command -v jq' +probe slack '[ -n "$CLOUD_API_URL" ] && [ -n "$CLOUD_API_TOKEN" ]' +probe provider_claude 'command -v claude' +probe provider_codex 'command -v codex' +probe provider_opencode 'command -v opencode' +probe provider_gemini 'command -v gemini' + +# Any provider at all is enough for the managed-agent paths. +if grep -q '^provider_.*=1$' "$CAPS"; then + echo "provider_any=1" >> "$CAPS" +else + echo "provider_any=0" >> "$CAPS" +fi -# Agent management -run_check "agent register" "relay agent register '$AGENT_NAME'" "." -run_check "agent list shows registered agent" \ - "relay agent list" "$AGENT_NAME" +cat "$CAPS" +exit 0 +`, + }); -# Workspace (active may 404 if workspace key is stale — just check command runs) -run_check "workspace active" "relay workspace active 2>&1 || true" "." -run_check "workspace list" "relay workspace list" "." + // ── Phase 4: Tier 1 — CLI health and command discovery ─────────────────── -# Node agent list (requires broker running) -run_check "node agent list" "relay node agent list 2>&1 || true" "." + wf.step('tier1-cli-health', { + type: 'deterministic', + dependsOn: ['capabilities'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail -# Fleet (broker only — no cloud state modified) -run_check "fleet status" "relay fleet status" "." -run_check "fleet nodes" "relay fleet nodes" "." +TIER="tier1" +LOG="${ARTIFACTS}/tier1.log" +${PRELUDE} + +echo "=== Tier 1: CLI Health and Command Discovery ===" | tee "$LOG" + +run_check "relay version" "relay version" "[0-9]\+\.[0-9]" +run_check "relay --help" "relay --help" "Usage" +run_check "relay status" "relay status" "." +run_check "relay telemetry status" "relay telemetry status" "." +run_check "relay workspace list" "relay workspace list" "." +run_check "relay update --help" "relay update --help" "Usage" +run_check "relay uninstall --help" "relay uninstall --help" "Usage" + +# Command-group discovery: a missing group means a whole feature category is +# unreachable, which no individual check below would attribute correctly. +for group in node cloud fleet workspace agent channel message integration capabilities skills reflex telemetry mcp; do + run_check "group discovery: $group" "relay $group --help" "Usage" +done -# Cloud auth reads (already logged in — no browser needed) -run_check "cloud whoami" "relay cloud whoami" "." -run_check "cloud session" "relay cloud session" "." +run_check "node subcommands" "relay node --help" "up" +run_check "node agent subcommands" "relay node agent --help" "spawn" +run_check "node workflow subcommands" "relay node workflow --help" "run" +run_check "message dm subcommands" "relay message dm --help" "send" +run_check "message inbox subcommands" "relay message inbox --help" "check" +run_check "integration webhook subs" "relay integration webhook --help" "create" -# Cleanup -relay agent remove "$AGENT_NAME" 2>/dev/null || true +finish_tier +exit 0 +`, + }); -echo "" | tee -a "$LOG" -echo "Tier 2 result: $PASS passed, $FAIL failed" | tee -a "$LOG" + // ── Phase 5: Tier 2 — broker lifecycle and node agents ─────────────────── -if [ "$FAIL" -gt 0 ]; then - echo "TIER2_FAIL" >> "$LOG" - exit 1 -else - echo "TIER2_PASS" >> "$LOG" - exit 0 -fi + wf.step('tier2-broker', { + type: 'deterministic', + dependsOn: ['tier1-cli-health'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="tier2" +LOG="${ARTIFACTS}/tier2.log" +${PRELUDE} + +echo "=== Tier 2: Broker Lifecycle and Node Agents ===" | tee "$LOG" + +run_check "node status shows running" "relay node status" "running" +run_check "node metrics" "relay node metrics" "." +run_check "node deadletters" "relay node deadletters" "." +run_check "node deadletters --json" "relay node deadletters --json" "." +run_check "node agent list" "relay node agent list" "." +run_check "fleet status" "relay fleet status" "." + +# 'relay node tail' streams until interrupted; bound it and accept a timeout +# kill as success. Without the bound this step hangs until the run times out. +# Only a clean exit (0) or the timeout's own kill (124/137) counts. The old +# "-le 124" also accepted exit 1, so a removed or broken 'node tail' still +# scored PASS — precisely the false green this file exists to prevent. +run_check "node tail (bounded)" \ + "timeout 5 relay node tail >/dev/null 2>&1; _rc=\$?; case \$_rc in 0|124|137) echo tail-ok ;; *) echo \"tail-failed rc=\$_rc\" ;; esac" "tail-ok" + +# Redeliver against an empty queue is a no-op, so this exercises the command +# path without mutating real delivery state. +run_check "node redeliver --all (empty queue)" \ + "relay node redeliver --all 2>&1 || true" "." + +finish_tier +exit 0 `, }); - // ── Phase 4: Tier 3 — Channel messaging round-trip ────────────────────── + // ── Phase 6: Tier 3 — channels, messages, threads, reactions, inbox ────── - wf.step('tier3-channel-messaging', { + wf.step('tier3-messaging', { type: 'deterministic', - dependsOn: ['tier2-broker-agents'], + dependsOn: ['tier2-broker'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 +TIER="tier3" LOG="${ARTIFACTS}/tier3.log" +${PRELUDE} + AGENT="vf-msg-${SUFFIX}" CHANNEL="vf-channel-${SUFFIX}" -MSG_TEXT="verify-$(date +%s)" -echo "=== Tier 3: Channel Messaging Round-Trip ===" | tee "$LOG" +MSG_TEXT="verify-msg-${SUFFIX}" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +echo "=== Tier 3: Channels, Messages, Threads, Reactions, Inbox ===" | tee "$LOG" -# Register a test agent and get its token -TOKEN=$(relay agent register "$AGENT" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +# Parse the JSON token field rather than matching any long-ish string: +# "agent register" on an existing name prints that the agent already exists, +# and a 20+ char agent name matched the old heuristic, so a failed registration +# silently yielded the agent own name as its token. Every downstream call then +# failed for reasons that had nothing to do with the feature under test. +TOKEN=$(relay agent register "$AGENT" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -z "$TOKEN" ]; then + record "$TIER" "agent register + token" fail "no token returned from relay agent register" echo " FAIL agent register (no token returned)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -else - echo " PASS agent register + token" | tee -a "$LOG" - PASS=$((PASS + 1)) + echo " Remaining tier 3 checks cannot run without an agent token." | tee -a "$LOG" + for remaining in "channel create" "channel list" "channel join" "channel set_topic" \ + "message post" "message list" "message reply" "message get_thread" \ + "reaction add" "reaction remove" "inbox check" "inbox mark_read" \ + "message search" "channel leave" "channel archive"; do + skip_check "$remaining" "agent registration failed — no token" + done + finish_tier + exit 0 fi + +echo " PASS agent register + token" | tee -a "$LOG" +record "$TIER" "agent register + token" pass "" export RELAY_AGENT_TOKEN="$TOKEN" -# Channel operations (token passed via RELAY_AGENT_TOKEN env var) -run_check "channel create" "relay channel create '$CHANNEL'" "." -run_check "channel list" "relay channel list" "$CHANNEL" -run_check "channel join" "relay channel join '$CHANNEL'" "." -run_check "channel set_topic" "relay channel set_topic '$CHANNEL' 'verify topic'" "." +run_check "agent list shows agent" "relay agent list" "$AGENT" -# Message operations (positional args: ) -run_check "message post" "relay message post '$CHANNEL' '$MSG_TEXT'" "." -run_check "message list" "relay message list '$CHANNEL' --limit 5" "$MSG_TEXT" +run_check "channel create" "relay channel create '$CHANNEL'" "." +run_check "channel list" "relay channel list" "$CHANNEL" +run_check "channel join" "relay channel join '$CHANNEL'" "." +run_check "channel set_topic" "relay channel set_topic '$CHANNEL' 'verify topic'" "." + +run_check "message post" "relay message post '$CHANNEL' '$MSG_TEXT'" "." +# Asserting the posted text reads back is the actual round-trip proof; a bare +# "command exited 0" would pass even if the message were dropped. +run_check "message list round-trip" "relay message list '$CHANNEL' --limit 5" "$MSG_TEXT" -# Get the message id for thread operations MSG_ID=$(relay message list "$CHANNEL" 2>/dev/null | grep -o '"messageId": *"[^"]*"' | head -1 | sed 's/.*"messageId": *"//;s/".*//' || echo "") if [ -n "$MSG_ID" ]; then run_check "message reply" "relay message reply '$MSG_ID' 'thread-reply-${SUFFIX}'" "." - run_check "message get_thread" "relay message get_thread '$MSG_ID'" "." - run_check "reaction add" "relay message reaction add '$MSG_ID' thumbsup" "." - run_check "reaction remove" "relay message reaction remove '$MSG_ID' thumbsup" "." - run_check "inbox check" "relay message inbox check" "." - run_check "inbox mark_read" "relay message inbox mark_read '$MSG_ID'" "." + run_check "message get_thread" "relay message get_thread '$MSG_ID'" "thread-reply-${SUFFIX}" + run_check "reaction add" "relay message reaction add '$MSG_ID' thumbsup" "." + run_check "reaction remove" "relay message reaction remove '$MSG_ID' thumbsup" "." + run_check "inbox check" "relay message inbox check" "." + run_check "inbox mark_read" "relay message inbox mark_read '$MSG_ID'" "." else - echo " SKIP thread/reaction/inbox checks (no message id)" | tee -a "$LOG" + for remaining in "message reply" "message get_thread" "reaction add" "reaction remove" \ + "inbox check" "inbox mark_read"; do + skip_check "$remaining" "could not extract a messageId from message list output" + done fi -run_check "message search" "relay message search '$MSG_TEXT'" "$MSG_TEXT" - -# Channel leave + archive -run_check "channel leave" "relay channel leave '$CHANNEL'" "." -run_check "channel archive" "relay channel archive '$CHANNEL'" "." - -# Cleanup -relay agent remove "$AGENT" 2>/dev/null || true - -echo "" | tee -a "$LOG" -echo "Tier 3 result: $PASS passed, $FAIL failed" | tee -a "$LOG" - -if [ "$FAIL" -gt 0 ]; then - echo "TIER3_FAIL" >> "$LOG" - exit 1 +run_check "message search" "relay message search '$MSG_TEXT'" "$MSG_TEXT" +run_check "channel leave" "relay channel leave '$CHANNEL'" "." +run_check "channel archive" "relay channel archive '$CHANNEL'" "." + +# Assert the removal actually took effect rather than assuming it did. A live +# run found "agent remove" failing for any agent that had sent a message, +# leaking a raw SQL error, which left test identities accumulating in the +# workspace while every tier still reported clean. +run_check "agent remove" "relay agent remove '$AGENT'" "." +if relay agent list 2>&1 | grep "\"name\": \"$AGENT\"" >/dev/null; then + echo " FAIL agent gone after remove" | tee -a "$LOG" + record "$TIER" "agent gone after remove" fail "$AGENT is still listed after a remove that reported success" else - echo "TIER3_PASS" >> "$LOG" - exit 0 + echo " PASS agent gone after remove" | tee -a "$LOG" + record "$TIER" "agent gone after remove" pass "" fi + +finish_tier +exit 0 `, }); - // ── Phase 5: Tier 4 — Cross-agent DMs and invite ──────────────────────── + // ── Phase 7: Tier 4 — cross-agent coordination ─────────────────────────── wf.step('tier4-cross-agent', { type: 'deterministic', - dependsOn: ['tier3-channel-messaging'], + dependsOn: ['tier3-messaging'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 +TIER="tier4" LOG="${ARTIFACTS}/tier4.log" +${PRELUDE} + AGENT_A="vf-alice-${SUFFIX}" AGENT_B="vf-bob-${SUFFIX}" CHANNEL="vf-private-${SUFFIX}" -echo "=== Tier 4: Cross-Agent DMs + Channel Invite ===" | tee "$LOG" -run_check() { - local name="$1" - local cmd="$2" - local expect="$3" - if output=$(eval "$cmd" 2>&1) && echo "$output" | grep -qi "$expect"; then - echo " PASS $name" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL $name (expected: $expect)" | tee -a "$LOG" - echo " output: $(echo "$output" | head -3)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi -} +echo "=== Tier 4: Cross-Agent DMs, Invites, Read Receipts, Files ===" | tee "$LOG" -# Register two agents -TOKEN_A=$(relay agent register "$AGENT_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_B=$(relay agent register "$AGENT_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") +TOKEN_A=$(relay agent register "$AGENT_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_B=$(relay agent register "$AGENT_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") if [ -z "$TOKEN_A" ] || [ -z "$TOKEN_B" ]; then - echo " FAIL register two agents (tokens missing)" | tee -a "$LOG" - FAIL=$((FAIL + 2)) -else - echo " PASS register agent A ($AGENT_A)" | tee -a "$LOG" - echo " PASS register agent B ($AGENT_B)" | tee -a "$LOG" - PASS=$((PASS + 2)) + record "$TIER" "register two agents" fail "token A empty=$([ -z "$TOKEN_A" ] && echo yes || echo no), token B empty=$([ -z "$TOKEN_B" ] && echo yes || echo no)" + echo " FAIL register two agents" | tee -a "$LOG" + for remaining in "dm send A to B" "dm list" "channel invite" "dm send_group" \ + "inbox get_readers" "message file upload"; do + skip_check "$remaining" "two-agent registration failed" + done + finish_tier + exit 0 fi -# DM: A sends to B (capture conversationId for dm list) -DM_TEXT="dm-verify-$(date +%s)" -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CHANNEL" 2>/dev/null || true -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CHANNEL" 2>/dev/null || true +echo " PASS register agent A ($AGENT_A)" | tee -a "$LOG" +echo " PASS register agent B ($AGENT_B)" | tee -a "$LOG" +record "$TIER" "register agent A" pass "" +record "$TIER" "register agent B" pass "" -if RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "$AGENT_B" "$DM_TEXT" > /tmp/dm-send-out.txt 2>&1; then - DM_SEND_OUT=$(cat /tmp/dm-send-out.txt) - echo " PASS dm send A→B" | tee -a "$LOG" - PASS=$((PASS + 1)) +DM_TEXT="dm-verify-${SUFFIX}" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CHANNEL" >/dev/null 2>&1 || true +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CHANNEL" >/dev/null 2>&1 || true + +DM_OUT_FILE="${ARTIFACTS}/dm-send.out" +if RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "$AGENT_B" "$DM_TEXT" > "$DM_OUT_FILE" 2>&1; then + echo " PASS dm send A to B" | tee -a "$LOG" + record "$TIER" "dm send A to B" pass "" else - DM_SEND_OUT=$(cat /tmp/dm-send-out.txt) - echo " FAIL dm send A→B (exit code non-zero)" | tee -a "$LOG" - echo " output: $DM_SEND_OUT" >> "$LOG" - FAIL=$((FAIL + 1)) + echo " FAIL dm send A to B" | tee -a "$LOG" + record "$TIER" "dm send A to B" fail "$(cat "$DM_OUT_FILE")" fi -CONV_ID=$(echo "$DM_SEND_OUT" | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") -# B lists DMs using conversationId from send response +DM_SEND_OUT=$(cat "$DM_OUT_FILE" 2>/dev/null || echo "") +CONV_ID=$(printf '%s' "$DM_SEND_OUT" | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") +DM_ID=$(printf '%s' "$DM_SEND_OUT" | grep -oE '"(messageId|id)": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + if [ -n "$CONV_ID" ]; then - if RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV_ID" 2>&1 | grep -qi "."; then - echo " PASS dm list (B sees DM from A)" | tee -a "$LOG" - PASS=$((PASS + 1)) - else - echo " FAIL dm list (B could not list conversation $CONV_ID)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) - fi + # B must see A's exact text, not merely a listable conversation. + run_check "dm list (B sees A's text)" \ + "RELAY_AGENT_TOKEN='$TOKEN_B' relay message dm list '$CONV_ID'" "$DM_TEXT" else - echo " FAIL dm list (no conversationId in send response — cannot verify receipt)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "dm list (B sees A's text)" "no conversationId in dm send response" fi -# Channel invite B into A's channel -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite "$CHANNEL" "$AGENT_B" 2>&1 | tee -a "$LOG" > /tmp/invite-out.txt -if grep -qi "." /tmp/invite-out.txt; then - echo " PASS channel invite B into private channel" | tee -a "$LOG" - PASS=$((PASS + 1)) +run_check "channel invite B into private channel" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay channel invite '$CHANNEL' '$AGENT_B'" "." +run_check "dm send_group" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message dm send_group 'group-dm-${SUFFIX}' --to '$AGENT_B'" "." + +if [ -n "$DM_ID" ]; then + run_check "inbox mark_read (B)" \ + "RELAY_AGENT_TOKEN='$TOKEN_B' relay message inbox mark_read '$DM_ID'" "." + run_check "inbox get_readers (A sees B)" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message inbox get_readers '$DM_ID'" "$AGENT_B" else - echo " FAIL channel invite B into private channel" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "inbox mark_read (B)" "no message id in dm send response" + skip_check "inbox get_readers (A sees B)" "no message id in dm send response" fi -# Group DM (positional , then --to ) -RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group-dm-$(date +%s)" --to "$AGENT_B" 2>&1 | grep -qi "." && { - echo " PASS dm send_group" | tee -a "$LOG" - PASS=$((PASS + 1)) -} || { - echo " FAIL dm send_group" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -} - -# Cleanup -RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CHANNEL" 2>/dev/null || true +UPLOAD_FILE="${ARTIFACTS}/upload-fixture.txt" +echo "verify-upload-${SUFFIX}" > "$UPLOAD_FILE" +# Signature is "upload --channel " — the channel is an option, +# not a positional. A live run caught this check passing the channel first. +# --text is passed explicitly even though --help documents it as defaulting to +# "": the API rejects an empty text with "text is required", so the documented +# default cannot succeed. Tracked as a product finding, not worked around +# silently — the check exercises the path that actually works. +run_check "message file upload" \ + "RELAY_AGENT_TOKEN='$TOKEN_A' relay message file upload '$UPLOAD_FILE' --channel '$CHANNEL' --text 'verify upload ${SUFFIX}'" "." + +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CHANNEL" >/dev/null 2>&1 || true relay agent remove "$AGENT_A" 2>/dev/null || true relay agent remove "$AGENT_B" 2>/dev/null || true -echo "" | tee -a "$LOG" -echo "Tier 4 result: $PASS passed, $FAIL failed" | tee -a "$LOG" - -if [ "$FAIL" -gt 0 ]; then - echo "TIER4_FAIL" >> "$LOG" - exit 1 -else - echo "TIER4_PASS" >> "$LOG" - exit 0 -fi +finish_tier +exit 0 `, }); - // ── Phase 6: Critical paths ────────────────────────────────────────────── + // ── Phase 8: Tier 5 — cloud, fleet, integrations, skills, reflex ───────── // - // Runs the 4 non-cloud critical paths from critical-paths.md in sequence. - // If any critical path fails, the whole check is FAIL regardless of tiers. + // Everything here needs real cloud auth. Each check is gated so an + // unauthenticated sandbox produces SKIPs with a cause rather than a wall of + // failures that say nothing about the product. - wf.step('critical-paths', { + wf.step('tier5-cloud', { type: 'deterministic', dependsOn: ['tier4-cross-agent'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail -PASS=0 -FAIL=0 -LOG="${ARTIFACTS}/critical-paths.log" -SUFFIX2="cp-${SUFFIX}" -echo "=== Critical Paths ===" | tee "$LOG" -echo "See .agentworkforce/features/critical-paths.md for definitions." | tee -a "$LOG" -echo "" | tee -a "$LOG" +TIER="tier5" +LOG="${ARTIFACTS}/tier5.log" +${PRELUDE} -# ── Critical Path 1: Broker + Agent Registration ────────────────────────── -echo "-- CP1: Broker + Agent Registration --" | tee -a "$LOG" -CP1=0 +echo "=== Tier 5: Cloud, Fleet, Integrations, Skills, Reflex ===" | tee "$LOG" -relay status 2>&1 | grep -qi "running" && { - echo " PASS broker is running" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || { - echo " FAIL broker is not running" | tee -a "$LOG" -} +if ! have_cap cloud; then + echo " Cloud capability absent — every tier 5 check is a stated SKIP." | tee -a "$LOG" +fi -CP1_AGENT="cp1-${SUFFIX2}" -relay agent register "$CP1_AGENT" 2>&1 | grep -qi "." && { - echo " PASS agent register" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || echo " FAIL agent register" | tee -a "$LOG" +gated_check cloud "cloud whoami" "relay cloud whoami" "." +gated_check cloud "cloud session" "relay cloud session" "." +skip_check "cloud status" "takes a required ; no disposable cloud run is made (see the cloud run skip)" +gated_check cloud "cloud schedules" "relay cloud schedules" "." +skip_check "cloud logs" "takes a required ; no disposable cloud run is made (see the cloud run skip)" +gated_check cloud "cloud sync --help" "relay cloud sync --help" "Usage" + +# Deliberately not exercised: 'cloud run' bills real sandbox time, and +# cloud login/connect/enroll need an interactive browser. Recorded as +# skips so the report shows the coverage hole instead of hiding it. +skip_check "cloud run" "spends real cloud budget — excluded from scheduled verification" +skip_check "cloud login" "requires interactive browser auth" +skip_check "cloud connect" "requires interactive browser auth" +skip_check "cloud enroll" "requires interactive browser auth" + +gated_check cloud "fleet nodes" "relay fleet nodes" "." +gated_check cloud "fleet config" "relay fleet config" "." +gated_check cloud "fleet inherit" "relay fleet inherit --help" "Usage" +skip_check "fleet enable" "mutates workspace cloud state" +skip_check "fleet disable" "mutates workspace cloud state" + +# fleet spawn/release were undocumented in the manifest until this change and +# are still unverified: spawning burns provider credits on a remote node. +skip_check "fleet spawn" "spawns a billed remote agent — needs a dedicated fixture node" +skip_check "fleet release" "depends on fleet spawn" + +gated_check cloud "reflex status" "relay reflex status" "." +skip_check "reflex on" "mutates history sync state for the workspace" +skip_check "reflex off" "mutates history sync state for the workspace" + +# The endpoint takes the channel to deliver into and returns a generated URL, +# and the channel must already exist, so create it before registering the hook. +WEBHOOK_CHANNEL="vf-hook-${SUFFIX}" +WEBHOOK_AGENT="vf-hookagent-${SUFFIX}" +WEBHOOK_TOKEN=$(relay agent register "$WEBHOOK_AGENT" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || true) +RELAY_AGENT_TOKEN="$WEBHOOK_TOKEN" relay channel create "$WEBHOOK_CHANNEL" >/dev/null 2>&1 || true +if have_cap cloud; then + run_check "integration webhook list" "relay integration webhook list" "." + run_check "integration subscription list" "relay integration subscription list" "." + # create returns the generated webhook; trigger and delete take its id, so the + # id has to be carried between them rather than reusing the channel name. + WEBHOOK_OUT="${ARTIFACTS}/webhook-create.json" + if relay integration webhook create "$WEBHOOK_CHANNEL" > "$WEBHOOK_OUT" 2>&1; then + echo " PASS integration webhook create" | tee -a "$LOG" + record "$TIER" "integration webhook create" pass "" + WEBHOOK_ID=$(grep -o '"webhookId": *"[^"]*"' "$WEBHOOK_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || true) + if [ -n "$WEBHOOK_ID" ]; then + run_check "integration webhook trigger" "relay integration webhook trigger '$WEBHOOK_ID' 2>&1 || true" "." + run_check "integration webhook delete" "relay integration webhook delete '$WEBHOOK_ID'" "." + else + skip_check "integration webhook trigger" "no webhookId in create response" + skip_check "integration webhook delete" "no webhookId in create response" + fi + else + record "$TIER" "integration webhook create" fail "relay integration webhook create exited non-zero" + echo " FAIL integration webhook create" | tee -a "$LOG" + skip_check "integration webhook trigger" "webhook create failed" + skip_check "integration webhook delete" "webhook create failed" + fi +else + for remaining in "integration webhook list" "integration subscription list" \ + "integration webhook create" "integration webhook trigger" "integration webhook delete"; do + skip_check "$remaining" "requires capability: cloud" + done +fi -relay agent list 2>&1 | grep -qi "$CP1_AGENT" && { - echo " PASS agent list shows registered agent" | tee -a "$LOG" - CP1=$((CP1 + 1)) -} || echo " FAIL agent list did not show agent" | tee -a "$LOG" +RELAY_AGENT_TOKEN="$WEBHOOK_TOKEN" relay channel archive "$WEBHOOK_CHANNEL" >/dev/null 2>&1 || true +relay agent remove "$WEBHOOK_AGENT" >/dev/null 2>&1 || true -relay agent remove "$CP1_AGENT" 2>/dev/null || true +gated_check cloud "capabilities list" "relay capabilities list" "." +gated_check cloud "skills add --help" "relay skills add --help" "Usage" -if [ "$CP1" -eq 3 ]; then - echo " PASS CP1 complete" | tee -a "$LOG" - PASS=$((PASS + 1)) -else - echo " FAIL CP1 incomplete ($CP1/3 checks passed)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -fi - -echo "" | tee -a "$LOG" - -# ── Critical Path 2: Channel Messaging ──────────────────────────────────── -echo "-- CP2: Channel Messaging Round-Trip --" | tee -a "$LOG" -CP2=0 -CP2_AGENT="cp2-${SUFFIX2}" -CP2_CH="cp2-ch-${SUFFIX2}" -CP2_MSG="cp2-msg-$(date +%s)" - -TOKEN_CP2=$(relay agent register "$CP2_AGENT" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -if [ -n "$TOKEN_CP2" ]; then - CP2=$((CP2 + 1)) - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel create "$CP2_CH" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel join "$CP2_CH" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay message post "$CP2_CH" "$CP2_MSG" 2>&1 | grep -qi "." && CP2=$((CP2 + 1)) || true - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay message list "$CP2_CH" --limit 3 2>&1 | grep -qi "$CP2_MSG" && { - echo " PASS posted message appears in list" | tee -a "$LOG" - CP2=$((CP2 + 1)) - } || echo " FAIL posted message not in list" | tee -a "$LOG" - RELAY_AGENT_TOKEN="$TOKEN_CP2" relay channel archive "$CP2_CH" 2>/dev/null || true - relay agent remove "$CP2_AGENT" 2>/dev/null || true -fi - -if [ "$CP2" -ge 4 ]; then - echo " PASS CP2 complete ($CP2/5 checks)" | tee -a "$LOG" - PASS=$((PASS + 1)) +finish_tier +exit 0 +`, + }); + + // ── Phase 9: Tier 6 — managed agents, harnesses, PTY, SDK ──────────────── + + wf.step('tier6-harnesses', { + type: 'deterministic', + dependsOn: ['tier5-cloud'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="tier6" +LOG="${ARTIFACTS}/tier6.log" +${PRELUDE} + +echo "=== Tier 6: Managed Agents, Harnesses, PTY, SDK ===" | tee "$LOG" + +# Discovery works without credentials and catches a removed subcommand. +run_check "node agent spawn --help" "relay node agent spawn --help" "Usage" +run_check "node agent release --help" "relay node agent release --help" "Usage" +run_check "node agent attach --help" "relay node agent attach --help" "Usage" +run_check "node agent set-model --help" "relay node agent set-model --help" "Usage" +run_check "node agent new --help" "relay node agent new --help" "Usage" +run_check "node agent message hold --help" "relay node agent message hold --help" "Usage" + +PROVIDER="" +for candidate in claude codex opencode gemini; do + if command -v "$candidate" >/dev/null 2>&1; then + PROVIDER="$candidate" + break + fi +done + +if [ -z "$PROVIDER" ]; then + echo " No provider CLI on PATH — managed-agent lifecycle is a stated SKIP." | tee -a "$LOG" + for remaining in "node agent spawn" "node agent list shows worker" \ + "node agent message hold" "node agent message auto" "node agent release"; do + skip_check "$remaining" "no provider CLI (claude/codex/opencode/gemini) on PATH" + done else - echo " FAIL CP2 incomplete ($CP2/5 checks)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) -fi - -echo "" | tee -a "$LOG" - -# ── Critical Path 3: MCP Server Starts ──────────────────────────────────── -echo "-- CP3: MCP Server --" | tee -a "$LOG" -# We can only verify the MCP server starts without crashing on startup -# Full tool invocation requires a connected harness (tier 6) -# Capture output first to avoid pipefail interaction with grep -MCP_HELP=$(relay mcp --help 2>&1 || true) -echo " mcp help output: $(echo "$MCP_HELP" | head -1)" | tee -a "$LOG" -if echo "$MCP_HELP" | grep -qiE "mcp|server|stdio|relay|Usage"; then - echo " PASS relay mcp --help responds" | tee -a "$LOG" - PASS=$((PASS + 1)) + echo " Using provider: $PROVIDER" | tee -a "$LOG" + WORKER="vf-worker-${SUFFIX}" + if relay node agent spawn "$PROVIDER" --name "$WORKER" \ + --task 'Reply verify-ok and exit' --spawn-mode task-exit --exit-after-task >/dev/null 2>&1; then + echo " PASS node agent spawn" | tee -a "$LOG" + record "$TIER" "node agent spawn" pass "" + run_check "node agent list shows worker" "relay node agent list" "$WORKER" + run_check "node agent message hold" "relay node agent message hold '$WORKER'" "." + run_check "node agent message auto" "relay node agent message auto '$WORKER'" "." + run_check "node agent release" "relay node agent release '$WORKER'" "." + else + record "$TIER" "node agent spawn" fail "spawn of provider $PROVIDER exited non-zero" + echo " FAIL node agent spawn ($PROVIDER)" | tee -a "$LOG" + for remaining in "node agent list shows worker" "node agent message hold" \ + "node agent message auto" "node agent release"; do + skip_check "$remaining" "spawn failed" + done + fi +fi + +# SDK: presence of the package manifest only. A full SDK round trip belongs in +# the SDK package's own tests, not in a CLI feature sweep. +if [ -d packages/sdk ]; then + run_check "sdk package resolves" "test -f packages/sdk/package.json && echo sdk-ok" "sdk-ok" else - echo " FAIL relay mcp --help returned no recognizable output" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + skip_check "sdk package resolves" "packages/sdk not present in this checkout" fi -echo "" | tee -a "$LOG" +skip_check "browser primitive" "requires a provisioned browser session (tier 6 interactive)" +skip_check "python sdk" "verified in the python-sdk repo test suite" +skip_check "swift sdk" "verified in the swift-sdk repo test suite" -# ── Critical Path 4: DM Cross-Agent ─────────────────────────────────────── -echo "-- CP4: DM Between Two Agents --" | tee -a "$LOG" -CP4=0 -CP4_A="cp4a-${SUFFIX2}" -CP4_B="cp4b-${SUFFIX2}" +finish_tier +exit 0 +`, + }); + + // ── Phase 10: critical paths 1-6 ───────────────────────────────────────── -TOKEN_CP4A=$(relay agent register "$CP4_A" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") -TOKEN_CP4B=$(relay agent register "$CP4_B" 2>&1 | grep -oE '[A-Za-z0-9_-]{20,}' | tail -1 || echo "") + wf.step('critical-paths', { + type: 'deterministic', + dependsOn: ['tier6-harnesses'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +TIER="critical-paths" +LOG="${ARTIFACTS}/critical-paths.log" +${PRELUDE} + +echo "=== Critical Paths (see .agentworkforce/features/critical-paths.md) ===" | tee "$LOG" + +# ── CP1: local broker lifecycle ────────────────────────────────────────── +# Uses an isolated state dir so bringing the broker down cannot disturb the +# broker the remaining paths depend on. +echo "-- CP1: Local Broker Lifecycle --" | tee -a "$LOG" +CP1_STATE="${ARTIFACTS}/cp1-state" +mkdir -p "$CP1_STATE" +CP1=0 +relay node up --background --no-spawn --state-dir "$CP1_STATE" >/dev/null 2>&1 || true +sleep 2 +relay node status --state-dir "$CP1_STATE" 2>&1 | grep -i running >/dev/null && CP1=$((CP1 + 1)) +relay node metrics >/dev/null 2>&1 && CP1=$((CP1 + 1)) +relay node deadletters --json >/dev/null 2>&1 && CP1=$((CP1 + 1)) +relay node down --state-dir "$CP1_STATE" >/dev/null 2>&1 || true +sleep 2 +# "grep -v" succeeds when ANY line lacks the pattern, so a multi-line status +# that still says running satisfied it. Negate the match over the whole output. +if ! relay node status --state-dir "$CP1_STATE" 2>&1 | grep -i running >/dev/null; then + CP1=$((CP1 + 1)) +fi +if [ "$CP1" -ge 4 ]; then + echo " PASS CP1 ($CP1/4)" | tee -a "$LOG" + record "$TIER" "CP1 broker lifecycle" pass "" +else + echo " FAIL CP1 ($CP1/4)" | tee -a "$LOG" + record "$TIER" "CP1 broker lifecycle" fail "only $CP1 of 4 lifecycle assertions held" +fi -if [ -n "$TOKEN_CP4A" ] && [ -n "$TOKEN_CP4B" ]; then - CP4=$((CP4 + 1)) - DM_MSG="dm-cp4-$(date +%s)" - if RELAY_AGENT_TOKEN="$TOKEN_CP4A" relay message dm send "$CP4_B" "$DM_MSG" > /tmp/cp4-dm-out.txt 2>&1; then - CP4=$((CP4 + 1)) +# ── CP2: cross-agent channel message ──────────────────────────────────── +echo "-- CP2: Cross-Agent Channel Message --" | tee -a "$LOG" +CP2_A="cp2a-${SUFFIX}" +CP2_B="cp2b-${SUFFIX}" +CP2_CH="cp2-ch-${SUFFIX}" +CP2_MSG="cp2-msg-${SUFFIX}" +TOKEN_2A=$(relay agent register "$CP2_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_2B=$(relay agent register "$CP2_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +if [ -n "$TOKEN_2A" ] && [ -n "$TOKEN_2B" ]; then + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel create "$CP2_CH" >/dev/null 2>&1 || true + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel invite "$CP2_CH" "$CP2_B" >/dev/null 2>&1 || true + RELAY_AGENT_TOKEN="$TOKEN_2A" relay message post "$CP2_CH" "$CP2_MSG" >/dev/null 2>&1 || true + if RELAY_AGENT_TOKEN="$TOKEN_2B" relay message list "$CP2_CH" --limit 10 2>&1 | grep "$CP2_MSG" >/dev/null; then + echo " PASS CP2 (B read A's exact text)" | tee -a "$LOG" + record "$TIER" "CP2 cross-agent channel message" pass "" else - echo " dm send output: $(cat /tmp/cp4-dm-out.txt)" >> "$LOG" + echo " FAIL CP2 (B did not see A's text)" | tee -a "$LOG" + record "$TIER" "CP2 cross-agent channel message" fail "agent B could not read the message agent A posted to $CP2_CH" fi - CONV4_ID=$(cat /tmp/cp4-dm-out.txt | grep -o '"conversationId": *"[^"]*"' | head -1 | sed 's/.*"conversationId": *"//;s/".*//' || echo "") - if [ -n "$CONV4_ID" ]; then - RELAY_AGENT_TOKEN="$TOKEN_CP4B" relay message dm list "$CONV4_ID" 2>&1 | grep -qi "." && { - echo " PASS B can list DMs" | tee -a "$LOG" - CP4=$((CP4 + 1)) - } || echo " FAIL B cannot list DMs" | tee -a "$LOG" + RELAY_AGENT_TOKEN="$TOKEN_2A" relay channel archive "$CP2_CH" >/dev/null 2>&1 || true +else + skip_check "CP2 cross-agent channel message" "could not register two agents" +fi +relay agent remove "$CP2_A" 2>/dev/null || true +relay agent remove "$CP2_B" 2>/dev/null || true + +# ── CP3: managed local agent ──────────────────────────────────────────── +echo "-- CP3: Managed Local Agent --" | tee -a "$LOG" +CP3_PROVIDER="" +for candidate in claude codex opencode gemini; do + command -v "$candidate" >/dev/null 2>&1 && CP3_PROVIDER="$candidate" && break +done +if [ -z "$CP3_PROVIDER" ]; then + skip_check "CP3 managed local agent" "no provider CLI on PATH; provider cost makes this a pre-provisioned check" +else + CP3_WORKER="cp3-${SUFFIX}" + if relay node agent spawn "$CP3_PROVIDER" --name "$CP3_WORKER" \ + --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task >/dev/null 2>&1 \ + && relay node agent list 2>&1 | grep "$CP3_WORKER" >/dev/null; then + relay node agent message hold "$CP3_WORKER" >/dev/null 2>&1 || true + relay node agent message auto "$CP3_WORKER" >/dev/null 2>&1 || true + relay node agent release "$CP3_WORKER" >/dev/null 2>&1 || true + echo " PASS CP3" | tee -a "$LOG" + record "$TIER" "CP3 managed local agent" pass "" else - echo " FAIL dm list (no conversationId in send response)" | tee -a "$LOG" + echo " FAIL CP3" | tee -a "$LOG" + record "$TIER" "CP3 managed local agent" fail "spawn or list did not show worker $CP3_WORKER with provider $CP3_PROVIDER" fi fi -relay agent remove "$CP4_A" 2>/dev/null || true -relay agent remove "$CP4_B" 2>/dev/null || true - -if [ "$CP4" -ge 2 ]; then - echo " PASS CP4 complete" | tee -a "$LOG" - PASS=$((PASS + 1)) +# ── CP4: MCP stdio round trip ─────────────────────────────────────────── +# A real JSON-RPC handshake. The old version only ran 'relay mcp --help', +# which proves argument parsing and nothing about the server. +echo "-- CP4: MCP Stdio Round Trip --" | tee -a "$LOG" +MCP_OUT="${ARTIFACTS}/cp4-mcp.json" +cat > "${ARTIFACTS}/cp4-mcp.mjs" <<'MCPEOF' +import { spawn } from 'node:child_process'; + +const child = spawn('relay', ['mcp'], { stdio: ['pipe', 'pipe', 'pipe'] }); +let buffer = ''; +let done = false; + +const finish = (payload) => { + if (done) return; + done = true; + child.kill('SIGTERM'); + console.log(JSON.stringify(payload)); + process.exit(0); +}; + +const timer = setTimeout(() => finish({ ok: false, error: 'timeout waiting for tools/list' }), 25000); +timer.unref(); + +child.stdout.on('data', (chunk) => { + buffer += chunk.toString(); + for (const line of buffer.split('\n')) { + const trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + const msg = JSON.parse(trimmed); + if (msg.id === 2 && msg.result?.tools) { + clearTimeout(timer); + finish({ ok: true, tools: msg.result.tools.map((t) => t.name).sort() }); + } + } catch { + // partial frame + } + } +}); +child.on('error', (err) => finish({ ok: false, error: err.message })); +child.on('exit', (code) => finish({ ok: false, error: 'mcp server exited with code ' + code })); + +const send = (msg) => child.stdin.write(JSON.stringify(msg) + '\n'); +send({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'verify-features', version: '1.0.0' }, + }, +}); +send({ jsonrpc: '2.0', method: 'notifications/initialized' }); +send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }); +MCPEOF + +if node "${ARTIFACTS}/cp4-mcp.mjs" > "$MCP_OUT" 2>/dev/null && grep -q '"ok":true' "$MCP_OUT"; then + TOOL_COUNT=$(grep -o '"' "$MCP_OUT" | wc -l | tr -d ' ') + # Assert the tools we depend on are actually listed, not just that some + # list came back. + MCP_MISSING="" + for tool in register_agent create_channel post_message list_messages send_dm check_inbox; do + grep -q "\"$tool\"" "$MCP_OUT" || MCP_MISSING="$MCP_MISSING $tool" + done + if [ -z "$MCP_MISSING" ]; then + echo " PASS CP4 (tools/list returned the required tool set)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" pass "" + else + echo " FAIL CP4 (missing tools:$MCP_MISSING)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" fail "tools/list is missing:$MCP_MISSING" + fi else - echo " FAIL CP4 incomplete ($CP4/3 checks)" | tee -a "$LOG" - FAIL=$((FAIL + 1)) + echo " FAIL CP4 (no tools/list response)" | tee -a "$LOG" + record "$TIER" "CP4 mcp stdio round trip" fail "$(head -c 300 "$MCP_OUT" 2>/dev/null || echo 'no output from mcp probe')" fi -echo "" | tee -a "$LOG" -echo "Critical paths: $PASS/4 passed" | tee -a "$LOG" +# ── CP5: local workflow run, logs, sync ───────────────────────────────── +echo "-- CP5: Local Workflow Run, Logs, Sync --" | tee -a "$LOG" +CP5_DIR="${ARTIFACTS}/cp5" +mkdir -p "$CP5_DIR" +echo 'console.log("critical-workflow-ok")' > "$CP5_DIR/workflow.js" +CP5_JSON=$(relay node workflow run "$CP5_DIR/workflow.js" --json 2>&1 || echo "") +CP5_RUN_ID=$(printf '%s' "$CP5_JSON" | grep -o '"runId": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + +# "workflow run" is asynchronous: it returns status "running" with a run id and +# the sentinel only reaches the log later. Poll the log the way critical-path 5 +# documents (--follow), rather than asserting against the launch response, which +# can never contain the output. +CP5_SEEN=0 +if [ -n "$CP5_RUN_ID" ]; then + for _ in $(seq 1 20); do + if relay node workflow logs "$CP5_RUN_ID" 2>&1 | grep "critical-workflow-ok" >/dev/null; then + CP5_SEEN=1 + break + fi + sleep 1 + done +fi -if [ "$FAIL" -gt 0 ]; then - echo "CRITICAL_PATHS_FAIL" >> "$LOG" - exit 1 +if [ "$CP5_SEEN" -eq 1 ]; then + CP5_SYNC=$(relay node workflow sync "$CP5_RUN_ID" --dry-run --json 2>&1 || echo "") + if printf '%s' "$CP5_SYNC" | grep '"status": *"completed"' >/dev/null; then + echo " PASS CP5 (run $CP5_RUN_ID, sentinel + completed)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" pass "" + else + echo " FAIL CP5 (sentinel seen but run did not report completed)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "sync did not report completed: $(printf '%s' "$CP5_SYNC" | head -c 200)" + fi +elif [ -z "$CP5_RUN_ID" ]; then + echo " FAIL CP5 (no runId from workflow run)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "$(printf '%s' "$CP5_JSON" | head -c 300)" else - echo "CRITICAL_PATHS_PASS" >> "$LOG" - exit 0 + echo " FAIL CP5 (sentinel never appeared in the run log)" | tee -a "$LOG" + record "$TIER" "CP5 local workflow lifecycle" fail "run $CP5_RUN_ID produced no critical-workflow-ok within 20s" +fi + +# ── CP6: direct message and read receipt ──────────────────────────────── +echo "-- CP6: Direct Message and Read Receipt --" | tee -a "$LOG" +CP6_A="cp6a-${SUFFIX}" +CP6_B="cp6b-${SUFFIX}" +TOKEN_6A=$(relay agent register "$CP6_A" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +TOKEN_6B=$(relay agent register "$CP6_B" 2>&1 | grep -o '"token": *"[^"]*"' | head -1 | sed 's/.*: *"//;s/".*//' || echo "") +if [ -n "$TOKEN_6A" ] && [ -n "$TOKEN_6B" ]; then + CP6_MSG="cp6-dm-${SUFFIX}" + CP6_OUT="${ARTIFACTS}/cp6-dm.out" + RELAY_AGENT_TOKEN="$TOKEN_6A" relay message dm send "$CP6_B" "$CP6_MSG" > "$CP6_OUT" 2>&1 || true + CP6_CONV=$(grep -o '"conversationId": *"[^"]*"' "$CP6_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + CP6_ID=$(grep -oE '"(messageId|id)": *"[^"]*"' "$CP6_OUT" | head -1 | sed 's/.*: *"//;s/".*//' || echo "") + CP6=0 + if [ -n "$CP6_CONV" ]; then + RELAY_AGENT_TOKEN="$TOKEN_6B" relay message dm list "$CP6_CONV" 2>&1 | grep "$CP6_MSG" >/dev/null && CP6=$((CP6 + 1)) + fi + if [ -n "$CP6_ID" ]; then + RELAY_AGENT_TOKEN="$TOKEN_6B" relay message inbox mark_read "$CP6_ID" >/dev/null 2>&1 && CP6=$((CP6 + 1)) + RELAY_AGENT_TOKEN="$TOKEN_6A" relay message inbox get_readers "$CP6_ID" 2>&1 | grep "$CP6_B" >/dev/null && CP6=$((CP6 + 1)) + fi + if [ "$CP6" -ge 3 ]; then + echo " PASS CP6 ($CP6/3)" | tee -a "$LOG" + record "$TIER" "CP6 dm and read receipt" pass "" + else + echo " FAIL CP6 ($CP6/3)" | tee -a "$LOG" + record "$TIER" "CP6 dm and read receipt" fail "only $CP6 of 3 assertions held (conv=$CP6_CONV id=$CP6_ID)" + fi +else + skip_check "CP6 dm and read receipt" "could not register two agents" fi +relay agent remove "$CP6_A" 2>/dev/null || true +relay agent remove "$CP6_B" 2>/dev/null || true + +finish_tier +exit 0 `, }); - // ── Phase 7: Cleanup (always runs) ────────────────────────────────────── + // ── Phase 11: cleanup ──────────────────────────────────────────────────── wf.step('cleanup', { type: 'deterministic', dependsOn: ['critical-paths'], - captureOutput: false, + captureOutput: true, failOnError: false, - command: ` -# Remove any test agents that may have leaked (best-effort) -relay agent list 2>/dev/null | grep "^vf-\\|^cp[0-9]" | awk '{print $1}' | while read name; do - relay agent remove "$name" 2>/dev/null || true -done - -# Archive any test channels that may have leaked -relay channel list 2>/dev/null | grep "^vf-\\|^cp[0-9]" | awk '{print $1}' | while read name; do - relay channel archive "$name" 2>/dev/null || true -done + command: String.raw` +# Scope teardown to THIS run's suffix. The previous pattern matched the +# substrings "vf"/"cp" anywhere in the listing, so any pre-existing +# workspace agent or channel whose name happened to contain them — or an +# unrelated column of the JSON — would have been removed or archived. +SUFFIX="${SUFFIX}" + +relay agent list 2>/dev/null | grep -oE '"name": "[^"]*"' | sed 's/"name": "//;s/"$//' | + grep -E "^(vf|cp[0-9])[A-Za-z0-9_-]*-$SUFFIX\$" | while read -r name; do + relay agent remove "$name" >/dev/null 2>&1 || true + done + +relay channel list 2>/dev/null | grep -oE '"name": "[^"]*"' | sed 's/"name": "//;s/"$//' | + grep -E "^(vf|cp[0-9])[A-Za-z0-9_-]*-$SUFFIX\$" | while read -r name; do + relay channel archive "$name" >/dev/null 2>&1 || true + done echo "Cleanup complete" +exit 0 `, }); - // ── Phase 8: Collect results ───────────────────────────────────────────── - - wf.step('collect-results', { + // ── Phase 12: verdict ──────────────────────────────────────────────────── + // + // The authoritative aggregation. It exits 0 even on FAIL, on purpose: + // findReadySteps() in @relayflows/core only schedules a step whose + // dependencies are "completed" or "skipped", so a failing gate here would + // permanently block the Slack, issue, and PR steps below — the escalation + // path would silently never run. Enforcement is `enforce-verdict` plus the + // process exit code in main(). + + wf.step('verdict', { type: 'deterministic', dependsOn: ['cleanup'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail ARTIFACTS="${ARTIFACTS}" -SUMMARY="$ARTIFACTS/summary.txt" - -echo "=== Verification Run: ${RUN_ID} ===" > "$SUMMARY" -echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$SUMMARY" -echo "" >> "$SUMMARY" - -for tier in tier1 tier2 tier3 tier4; do - log="$ARTIFACTS/$tier.log" - if [ -f "$log" ]; then - result=$(grep -E "^(TIER|FAIL|PASS)" "$log" | tail -1 || echo "UNKNOWN") - counts=$(grep -E "result:" "$log" | tail -1 || echo "") - echo "[$tier] $result $counts" >> "$SUMMARY" - else - echo "[$tier] NOT_RUN" >> "$SUMMARY" +${ENV_DEFAULTS} + +# Published for the fix chain: attempt-fix is an agent step and cannot read the +# workflow's own constants, so the operator's autofix decision has to reach it +# through the filesystem. +echo "AUTOFIX=${AUTOFIX ? '1' : '0'}" > "$ARTIFACTS/autofix.env" + +node <<'VERDICTEOF' +const fs = require('node:fs'); + +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const ledgerPath = artifacts + '/checks.jsonl'; + +// Every tier that must have produced records. A tier missing from the ledger +// crashed before its first check; reporting that as "clean" is the exact +// failure mode this gate exists to prevent. +const EXPECTED_TIERS = [ + 'provenance', + 'tier1', + 'tier2', + 'tier3', + 'tier4', + 'tier5', + 'tier6', + 'critical-paths', +]; + +let checks = []; +let ledgerError = null; +// A line the reader cannot parse is a check whose result was lost. Counting +// them is the difference between "no failures" and "we could not tell", so +// they are tracked and fail the run rather than being quietly discarded. +let malformedLines = 0; +try { + const raw = fs + .readFileSync(ledgerPath, 'utf8') + .split('\n') + .filter((line) => line.trim().startsWith('{')); + for (const line of raw) { + try { + checks.push(JSON.parse(line)); + } catch { + malformedLines += 1; + } + } +} catch (err) { + ledgerError = err.message; +} + +const byTier = {}; +for (const tier of EXPECTED_TIERS) { + byTier[tier] = { pass: 0, fail: 0, skip: 0, failures: [] }; +} +for (const check of checks) { + const bucket = (byTier[check.tier] ??= { pass: 0, fail: 0, skip: 0, failures: [] }); + if (check.status === 'pass') bucket.pass++; + else if (check.status === 'fail') { + bucket.fail++; + bucket.failures.push({ check: check.check, reason: check.reason }); + } else if (check.status === 'skip') bucket.skip++; +} + +const notRun = EXPECTED_TIERS.filter( + (tier) => byTier[tier].pass + byTier[tier].fail + byTier[tier].skip === 0 +); + +const totals = { pass: 0, fail: 0, skip: 0 }; +for (const bucket of Object.values(byTier)) { + totals.pass += bucket.pass; + totals.fail += bucket.fail; + totals.skip += bucket.skip; +} + +const provenance = {}; +try { + for (const line of fs.readFileSync(artifacts + '/provenance.env', 'utf8').split('\n')) { + const [key, ...rest] = line.split('='); + if (key) provenance[key] = rest.join('='); + } +} catch { + // provenance step did not run; reported via notRun +} + +const reasons = []; +if (ledgerError) reasons.push('check ledger unreadable: ' + ledgerError); +if (notRun.length > 0) reasons.push('tiers produced no records: ' + notRun.join(', ')); +if (totals.fail > 0) reasons.push(totals.fail + ' check(s) failed'); +if (malformedLines > 0) { + reasons.push(malformedLines + ' unparseable ledger line(s) — check results were lost'); +} + +const verdict = { + runId: process.env.VERIFY_RUN_ID, + verdict: reasons.length === 0 ? 'PASS' : 'FAIL', + reasons, + totals, + malformedLines, + tiers: byTier, + tiersNotRun: notRun, + provenance, + generatedAt: new Date().toISOString(), +}; + +fs.writeFileSync(artifacts + '/verdict.json', JSON.stringify(verdict, null, 2)); + +console.log('=== Verdict: ' + verdict.verdict + ' ==='); +console.log( + 'checks: ' + totals.pass + ' passed, ' + totals.fail + ' failed, ' + totals.skip + ' skipped' +); +for (const [tier, bucket] of Object.entries(byTier)) { + const state = bucket.pass + bucket.fail + bucket.skip === 0 ? 'NOT_RUN' : bucket.fail > 0 ? 'FAIL' : 'PASS'; + console.log( + ' [' + tier + '] ' + state + ' ' + bucket.pass + 'p/' + bucket.fail + 'f/' + bucket.skip + 's' + ); + for (const failure of bucket.failures) { + console.log(' FAIL: ' + failure.check + ' — ' + String(failure.reason).slice(0, 160)); + } +} +if (reasons.length > 0) console.log('reasons: ' + reasons.join('; ')); +VERDICTEOF + +exit 0 +`, + }); + + // ── Phase 13: observability — PostHog ──────────────────────────────────── + + wf.step('emit-posthog', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} +${POSTHOG_FN} + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo " [posthog] no verdict.json — nothing to emit" + exit 0 +fi + +# Build the property payloads with node so the JSON is always well-formed. +node <<'PHEOF' > "$ARTIFACTS/posthog-payloads.txt" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const verdict = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const runProps = { + run_id: verdict.runId, + verdict: verdict.verdict, + checks_passed: verdict.totals.pass, + checks_failed: verdict.totals.fail, + checks_skipped: verdict.totals.skip, + tiers_not_run: verdict.tiersNotRun.join(',') || null, + cli_version: verdict.provenance.VERIFY_CLI_VERSION || null, + repo_version: verdict.provenance.VERIFY_REPO_VERSION || null, + git_sha: verdict.provenance.VERIFY_GIT_SHA || null, + reasons: verdict.reasons.join('; ') || null, +}; +console.log('relay_verify_run\t' + JSON.stringify(runProps)); + +// One event per failing check, bounded so a catastrophic run cannot emit +// thousands of events. +const failures = Object.entries(verdict.tiers).flatMap(([tier, bucket]) => + bucket.failures.map((f) => ({ tier, ...f })) +); +for (const failure of failures.slice(0, 50)) { + console.log( + 'relay_verify_check_failed\t' + + JSON.stringify({ + run_id: verdict.runId, + tier: failure.tier, + check: failure.check, + reason: String(failure.reason).slice(0, 300), + cli_version: verdict.provenance.VERIFY_CLI_VERSION || null, + }) + ); +} +PHEOF + +while IFS="$(printf '\t')" read -r event props; do + [ -n "$event" ] || continue + posthog_capture "$event" "$props" +done < "$ARTIFACTS/posthog-payloads.txt" + +exit 0 +`, + }); + + // ── Phase 14: infra escalation to NightCTO ─────────────────────────────── + // + // A broken *harness* is a different failure class from a broken feature. A + // product regression gets an issue and a PR; a harness that cannot run gets + // an operator alert, because otherwise the run just goes quiet and nobody + // notices that verification stopped happening. The pasted run showed exactly + // this class: an expired openai refresh token during bootstrap, logged as + // "non-fatal" and never surfaced to anyone. + + wf.step('escalate-infra', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +RUN_ID="${RUN_ID}" +${ENV_DEFAULTS} +VERIFY_CLI_VERSION=$(grep '^VERIFY_CLI_VERSION=' "$ARTIFACTS/provenance.env" 2>/dev/null | cut -d= -f2 || true) +if [ -z "$VERIFY_CLI_VERSION" ]; then VERIFY_CLI_VERSION="unknown"; fi +${NIGHTCTO_FN} + +ESCALATED=0 + +# 1. The verdict gate itself did not produce a verdict. +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + escalate_infra "relayflow.verify.verdict" "verdict_missing" \ + "verify-features produced no verdict.json — the verification pipeline did not complete" + ESCALATED=1 +fi + +# 2. Tiers that never recorded a single check: harness breakage, not a +# product signal. +if [ -f "$ARTIFACTS/verdict.json" ]; then + NOT_RUN=$(node -e 'const v=require("node:fs").readFileSync(process.argv[1],"utf8");process.stdout.write((JSON.parse(v).tiersNotRun||[]).join(","))' "$ARTIFACTS/verdict.json" 2>/dev/null || echo "") + if [ -n "$NOT_RUN" ]; then + escalate_infra "relayflow.verify.tier_not_run" "tier_not_run" \ + "verify-features tiers produced no records: $NOT_RUN" + ESCALATED=1 fi -done +fi -cp_log="$ARTIFACTS/critical-paths.log" -if [ -f "$cp_log" ]; then - cp_result=$(grep -E "^CRITICAL" "$cp_log" | tail -1 || echo "UNKNOWN") - echo "[critical-paths] $cp_result" >> "$SUMMARY" -else - echo "[critical-paths] NOT_RUN" >> "$SUMMARY" +# 3. The broker never came up, so nothing below tier 1 could be meaningful. +if grep -q "SETUP_FAIL" "$ARTIFACTS/setup.log" 2>/dev/null; then + escalate_infra "relayflow.verify.setup" "broker_start_failed" \ + "verify-features could not start the local broker within 30s" + ESCALATED=1 fi -echo "" >> "$SUMMARY" -echo "Artifact files:" >> "$SUMMARY" -ls -la "$ARTIFACTS/" >> "$SUMMARY" +# 4. No provider CLI at all. Bootstrap logs an expired provider credential as +# "non-fatal" and continues, which silently converts tier 6 and CP3 into +# permanent SKIPs — verification quietly stops covering managed agents. +if grep -q "^provider_any=0$" "$ARTIFACTS/caps.env" 2>/dev/null; then + escalate_infra "relayflow.verify.providers" "no_provider_cli" \ + "verify-features found no provider CLI on PATH — tier 6 and CP3 cannot be verified in this environment" + ESCALATED=1 +fi + +if [ "$ESCALATED" -eq 0 ]; then + echo " No harness-level failures to escalate." +fi -cat "$SUMMARY" +exit 0 `, }); - // ── Phase 9: Post verification report ─────────────────────────────────── + // ── Phase 15: Slack escalation ─────────────────────────────────────────── // - // Posts a structured PASS/FAIL report to #relay-health. - // Must be fast — just read logs and post. No code exploration, no waiting. + // Posts on FAIL through the relayflows Slack primitive, pinned to its + // cloud-relay runtime so delivery always goes via the workspace's Slack + // integration rather than a bot token. When CLOUD_API_URL/CLOUD_API_TOKEN are + // absent the primitive throws auth_token_missing and the step echoes the + // undelivered payload — a silent no-op must never be mistakable for a + // delivered alert. + + wf.step('slack-alert', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} +${SLACK_POST_FN} + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "SLACK_SKIPPED: no verdict.json to report" + exit 0 +fi + +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") + +if [ "$VERDICT" = "PASS" ]; then + echo "SLACK_SKIPPED: verdict is PASS — not alerting" + exit 0 +fi + +node <<'SLACKEOF' > "$ARTIFACTS/slack-message.txt" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const v = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const lines = []; +lines.push(':rotating_light: *Feature verification FAILED* — ${BT}' + v.runId + '${BT}'); +lines.push( + 'CLI ' + + (v.provenance.VERIFY_CLI_VERSION || '?') + + ' / repo ' + + (v.provenance.VERIFY_REPO_VERSION || '?') + + ' @ ' + + (v.provenance.VERIFY_GIT_SHA || '?') +); +lines.push( + v.totals.pass + ' passed, *' + v.totals.fail + ' failed*, ' + v.totals.skip + ' skipped' +); +lines.push(''); +for (const [tier, bucket] of Object.entries(v.tiers)) { + const total = bucket.pass + bucket.fail + bucket.skip; + const state = total === 0 ? 'NOT_RUN' : bucket.fail > 0 ? 'FAIL' : 'PASS'; + lines.push( + '• ${BT}' + tier + '${BT} ' + state + ' — ' + bucket.pass + 'p/' + bucket.fail + 'f/' + bucket.skip + 's' + ); + for (const f of bucket.failures.slice(0, 5)) { + lines.push(' ↳ ' + f.check + ': ' + String(f.reason).slice(0, 180)); + } +} +if (v.tiersNotRun.length > 0) { + lines.push(''); + lines.push('*Tiers that produced no records:* ' + v.tiersNotRun.join(', ')); +} +lines.push(''); +lines.push('Artifacts: ${BT}' + artifacts + '${BT}'); +process.stdout.write(lines.join('\n')); +SLACKEOF + +slack_post "$CHANNEL" "$ARTIFACTS/slack-message.txt" || true +exit 0 +`, + }); + + // ── Phase 16: relay-native report ──────────────────────────────────────── wf.step('report', { agent: 'reporter', - dependsOn: ['collect-results'], + dependsOn: ['verdict'], task: `You are an automated feature health agent for agent-relay. -First, check whether the verification artifact directory exists. Look for the directory "${ARTIFACTS}". +Read ${VERDICT_FILE}. It is the authoritative result — do NOT re-derive the +verdict from the individual tier logs, and do NOT run any relay commands to +re-test anything. -If the directory does NOT exist or NONE of the expected log files are present: - Post this message to relay-health: +If ${VERDICT_FILE} does not exist, post exactly this to the relay-health channel +and then finish: ## Feature Verification: ${RUN_ID} -**Overall: NOT_RUN** +**Overall: NOT_RUN** — the verification pipeline produced no verdict. +This is a harness failure, not a product result. Check the workflow logs. -| Check | Result | Notes | -|-------|--------|-------| -| V1: CLI Health | NOT_RUN | verification pipeline did not produce artifacts | -| V2: Broker + Agents | NOT_RUN | | -| V3: Channel Messaging | NOT_RUN | | -| V4: Cross-Agent DMs | NOT_RUN | | -| V5: Critical Paths | NOT_RUN | | +Otherwise post a message to the relay-health channel in this format, filling it +in from verdict.json: -**Failures:** verification pipeline did not run — check broker startup and workflow logs -**Next:** Run \`relay node workflow run workflows/verify-features.ts\` or debug broker startup +## Feature Verification: ${RUN_ID} -Then finish immediately — do not attempt to read missing files. +**Overall: ** ( passed, failed, skipped) +CLI / repo -If the directory DOES exist and has log files, read these artifact files directly: -- ${ARTIFACTS}/summary.txt -- ${ARTIFACTS}/tier1.log -- ${ARTIFACTS}/tier2.log -- ${ARTIFACTS}/tier3.log -- ${ARTIFACTS}/tier4.log -- ${ARTIFACTS}/critical-paths.log +| Tier | Result | p/f/s | Notes | +|------|--------|-------|-------| +| one row per tier in verdict.json "tiers" | -Then post a message to the relay-health channel with this format: +**Failures:** every entry in each tier's "failures" array, as +\`: \`. Write "None" only if there are genuinely zero. +**Skipped:** the skip count per tier — state plainly that skipped checks were +NOT verified. +**Tiers not run:** verdict.json "tiersNotRun", or "None". -## Feature Verification: ${RUN_ID} +Be accurate over brief. Never describe a skipped check as passing.`, + }); + + // ── Phase 17: GitHub issue ─────────────────────────────────────────────── + + wf.step('file-issue', { + type: 'deterministic', + dependsOn: ['verdict'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +AUTOFIX="${AUTOFIX ? '1' : '0'}" +${ENV_DEFAULTS} + +if [ "$AUTOFIX" != "1" ]; then + echo "ISSUE_SKIPPED: VERIFY_AUTOFIX=0" + exit 0 +fi +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "ISSUE_SKIPPED: no verdict.json" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh CLI not available" + exit 0 +fi +if ! gh auth status >/dev/null 2>&1; then + echo "ISSUE_SKIPPED: gh is not authenticated" + exit 0 +fi -**Overall: PASS** or **Overall: FAIL** +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") +if [ "$VERDICT" = "PASS" ]; then + echo "ISSUE_SKIPPED: verdict is PASS" + exit 0 +fi -| Check | Result | Notes | -|-------|--------|-------| -| V1: CLI Health | PASS/FAIL | brief note if failed | -| V2: Broker + Agents | PASS/FAIL | | -| V3: Channel Messaging | PASS/FAIL | | -| V4: Cross-Agent DMs | PASS/FAIL | | -| V5: Critical Paths | PASS/FAIL | | +node <<'ISSUEEOF' > "$ARTIFACTS/issue-body.md" +const fs = require('node:fs'); +const artifacts = process.env.VERIFY_ARTIFACTS || '.workflow-artifacts/verify-features'; +const v = JSON.parse(fs.readFileSync(artifacts + '/verdict.json', 'utf8')); + +const out = []; +out.push('Automated feature verification failed.'); +out.push(''); +out.push('- Run: ${BT}' + v.runId + '${BT}'); +out.push('- CLI under test: ${BT}' + (v.provenance.VERIFY_CLI_VERSION || '?') + '${BT}'); +out.push('- Repo version: ${BT}' + (v.provenance.VERIFY_REPO_VERSION || '?') + '${BT}'); +out.push('- Git sha: ${BT}' + (v.provenance.VERIFY_GIT_SHA || '?') + '${BT}'); +out.push('- Totals: ' + v.totals.pass + ' passed, ' + v.totals.fail + ' failed, ' + v.totals.skip + ' skipped'); +out.push(''); +out.push('## Failing checks'); +out.push(''); +let any = false; +for (const [tier, bucket] of Object.entries(v.tiers)) { + if (bucket.failures.length === 0) continue; + any = true; + out.push('### ' + tier); + for (const f of bucket.failures) { + out.push('- **' + f.check + '** — ' + String(f.reason).slice(0, 500)); + } + out.push(''); +} +if (!any) out.push('_No individual check failures; see the reasons below._'); +out.push('## Verdict reasons'); +out.push(''); +for (const reason of v.reasons) out.push('- ' + reason); +if (v.tiersNotRun.length > 0) { + out.push(''); + out.push('## Tiers that produced no records'); + out.push(''); + out.push('These crashed before their first check — treat as harness breakage, not a clean pass.'); + out.push(''); + for (const tier of v.tiersNotRun) out.push('- ${BT}' + tier + '${BT}'); +} +out.push(''); +out.push('---'); +out.push('Filed by ${BT}workflows/verify-features.ts${BT}. Artifacts: ${BT}' + artifacts + '${BT}'); +process.stdout.write(out.join('\n')); +ISSUEEOF + +TITLE="Feature verification failed: ${RUN_ID}" +ISSUE_URL=$(gh issue create --title "$TITLE" --body-file "$ARTIFACTS/issue-body.md" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || echo "") + +if [ -n "$ISSUE_URL" ]; then + echo "$ISSUE_URL" > "$ARTIFACTS/issue-url.txt" + echo "ISSUE_CREATED: $ISSUE_URL" +else + echo "ISSUE_FAILED: gh issue create produced no URL" +fi -**Failures:** list specific failed commands, or "None" -**Next:** "System healthy" if all pass, or what to debug if not +exit 0 +`, + }); -Do NOT explore the codebase, run builds, or test CLI commands. Just read the files and post.`, + // ── Phase 18: attempt a fix ────────────────────────────────────────────── + + wf.step('attempt-fix', { + agent: 'fixer', + dependsOn: ['file-issue'], + task: `Fix the underlying defect behind a feature verification failure. + +## FIRST: check whether there is anything to do + +Read ${ARTIFACTS}/autofix.env and ${VERDICT_FILE}. + +- If ${ARTIFACTS}/autofix.env contains \`AUTOFIX=0\`, the operator has disabled + the fix path. Write "Autofix disabled." to ${ARTIFACTS}/fix-summary.md and + STOP IMMEDIATELY. +- If ${VERDICT_FILE} does not exist, or its "verdict" field is "PASS", there is + NOTHING to fix. Write "No failures — nothing to fix." to + ${ARTIFACTS}/fix-summary.md and STOP IMMEDIATELY. +- In either case: do not read other files, do not create a branch, do not edit + anything, do not run any commands. +- Only if autofix is enabled AND "verdict" is "FAIL" do you continue. + +(The workflow engine has no conditional steps, so this step is scheduled on +every run. The early exit above is what keeps a green run from spending model +time and, more importantly, from letting an agent loose on a clean tree.) + +## The task + +Read ${VERDICT_FILE} for the authoritative list of failing checks, and the +per-tier logs in ${ARTIFACTS}/ for detail. + +## Hard rules + +1. NEVER commit or push to \`main\`. Work on a branch named exactly + \`${FIX_BRANCH}\`. Create it from the current HEAD. +2. Fix the ROOT CAUSE in product code. Do NOT make a check pass by weakening, + deleting, or loosening the check itself. If a check is genuinely wrong (it + tests a command that was intentionally removed, or asserts something the + product never promised), you may correct the check — but you must say so + explicitly and justify it in your summary. +3. Do NOT reduce the number of checks in \`workflows/verify-features.ts\`. A + later gate counts them and will reject the branch if the count drops. +4. Re-run the specific failing tier after your fix and paste the real output as + evidence. A fix with no re-run is not a fix. +5. If a failure is environmental (no cloud login, no provider CLI, expired + credentials), do NOT patch code. Report it as environmental and change + nothing — those cases are already recorded as SKIPs or escalated to NightCTO. + +## Deliverables + +Write a file \`${ARTIFACTS}/fix-summary.md\` containing: + +- \`Root cause:\` one paragraph per distinct failure, naming the file and symbol. +- \`Change:\` what you edited and why it addresses the root cause. +- \`Evidence:\` the exact command you re-ran and its output. +- \`Environmental:\` any failures you deliberately did not patch, and why. +- \`Assertions changed:\` any verification check you modified, with justification, + or "none". + +Commit your work on the branch. Do NOT open a pull request — a later step does +that. Do NOT merge anything.`, }); - // ── Phase 10: Analyze for improvements and drift ───────────────────────── + // ── Phase 19: fix integrity gate ───────────────────────────────────────── // - // Reads logs to identify feature drift, broken surfaces, or improvement - // opportunities. Produces an analysis artifact. Runs as deterministic - // to avoid opencode agent spawning failures. + // The one guard that makes the auto-fix path trustworthy. A fixer agent's + // cheapest route to green is always to weaken the assertion, so this + // compares the check count before and after and refuses a branch that + // shrank coverage. - wf.step('analyze-improvements', { + wf.step('fix-integrity', { type: 'deterministic', - dependsOn: ['collect-results'], + dependsOn: ['attempt-fix'], captureOutput: true, failOnError: false, - command: ` + command: String.raw` set -uo pipefail ARTIFACTS="${ARTIFACTS}" -SUMMARY="$ARTIFACTS/summary.txt" -REPORT="$ARTIFACTS/improvements-report.txt" -MANIFEST=".agentworkforce/features/manifest.yaml" - -echo "=== Improvement Analysis: ${RUN_ID} ===" > "$REPORT" -echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$REPORT" -echo "" >> "$REPORT" - -# Check if artifacts exist -if [ ! -d "$ARTIFACTS" ] || [ -z "$(ls -A "$ARTIFACTS"/*.log 2>/dev/null)" ]; then - echo "Status: NO_ARTIFACTS" >> "$REPORT" - echo "No verification artifacts found — pipeline did not produce results." >> "$REPORT" - cat "$REPORT" +WORKFLOW_FILE="workflows/verify-features.ts" + +# This step is scheduled unconditionally (no conditional steps in the engine), +# so a green run reaches it with nothing to check. Exit before the git +# assertions below, which would otherwise report a misleading INTEGRITY_FAIL +# just because HEAD is still main on a clean run. +if grep -q '^AUTOFIX=0$' "$ARTIFACTS/autofix.env" 2>/dev/null; then + echo "INTEGRITY_NOT_APPLICABLE: autofix disabled, no fix was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/fix-integrity.env" exit 0 fi -FOUND=0 +if [ -f "$ARTIFACTS/verdict.json" ]; then + VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json" 2>/dev/null || true) + if [ "$VERDICT" = "PASS" ]; then + echo "INTEGRITY_NOT_APPLICABLE: verdict is PASS, no fix was attempted" + echo "INTEGRITY=not-applicable" > "$ARTIFACTS/fix-integrity.env" + exit 0 + fi +fi -# ── Check each tier and critical paths for failures ──────────────────────── -for tier in tier1 tier2 tier3 tier4; do - log="$ARTIFACTS/$tier.log" - if [ -f "$log" ]; then - if grep -q "FAIL" "$log" 2>/dev/null; then - FOUND=$((FOUND + 1)) - failed=$(grep "FAIL" "$log" | head -5) - echo "[$tier] FAILURES found:" >> "$REPORT" - echo "$failed" | sed 's/^/ /' >> "$REPORT" - echo "" >> "$REPORT" - fi +if ! git rev-parse --git-dir >/dev/null 2>&1; then + echo "INTEGRITY_SKIPPED: not a git repository" + echo "INTEGRITY=skipped" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +CURRENT=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +echo "Current branch: $CURRENT" + +if [ "$CURRENT" = "main" ]; then + echo "INTEGRITY_FAIL: the fixer left the repo on main — refusing to proceed" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +# 'grep -c' prints 0 and exits 1 when there are no matches, so the fallback +# must be '|| true' — '|| echo 0' would emit a second line and break the +# numeric comparison below. POSIX bracket classes, not \s, for portability. +CHECK_PATTERN='^[[:space:]]*(run_check|gated_check|skip_check|record) ' + +count_checks_at() { + git show "$1:$WORKFLOW_FILE" 2>/dev/null | grep -cE "$CHECK_PATTERN" || true +} + +BASE_REF=$(git merge-base HEAD origin/main 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || true) +if [ -z "$BASE_REF" ]; then + echo "INTEGRITY_SKIPPED: cannot resolve a base ref to diff against" + echo "INTEGRITY=skipped" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +BEFORE=$(count_checks_at "$BASE_REF") +AFTER=$(grep -cE "$CHECK_PATTERN" "$WORKFLOW_FILE" 2>/dev/null || true) +if [ -z "$BEFORE" ]; then BEFORE=0; fi +if [ -z "$AFTER" ]; then AFTER=0; fi + +echo "Verification call sites: $BEFORE (base $BASE_REF) -> $AFTER (working tree)" + +if [ "$AFTER" -lt "$BEFORE" ]; then + echo "INTEGRITY_FAIL: the fix removed $((BEFORE - AFTER)) verification call site(s)." + echo "A fix that deletes checks has not fixed anything." + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +if [ ! -f "$ARTIFACTS/fix-summary.md" ]; then + echo "INTEGRITY_FAIL: fixer produced no fix-summary.md, so there is no root cause on record" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +if ! grep -qi "Root cause:" "$ARTIFACTS/fix-summary.md"; then + echo "INTEGRITY_FAIL: fix-summary.md has no 'Root cause:' section" + echo "INTEGRITY=fail" > "$ARTIFACTS/fix-integrity.env" + exit 0 +fi + +echo "INTEGRITY_OK" +echo "INTEGRITY=ok" > "$ARTIFACTS/fix-integrity.env" +exit 0 +`, + }); + + // ── Phase 20: draft PR ─────────────────────────────────────────────────── + + wf.step('open-pr', { + type: 'deterministic', + dependsOn: ['fix-integrity'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/fix-integrity.env" ]; then + echo "PR_SKIPPED: no integrity result — the fix path did not complete" + exit 0 +fi + +. "$ARTIFACTS/fix-integrity.env" + +if [ "$INTEGRITY" != "ok" ]; then + echo "PR_SKIPPED: integrity gate returned '$INTEGRITY' — not opening a PR" + exit 0 +fi +if ! command -v gh >/dev/null 2>&1 || ! gh auth status >/dev/null 2>&1; then + echo "PR_SKIPPED: gh unavailable or unauthenticated" + exit 0 +fi + +BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$BRANCH" = "main" ]; then + echo "PR_SKIPPED: refusing to open a PR from main" + exit 0 +fi + +if git diff --quiet HEAD 2>/dev/null && [ -z "$(git log origin/main..HEAD --oneline 2>/dev/null)" ]; then + echo "PR_SKIPPED: no commits on $BRANCH relative to origin/main" + exit 0 +fi + +git push -u origin "$BRANCH" 2>&1 | tail -3 || { + echo "PR_FAILED: could not push $BRANCH" + exit 0 +} + +ISSUE_REF="" +if [ -f "$ARTIFACTS/issue-url.txt" ]; then + ISSUE_REF="Fixes $(cat "$ARTIFACTS/issue-url.txt")" +fi + +{ + echo "## Automated fix for feature verification failure" + echo "" + echo "Run: \`${RUN_ID}\`" + echo "" + [ -n "$ISSUE_REF" ] && echo "$ISSUE_REF" && echo "" + echo "### Fixer summary" + echo "" + cat "$ARTIFACTS/fix-summary.md" 2>/dev/null || echo "_No fix summary produced._" + echo "" + echo "### Verdict at time of failure" + echo "" + echo '${BT}${BT}${BT}' + node -e 'const v=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));console.log(v.totals.pass+" passed, "+v.totals.fail+" failed, "+v.totals.skip+" skipped");for(const[t,b]of Object.entries(v.tiers)){if(b.failures.length)for(const f of b.failures)console.log("["+t+"] "+f.check+": "+String(f.reason).slice(0,200))}' "$ARTIFACTS/verdict.json" 2>/dev/null || echo "unavailable" + echo '${BT}${BT}${BT}' + echo "" + echo "### Review notes" + echo "" + echo "- Opened as a **draft** by \`workflows/verify-features.ts\`. Not auto-merged." + echo "- The integrity gate confirmed this branch does not reduce the number of" + echo " verification call sites, but a human still needs to confirm the fix" + echo " addresses the root cause rather than the symptom." +} > "$ARTIFACTS/pr-body.md" + +PR_URL=$(gh pr create --draft --title "fix: feature verification failures (${RUN_ID})" \ + --body-file "$ARTIFACTS/pr-body.md" --base main --head "$BRANCH" 2>&1 | grep -oE 'https://[^ ]+' | head -1 || echo "") + +if [ -n "$PR_URL" ]; then + echo "$PR_URL" > "$ARTIFACTS/pr-url.txt" + echo "PR_CREATED: $PR_URL" + if [ -f "$ARTIFACTS/issue-url.txt" ]; then + gh issue comment "$(cat "$ARTIFACTS/issue-url.txt")" \ + --body "Draft fix PR opened: $PR_URL" >/dev/null 2>&1 || true fi -done +else + echo "PR_FAILED: gh pr create produced no URL" +fi -if [ -f "$ARTIFACTS/critical-paths.log" ]; then - if grep -q "FAIL" "$ARTIFACTS/critical-paths.log" 2>/dev/null; then - FOUND=$((FOUND + 1)) - failed=$(grep "FAIL" "$ARTIFACTS/critical-paths.log" | head -10) - echo "[critical-paths] FAILURES found:" >> "$REPORT" - echo "$failed" | sed 's/^/ /' >> "$REPORT" - echo "" >> "$REPORT" - - if grep -q "relay mcp.*no recognizable output" "$ARTIFACTS/critical-paths.log" 2>/dev/null; then - echo " [mcp-server-start] CP3 MCP help check failed — verify relay mcp --help returns expected output" >> "$REPORT" - echo "" >> "$REPORT" - fi +exit 0 +`, + }); + + // ── Phase 21: Slack follow-up with issue and PR links ──────────────────── + + wf.step('slack-followup', { + type: 'deterministic', + dependsOn: ['open-pr'], + captureOutput: true, + failOnError: false, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" +CHANNEL="${SLACK_CHANNEL}" +${ENV_DEFAULTS} +${SLACK_POST_FN} + +ISSUE=$(cat "$ARTIFACTS/issue-url.txt" 2>/dev/null || true) +PR=$(cat "$ARTIFACTS/pr-url.txt" 2>/dev/null || true) + +if [ -z "$ISSUE" ] && [ -z "$PR" ]; then + echo "FOLLOWUP_SKIPPED: no issue or PR to report" + exit 0 +fi + +FOLLOWUP="$ARTIFACTS/slack-followup.txt" +{ + echo "Follow-up for \`${RUN_ID}\`:" + [ -n "$ISSUE" ] && echo "• Issue: $ISSUE" + if [ -n "$PR" ]; then + echo "• Draft fix PR: $PR" + else + echo "• No fix PR — the fix attempt did not pass the integrity gate. Needs a human." fi +} > "$FOLLOWUP" + +slack_post "$CHANNEL" "$FOLLOWUP" || true +exit 0 +`, + }); + + // ── Phase 22: enforce the verdict ──────────────────────────────────────── + // + // Terminal step, and the only one that fails on a bad result. Nothing + // depends on it, so its failure cannot block the escalation steps above — + // it exists to colour the run red after all reporting has happened. + // + // Deliberately does NOT depend on the fix chain (attempt-fix → fix-integrity + // → open-pr → slack-followup). A step failure marks its whole downstream + // subtree skipped (markDownstreamSkipped is transitive), so hanging + // enforcement off the fix chain would mean any hiccup while opening a PR + // silently swallowed the red verdict. Reporting and alerting are the only + // things enforcement waits on. + + wf.step('enforce-verdict', { + type: 'deterministic', + dependsOn: ['report', 'emit-posthog', 'escalate-infra', 'slack-alert'], + captureOutput: true, + failOnError: true, + command: String.raw` +set -uo pipefail + +ARTIFACTS="${ARTIFACTS}" + +if [ ! -f "$ARTIFACTS/verdict.json" ]; then + echo "ENFORCE: no verdict.json — treating as FAIL (harness did not complete)" + exit 2 fi -if [ "$FOUND" -eq 0 ]; then - echo "Result: NO_IMPROVEMENTS_NEEDED" >> "$REPORT" - echo "" >> "$REPORT" - echo "All tiers and critical paths passed. No drift or improvements identified." >> "$REPORT" -else - echo "Result: IMPROVEMENTS_FOUND" >> "$REPORT" - echo "" >> "$REPORT" - echo "Summary: $FOUND issue(s) above. Review the verification logs for details." >> "$REPORT" - echo "To fix: review each FAIL and correct the command/check or the test expectation." >> "$REPORT" +VERDICT=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).verdict)' "$ARTIFACTS/verdict.json") +node -e 'const v=JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8"));console.log("verdict="+v.verdict);console.log("reasons="+(v.reasons.join("; ")||"none"))' "$ARTIFACTS/verdict.json" + +if [ "$VERDICT" = "PASS" ]; then + echo "ENFORCE: PASS" + exit 0 fi -cat "$REPORT" +echo "ENFORCE: FAIL — see verdict.json" +exit 1 `, }); - await wf.run(); + const result = await wf.run(); + + // A dry run plans the graph without executing it, so there is no verdict to + // read and nothing to enforce. Bail before the checks below, which would + // otherwise report a validated plan as harness breakage. + if (process.env.DRY_RUN || !('status' in result)) { + return; + } + + // ── Post-run: make the process exit code tell the truth ────────────────── + // + // The previous version awaited run() and discarded the result, so a run with + // four failing checks still exited 0 and every scheduler saw success. Read + // the verdict directly rather than trusting the runner's row status, and + // fail closed when the verdict is missing. + + let verdict: { verdict?: string; reasons?: string[]; totals?: Record } | null = null; + if (existsSync(VERDICT_FILE)) { + try { + verdict = JSON.parse(readFileSync(VERDICT_FILE, 'utf8')); + } catch (err) { + console.error(`[verify-features] verdict.json is unreadable: ${(err as Error).message}`); + } + } + + if (!verdict) { + console.error( + `[verify-features] no usable verdict at ${VERDICT_FILE} — treating this run as FAILED. ` + + `The verification pipeline did not complete; this is harness breakage, not a clean run.` + ); + process.exitCode = 2; + return; + } + + const totals = verdict.totals ?? {}; + console.log( + `[verify-features] verdict=${verdict.verdict} ` + + `pass=${totals.pass ?? '?'} fail=${totals.fail ?? '?'} skip=${totals.skip ?? '?'} ` + + `(workflow row status: ${'status' in result ? String(result.status) : 'unknown'})` + ); + + if (verdict.verdict !== 'PASS') { + console.error(`[verify-features] FAILED: ${(verdict.reasons ?? []).join('; ')}`); + process.exitCode = 1; + } } -main(); +main().catch((err) => { + // A throw here means the harness itself broke — the runner could not even + // produce a result. That is the NightCTO escalation case, and it must not + // exit 0. + console.error(`[verify-features] harness failure: ${err instanceof Error ? err.stack : String(err)}`); + process.exitCode = 2; +});