diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.test.ts b/.agentworkforce/agents/relay-feature-guardian/agent.test.ts index d08570397..7d7d7a249 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.test.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.test.ts @@ -330,6 +330,25 @@ describe('relay-feature-guardian runtime paths', () => { expect(persona.inputs.SLACK_CHANNEL.default).toBe('C0AEKNLDNKW'); }); + it('falls back with every declared MCP surface when quiz generation fails', async () => { + const transport = new IdempotentSlackTransport(); + const restore = bindPreviewTransport(transport); + const mcpOnlyManifest = manifest.replace(' cli: relay node up', ' mcp: create_workspace'); + const { ctx } = exactStateContext(JSON.stringify(progressState(0)), mcpOnlyManifest); + ctx.llm.complete = vi.fn(async () => { + throw new Error('simulated quiz model failure'); + }); + + try { + await guardian.handler(ctx, { type: 'cron.tick' } as never); + const text = (transport.attempts[0]?.body as { text: string }).text; + expect(text).toContain('MCP tool: create_workspace'); + expect(text).not.toContain('CLI command:'); + } finally { + restore(); + } + }); + it('uses a dedicated low-reasoning model path instead of shared subscription quota', () => { expect(persona).toMatchObject({ harness: 'opencode', diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.ts b/.agentworkforce/agents/relay-feature-guardian/agent.ts index 2e1c004bf..fce38661a 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.ts @@ -29,10 +29,11 @@ type Criticality = 'critical' | 'hot' | 'standard'; interface ManifestFeature { id: string; name: string; - cli: string; + cli?: string; description: string; verify_tier: number; mcp?: string; + mcp_prompt?: string; location?: string; } @@ -53,11 +54,12 @@ interface Manifest { interface Feature { id: string; name: string; - cli: string; + cli?: string; desc: string; tier: number; criticality: Criticality; mcp?: string; + mcpPrompt?: string; } const MANIFEST_RELPATH = '.agentworkforce/features/manifest.yaml'; @@ -88,6 +90,7 @@ async function loadFeatures(ctx: WorkforceCtx): Promise { tier: f.verify_tier, criticality: category.criticality, mcp: f.mcp, + mcpPrompt: f.mcp_prompt, }); } } @@ -543,27 +546,32 @@ function pickNextFeature(features: Feature[], checkedIds: Set): Feature // ── quiz generation ─────────────────────────────────────────────────────────── async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise { - const mcpNote = feature.mcp ? `\nMCP tool: \`${feature.mcp}\`` : ''; + const surface = [ + feature.cli ? `CLI command: ${feature.cli}` : null, + feature.mcp ? `MCP tool: ${feature.mcp}` : null, + feature.mcpPrompt ? `MCP prompt: ${feature.mcpPrompt}` : null, + ] + .filter((entry): entry is string => entry !== null) + .join('\n'); const tierLabel = - feature.tier === 1 - ? 'no broker needed' - : feature.tier === 2 - ? 'broker required' - : feature.tier === 3 - ? 'broker + agent token' - : feature.tier === 4 - ? 'broker + two agents' - : 'cloud auth required'; + { + 1: 'isolated local CLI/filesystem', + 2: 'local broker required', + 3: 'hosted workspace + agent token', + 4: 'hosted workspace + two agents', + 5: 'authenticated disposable external service', + 6: 'interactive or pre-provisioned integration', + }[feature.tier] ?? 'see feature procedure'; const prompt = [ 'You are the Relay Feature Guardian, a proactive Slack bot for the Agent Relay team.', - 'Write a brief, conversational Slack message (3-5 sentences, no markdown headers) asking the team to confirm whether a specific CLI feature is working as intended.', - 'Be specific: name the feature, describe what it should do, show the CLI command, and ask if it behaves this way or if anything has drifted.', + 'Write a brief, conversational Slack message (3-5 sentences, no markdown headers) asking the team to confirm whether a specific feature is working as intended. The feature can be a CLI command or an MCP tool/prompt.', + 'Be specific: name the feature, describe what it should do, show the relevant CLI command or MCP tool/prompt, and ask if it behaves this way or if anything has drifted.', 'End with: "React ✅ if working as expected, 🔧 if something is off, or ❓ if untested."', 'Keep it casual and direct — this is an internal team check.', '', `Feature: ${feature.name}`, - `CLI: ${feature.cli}${mcpNote}`, + surface, `What it should do: ${feature.desc}`, `Verify tier: ${feature.tier} (${tierLabel})`, `Criticality: ${feature.criticality}`, @@ -576,7 +584,7 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise return [ `🔍 *Relay Feature Check: ${feature.name}*`, ``, - `\`${feature.cli}\`${mcpNote}`, + surface, ``, `This should: ${feature.desc}`, ``, diff --git a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts new file mode 100644 index 000000000..1375741b9 --- /dev/null +++ b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts @@ -0,0 +1,195 @@ +import { readFileSync } from 'node:fs'; + +import { parse } from 'yaml'; +import { describe, expect, it } from 'vitest'; + +type Feature = { id: string; cli?: string; mcp?: string; mcp_prompt?: string; location?: string }; +type Category = { features?: Feature[] }; +type Manifest = { + version: string; + verification: { document: string; categories: Record }; + categories: Record; +}; + +const manifestPath = new URL('../../features/manifest.yaml', import.meta.url); +const proceduresPath = new URL('../../features/verify/procedures.md', import.meta.url); +const manifest = parse(readFileSync(manifestPath, 'utf8')) as Manifest; +const procedures = readFileSync(proceduresPath, 'utf8'); +const features = Object.values(manifest.categories).flatMap((category) => category.features ?? []); + +describe('feature manifest contract', () => { + it('maps every category to a detailed verification procedure', () => { + expect(manifest.version).toBe('1.1'); + expect(manifest.verification.document).toBe('.agentworkforce/features/verify/procedures.md'); + + for (const category of Object.keys(manifest.categories)) { + const procedure = manifest.verification.categories[category]; + expect(procedure, `${category} needs a verification procedure`).toMatch(/^[a-z0-9-]+$/); + expect(procedures).toContain(`## ${procedure}`); + } + }); + + it('has unique feature ids and a concrete CLI, MCP, or prompt surface', () => { + const ids = features.map((feature) => feature.id); + expect(new Set(ids).size).toBe(ids.length); + for (const feature of features) { + expect(feature.id).toMatch(/^[a-z0-9-]+$/); + expect(Boolean(feature.cli || feature.mcp || feature.mcp_prompt || feature.location)).toBe(true); + } + }); + + it('covers every public Commander leaf command', () => { + const commands = features.flatMap((feature) => (feature.cli ? [feature.cli] : [])); + const expected = [ + 'relay node up', + 'relay node down', + 'relay node status', + 'relay node metrics', + 'relay node deadletters', + 'relay node redeliver', + 'relay node tail', + 'relay node agent list', + 'relay node agent spawn', + 'relay node agent new', + 'relay node agent release', + 'relay node agent set-model', + 'relay node agent attach', + 'relay node agent message flush', + 'relay node agent message hold', + 'relay node agent message auto', + 'relay node workflow run', + 'relay node workflow logs', + 'relay node workflow sync', + 'relay version', + 'relay update', + 'relay uninstall', + 'relay status', + 'relay telemetry enable', + 'relay telemetry disable', + 'relay telemetry status', + 'relay cloud login', + 'relay cloud logout', + 'relay cloud session', + 'relay cloud whoami', + 'relay cloud connect', + 'relay cloud enroll', + 'relay cloud run', + 'relay cloud schedule', + 'relay cloud schedules', + 'relay cloud status', + 'relay cloud logs', + 'relay cloud sync', + 'relay cloud cancel', + 'relay cloud worker register', + 'relay cloud worker start', + 'relay cloud worker status', + 'relay cloud worker logs', + 'relay reflex on', + 'relay reflex off', + 'relay reflex status', + 'relay workspace active', + 'relay workspace create', + 'relay workspace list', + 'relay workspace set_key', + 'relay workspace join', + 'relay workspace switch', + 'relay agent register', + 'relay agent list', + 'relay agent add', + 'relay agent remove', + 'relay channel create', + 'relay channel list', + 'relay channel join', + 'relay channel leave', + 'relay channel invite', + 'relay channel set_topic', + 'relay channel archive', + 'relay message post', + 'relay message list', + 'relay message reply', + 'relay message get_thread', + 'relay message search', + 'relay message dm send', + 'relay message dm list', + 'relay message dm send_group', + 'relay message reaction add', + 'relay message reaction remove', + 'relay message inbox check', + 'relay message inbox mark_read', + 'relay message inbox get_readers', + 'relay message file upload', + 'relay integration subscribe', + 'relay integration unsubscribe', + 'relay integration webhook create', + 'relay integration webhook list', + 'relay integration webhook delete', + 'relay integration webhook trigger', + 'relay integration webhook create-inbound', + 'relay integration webhook list-inbound', + 'relay integration webhook delete-inbound', + 'relay integration subscription create', + 'relay integration subscription list', + 'relay integration subscription get', + 'relay integration subscription delete', + 'relay capabilities register', + 'relay capabilities list', + 'relay capabilities delete', + 'relay skills add', + 'relay mcp', + 'relay fleet nodes', + 'relay fleet config', + 'relay fleet enable', + 'relay fleet disable', + 'relay fleet inherit', + 'relay fleet status', + ]; + + for (const command of expected) { + expect( + commands.some((documented) => documented.startsWith(command)), + `${command} is missing` + ).toBe(true); + } + expect(commands).not.toContain('relay metrics'); + expect(commands).not.toContain('relay deadletters'); + expect(commands).not.toContain('relay redeliver [id]'); + }); + + it('covers every static MCP tool registered by the server', () => { + const tools = new Set(features.flatMap((feature) => (feature.mcp ? [feature.mcp] : []))); + const expected = [ + 'create_workspace', + 'set_workspace_key', + 'register_agent', + 'list_agents', + 'query_nodes', + 'add_agent', + 'spawn', + 'remove_agent', + 'create_channel', + 'list_channels', + 'join_channel', + 'leave_channel', + 'invite_to_channel', + 'set_channel_topic', + 'archive_channel', + 'post_message', + 'list_messages', + 'reply_to_thread', + 'get_message_thread', + 'send_dm', + 'list_dms', + 'send_group_dm', + 'add_reaction', + 'remove_reaction', + 'search_messages', + 'check_inbox', + 'mark_message_read', + 'get_message_readers', + 'list_actions', + 'invoke_action', + 'submit_result', + ]; + expect([...tools].sort()).toEqual(expected.sort()); + }); +}); diff --git a/.agentworkforce/features/critical-paths.md b/.agentworkforce/features/critical-paths.md index 7569e90ac..41c5fe421 100644 --- a/.agentworkforce/features/critical-paths.md +++ b/.agentworkforce/features/critical-paths.md @@ -1,138 +1,104 @@ # Critical Paths -The features and sequences that must work for the product to function. These are the first things to verify after any change, and the last things to break. +These are the product sequences to run first after a related change. They use the exact public surface recorded in `manifest.yaml`; detailed fixtures, assertions, and cleanup live in `verify/procedures.md`. ---- - -## Path 1: Broker + Agent Registration (Foundation) - -Everything depends on this. If it breaks, nothing works. +## Path 1: Local Broker Lifecycle ```bash -relay node up --background -relay agent register → produces a token -relay agent list → shows the registered agent -relay status → shows broker running with agent count +relay node up --background --no-spawn +relay node status +relay node metrics +relay node deadletters --json +relay node down +relay node status ``` -**What breaks if this fails:** All messaging, local agent orchestration, MCP tools, workflow execution. +The first status must report running and the last must report stopped. Use an isolated project/state directory and never stop all brokers system-wide. ---- +## Path 2: Cross-Agent Channel Message -## Path 2: Channel Messaging (Core Coordination Loop) - -The primary way agents communicate. +**Prerequisite:** disposable hosted workspace; `TOKEN_A` must be the token returned for +`critical-a`, and `TOKEN_B` must be the token returned for `critical-b`. ```bash -relay node up --background -relay agent register alice → TOKEN_A -relay agent register bob → TOKEN_B - -# As alice: -RELAY_AGENT_TOKEN= relay channel create team -RELAY_AGENT_TOKEN= relay channel join team -RELAY_AGENT_TOKEN= relay message post team "hello from alice" - -# Verify bob receives it: -RELAY_AGENT_TOKEN= relay channel join team -RELAY_AGENT_TOKEN= relay message list team --limit 1 -# → should show alice's message +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create critical-path +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite critical-path critical-b +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post critical-path 'critical-path-message')" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list critical-path --limit 10 +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive critical-path ``` -**What breaks if this fails:** All multi-agent coordination, workflow communication, the entire product premise. - ---- +Agent B must see the exact text from A. Remove both test identities after archiving the channel. -## Path 3: Local Agent Spawn + Message Injection +## Path 3: Managed Local Agent -The mechanism for spawning real AI agents and injecting messages into them. +**Prerequisite:** a locally installed and authenticated provider CLI. ```bash -relay node up --background -relay node agent spawn claude --name worker-1 -relay node agent list → shows worker-1 as active -relay node agent message hold worker-1 -relay node agent message flush worker-1 -relay node agent release worker-1 +PROVIDER="${PROVIDER:-claude}" # installed and authenticated fixture provider +relay node up --background --no-spawn +relay node agent spawn "$PROVIDER" --name critical-worker --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task +relay node agent list +relay node agent message hold critical-worker +relay node agent message auto critical-worker +relay node agent release critical-worker ``` -**What breaks if this fails:** All multi-agent orchestration workflows, the local workflow engine. +Assert that the worker appears, receives its bounded task, and is released. Provider credentials/cost make this a pre-provisioned integration check, not a generic CI test. ---- +## Path 4: MCP Stdio Round Trip -## Path 4: MCP Server Integration +**Prerequisite:** disposable hosted workspace and an MCP JSON-RPC client. -The path used when an agent operates inside a harness (Claude Code, Cursor, etc.). - -```bash -relay mcp → starts MCP server on stdio -# From a harness MCP call: -list_channels → returns channel list -post_message(channel, text) → posts message -list_messages(channel) → returns messages including the one just posted +```text +start relay mcp over stdio +initialize → tools/list → prompts/list +set_workspace_key or create_workspace → register_agent(A and B) +create_channel → post_message → list_messages +reply_to_thread → get_message_thread +send_dm → list_dms → check_inbox → mark_message_read → get_message_readers +close the stdio child ``` -**What breaks if this fails:** All usage from within harness tools (the primary user workflow for most users). - ---- +Assert every static MCP tool in the manifest is listed and that the returned content reflects the values written by the test. The server itself does not require a local broker, but the messaging tools require a workspace and agent identities. -## Path 5: Workflow Execution (Local) - -A workflow YAML or JS file drives multiple agents to complete a task. +## Path 5: Local Workflow Run, Logs, and Sync ```bash -relay node up --background -relay node workflow run examples/basic-workflow.yaml -# → spawns agents, coordinates them, produces output, terminates cleanly +echo 'console.log("critical-workflow-ok")' > workflow.js +RUN_JSON="$(relay node workflow run workflow.js --json)" +RUN_ID="$(jq -er .runId <<<"$RUN_JSON")" +relay node workflow logs "$RUN_ID" --follow --json +relay node workflow sync "$RUN_ID" --dry-run --json ``` -**What breaks if this fails:** The primary value proposition for multi-agent task execution. +Assert completed status and the sentinel output. Run it in a disposable project because local workflow records are retained. ---- +## Path 6: Direct Message and Read Receipt -## Path 6: Direct Messaging Between Agents +**Prerequisite:** disposable hosted workspace; `TOKEN_A` must be the token returned for +`critical-a`, and `TOKEN_B` must be the token returned for `critical-b`. ```bash -relay node up --background -relay agent register orchestrator → TOKEN_O -relay agent register worker → TOKEN_W - -# Send DM and capture conversationId from JSON output: -RELAY_AGENT_TOKEN= relay message dm send worker "your task" -# dm list requires the conversationId returned by send -# RELAY_AGENT_TOKEN= relay message dm list +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send critical-b 'critical-dm')" +DM_ID="$(jq -er '.id // .messageId' <<<"$DM")" +CONV_ID="$(jq -er '.conversationId // .conversation_id' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV_ID" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox mark_read "$DM_ID" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message inbox get_readers "$DM_ID" ``` -**What breaks if this fails:** Lead/worker orchestration patterns, any workflow that routes tasks via DM. - ---- - -## Hot Paths (Sensitive, Frequently Touched) - -These are not foundational but are exercised constantly and failures are immediately noticeable. - -| Path | Risk | Code Area | -| ----------------------------- | ---------------------------------------- | ------------------------------------------------------------ | -| PTY message injection timing | Race conditions in TMUX wrapper | crates/relay-pty/, packages/harness-driver/ | -| Dead letter queue + redeliver | Messages lost silently | crates/broker/src/queue.rs | -| Agent token auth validation | Auth bypass or silent rejection | crates/broker/src/auth.rs (or equivalent) | -| Message delivery ordering | Out-of-order delivery corrupts workflows | crates/broker/ | -| Hold/flush state machine | Stuck agents that never receive messages | crates/broker/, packages/cli/src/cli/commands/local-agent.ts | -| MCP tool schema validation | Tool calls rejected silently by harness | packages/cli/src/cli/mcp/ | - ---- - -## What an Agent Should Check First +Assert B reads the exact DM and A sees B in readers. Remove the disposable agents afterward. -When unsure if the system is healthy, run this sequence in order: +## Fast Health Triage ```bash -relay version # CLI is installed and not corrupt -relay status # Broker state (running or not) -relay node up --background # Start if not running -relay status # Confirm running -relay agent list # Workspace is reachable and has agents -relay message list general --limit 1 # Messaging works end to end +relay version +relay status +relay node status +relay node up --background --no-spawn +relay node status +relay node metrics ``` -If any step fails, diagnose before proceeding. The failure at each step points to a specific subsystem. +If any command fails, diagnose that subsystem before testing higher-level paths. Do not treat top-level `relay status` as the broker statistics command; `relay node status` is the direct broker check. diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 3144d68a8..3781b9a56 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1,5 +1,5 @@ -version: '1.0' -updated: '2026-07-16' +version: '1.1' +updated: '2026-07-20' # Every user-facing feature in this repo, categorized and scored. # @@ -8,13 +8,47 @@ updated: '2026-07-16' # hot - commonly used path; verify for any related change # standard - useful feature; verify periodically # -# verify_tier (what is required to run verification): -# 1 - nothing (version, help, telemetry toggle, workspace list) -# 2 - broker running (relay node up --background) -# 3 - broker running + at least one agent registered with token -# 4 - broker running + two or more agents registered -# 5 - cloud auth required (relay cloud login) -# 6 - manual / browser only +# verify_tier (the primary environment needed to run verification): +# 1 - isolated local CLI/filesystem +# 2 - local broker running +# 3 - hosted workspace plus one registered agent token +# 4 - hosted workspace plus two or more registered agent identities +# 5 - authenticated, disposable hosted workspace or external test service +# 6 - interactive provider/browser/PTY or a pre-provisioned external system +# +# Tier alone is deliberately not a test plan. Every category is mapped below +# to a procedure with precise prerequisites, setup, assertions, cleanup, and +# any automation limitation. Procedures are in verify/procedures.md. +verification: + document: .agentworkforce/features/verify/procedures.md + categories: + broker: broker-lifecycle + agent-management: workspace-agents-and-capabilities + messaging-channels: channel-messaging + messaging-messages: message-round-trip + messaging-dm: direct-messages + messaging-reactions: reactions-and-read-status + messaging-inbox: reactions-and-read-status + local-agents: local-agent-lifecycle + local-workflows: local-workflow-lifecycle + cloud: cloud-workflows + cloud-workers: cloud-workers + fleet: fleet-management + workspace: workspace-management + skills: skills-installation + integration: integrations-and-webhooks + reflex: reflex-history + mcp: mcp-stdio + harnesses: harnesses + sdk: typescript-sdk + python-sdk: python-sdk + swift-sdk: swift-sdk + opencode-plugin: opencode-plugin + codex-relay-skill: codex-relay-skill + gemini-relay-extension: gemini-relay-extension + setup: cli-maintenance + telemetry: telemetry + node: node-command-discovery categories: broker: @@ -38,28 +72,28 @@ categories: - id: broker-status name: Broker Status - cli: relay status - description: Check whether the broker is running and show agent/queue stats - location: packages/cli/src/cli/commands/core.ts - verify_tier: 1 + cli: relay node status [--wait-for ] + description: Check whether the local broker daemon is running and report its agent and delivery state + location: packages/cli/src/cli/commands/core.ts, packages/cli/src/cli/commands/node.ts + verify_tier: 2 - id: broker-metrics name: Broker Metrics - cli: relay metrics + cli: relay node metrics [--agent ] description: Show resource usage for broker and its agents location: packages/cli/src/cli/commands/core.ts verify_tier: 2 - id: broker-deadletters name: Dead Letter Queue - cli: relay deadletters + cli: relay node deadletters [--json] description: List terminally-failed message deliveries retained in the broker location: packages/cli/src/cli/commands/core.ts verify_tier: 2 - id: broker-redeliver name: Redeliver Messages - cli: relay redeliver [id] + cli: relay node redeliver ( | --all) description: Requeue dead-letter messages through the normal delivery path location: packages/cli/src/cli/commands/core.ts verify_tier: 2 @@ -71,38 +105,59 @@ categories: features: - id: agent-register name: Register Agent - cli: relay agent register + cli: relay agent register [--type ] [--persona ] description: Register a new agent and print its auth token location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-add name: Add Agent - cli: relay agent add - description: Add an agent to the workspace (interactive) + cli: relay agent add [--type ] + description: Register an additional agent in the workspace location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-list name: List Agents - cli: relay agent list + cli: relay agent list [--status ] description: Show all agents in the workspace with status location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-remove name: Remove Agent cli: relay agent remove description: Remove an agent from the workspace location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-capabilities name: List Capabilities - cli: relay capabilities + cli: relay capabilities list description: Show available agent capabilities location: packages/cli/src/cli/commands/capabilities.ts - verify_tier: 2 + verify_tier: 3 + + - id: capabilities-register + name: Register Capability + cli: relay capabilities register --description --handler + description: Register a named capability and the agent that handles it + location: packages/cli/src/cli/commands/capabilities.ts + verify_tier: 3 + + - id: capabilities-delete + name: Delete Capability + cli: relay capabilities delete + description: Delete a previously registered capability + location: packages/cli/src/cli/commands/capabilities.ts + verify_tier: 3 + + - id: system-status + name: Composite System Status + cli: relay status + description: Show the current project path, local broker state, and Cloud-login state + location: packages/cli/src/cli/commands/status.ts + verify_tier: 1 messaging-channels: name: Channel Messaging @@ -111,7 +166,7 @@ categories: features: - id: channel-create name: Create Channel - cli: relay channel create + cli: relay channel create [--topic ] mcp: create_channel description: Create a new messaging channel location: packages/cli/src/cli/commands/channel.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -119,7 +174,7 @@ categories: - id: channel-list name: List Channels - cli: relay channel list + cli: relay channel list [--archived] mcp: list_channels description: Show all channels in the workspace location: packages/cli/src/cli/commands/channel.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -172,7 +227,7 @@ categories: features: - id: message-post name: Post Message - cli: relay message post + cli: relay message post mcp: post_message description: Send a message to a channel as the current agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -180,7 +235,7 @@ categories: - id: message-list name: List Messages - cli: relay message list + cli: relay message list [--limit ] mcp: list_messages description: Retrieve message history from a channel location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -188,28 +243,35 @@ categories: - id: message-reply name: Reply to Message (Threads) - cli: relay message reply - mcp: reply_to_message + cli: relay message reply + mcp: reply_to_thread description: Reply to a message, creating or extending a thread location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 - id: message-get-thread name: Get Thread - cli: relay message get_thread - mcp: get_thread + cli: relay message get_thread + mcp: get_message_thread description: Retrieve all messages in a thread location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 - id: message-search name: Search Messages - cli: relay message search + cli: relay message search [--channel ] [--from ] [--limit ] mcp: search_messages description: Search for messages by content across the workspace location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 + - id: message-file-upload + name: Upload File Attachment + cli: relay message file upload --channel [--text ] + description: Send a channel message containing a local file attachment + location: packages/cli/src/cli/commands/message.ts + verify_tier: 3 + messaging-dm: name: Direct Messages description: Private messaging between agents @@ -217,23 +279,29 @@ categories: features: - id: dm-send name: Send DM - cli: relay message dm send + cli: relay message dm send mcp: send_dm description: Send a private direct message to another agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 - id: dm-list - name: List DMs - cli: relay message dm list - mcp: list_dms - description: List direct message conversations for the current agent + name: List Messages in a DM Conversation + cli: relay message dm list [--limit ] + description: List the messages in a known direct-message conversation location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 + - id: dm-list-conversations + name: List DM Conversations + mcp: list_dms + description: List direct-message conversations visible to the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 4 + - id: dm-send-group name: Send Group DM - cli: relay message dm send_group + cli: relay message dm send_group --to mcp: send_group_dm description: Create a group DM and send the first message to multiple agents location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -246,19 +314,19 @@ categories: features: - id: reaction-add name: Add Reaction - cli: relay message reaction add + cli: relay message reaction add mcp: add_reaction description: Add an emoji reaction to a message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: reaction-remove name: Remove Reaction - cli: relay message reaction remove + cli: relay message reaction remove mcp: remove_reaction description: Remove an emoji reaction from a message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 messaging-inbox: name: Inbox & Read Status @@ -267,24 +335,24 @@ categories: features: - id: inbox-check name: Check Inbox - cli: relay message inbox check - mcp: list_inbox + cli: relay message inbox check [--limit ] + mcp: check_inbox description: Show unread messages, mentions, and DMs directed to this agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: inbox-mark-read name: Mark Message Read - cli: relay message inbox mark_read + cli: relay message inbox mark_read mcp: mark_message_read description: Mark a message or thread as read for the current agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: inbox-get-readers name: Get Readers - cli: relay message inbox get_readers - mcp: get_readers + cli: relay message inbox get_readers + mcp: get_message_readers description: Show which agents have read a specific message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 @@ -296,17 +364,17 @@ categories: features: - id: local-agent-spawn name: Spawn Agent - cli: relay node agent spawn --name + cli: relay node agent spawn [--name ] [--task ] [--spawn-mode task-exit] [--exit-after-task] description: Launch an AI agent process attached to the local broker location: packages/cli/src/cli/commands/local-agent.ts verify_tier: 2 - id: local-agent-new name: Spawn and Attach - cli: relay node agent new --name + cli: relay node agent new [--name ] description: Spawn an agent and immediately attach your terminal to it location: packages/cli/src/cli/commands/local-agent.ts - verify_tier: 2 + verify_tier: 6 - id: local-agent-release name: Release Agent @@ -325,7 +393,7 @@ categories: - id: local-agent-set-model name: Switch Agent Model cli: relay node agent set-model - description: Change the AI model a running agent uses (sends /model to TUI) + description: Request a best-effort /model switch from a running agent's provider TUI location: packages/cli/src/cli/commands/local-agent.ts verify_tier: 2 @@ -334,7 +402,7 @@ categories: cli: relay node agent attach description: Interactively connect terminal to a running agent (drive/view/passthrough) location: packages/cli/src/cli/commands/local-agent.ts - verify_tier: 2 + verify_tier: 6 - id: local-agent-flush name: Flush Agent Messages @@ -361,7 +429,7 @@ categories: name: Tail Broker Events cli: relay node tail [--agent ] description: Stream live broker events, optionally filtered to one agent - location: packages/cli/src/cli/commands/local-agent.ts + location: packages/cli/src/cli/commands/local-agent.ts, packages/cli/src/cli/commands/node.ts verify_tier: 2 local-workflows: @@ -371,10 +439,24 @@ categories: features: - id: local-workflow-run name: Run Local Workflow - cli: relay node workflow run - description: Execute a workflow YAML or TypeScript/JS file on the local broker + cli: relay node workflow run [--file-type ] [--json] + description: Run a local YAML, TypeScript, JavaScript, Python, or shell workflow and record its result locally location: packages/cli/src/cli/commands/local-workflow.ts - verify_tier: 2 + verify_tier: 1 + + - id: local-workflow-logs + name: Read Local Workflow Logs + cli: relay node workflow logs [--follow] [--poll-interval ] [--offset ] [--json] + description: Read or follow the locally recorded output of a workflow run + location: packages/cli/src/cli/commands/local-workflow.ts + verify_tier: 1 + + - id: local-workflow-sync + name: Sync Local Workflow Result + cli: relay node workflow sync [--dry-run] [--json] + description: Report the local synchronization state for a completed workflow run + location: packages/cli/src/cli/commands/local-workflow.ts + verify_tier: 1 cloud: name: Cloud Features @@ -400,74 +482,74 @@ categories: cli: relay cloud whoami description: Show current cloud authentication status (readable if already logged in) location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-session name: Cloud Session cli: relay cloud session description: Show canonical cloud session details (readable if already logged in) location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-connect name: Connect Provider - cli: relay cloud connect + cli: relay cloud connect [--language ] [--timeout ] description: Connect a provider via interactive SSH session location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 5 + verify_tier: 6 - id: cloud-enroll name: Enroll Fleet Node - cli: relay cloud enroll + cli: relay cloud enroll (--token | --workspace ) [--name ] [--max-agents ] [--json] description: Register this machine as a cloud-managed fleet node location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-run name: Submit Cloud Workflow - cli: relay cloud run + cli: relay cloud run [--file-type ] [--no-sync-code] [--json] description: Submit a workflow for remote cloud execution location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-schedule name: Schedule Cloud Workflow - cli: relay cloud schedule + cli: relay cloud schedule (--cron | --at ) [--timezone ] [--json] description: Schedule a repeatable cloud workflow run with cron location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-schedules name: List Cloud Schedules - cli: relay cloud schedules + cli: relay cloud schedules [--json] description: List all scheduled workflow runs location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-status name: Cloud Run Status - cli: relay cloud status + cli: relay cloud status [--json] description: Fetch status of a workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-logs name: Cloud Run Logs - cli: relay cloud logs + cli: relay cloud logs [--follow] [--agent ] [--json] description: Read logs from a workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-sync name: Sync Cloud Results - cli: relay cloud sync + cli: relay cloud sync [--dir ] [--dry-run] description: Download and apply code changes from a completed cloud workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-cancel name: Cancel Cloud Run - cli: relay cloud cancel + cli: relay cloud cancel [--json] description: Cancel a running cloud workflow location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 @@ -479,31 +561,31 @@ categories: features: - id: cloud-worker-register name: Register Cloud Worker - cli: relay cloud worker register + cli: relay cloud worker register --token --name [--base-url ] [--json] description: Register this machine as a cloud worker node location: packages/cli/src/cli/commands/cloud-worker.ts verify_tier: 5 - id: cloud-worker-start name: Start Cloud Worker - cli: relay cloud worker start + cli: relay cloud worker start [--worker-id | --name ] [--daemon] [--once] description: Start the cloud worker service on this machine location: packages/cli/src/cli/commands/cloud-worker.ts verify_tier: 5 - id: cloud-worker-status name: Cloud Worker Status - cli: relay cloud worker status - description: Show local cloud worker daemon status (reads local state, no cloud call) + cli: relay cloud worker status [--worker-id | --name ] [--json] + description: Show the state of a previously registered local Cloud worker location: packages/cli/src/cli/commands/cloud-worker.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-worker-logs name: Cloud Worker Logs - cli: relay cloud worker logs + cli: relay cloud worker logs [--worker-id | --name ] [--follow] description: Read local cloud worker daemon logs location: packages/cli/src/cli/commands/cloud-worker.ts - verify_tier: 2 + verify_tier: 5 fleet: name: Fleet Management @@ -515,14 +597,14 @@ categories: cli: relay fleet nodes description: Show all fleet nodes registered in the workspace (returns empty list if none configured) location: packages/cli/src/cli/commands/fleet.ts - verify_tier: 2 + verify_tier: 5 - id: fleet-config name: Fleet Config cli: relay fleet config - description: Show workspace fleet node configuration (reads local config, no cloud needed) + description: Show the workspace fleet-node configuration location: packages/cli/src/cli/commands/fleet.ts - verify_tier: 2 + verify_tier: 5 - id: fleet-enable name: Enable Fleet @@ -560,16 +642,16 @@ categories: - id: workspace-active name: Show Active Workspace cli: relay workspace active - description: Show the currently active workspace + description: Resolve and show the canonical Cloud identifiers for the active workspace location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 5 - id: workspace-create name: Create Workspace - cli: relay workspace create + cli: relay workspace create [--base-url ] description: Create a new workspace and store its key location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 5 - id: workspace-list name: List Workspaces @@ -580,24 +662,24 @@ categories: - id: workspace-set-key name: Store Workspace Key - cli: relay workspace set_key + cli: relay workspace set_key description: Store a workspace key under a name location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 - id: workspace-join name: Join Workspace - cli: relay workspace join + cli: relay workspace join description: Join a workspace by key and make it active location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 - id: workspace-switch name: Switch Workspace cli: relay workspace switch description: Switch the active workspace to a stored one location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 skills: name: Skills Management @@ -609,7 +691,7 @@ categories: cli: relay skills add description: Install the /orchestrate skill into one or more AI harnesses location: packages/cli/src/cli/commands/skills.ts - verify_tier: 1 + verify_tier: 5 integration: name: Integrations & Webhooks @@ -618,94 +700,101 @@ categories: features: - id: integration-subscribe name: Subscribe Integration - cli: relay integration subscribe [provider] + cli: relay integration subscribe --resource --to <@agent|#channel> [--no-input] description: Subscribe a relay recipient to a relayfile integration location: packages/cli/src/cli/commands/integration.ts - verify_tier: 2 + verify_tier: 6 - id: integration-unsubscribe name: Unsubscribe Integration - cli: relay integration unsubscribe + cli: relay integration unsubscribe --resource description: Remove a relayfile integration subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 2 + verify_tier: 6 + + - id: integration-list-bindings + name: List Relayfile Integration Bindings + cli: relay integration subscribe --list + description: List active Relayfile bindings and their Relay resources + location: packages/cli/src/cli/commands/integration.ts + verify_tier: 5 - id: webhook-create name: Create Webhook - cli: relay integration webhook create + cli: relay integration webhook create [--event ] description: Register an outbound webhook location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-list name: List Webhooks cli: relay integration webhook list description: Show all registered webhooks location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-delete name: Delete Webhook cli: relay integration webhook delete description: Remove a registered webhook location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-trigger name: Trigger Webhook - cli: relay integration webhook trigger + cli: relay integration webhook trigger [--payload ] description: Manually fire a webhook for testing location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-create name: Create Inbound Webhook - cli: relay integration webhook create-inbound + cli: relay integration webhook create-inbound [--name ] description: Create an endpoint external services POST to, delivering into a channel location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-list name: List Inbound Webhooks cli: relay integration webhook list-inbound description: Show all inbound webhook endpoints location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-delete name: Delete Inbound Webhook - cli: relay integration webhook delete-inbound + cli: relay integration webhook delete-inbound description: Remove an inbound webhook endpoint location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-create name: Create Event Subscription - cli: relay integration subscription create + cli: relay integration subscription create [--filter ] [--url ] [--secret ] description: Subscribe to workspace events location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-list name: List Subscriptions cli: relay integration subscription list description: Show all event subscriptions location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-get name: Get Subscription cli: relay integration subscription get description: Show details for a specific subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-delete name: Delete Subscription cli: relay integration subscription delete description: Remove an event subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 reflex: name: Reflex @@ -717,21 +806,21 @@ categories: cli: relay reflex on description: Enable reflex history sync so agents receive relay activity context location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 6 - id: reflex-off name: Reflex Off cli: relay reflex off description: Disable reflex history sync location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 1 - id: reflex-status name: Reflex Status cli: relay reflex status description: Show whether reflex history sync is enabled location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 1 mcp: name: MCP Server Mode @@ -743,98 +832,231 @@ categories: cli: relay mcp description: Run agent-relay as an MCP server so harnesses can use relay tools location: packages/cli/src/cli/agent-relay-mcp.ts - verify_tier: 2 + verify_tier: 1 + + - id: mcp-create-workspace + name: MCP create_workspace + mcp: create_workspace + description: Create a workspace and retain its key in the MCP session + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-set-workspace-key + name: MCP set_workspace_key + mcp: set_workspace_key + description: Set the workspace key used by the MCP session + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 - id: mcp-register-agent name: MCP register_agent mcp: register_agent description: Register a new agent from within a harness - location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 2 + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 - id: mcp-list-agents name: MCP list_agents mcp: list_agents description: List workspace agents from within a harness - location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 2 + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-query-nodes + name: MCP query_nodes + mcp: query_nodes + description: Query fleet nodes in the current workspace + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-add-agent + name: MCP add_agent + mcp: add_agent + description: Spawn a managed worker through the workspace API + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 + + - id: mcp-spawn + name: MCP spawn + mcp: spawn + description: Invoke the workspace spawn action for an agent identity + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 + + - id: mcp-remove-agent + name: MCP remove_agent + mcp: remove_agent + description: Release a managed worker from active duty + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 - id: mcp-create-channel name: MCP create_channel mcp: create_channel description: Create a channel from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-channels name: MCP list_channels mcp: list_channels description: List channels from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-join-channel + name: MCP join_channel + mcp: join_channel + description: Join a channel as the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-leave-channel + name: MCP leave_channel + mcp: leave_channel + description: Leave a channel as the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-invite-to-channel + name: MCP invite_to_channel + mcp: invite_to_channel + description: Invite an agent to a channel + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-set-channel-topic + name: MCP set_channel_topic + mcp: set_channel_topic + description: Update a channel topic + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-archive-channel + name: MCP archive_channel + mcp: archive_channel + description: Archive a channel + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-post-message name: MCP post_message mcp: post_message description: Post a message from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-messages name: MCP list_messages mcp: list_messages description: Retrieve channel history from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-reply-to-message - name: MCP reply_to_message - mcp: reply_to_message + name: MCP reply_to_thread + mcp: reply_to_thread description: Reply in a thread from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-get-message-thread + name: MCP get_message_thread + mcp: get_message_thread + description: Retrieve a message thread from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-send-dm name: MCP send_dm mcp: send_dm description: Send a DM from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 4 + verify_tier: 5 + + - id: mcp-list-dms + name: MCP list_dms + mcp: list_dms + description: List direct-message conversations for the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-send-group-dm + name: MCP send_group_dm + mcp: send_group_dm + description: Create a group direct-message conversation and post its first message + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-add-reaction name: MCP add_reaction mcp: add_reaction description: Add a reaction from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-remove-reaction + name: MCP remove_reaction + mcp: remove_reaction + description: Remove a reaction from a message from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-search-messages name: MCP search_messages mcp: search_messages description: Search messages from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-inbox - name: MCP list_inbox - mcp: list_inbox + name: MCP check_inbox + mcp: check_inbox description: Check unread inbox from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-mark-message-read + name: MCP mark_message_read + mcp: mark_message_read + description: Mark a message read from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-get-message-readers + name: MCP get_message_readers + mcp: get_message_readers + description: List the agents that read a message from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-invoke-action name: MCP invoke_action mcp: invoke_action description: Fire an action from within a harness location: packages/cli/src/cli/mcp/action-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-actions name: MCP list_actions mcp: list_actions description: List available actions from within a harness location: packages/cli/src/cli/mcp/action-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-system-prompt + name: MCP system prompt + mcp_prompt: system + description: Retrieve Agent Relay's default collaboration instructions through the MCP prompts interface + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 1 + + - id: mcp-submit-result + name: MCP submit_result + mcp: submit_result + description: Submit a structured task result when AGENT_RELAY_RESULT_URL and AGENT_RELAY_RESULT_TOKEN configure the callback + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 harnesses: name: AI Harness Integrations @@ -845,19 +1067,19 @@ categories: name: Claude Code Harness description: Run agents using Claude Code (claude CLI) with PTY injection location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-codex name: Codex Harness description: Run agents using OpenAI Codex CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-gemini name: Gemini Harness description: Run agents using Google Gemini CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-cursor name: Cursor Harness @@ -869,31 +1091,37 @@ categories: name: Aider Harness description: Run agents using the Aider code editing tool location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-goose name: Goose Harness description: Run agents using Block's Goose automation tool location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-grok name: Grok Harness description: Run agents using xAI Grok CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-opencode name: OpenCode Harness description: Run agents using the OpenCode extension location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 + + - id: harness-droid + name: Droid Harness + description: Run agents using Factory's Droid CLI + location: packages/harnesses/src/index.ts + verify_tier: 6 - id: harness-custom name: Custom Harness description: Define a bespoke harness via the defineHarness contract location: packages/harnesses/ - verify_tier: 2 + verify_tier: 3 sdk: name: SDK (Programmatic API) @@ -920,10 +1148,197 @@ categories: - id: sdk-delivery name: SDK Delivery Control - description: Control message delivery hold/flush/auto programmatically + description: Process inbox work with DeliveryRunner acknowledgements, failures, and deferrals location: packages/sdk/src/ verify_tier: 3 + - id: sdk-workspace + name: SDK Workspace and Identity + description: Create or reconnect to workspaces and register agent or human participants + location: packages/sdk/src/agent-relay.ts, packages/sdk/src/facade.ts + verify_tier: 5 + + - id: sdk-realtime + name: SDK Realtime Listeners + description: Consume workspace events through listeners, event fan-in, and observer mode + location: packages/sdk/src/listeners.ts, packages/sdk/src/messaging/ + verify_tier: 5 + + - id: sdk-integrations + name: SDK Nodes and Integrations + description: Manage fleet nodes, triggers, event subscriptions, and inbound webhooks programmatically + location: packages/sdk/src/facade.ts, packages/sdk/src/messaging/ + verify_tier: 5 + + - id: sdk-harness-contract + name: SDK Harness and Session Contracts + description: Define custom harnesses and session capabilities for Relay-connected agents + location: packages/sdk/src/session/, packages/sdk/src/agent-relay.ts + verify_tier: 3 + + python-sdk: + name: Python SDK + description: Python orchestration, Relay communication, adapters, and workflow building + criticality: hot + features: + - id: python-sdk-orchestration + name: Python AgentRelay Orchestration + description: Spawn, message, release, and wait for managed Python AgentRelay agents + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + - id: python-sdk-communication + name: Python Relay Communication + description: Coordinate agents through the Relay communication interface + location: packages/sdk-py/src/agent_relay/ + verify_tier: 3 + + - id: python-sdk-adapters + name: Python Relay Adapters + description: Attach Relay behavior to supported Python agent frameworks with on_relay adapters + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + - id: python-sdk-workflows + name: Python Workflow Builder + description: Define Python workflow templates and multi-agent workflow steps + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + swift-sdk: + name: Swift SDKs + description: Swift clients for hosted Agent Relay and the local broker + criticality: standard + features: + - id: swift-sdk-hosted + name: AgentRelaySDK + description: Use the hosted Swift Agent Relay client for workspaces, messaging, and events + location: packages/sdk-swift/Sources/ + verify_tier: 5 + + - id: swift-sdk-broker + name: AgentRelayBrokerSDK + description: Use the local-broker Swift SDK for broker-backed agent coordination + location: packages/sdk-swift/Sources/ + verify_tier: 2 + + opencode-plugin: + name: OpenCode Relay Plugin + description: OpenCode-native Relaycast tools and session lifecycle hooks + criticality: hot + features: + - id: opencode-relay-connect + name: OpenCode relay_connect + description: Connect an OpenCode session to an Agent Relay workspace + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-send + name: OpenCode relay_send + description: Send a direct message from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-inbox + name: OpenCode relay_inbox + description: Check the Relay inbox from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-agents + name: OpenCode relay_agents + description: List Relay agents from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-post + name: OpenCode relay_post + description: Post a channel message from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-spawn + name: OpenCode relay_spawn + description: Spawn a Relay-connected OpenCode worker + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 6 + + - id: opencode-relay-dismiss + name: OpenCode relay_dismiss + description: Stop and release a spawned OpenCode worker + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 6 + + - id: opencode-relay-hooks + name: OpenCode Relay Lifecycle Hooks + description: Preserve Relay state across compaction, poll while idle, and clean up workers on session end + location: plugins/opencode-relay-plugin/src/hooks.ts + verify_tier: 6 + + codex-relay-skill: + name: Codex Relay Skill + description: Codex-native Relay coordination skill, MCP setup, hooks, and relay-worker template + criticality: hot + features: + - id: codex-relay-install + name: Install Codex Relay Skill + description: Install the project or user-scoped skill and idempotently configure Codex Relay MCP support + location: plugins/codex-relay-skill/scripts/setup.sh + verify_tier: 5 + + - id: codex-relay-worker + name: Codex relay-worker + description: Delegate a bounded task to the Relay-aware Codex worker template + location: plugins/codex-relay-skill/codex-config/relay-worker.toml + verify_tier: 6 + + - id: codex-relay-hooks + name: Codex Relay Hooks + description: Connect on session start, inject inbox state, and protect completion while messages remain unread + location: plugins/codex-relay-skill/hooks/ + verify_tier: 6 + + gemini-relay-extension: + name: Gemini CLI Relay Extension + description: Gemini-native Relay MCP integration, subagents, hooks, and orchestration commands + criticality: hot + features: + - id: gemini-relay-install + name: Install Gemini Relay Extension + description: Install the Relay extension into Gemini CLI and connect a Relay workspace + location: plugins/gemini-relay-extension/gemini-extension.json + verify_tier: 6 + + - id: gemini-relay-status + name: Gemini /relay:status + description: Show Relay connection and team status from Gemini CLI + location: plugins/gemini-relay-extension/commands/status/ + verify_tier: 6 + + - id: gemini-relay-team + name: Gemini /relay:team + description: Delegate a coordinated team task to Relay-aware Gemini workers + location: plugins/gemini-relay-extension/commands/team/ + verify_tier: 6 + + - id: gemini-relay-fanout + name: Gemini /relay:fanout + description: Fan out independent work to Relay-aware Gemini workers + location: plugins/gemini-relay-extension/commands/fanout/ + verify_tier: 6 + + - id: gemini-relay-workers + name: Gemini Relay Worker Roles + description: Run the relay-worker, relay-researcher, or relay-reviewer Gemini subagent roles + location: plugins/gemini-relay-extension/agents/ + verify_tier: 6 + + - id: gemini-relay-hooks + name: Gemini Relay Lifecycle Hooks + description: Auto-connect, poll/inject inbox content, guard stop, and clean up session workers + location: plugins/gemini-relay-extension/hooks/ + verify_tier: 6 + setup: name: Setup & Health description: Initial setup, diagnostics, and CLI lifecycle @@ -938,11 +1353,18 @@ categories: - id: cli-update name: Update CLI - cli: relay update + cli: relay update [--check] description: Check for and install CLI updates location: packages/cli/src/cli/commands/core.ts verify_tier: 1 + - id: cli-uninstall + name: Uninstall CLI + cli: relay uninstall [--keep-data] [--force] [--dry-run] + description: Remove Agent Relay runtime data, configuration, and global binaries + location: packages/cli/src/cli/commands/core.ts + verify_tier: 6 + telemetry: name: Telemetry description: Usage analytics opt-in/out @@ -952,21 +1374,21 @@ categories: name: Enable Telemetry cli: relay telemetry enable description: Opt in to usage telemetry reporting - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 - id: telemetry-disable name: Disable Telemetry cli: relay telemetry disable description: Opt out of usage telemetry reporting - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 - id: telemetry-status name: Telemetry Status cli: relay telemetry status description: Show current telemetry opt-in/out status - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 node: @@ -976,14 +1398,14 @@ categories: features: - id: node-up name: Node Up - cli: relay node up - description: Start this node's local broker (use --background to run as daemon) + cli: relay node up [--no-spawn] [--config ] + description: Start this node in the foreground, optionally serving a node-definition file location: packages/cli/src/cli/commands/node.ts verify_tier: 1 - id: node-workflow - name: Node Workflow - cli: relay node workflow - description: Run and inspect workflows on this node + name: Workflow Command Discovery + cli: relay node workflow --help + description: Display the available local workflow commands; run, logs, and sync are cataloged separately location: packages/cli/src/cli/commands/node.ts - verify_tier: 2 + verify_tier: 1 diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 260004fb6..5d4cfbf70 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -1,402 +1,529 @@ # Feature Verification Procedures -How an agent verifies that each feature works from a user perspective. Organized by tier (what's required to run). Always run lower tiers first — they establish prerequisites for higher ones. +This document is the executable companion to `../manifest.yaml`. Its `verification.categories` map assigns every feature in a category to one procedure below. The tier orders work, but never replaces the stated prerequisites, assertions, cleanup, or automation limitation. ---- +Use `relay` (or the equivalent `agent-relay` binary). Run all mutations in a disposable project, workspace, account, configuration directory, or provider fixture. Never use global `update`, non-dry-run `sync`/`uninstall`, `node down --all`, or production credentials in an unattended test. -## Tier 1 — No dependencies +## Shared fixtures -Features that can be verified with just the CLI installed. Run these first. - -### CLI Health +Start every procedure in a Bash shell with an isolated run identifier, strict +failure handling, and cleanup. The trap safely cleans only resources named for +this run, whether the procedure uses a broker, hosted APIs, or both: ```bash -relay version -# → prints version string like "10.x.x" - -relay --help -# → shows command list without error +set -Eeuo pipefail +RUN_ID="${CI_RUN_ID:-${GITHUB_RUN_ID:-local}}" +RUN_RANDOM="$(od -An -N6 -tx1 /dev/urandom | tr -d '[:space:]')" +RUN="feature-${RUN_ID}-$(date +%s)-$RUN_RANDOM" +TMP="$(mktemp -d)" +TAIL_PID="" +CLOUD_WORKER_PID="" +CLOUD_WORKER_ID="" +cleanup_cloud_worker_daemon() { + [[ -n "$CLOUD_WORKER_PID" && -n "$CLOUD_WORKER_ID" ]] || return 0 + kill -0 "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + + local command + command="$(ps -p "$CLOUD_WORKER_PID" -o command= 2>/dev/null || true)" + if ! printf '%s\n' "$command" | grep -F -- "cloud worker start --worker-id $CLOUD_WORKER_ID" >/dev/null || \ + ! printf '%s\n' "$command" | grep -F -- '--foreground-child' >/dev/null; then + printf 'Refusing to terminate unexpected cloud-worker process %s\n' "$CLOUD_WORKER_PID" >&2 + return 1 + fi + + kill -TERM "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + for _ in 1 2 3 4 5; do + kill -0 "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + sleep 1 + done + + # Recheck the command before escalating in case the PID was reused. + command="$(ps -p "$CLOUD_WORKER_PID" -o command= 2>/dev/null || true)" + if printf '%s\n' "$command" | grep -F -- "cloud worker start --worker-id $CLOUD_WORKER_ID" >/dev/null && \ + printf '%s\n' "$command" | grep -F -- '--foreground-child' >/dev/null; then + kill -KILL "$CLOUD_WORKER_PID" 2>/dev/null || true + fi +} +cleanup() { + local status=$? + if [[ -n "$TAIL_PID" ]]; then + kill "$TAIL_PID" 2>/dev/null || true + wait "$TAIL_PID" 2>/dev/null || true + fi + cleanup_cloud_worker_daemon || true + relay agent remove "audit-extra-$RUN" 2>/dev/null || true + relay agent remove "audit-c-$RUN" 2>/dev/null || true + relay agent remove "audit-b-$RUN" 2>/dev/null || true + relay agent remove "audit-a-$RUN" 2>/dev/null || true + relay node down --force 2>/dev/null || true + rm -rf "$TMP" || true + exit "$status" +} +trap cleanup EXIT +``` -relay telemetry status -# → prints "enabled" or "disabled" +For a local broker, use the shared temporary project and only stop the broker it +created: -relay workspace list -# → prints list (may be empty) without error +```bash +cd "$TMP" +relay node up --background --no-spawn ``` -### Setup and Doctor +For hosted APIs, use explicit disposable credentials rather than an operator's active workspace: ```bash -relay version -# → confirms CLI is installed (relay setup does not exist) - -relay node --help -# → shows node subcommand options without error +export RELAY_WORKSPACE_KEY='' +export RELAY_BASE_URL='' +A_JSON="$(relay agent register "audit-a-$RUN")"; TOKEN_A="$(jq -er .token <<<"$A_JSON")" +B_JSON="$(relay agent register "audit-b-$RUN")"; TOKEN_B="$(jq -er .token <<<"$B_JSON")" ``` -Pass criteria: all commands exit 0, output is non-empty and sensibly formatted. +SDK-backed CLI commands emit JSON already; do not add an unsupported `--json` flag. ---- +## Externally provisioned fixtures -## Tier 2 — Broker running +Two procedures need fixtures that the Relay CLI cannot create. Create only the +fixture required by the procedure you are about to run; its command block +performs the corresponding fail-fast check. -Start the broker first: `relay node up --background` +```bash +# A Cloud *worker* enrollment token minted for the disposable Cloud workspace. +# This is not the node-enrollment token used by `relay cloud enroll`. +export WORKER_TOKEN='' + +# A unique, internet-reachable receiver owned by the test harness. It must +# accept the webhook POST, retain request headers and body for assertion, and +# return a 2xx response. Localhost is suitable only when the tested Relay API +# is local; hosted APIs require a tunneled or CI-provisioned receiver. +export CAPTURE_URL='https://controlled-test-receiver.example/unique-run-path' +# Fetch only this run's captured requests from the same controlled receiver. +export CAPTURE_FETCH_URL='https://controlled-test-receiver.example/unique-run-path/requests' +``` -Confirm it started: `relay status` should show "running". +Do not substitute a public request-bin or a shared endpoint: webhook payloads +and signatures are test data that must remain isolated. Each run gets its own +receiver path and destroys it after the webhook/subscription cleanup below. -### Broker Lifecycle +## broker-lifecycle -```bash -relay node up --background -relay status -# → shows "running", agent count, queue stats +**Features:** `broker-up`, `broker-down`, `broker-status`, `broker-metrics`, `broker-deadletters`, `broker-redeliver`. + +**Prerequisites:** local fixture and broker binary. -relay metrics -# → shows memory, CPU, message throughput +```bash +relay node status | grep -Eiq 'running|stopped' +relay node metrics | jq -e 'type == "object"' +relay node deadletters --json | jq -e 'type == "object" or type == "array"' +relay status | grep -Eiq 'Local broker:' +relay node down; relay node status | grep -Eiq 'stopped|not running' +``` -relay deadletters -# → shows empty list or existing dead letters (not an error) +Assert daemon state and structured metrics/deadletters, then restart only when a later local test needs it. A successful redelivery needs a real dead-letter addressed to a live worker; the public CLI cannot seed one. Use the broker integration fixture, run exactly one of `relay node redeliver ` or `relay node redeliver --all`, and assert the delivery leaves the dead-letter queue. -relay node down -relay status -# → shows "stopped" or "not running" +## workspace-agents-and-capabilities -relay node up --background # restart for subsequent tests -``` +**Features:** `agent-register`, `agent-add`, `agent-list`, `agent-remove`, `agent-capabilities`, `capabilities-register`, `capabilities-delete`, `system-status`. -### Agent Management +**Prerequisites:** hosted fixture; a local broker is not the dependency. ```bash -# Register agents -relay agent register verify-agent-1 -# → prints token: RELAY_AGENT_TOKEN= +CAP="audit-cap-$RUN" +relay agent list | jq -e --arg n "audit-a-$RUN" '.[] | select(.name == $n)' +relay agent add "audit-extra-$RUN" | jq -e '.token' +relay capabilities register "$CAP" --description 'audit capability' --handler "audit-a-$RUN" +relay capabilities list | jq -e --arg c "$CAP" '.[] | select(.command == $c)' +relay capabilities delete "$CAP" +relay capabilities list | jq -e --arg c "$CAP" 'all(.[]; .command != $c)' +relay agent remove "audit-extra-$RUN" +relay agent list | jq -e --arg n "audit-extra-$RUN" 'all(.[]; .name != $n)' +relay status | grep -Eiq 'Workspace:|Local broker:|Cloud:' +``` -relay agent list -# → shows verify-agent-1 in list +Assert list visibility after every create and absence after every delete; the shared exit trap removes all disposable identities. -relay agent remove verify-agent-1 -relay agent list -# → verify-agent-1 no longer appears -``` +## channel-messaging -### Local Agent Orchestration +**Features:** `channel-create`, `channel-list`, `channel-join`, `channel-leave`, `channel-invite`, `channel-set-topic`, `channel-archive`. + +**Prerequisites:** hosted fixture with two identities. ```bash -relay node agent list -# → shows empty list or running agents +CH="audit-channel-$RUN" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" --topic 'first topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list | jq -e --arg n "$CH" \ + '.[] | select(.name == $n)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel set_topic "$CH" 'second topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list | jq -e --arg n "$CH" --arg topic 'second topic' \ + '.[] | select(.name == $n and .topic == $topic)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite "$CH" "audit-b-$RUN" +RELAY_AGENT_TOKEN="$TOKEN_B" relay channel list | jq -e --arg n "$CH" --arg agent "audit-b-$RUN" \ + '.[] | select( + .name == $n and + ((.members // []) | map(if type == "string" then . else (.agentName // .name) end) | index($agent)) + )' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel leave "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list --archived | jq -e --arg n "$CH" '.[] | select(.name == $n)' +``` -# If claude harness is available: -relay node agent spawn claude --name test-worker -relay node agent list -# → shows test-worker with status active +Assert the new topic and invited-agent membership, not just success output. Archive the test channel and remove test identities. -relay node agent message hold test-worker -# → message delivery paused +## message-round-trip -relay node agent message auto test-worker -# → message delivery resumed +**Features:** `message-post`, `message-list`, `message-reply`, `message-get-thread`, `message-search`, `message-file-upload`. -relay node agent release test-worker -relay node agent list -# → test-worker no longer appears +**Prerequisites:** hosted fixture and a channel owned by A. + +```bash +CH="audit-messages-$RUN"; TEXT="audit message $RUN" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post "$CH" "$TEXT")"; MSG_ID="$(jq -er '.id // .messageId' <<<"$POST")" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message list "$CH" --limit 10 | jq -e --arg t "$TEXT" '.[] | select(.text == $t)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message reply "$MSG_ID" 'audit reply' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message get_thread "$MSG_ID" | jq -e 'tostring | contains("audit reply")' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message search "$RUN" --channel "$CH" --from "audit-a-$RUN" | jq -e 'tostring | contains("audit message")' +echo "attachment $RUN" > "$TMP/attachment.txt" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message file upload "$TMP/attachment.txt" --channel "$CH" --text 'attachment audit' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message list "$CH" --limit 20 | jq -e 'tostring | contains("attachment audit")' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" ``` -### Local Workflow +Assert that the returned parent ID links the reply, search filters return the unique text, and the attachment is listed. Remove the temp file, archive, and let the shared exit trap remove identities. + +## direct-messages + +**Features:** `dm-send`, `dm-list`, `dm-list-conversations`, `dm-send-group`. + +**Prerequisites:** hosted fixture; group DM needs third identity C. ```bash -# Requires a minimal workflow file; check examples/ or create one: -cat > /tmp/test-workflow.yaml << 'EOF' -version: "1" -swarm: - agents: - - name: test-agent - harness: claude - workflows: - - name: health-check - steps: - - agent: test-agent - prompt: "Reply with just: OK" -EOF - -relay node workflow run /tmp/test-workflow.yaml -# → executes without crashing, agent responds +C_JSON="$(relay agent register "audit-c-$RUN")"; TOKEN_C="$(jq -er .token <<<"$C_JSON")" +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "dm $RUN")"; CONV="$(jq -er '.conversationId // .conversation_id' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV" | jq -e --arg t "dm $RUN" 'tostring | contains($t)' +GROUP="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group $RUN" --to "audit-b-$RUN" "audit-c-$RUN")"; GROUP_CONV="$(jq -er '.conversationId // .conversation_id' <<<"$GROUP")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$GROUP_CONV" | jq -e --arg t "group $RUN" 'tostring | contains($t)' +RELAY_AGENT_TOKEN="$TOKEN_C" relay message dm list "$GROUP_CONV" | jq -e --arg t "group $RUN" 'tostring | contains($t)' +relay agent remove "audit-c-$RUN" ``` -### Workspace +Use `list_dms` through the MCP client and assert the same conversation appears. Assert both group recipients see it before cleanup. -```bash -relay workspace active -# → prints active workspace name/id +## reactions-and-read-status -relay workspace list -# → lists stored workspaces -``` +**Features:** `reaction-add`, `reaction-remove`, `inbox-check`, `inbox-mark-read`, `inbox-get-readers`. -### Fleet Status +**Prerequisites:** hosted fixture with two identities. ```bash -relay fleet status -# → shows local broker status and provider attachment (even if no fleet configured) +CH="audit-read-$RUN"; RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" +RELAY_AGENT_TOKEN="$TOKEN_B" relay channel join "$CH" +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post "$CH" "read $RUN")"; MSG_ID="$(jq -er '.id // .messageId' <<<"$POST")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction add "$MSG_ID" thumbsup +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list "$CH" --limit 10 | jq -e --arg id "$MSG_ID" ' + .[] | select((.id == $id) or (.messageId == $id)) + | .reactions[]? | select(.emoji == "thumbsup" and (.count // 0) >= 1) +' +RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction remove "$MSG_ID" thumbsup +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list "$CH" --limit 10 | jq -e --arg id "$MSG_ID" ' + [ .[] | select((.id == $id) or (.messageId == $id)) + | .reactions[]? | select(.emoji == "thumbsup" and (.count // 0) > 0) ] + | length == 0 +' +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "inbox $RUN")"; DM_ID="$(jq -er '.id // .messageId' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox check | jq -e --arg t "inbox $RUN" 'tostring | contains($t)' +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox mark_read "$DM_ID" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message inbox get_readers "$DM_ID" | jq -e --arg n "audit-b-$RUN" 'tostring | contains($n)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" ``` -Pass criteria: each command exits 0, output matches expected description. +Assert the reaction is observable after add and absent after remove, and B's unread message becomes A-visible read receipt. Archive and remove identities. ---- +## local-agent-lifecycle -## Tier 3 — Broker running + agent token +**Features:** `local-agent-spawn`, `local-agent-new`, `local-agent-release`, `local-agent-list`, `local-agent-set-model`, `local-agent-attach`, `local-agent-flush`, `local-agent-hold`, `local-agent-auto`, `local-agent-tail`. -Register at least one agent (`relay agent register `) and export its token: +**Prerequisites:** local broker; spawn/new/set-model need an installed authenticated provider CLI and can incur cost. ```bash -export RELAY_AGENT_TOKEN=$(relay agent register verify-agent | jq -r '.token') +PROVIDER="${PROVIDER:-claude}" # installed and authenticated fixture provider +TASK_AGENT="audit-task-$RUN" +TASK_TAIL="$TMP/$TASK_AGENT.tail" +relay node agent list | jq -e 'type == "array"' +relay node tail --agent "$TASK_AGENT" >"$TASK_TAIL" 2>&1 & TAIL_PID=$! +relay node agent spawn "$PROVIDER" --name "$TASK_AGENT" --task 'Reply with exactly relay-e2e-ok, then exit.' --spawn-mode task-exit --exit-after-task +for _ in $(seq 1 60); do + grep -Fq 'relay-e2e-ok' "$TASK_TAIL" && break + sleep 1 +done +grep -Fq 'relay-e2e-ok' "$TASK_TAIL" +kill "$TAIL_PID" 2>/dev/null || true; wait "$TAIL_PID" 2>/dev/null || true; TAIL_PID="" +AGENT="audit-worker-$RUN" +relay node agent spawn "$PROVIDER" --name "$AGENT" +relay node agent list | jq -e --arg n "$AGENT" '.[] | select(.name == $n)' +relay node agent message hold "$AGENT" | jq -e 'tostring | test("manual|hold"; "i")' +relay node agent message auto "$AGENT" | jq -e 'tostring | test("auto"; "i")' +relay node agent release "$AGENT" ``` -Or pass `--token ` to commands that support it. +The bounded task-exit worker proves task output through `relay node tail` before it exits; the separate interactive worker proves hold, auto, and release. `new` and `attach` are attended PTY checks; verify `drive`, `view`, and `passthrough`. A model switch proves broker acceptance only, not provider-side model change. -### Channel Operations +## local-workflow-lifecycle + +**Features:** `local-workflow-run`, `local-workflow-logs`, `local-workflow-sync`. + +**Prerequisites:** disposable local project and selected workflow runtime; no broker. ```bash -# Create -relay channel create verify-test-channel -# → success message - -# List -relay channel list -# → shows verify-test-channel - -# Join -relay channel join verify-test-channel -# → success message - -# Set topic -relay channel set_topic verify-test-channel "verification test" -# → success message - -# Leave -relay channel leave verify-test-channel -# → success message - -# Archive -relay channel archive verify-test-channel -# → success message -relay channel list -# → verify-test-channel absent (or present with archived flag if --archived passed) +echo 'console.log("relay-workflow-e2e")' > "$TMP/workflow.js" +RUN_JSON="$(relay node workflow run "$TMP/workflow.js" --json)"; RUN_ID="$(jq -er .runId <<<"$RUN_JSON")" +relay node workflow logs "$RUN_ID" --follow --json | jq -e 'tostring | contains("relay-workflow-e2e")' +relay node workflow sync "$RUN_ID" --dry-run --json | jq -e 'type == "object"' ``` -### Message Operations +Assert completed status and log content. Local run records have no delete command; remove only the disposable state/project directory. + +## cloud-workflows + +**Features:** `cloud-login`, `cloud-logout`, `cloud-whoami`, `cloud-session`, `cloud-connect`, `cloud-enroll`, `cloud-run`, `cloud-schedule`, `cloud-schedules`, `cloud-status`, `cloud-logs`, `cloud-sync`, `cloud-cancel`. + +**Prerequisites:** dedicated Cloud account/workspace. Login and connect are browser/SSH interactive; enrollment needs a one-time test token or test-workspace mint. ```bash -# Post -relay channel create verify-msgs -relay channel join verify-msgs -relay message post verify-msgs "verification message $(date +%s)" -# → success - -# List and confirm delivery -relay message list verify-msgs --limit 1 -# → shows the message just posted with correct text - -# Reply (create thread) -MSG_ID=$(relay message list verify-msgs --limit 1 --json | jq -r '.[0].id') -relay message reply "$MSG_ID" "thread reply" -# → success - -# Get thread -relay message get_thread --message-id "$MSG_ID" -# → shows original message + the reply - -# Search -relay message search --query "verification" -# → returns at least one result containing the posted message - -# Inbox -relay message inbox check -# → shows unread count (may be 0 if no messages directed to this agent) - -# Mark read -relay message inbox mark_read --message-id "$MSG_ID" -# → success +relay cloud whoami +relay cloud session --json | jq -e '.apiUrl' +RUN_JSON="$(relay cloud run "$TMP/noop.yaml" --no-sync-code --json)"; CLOUD_RUN="$(jq -er '.runId // .id' <<<"$RUN_JSON")" +relay cloud status "$CLOUD_RUN" --json | jq -e 'type == "object"' +relay cloud logs "$CLOUD_RUN" --follow --json | jq -e 'type == "object"' +relay cloud sync "$CLOUD_RUN" --dry-run ``` -### Reactions +Use a no-op workflow, long disposable workflow for cancel, and only `sync --dry-run`. Schedules lack a CLI delete, so create them only in an isolated workspace and remove through Cloud control plane. Logout only an isolated account. -```bash -relay message reaction add --message-id "$MSG_ID" --emoji thumbsup -# → success +## cloud-workers -relay message reaction remove --message-id "$MSG_ID" --emoji thumbsup -# → success -``` +**Features:** `cloud-worker-register`, `cloud-worker-start`, `cloud-worker-status`, `cloud-worker-logs`. -### Webhooks (requires auth token) +**Prerequisites:** dedicated enrollment token and isolated worker state. ```bash -relay integration webhook list -# → success (empty or populated) +: "${WORKER_TOKEN:?See Externally provisioned fixtures}" +export AGENT_RELAY_HOME="$TMP/cloud-worker-state" +WORKER_NAME="audit-worker-$RUN" +REGISTERED="$(relay cloud worker register --token "$WORKER_TOKEN" --name "$WORKER_NAME" --json)" +WORKER_ID="$(jq -er '.workerId | strings | select(length > 0)' <<<"$REGISTERED")" +relay cloud worker start --worker-id "$WORKER_ID" --daemon +WORKER_STATUS="$(relay cloud worker status --worker-id "$WORKER_ID" --json)" +CLOUD_WORKER_PID="$(jq -er '.localDaemon.pid | numbers | select(. > 0)' <<<"$WORKER_STATUS")" +CLOUD_WORKER_ID="$WORKER_ID" +CLOUD_WORKER_LOG_PATH="$(jq -er '.localDaemon.logPath | strings | select(length > 0)' <<<"$WORKER_STATUS")" +jq -e '.localDaemon.running == true' <<<"$WORKER_STATUS" +test -r "$CLOUD_WORKER_LOG_PATH" +relay cloud worker logs --worker-id "$WORKER_ID" >/dev/null +cleanup_cloud_worker_daemon +CLOUD_WORKER_PID=""; CLOUD_WORKER_ID="" +``` -HOOK_ID=$(relay integration webhook create https://example.com/hook --event message.created | jq -r '.id') -# → prints webhook id +The CLI persists the daemon PID and log path in its isolated local worker state, so the procedure derives both through `status --json` rather than parsing command output. Cleanup checks that the PID is still the `--foreground-child` process for this worker ID before sending `TERM` (then `KILL` only after the same check); it never kills a PID merely because it was recorded. There is no worker stop or deregister CLI command: this terminates only the local daemon, while its local record and remote registration remain for the disposable test workspace/Cloud control plane to clean up. For a fixture with one bounded assignment, add `--once` to `start`; it exits after that assignment, but retain the cleanup check in case it is still running. -relay integration webhook delete "$HOOK_ID" -# → success -``` +## fleet-management + +**Features:** `fleet-nodes`, `fleet-config`, `fleet-enable`, `fleet-disable`, `fleet-inherit`, `fleet-status`. -### Subscriptions +**Prerequisites:** disposable workspace; `fleet-status` additionally benefits from a local broker. ```bash -relay integration subscription list -# → success (empty or populated) +relay fleet nodes | jq -e '.nodes' +BEFORE="$(relay fleet config)" +relay fleet enable; relay fleet config | jq -e 'type == "object"' +relay fleet disable; relay fleet inherit; relay fleet status | jq -e '.broker' ``` -Pass criteria: all channel/message round-trips show data that matches what was written. +Snapshot and restore configuration or discard the workspace. For full two-node dispatch/enrollment coverage, run `npm run test:e2e` with `tests/e2e/fleet/README.md` prerequisites. ---- +## workspace-management -## Tier 4 — Broker running + two agents +**Features:** `workspace-active`, `workspace-create`, `workspace-list`, `workspace-set-key`, `workspace-join`, `workspace-switch`. -Register two agents and test cross-agent features: +**Prerequisites:** isolated `AGENT_RELAY_HOME`; active/create also require Cloud API/auth. ```bash -export TOKEN_A=$(relay agent register verify-alice | jq -r '.token') -export TOKEN_B=$(relay agent register verify-bob | jq -r '.token') +export AGENT_RELAY_HOME="$(mktemp -d)" +relay workspace set_key audit-one rk_live_example +relay workspace join audit-two rk_live_example_two +relay workspace switch audit-one +relay workspace list | jq -e '.active == "audit-one"' +rm -rf "$AGENT_RELAY_HOME" ``` -### Channel Invite +Create remote workspaces only in a test tenant, assert `workspace active --json` returns canonical IDs, and delete them through the control plane. -```bash -RELAY_AGENT_TOKEN=$TOKEN_A relay channel create private-verify -RELAY_AGENT_TOKEN=$TOKEN_A relay channel join private-verify -RELAY_AGENT_TOKEN=$TOKEN_A relay channel invite private-verify verify-bob -# → success +## skills-installation -RELAY_AGENT_TOKEN=$TOKEN_B relay channel list -# → shows private-verify as a channel bob is in -``` +**Features:** `skills-add`. -### Direct Messages +**Prerequisites:** network and disposable project/configuration. ```bash -# Send DM and capture conversationId from JSON output: -DM_RESPONSE=$(RELAY_AGENT_TOKEN=$TOKEN_A relay message dm send verify-bob "hello bob") -CONV_ID=$(echo "$DM_RESPONSE" | jq -r '.conversationId') -# → success; conversationId captured +mkdir -p "$TMP/skills-project"; cd "$TMP/skills-project" +relay skills add --local --harness codex +test -e .agents/skills/orchestrate/SKILL.md +``` -# Verify recipient can list the conversation: -RELAY_AGENT_TOKEN=$TOKEN_B relay message dm list "$CONV_ID" -# → shows the DM from verify-alice +Assert the downloaded skill and delete only this disposable project. Do not run `--global` in automation. -RELAY_AGENT_TOKEN=$TOKEN_A relay message dm send_group "group hello" --to verify-bob -# → success -``` +## integrations-and-webhooks -### Read Receipts +**Features:** `integration-subscribe`, `integration-unsubscribe`, `integration-list-bindings`, all `webhook-*`, and all `subscription-*` entries. -```bash -MSG_ID=$(RELAY_AGENT_TOKEN=$TOKEN_A relay message list private-verify --limit 1 --json | jq -r '.[0].id') -RELAY_AGENT_TOKEN=$TOKEN_B relay message inbox mark_read --message-id "$MSG_ID" +**Prerequisites:** disposable workspace, identity, externally reachable controlled receiver. Relayfile also needs compatible authenticated daemon, connected provider, real provider resource, and provider observation API; first connection can require browser OAuth. The test-owned receiver must expose `CAPTURE_URL` for delivery and a run-scoped `CAPTURE_FETCH_URL` that returns `{"requests":[{"body":,"headers":{"lowercase-header-name":"string value"}}]}`. It must retain requests until this test reads them. -RELAY_AGENT_TOKEN=$TOKEN_A relay message inbox get_readers --message-id "$MSG_ID" -# → shows verify-bob has read the message +```bash +: "${CAPTURE_URL:?See Externally provisioned fixtures}" +: "${CAPTURE_FETCH_URL:?GET endpoint returning the current run captured requests}" +HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' +DELIVERED=false +for _ in $(seq 1 15); do + CAPTURE="$(curl --fail --silent --show-error "$CAPTURE_FETCH_URL")" + if jq -e ' + .requests[] + | select(.body == {"audit": true}) + | select( + [.headers | to_entries[] + | select(.key | contains("signature")) + | select(.value | (type == "string" and length > 0))] + | length > 0 + ) + ' <<<"$CAPTURE" >/dev/null; then + DELIVERED=true + break + fi + sleep 1 +done +test "$DELIVERED" = true +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook list | jq -e --arg id "$HOOK_ID" 'tostring | contains($id)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook delete "$HOOK_ID" ``` -### Cleanup +The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. + +## reflex-history + +**Features:** `reflex-on`, `reflex-off`, `reflex-status`. + +**Prerequisites:** isolated home; `on` is interactive and can use Cloud login. ```bash -relay agent remove verify-alice -relay agent remove verify-bob +echo y | relay reflex on; relay reflex status | grep -Eiq 'on' +relay reflex off; relay reflex status | grep -Eiq 'off' ``` -Pass criteria: messages posted by agent A appear in agent B's list, DMs route correctly. +Run only against a disposable home/configuration and assert cleanup leaves Reflex off. ---- +## mcp-stdio -## Tier 5 — Cloud auth required +**Features:** `mcp-server-start` and all `mcp-*` tool/prompt entries. -Requires `relay cloud login` to have been completed. +**Prerequisites:** Node MCP SDK client, disposable hosted workspace, registered identities. `add_agent`, `spawn`, and `submit_result` need real provider/callback fixtures. -### Auth Checks +Launch `relay mcp` as stdio child. Initialize it, call `tools/list` and `prompts/list`, and assert all 31 static tool names in the manifest plus prompt `system`. Call `set_workspace_key`/`register_agent` for A and B; repeat channel/message/thread/DM/reaction/read assertions through MCP fields (`message_id`, `include_archived`, `participants`, `as`). Register an action before `list_actions`/`invoke_action`. Spawn only a disposable worker then `remove_agent`. Configure result callback environment before testing conditional `submit_result`, assert receiver payload, and close the child cleanly. -```bash -relay cloud whoami -# → shows authenticated user/org +## harnesses -relay cloud session -# → shows session details (workspace, expiry, etc.) -``` +**Features:** all `harness-*` entries. + +**Prerequisites:** named provider CLI installed/authenticated, disposable project, and budget; custom harness only needs the SDK contract. + +Run the bounded local-agent spawn test for each installed provider with `--task`, `--spawn-mode task-exit`, and `--exit-after-task`; assert list, sentinel response, and release. PTY providers need attended verification. For a custom harness, run `npm test --workspace @agent-relay/harnesses` plus a `defineHarness` create/send/release fixture. + +## typescript-sdk + +**Features:** all `sdk-*` entries. -### Cloud Workflow Run +**Prerequisites:** local engine or disposable hosted workspace. ```bash -relay cloud run examples/basic-workflow.yaml -# → prints run ID +node tests/integration/sdk/v8-api-smoke.mjs +``` + +Assert bootstrap/reconnect, participants, channel/thread/reaction/DM/group-DM/listener/action/DeliveryRunner/webhook/node behavior. Release all identities and clean up remote workspace through its control plane; the smoke script does not delete it. -RUN_ID= -relay cloud status "$RUN_ID" -# → shows run status (queued, running, completed) +## python-sdk -relay cloud logs "$RUN_ID" -# → shows log output from the run +**Features:** all `python-sdk-*` entries. + +```bash +cd packages/sdk-py && python -m pytest ``` -### Schedules +For true E2E, create a disposable Relay, send/receive unique message, then release agents and shut down. Adapter/workflow tests require their provider CLI and must clean up spawned work. + +## swift-sdk + +**Features:** `swift-sdk-hosted`, `swift-sdk-broker`. ```bash -relay cloud schedule examples/basic-workflow.yaml --cron "0 * * * *" -# → prints schedule ID +cd packages/sdk-swift && swift test +``` + +Run hosted and local-broker round trips, assert receipt/listener behavior, then release identities and clean up the disposable workspace. + +## opencode-plugin -relay cloud schedules -# → shows the schedule just created +**Features:** all `opencode-relay-*` entries. + +```bash +npm --prefix plugins/opencode-relay-plugin test ``` -Pass criteria: cloud commands return data consistent with the authenticated account. +Connect two disposable OpenCode sessions, prove each native tool, spawn one worker, use `relay_dismiss`, and assert idle/compaction/end hooks preserve then clean state. + +## codex-relay-skill + +**Features:** all `codex-relay-*` entries. ---- +Copy the skill into a temporary `.agents/skills/agent-relay`, run setup twice, and diff resulting `.codex/config.toml`, hooks, and worker template to prove idempotence. In disposable Codex, require a relay-worker ACK/STATUS/DONE via MCP and assert hooks connect, surface inbox, and protect completion with unread work. -## Tier 6 — Manual / Browser +## gemini-relay-extension -These cannot be automated from CLI. A human must verify them. +**Features:** all `gemini-relay-*` entries. -| Feature | How to Verify | -| ------------------------------------ | --------------------------------------------------------------------------- | -| `relay cloud login` | Open browser OAuth flow, complete auth, check `relay cloud whoami` succeeds | -| Cursor harness | Open Cursor, run cursor-agent, confirm PTY injection works | -| Web dashboard (relay-dashboard repo) | Navigate to dashboard, confirm agents/channels visible | -| `relay cloud connect` | SSH session must complete provider auth interactively | -| `relay cloud enroll` | Machine enrollment requires cloud account and browser confirmation | +Install with `gemini extensions install AgentWorkforce/relay`; with experimental agents and disposable workspace run `/relay:status`, `/relay:team `, `/relay:fanout `. Assert worker ACK/DONE and session-start/after-tool/before-model/stop/session-end hook behavior, including cleanup. ---- +## cli-maintenance -## Verification Checklist by Change Type +**Features:** `cli-version`, `cli-update`, `cli-uninstall`. -Use this to determine which tiers to run: +```bash +relay version | grep -E '.' +relay update --check +relay uninstall --dry-run --keep-data +``` + +Assert version/update/removal-plan output. Mutating update/uninstall requires isolated OS/container and attended approval. + +## telemetry + +**Features:** `telemetry-enable`, `telemetry-disable`, `telemetry-status`. + +```bash +export AGENT_RELAY_DATA_DIR="$(mktemp -d)" +relay telemetry disable; relay telemetry status | grep -Eiq 'Enabled: No' +relay telemetry enable; relay telemetry status | grep -Eiq 'Enabled: Yes' +rm -rf "$AGENT_RELAY_DATA_DIR" +``` -| Change area | Tiers to run | -| ------------------------------------------------------------------ | ------------------------------------ | -| Broker Rust code (`crates/broker/`) | 1, 2, 3, 4 | -| PTY/harness code (`crates/relay-pty/`, `packages/harness-driver/`) | 2 (spawn tests) | -| CLI commands (`packages/cli/src/cli/commands/`) | 1 + tier matching the command | -| MCP tools (`packages/cli/src/cli/mcp/`) | 2, 3 (MCP server + tool calls) | -| SDK (`packages/sdk/`) | 3, 4 | -| Cloud client (`packages/cloud/`) | 5 | -| Any auth/token change | 2, 3, 4 | -| Message ordering/delivery | 3, 4 (post then list, confirm order) | -| Harness definitions (`packages/harnesses/`) | 2 (spawn that harness) | +Assert both persisted states; unset telemetry opt-out environment variables because they override stored preference. ---- +## node-command-discovery -## Quick Sanity Check (Run After Any Change) +**Features:** `node-up`, `node-workflow`. ```bash -relay version && relay status || relay node up --background && relay status -AGENT_NAME="quick-check-$(date +%s)" -export RELAY_AGENT_TOKEN=$(relay agent register "$AGENT_NAME" | jq -r '.token') -relay agent list -relay channel create quick-check-ch -relay channel join quick-check-ch -relay message post quick-check-ch "sanity $(date)" -relay message list quick-check-ch --limit 1 -relay channel archive quick-check-ch -relay agent remove "$AGENT_NAME" +relay node up --help | grep -Eiq 'config|no-spawn' +relay node workflow --help | grep -Eiq 'run|logs|sync' ``` -All steps should succeed. Total runtime: under 10 seconds. +Use `broker-lifecycle` for start behavior and `local-workflow-lifecycle` for the three workflow leaves. This group procedure is discoverability only, not workflow execution. diff --git a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json index ec034c701..51283b464 100644 --- a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json @@ -35,6 +35,47 @@ "reasoning": "Panic may occur on a tokio worker thread and the process may abort before the async sender loop drains; a dedicated std::thread with its own runtime avoids nested-runtime panic, and the reqwest timeout bounds process teardown" }, "significance": "high" + }, + { + "ts": 1784578305956, + "type": "decision", + "content": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose: Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "raw": { + "question": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "chosen": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "alternatives": [], + "reasoning": "The manifest already contains stale aliases and incomplete argument syntax, so repository source is the authoritative contract." + }, + "significance": "high" + }, + { + "ts": 1784579530918, + "type": "reflection", + "content": "Parallel audits reconciled the manifest with Commander registrations, MCP tools, SDKs, harnesses, and plugins. The catalog now uses explicit category-to-procedure mappings with contract tests so CLI and MCP drift fails locally before review.", + "raw": { + "focalPoints": ["surface-accuracy", "e2e-verification", "drift-prevention"], + "adjustments": "Added executable procedures and a manifest contract test; retained explicit external/interactive limits rather than claiming unsupported automation.", + "confidence": 0.9 + }, + "significance": "high", + "tags": [ + "focal:surface-accuracy", + "focal:e2e-verification", + "focal:drift-prevention", + "confidence:0.9" + ] + }, + { + "ts": 1784581060376, + "type": "decision", + "content": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child: Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "raw": { + "question": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "chosen": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "alternatives": [], + "reasoning": "The daemon is detached and persisted locally; checking worker ID plus --foreground-child before TERM/KILL prevents a stale or reused PID from targeting an unrelated process." + }, + "significance": "high" } ] } diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index 3a5b1dad6..f93c12d71 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -22,11 +22,25 @@ Use when you need to verify that a specific feature or set of features works cor ### 1. Read the manifest to find the feature ```bash -# View all features in a category -grep -A 10 "category: messaging-messages" .agentworkforce/features/manifest.yaml - -# Find a feature by id -grep -A 8 "id: message-post" .agentworkforce/features/manifest.yaml +# Prefer a structural query: category/feature length and indentation can change. +if command -v yq >/dev/null 2>&1; then + yq '.categories."messaging-messages"' .agentworkforce/features/manifest.yaml + yq '.. | select(type == "!!map" and .id == "message-post")' .agentworkforce/features/manifest.yaml +else + # Fallback for environments without mikefarah/yq: stop at the next sibling + # category/feature instead of assuming a fixed number of following lines. + awk ' + $0 == " messaging-messages:" { in_category = 1 } + in_category && $0 != " messaging-messages:" && /^ [[:alnum:]][[:alnum:]-]*:$/ { exit } + in_category { print } + ' .agentworkforce/features/manifest.yaml + awk ' + $0 == " - id: message-post" { in_feature = 1 } + in_feature && $0 != " - id: message-post" && /^ - id: / { exit } + in_feature && /^ [[:alnum:]][[:alnum:]-]*:$/ { exit } + in_feature { print } + ' .agentworkforce/features/manifest.yaml +fi ``` The manifest tells you: @@ -34,37 +48,26 @@ The manifest tells you: - `criticality` — how important is this feature - `verify_tier` — what's required to verify it (1=nothing, 2=broker, 3=agent token, 4=two agents, 5=cloud, 6=manual) - `location` — which source files implement it +- `verification.categories` — the named procedure in `verify/procedures.md` that supplies prerequisites, commands, assertions, cleanup, and automation limits ### 2. Check verify tier requirements -| Tier | Requires | Setup command | -| ---- | -------------------- | -------------------------------------------------------------------------------- | -| 1 | Nothing | (none) | -| 2 | Broker running | `relay node up --background` | -| 3 | Broker + agent token | `relay node up --background && relay agent register ` | -| 4 | Broker + 2 agents | `relay node up --background && relay agent register a && relay agent register b` | -| 5 | Cloud auth | `relay cloud whoami` (must already be logged in) | -| 6 | Manual only | Human must verify in browser or interactive session | +| Tier | Primary environment | Important qualification | +| ---- | ----------------------------------------- | ------------------------------------------------------------------------------ | +| 1 | Isolated local CLI/filesystem | May still mutate local config; use a temp directory. | +| 2 | Local broker | Start with `relay node up --background --no-spawn`. | +| 3 | Hosted workspace + one agent | Requires explicit workspace key/API, not merely a broker. | +| 4 | Hosted workspace + two agents | Use separate actor tokens for cross-agent assertions. | +| 5 | Authenticated disposable external service | Use a test workspace, receiver, or account and clean it up. | +| 6 | Interactive/pre-provisioned integration | Browser, SSH, PTY, provider credentials, or external callback may be required. | ### 3. Run the verification -Follow `.agentworkforce/features/verify/procedures.md` for the relevant tier. Always start from the lowest tier that applies and work up. +Resolve the category through `verification.categories`, then follow the matching procedure in `.agentworkforce/features/verify/procedures.md`. Always run lower prerequisites first. **Quick sanity check after any change:** -```bash -relay version && \ -relay status || relay node up --background && \ -relay status && \ -export RELAY_AGENT_TOKEN=$(relay agent register quick-check | jq -r '.token') && \ -relay agent list && \ -relay channel create quick-check-ch && \ -relay channel join quick-check-ch && \ -relay message post quick-check-ch "sanity $(date)" && \ -relay message list quick-check-ch --limit 1 && \ -relay channel archive quick-check-ch && \ -relay agent remove quick-check -``` +Start with the `Fast Health Triage` sequence in `critical-paths.md`; use the hosted-workspace fixture in `procedures.md` before attempting channel or message checks. ### 4. Determine which features to verify for a given change @@ -80,13 +83,14 @@ relay agent remove quick-check ### 5. Check critical paths last -Always run the 5 critical paths from `critical-paths.md` before declaring a change verified: +Always run the applicable critical paths from `critical-paths.md` before declaring a change verified: -1. `broker-up` → `agent-register` → `agent-list` → `status` -2. Channel create → join → post → list (two agents) -3. `local agent spawn` → `hold` → `flush` → `release` -4. `relay mcp` → `list_channels` → `post_message` → `list_messages` -5. `local workflow run` (basic workflow) +1. Local broker lifecycle +2. Cross-agent channel message +3. Managed local-agent lifecycle (when a provider is available) +4. MCP stdio round trip +5. Local workflow run/logs/sync +6. Direct message and read receipt ## When to update the manifest @@ -105,9 +109,9 @@ Update `critical-paths.md` when: ## Example: verifying after a messaging change ```text -1. Read the manifest: grep "messaging-messages" manifest.yaml → verify_tier: 3 -2. Setup: relay node up --background && export RELAY_AGENT_TOKEN=$(relay agent register test-agent | jq -r '.token') -3. Follow tier 3 procedures: post → list → reply → get_thread → search -4. Run critical path 2 (channel messaging) with two agents -5. Clean up: relay agent remove test-agent +1. Read the manifest category and resolve its verification procedure. +2. Create the exact disposable fixture stated by that procedure. +3. Run every listed command and assert values written by the test are read back. +4. Run the applicable critical path with separate agents where required. +5. Perform the procedure's cleanup and prove the test resources are gone. ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index b8e16257f..d9706922b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `@agent-relay/cloud` now exposes project-aware workspace resolution, and SDK-backed CLI consumers prefer the workspace recorded by the broker in the current checkout over an unrelated machine-global active workspace. +## [Unreleased - Patch] + +### Changed + +- Feature verification catalog now records the exact CLI and MCP surfaces, adds previously unlisted SDK and plugin integrations, and maps every category to an end-to-end procedure with prerequisites, assertions, cleanup, and automation limits. + ## [10.6.6] - 2026-07-19 ### Fixed