From 57f2796cda40736766b32dd001ecaed5e13b6bc6 Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 20 Jul 2026 22:33:27 +0200 Subject: [PATCH 1/6] docs: audit feature verification catalog --- .../agents/relay-feature-guardian/agent.ts | 30 +- .../manifest-contract.test.ts | 195 +++++ .agentworkforce/features/critical-paths.md | 159 ++-- .agentworkforce/features/manifest.yaml | 732 ++++++++++++++---- .agentworkforce/features/verify/procedures.md | 523 ++++++------- .../active/traj_rtyte8r4g07t/trajectory.json | 35 +- .claude/skills/verify-features.md | 58 +- CHANGELOG.md | 6 +- 8 files changed, 1166 insertions(+), 572 deletions(-) create mode 100644 .agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.ts b/.agentworkforce/agents/relay-feature-guardian/agent.ts index 2e1c004bf..abbe2c586 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.ts @@ -29,10 +29,11 @@ type Criticality = 'critical' | 'hot' | 'standard'; interface ManifestFeature { id: string; name: string; - cli: string; + cli?: string; description: string; verify_tier: number; mcp?: string; + mcp_prompt?: string; location?: string; } @@ -53,11 +54,12 @@ interface Manifest { interface Feature { id: string; name: string; - cli: string; + cli?: string; desc: string; tier: number; criticality: Criticality; mcp?: string; + mcpPrompt?: string; } const MANIFEST_RELPATH = '.agentworkforce/features/manifest.yaml'; @@ -88,6 +90,7 @@ async function loadFeatures(ctx: WorkforceCtx): Promise { tier: f.verify_tier, criticality: category.criticality, mcp: f.mcp, + mcpPrompt: f.mcp_prompt, }); } } @@ -544,16 +547,17 @@ function pickNextFeature(features: Feature[], checkedIds: Set): Feature async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise { const mcpNote = feature.mcp ? `\nMCP tool: \`${feature.mcp}\`` : ''; + const mcpPromptNote = feature.mcpPrompt ? `\nMCP prompt: \`${feature.mcpPrompt}\`` : ''; + const surface = feature.cli ? `CLI: ${feature.cli}` : 'CLI: (MCP-only surface)'; const tierLabel = - feature.tier === 1 - ? 'no broker needed' - : feature.tier === 2 - ? 'broker required' - : feature.tier === 3 - ? 'broker + agent token' - : feature.tier === 4 - ? 'broker + two agents' - : 'cloud auth required'; + { + 1: 'isolated local CLI/filesystem', + 2: 'local broker required', + 3: 'hosted workspace + agent token', + 4: 'hosted workspace + two agents', + 5: 'authenticated disposable external service', + 6: 'interactive or pre-provisioned integration', + }[feature.tier] ?? 'see feature procedure'; const prompt = [ 'You are the Relay Feature Guardian, a proactive Slack bot for the Agent Relay team.', @@ -563,7 +567,7 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise 'Keep it casual and direct — this is an internal team check.', '', `Feature: ${feature.name}`, - `CLI: ${feature.cli}${mcpNote}`, + `${surface}${mcpNote}${mcpPromptNote}`, `What it should do: ${feature.desc}`, `Verify tier: ${feature.tier} (${tierLabel})`, `Criticality: ${feature.criticality}`, @@ -576,7 +580,7 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise return [ `🔍 *Relay Feature Check: ${feature.name}*`, ``, - `\`${feature.cli}\`${mcpNote}`, + `\`${feature.cli ?? feature.mcp ?? feature.mcpPrompt ?? feature.name}\`${mcpNote}${mcpPromptNote}`, ``, `This should: ${feature.desc}`, ``, diff --git a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts new file mode 100644 index 000000000..22b675301 --- /dev/null +++ b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts @@ -0,0 +1,195 @@ +import { readFileSync } from 'node:fs'; + +import { parse } from 'yaml'; +import { describe, expect, it } from 'vitest'; + +type Feature = { id: string; cli?: string; mcp?: string; mcp_prompt?: string; location?: string }; +type Category = { features: Feature[] }; +type Manifest = { + version: string; + verification: { document: string; categories: Record }; + categories: Record; +}; + +const manifestPath = new URL('../../features/manifest.yaml', import.meta.url); +const proceduresPath = new URL('../../features/verify/procedures.md', import.meta.url); +const manifest = parse(readFileSync(manifestPath, 'utf8')) as Manifest; +const procedures = readFileSync(proceduresPath, 'utf8'); +const features = Object.values(manifest.categories).flatMap((category) => category.features); + +describe('feature manifest contract', () => { + it('maps every category to a detailed verification procedure', () => { + expect(manifest.version).toBe('1.1'); + expect(manifest.verification.document).toBe('.agentworkforce/features/verify/procedures.md'); + + for (const category of Object.keys(manifest.categories)) { + const procedure = manifest.verification.categories[category]; + expect(procedure, `${category} needs a verification procedure`).toMatch(/^[a-z0-9-]+$/); + expect(procedures).toContain(`## ${procedure}`); + } + }); + + it('has unique feature ids and a concrete CLI, MCP, or prompt surface', () => { + const ids = features.map((feature) => feature.id); + expect(new Set(ids).size).toBe(ids.length); + for (const feature of features) { + expect(feature.id).toMatch(/^[a-z0-9-]+$/); + expect(Boolean(feature.cli || feature.mcp || feature.mcp_prompt || feature.location)).toBe(true); + } + }); + + it('covers every public Commander leaf command', () => { + const commands = features.flatMap((feature) => (feature.cli ? [feature.cli] : [])); + const expected = [ + 'relay node up', + 'relay node down', + 'relay node status', + 'relay node metrics', + 'relay node deadletters', + 'relay node redeliver', + 'relay node tail', + 'relay node agent list', + 'relay node agent spawn', + 'relay node agent new', + 'relay node agent release', + 'relay node agent set-model', + 'relay node agent attach', + 'relay node agent message flush', + 'relay node agent message hold', + 'relay node agent message auto', + 'relay node workflow run', + 'relay node workflow logs', + 'relay node workflow sync', + 'relay version', + 'relay update', + 'relay uninstall', + 'relay status', + 'relay telemetry enable', + 'relay telemetry disable', + 'relay telemetry status', + 'relay cloud login', + 'relay cloud logout', + 'relay cloud session', + 'relay cloud whoami', + 'relay cloud connect', + 'relay cloud enroll', + 'relay cloud run', + 'relay cloud schedule', + 'relay cloud schedules', + 'relay cloud status', + 'relay cloud logs', + 'relay cloud sync', + 'relay cloud cancel', + 'relay cloud worker register', + 'relay cloud worker start', + 'relay cloud worker status', + 'relay cloud worker logs', + 'relay reflex on', + 'relay reflex off', + 'relay reflex status', + 'relay workspace active', + 'relay workspace create', + 'relay workspace list', + 'relay workspace set_key', + 'relay workspace join', + 'relay workspace switch', + 'relay agent register', + 'relay agent list', + 'relay agent add', + 'relay agent remove', + 'relay channel create', + 'relay channel list', + 'relay channel join', + 'relay channel leave', + 'relay channel invite', + 'relay channel set_topic', + 'relay channel archive', + 'relay message post', + 'relay message list', + 'relay message reply', + 'relay message get_thread', + 'relay message search', + 'relay message dm send', + 'relay message dm list', + 'relay message dm send_group', + 'relay message reaction add', + 'relay message reaction remove', + 'relay message inbox check', + 'relay message inbox mark_read', + 'relay message inbox get_readers', + 'relay message file upload', + 'relay integration subscribe', + 'relay integration unsubscribe', + 'relay integration webhook create', + 'relay integration webhook list', + 'relay integration webhook delete', + 'relay integration webhook trigger', + 'relay integration webhook create-inbound', + 'relay integration webhook list-inbound', + 'relay integration webhook delete-inbound', + 'relay integration subscription create', + 'relay integration subscription list', + 'relay integration subscription get', + 'relay integration subscription delete', + 'relay capabilities register', + 'relay capabilities list', + 'relay capabilities delete', + 'relay skills add', + 'relay mcp', + 'relay fleet nodes', + 'relay fleet config', + 'relay fleet enable', + 'relay fleet disable', + 'relay fleet inherit', + 'relay fleet status', + ]; + + for (const command of expected) { + expect( + commands.some((documented) => documented.startsWith(command)), + `${command} is missing` + ).toBe(true); + } + expect(commands).not.toContain('relay metrics'); + expect(commands).not.toContain('relay deadletters'); + expect(commands).not.toContain('relay redeliver [id]'); + }); + + it('covers every static MCP tool registered by the server', () => { + const tools = new Set(features.flatMap((feature) => (feature.mcp ? [feature.mcp] : []))); + const expected = [ + 'create_workspace', + 'set_workspace_key', + 'register_agent', + 'list_agents', + 'query_nodes', + 'add_agent', + 'spawn', + 'remove_agent', + 'create_channel', + 'list_channels', + 'join_channel', + 'leave_channel', + 'invite_to_channel', + 'set_channel_topic', + 'archive_channel', + 'post_message', + 'list_messages', + 'reply_to_thread', + 'get_message_thread', + 'send_dm', + 'list_dms', + 'send_group_dm', + 'add_reaction', + 'remove_reaction', + 'search_messages', + 'check_inbox', + 'mark_message_read', + 'get_message_readers', + 'list_actions', + 'invoke_action', + 'submit_result', + ]; + expect([...tools].sort()).toEqual(expected.sort()); + }); +}); diff --git a/.agentworkforce/features/critical-paths.md b/.agentworkforce/features/critical-paths.md index 7569e90ac..992ea03cc 100644 --- a/.agentworkforce/features/critical-paths.md +++ b/.agentworkforce/features/critical-paths.md @@ -1,138 +1,101 @@ # Critical Paths -The features and sequences that must work for the product to function. These are the first things to verify after any change, and the last things to break. +These are the product sequences to run first after a related change. They use the exact public surface recorded in `manifest.yaml`; detailed fixtures, assertions, and cleanup live in `verify/procedures.md`. ---- - -## Path 1: Broker + Agent Registration (Foundation) - -Everything depends on this. If it breaks, nothing works. +## Path 1: Local Broker Lifecycle ```bash -relay node up --background -relay agent register → produces a token -relay agent list → shows the registered agent -relay status → shows broker running with agent count +relay node up --background --no-spawn +relay node status +relay node metrics +relay node deadletters --json +relay node down +relay node status ``` -**What breaks if this fails:** All messaging, local agent orchestration, MCP tools, workflow execution. +The first status must report running and the last must report stopped. Use an isolated project/state directory and never stop all brokers system-wide. ---- +## Path 2: Cross-Agent Channel Message -## Path 2: Channel Messaging (Core Coordination Loop) - -The primary way agents communicate. +**Prerequisite:** disposable hosted workspace with `TOKEN_A` and `TOKEN_B` from two registrations. ```bash -relay node up --background -relay agent register alice → TOKEN_A -relay agent register bob → TOKEN_B - -# As alice: -RELAY_AGENT_TOKEN= relay channel create team -RELAY_AGENT_TOKEN= relay channel join team -RELAY_AGENT_TOKEN= relay message post team "hello from alice" - -# Verify bob receives it: -RELAY_AGENT_TOKEN= relay channel join team -RELAY_AGENT_TOKEN= relay message list team --limit 1 -# → should show alice's message +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create critical-path +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite critical-path critical-b +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post critical-path 'critical-path-message')" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list critical-path --limit 10 +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive critical-path ``` -**What breaks if this fails:** All multi-agent coordination, workflow communication, the entire product premise. - ---- +Agent B must see the exact text from A. Remove both test identities after archiving the channel. -## Path 3: Local Agent Spawn + Message Injection +## Path 3: Managed Local Agent -The mechanism for spawning real AI agents and injecting messages into them. +**Prerequisite:** a locally installed and authenticated provider CLI. ```bash -relay node up --background -relay node agent spawn claude --name worker-1 -relay node agent list → shows worker-1 as active -relay node agent message hold worker-1 -relay node agent message flush worker-1 -relay node agent release worker-1 +relay node up --background --no-spawn +relay node agent spawn "$PROVIDER" --name critical-worker --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task +relay node agent list +relay node agent message hold critical-worker +relay node agent message auto critical-worker +relay node agent release critical-worker ``` -**What breaks if this fails:** All multi-agent orchestration workflows, the local workflow engine. +Assert that the worker appears, receives its bounded task, and is released. Provider credentials/cost make this a pre-provisioned integration check, not a generic CI test. ---- +## Path 4: MCP Stdio Round Trip -## Path 4: MCP Server Integration +**Prerequisite:** disposable hosted workspace and an MCP JSON-RPC client. -The path used when an agent operates inside a harness (Claude Code, Cursor, etc.). - -```bash -relay mcp → starts MCP server on stdio -# From a harness MCP call: -list_channels → returns channel list -post_message(channel, text) → posts message -list_messages(channel) → returns messages including the one just posted +```text +start relay mcp over stdio +initialize → tools/list → prompts/list +set_workspace_key or create_workspace → register_agent(A and B) +create_channel → post_message → list_messages +reply_to_thread → get_message_thread +send_dm → list_dms → check_inbox → mark_message_read → get_message_readers +close the stdio child ``` -**What breaks if this fails:** All usage from within harness tools (the primary user workflow for most users). - ---- +Assert every static MCP tool in the manifest is listed and that the returned content reflects the values written by the test. The server itself does not require a local broker, but the messaging tools require a workspace and agent identities. -## Path 5: Workflow Execution (Local) - -A workflow YAML or JS file drives multiple agents to complete a task. +## Path 5: Local Workflow Run, Logs, and Sync ```bash -relay node up --background -relay node workflow run examples/basic-workflow.yaml -# → spawns agents, coordinates them, produces output, terminates cleanly +echo 'console.log("critical-workflow-ok")' > workflow.js +RUN_JSON="$(relay node workflow run workflow.js --json)" +RUN_ID="$(jq -er .runId <<<"$RUN_JSON")" +relay node workflow logs "$RUN_ID" --follow --json +relay node workflow sync "$RUN_ID" --dry-run --json ``` -**What breaks if this fails:** The primary value proposition for multi-agent task execution. +Assert completed status and the sentinel output. Run it in a disposable project because local workflow records are retained. ---- +## Path 6: Direct Message and Read Receipt -## Path 6: Direct Messaging Between Agents +**Prerequisite:** disposable hosted workspace with two identities. ```bash -relay node up --background -relay agent register orchestrator → TOKEN_O -relay agent register worker → TOKEN_W - -# Send DM and capture conversationId from JSON output: -RELAY_AGENT_TOKEN= relay message dm send worker "your task" -# dm list requires the conversationId returned by send -# RELAY_AGENT_TOKEN= relay message dm list +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send critical-b 'critical-dm')" +DM_ID="$(jq -er '.id // .messageId' <<<"$DM")" +CONV_ID="$(jq -er '.conversationId // .conversation_id' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV_ID" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox mark_read "$DM_ID" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message inbox get_readers "$DM_ID" ``` -**What breaks if this fails:** Lead/worker orchestration patterns, any workflow that routes tasks via DM. - ---- - -## Hot Paths (Sensitive, Frequently Touched) - -These are not foundational but are exercised constantly and failures are immediately noticeable. - -| Path | Risk | Code Area | -| ----------------------------- | ---------------------------------------- | ------------------------------------------------------------ | -| PTY message injection timing | Race conditions in TMUX wrapper | crates/relay-pty/, packages/harness-driver/ | -| Dead letter queue + redeliver | Messages lost silently | crates/broker/src/queue.rs | -| Agent token auth validation | Auth bypass or silent rejection | crates/broker/src/auth.rs (or equivalent) | -| Message delivery ordering | Out-of-order delivery corrupts workflows | crates/broker/ | -| Hold/flush state machine | Stuck agents that never receive messages | crates/broker/, packages/cli/src/cli/commands/local-agent.ts | -| MCP tool schema validation | Tool calls rejected silently by harness | packages/cli/src/cli/mcp/ | - ---- - -## What an Agent Should Check First +Assert B reads the exact DM and A sees B in readers. Remove the disposable agents afterward. -When unsure if the system is healthy, run this sequence in order: +## Fast Health Triage ```bash -relay version # CLI is installed and not corrupt -relay status # Broker state (running or not) -relay node up --background # Start if not running -relay status # Confirm running -relay agent list # Workspace is reachable and has agents -relay message list general --limit 1 # Messaging works end to end +relay version +relay status +relay node status +relay node up --background --no-spawn +relay node status +relay node metrics ``` -If any step fails, diagnose before proceeding. The failure at each step points to a specific subsystem. +If any command fails, diagnose that subsystem before testing higher-level paths. Do not treat top-level `relay status` as the broker statistics command; `relay node status` is the direct broker check. diff --git a/.agentworkforce/features/manifest.yaml b/.agentworkforce/features/manifest.yaml index 3144d68a8..3781b9a56 100644 --- a/.agentworkforce/features/manifest.yaml +++ b/.agentworkforce/features/manifest.yaml @@ -1,5 +1,5 @@ -version: '1.0' -updated: '2026-07-16' +version: '1.1' +updated: '2026-07-20' # Every user-facing feature in this repo, categorized and scored. # @@ -8,13 +8,47 @@ updated: '2026-07-16' # hot - commonly used path; verify for any related change # standard - useful feature; verify periodically # -# verify_tier (what is required to run verification): -# 1 - nothing (version, help, telemetry toggle, workspace list) -# 2 - broker running (relay node up --background) -# 3 - broker running + at least one agent registered with token -# 4 - broker running + two or more agents registered -# 5 - cloud auth required (relay cloud login) -# 6 - manual / browser only +# verify_tier (the primary environment needed to run verification): +# 1 - isolated local CLI/filesystem +# 2 - local broker running +# 3 - hosted workspace plus one registered agent token +# 4 - hosted workspace plus two or more registered agent identities +# 5 - authenticated, disposable hosted workspace or external test service +# 6 - interactive provider/browser/PTY or a pre-provisioned external system +# +# Tier alone is deliberately not a test plan. Every category is mapped below +# to a procedure with precise prerequisites, setup, assertions, cleanup, and +# any automation limitation. Procedures are in verify/procedures.md. +verification: + document: .agentworkforce/features/verify/procedures.md + categories: + broker: broker-lifecycle + agent-management: workspace-agents-and-capabilities + messaging-channels: channel-messaging + messaging-messages: message-round-trip + messaging-dm: direct-messages + messaging-reactions: reactions-and-read-status + messaging-inbox: reactions-and-read-status + local-agents: local-agent-lifecycle + local-workflows: local-workflow-lifecycle + cloud: cloud-workflows + cloud-workers: cloud-workers + fleet: fleet-management + workspace: workspace-management + skills: skills-installation + integration: integrations-and-webhooks + reflex: reflex-history + mcp: mcp-stdio + harnesses: harnesses + sdk: typescript-sdk + python-sdk: python-sdk + swift-sdk: swift-sdk + opencode-plugin: opencode-plugin + codex-relay-skill: codex-relay-skill + gemini-relay-extension: gemini-relay-extension + setup: cli-maintenance + telemetry: telemetry + node: node-command-discovery categories: broker: @@ -38,28 +72,28 @@ categories: - id: broker-status name: Broker Status - cli: relay status - description: Check whether the broker is running and show agent/queue stats - location: packages/cli/src/cli/commands/core.ts - verify_tier: 1 + cli: relay node status [--wait-for ] + description: Check whether the local broker daemon is running and report its agent and delivery state + location: packages/cli/src/cli/commands/core.ts, packages/cli/src/cli/commands/node.ts + verify_tier: 2 - id: broker-metrics name: Broker Metrics - cli: relay metrics + cli: relay node metrics [--agent ] description: Show resource usage for broker and its agents location: packages/cli/src/cli/commands/core.ts verify_tier: 2 - id: broker-deadletters name: Dead Letter Queue - cli: relay deadletters + cli: relay node deadletters [--json] description: List terminally-failed message deliveries retained in the broker location: packages/cli/src/cli/commands/core.ts verify_tier: 2 - id: broker-redeliver name: Redeliver Messages - cli: relay redeliver [id] + cli: relay node redeliver ( | --all) description: Requeue dead-letter messages through the normal delivery path location: packages/cli/src/cli/commands/core.ts verify_tier: 2 @@ -71,38 +105,59 @@ categories: features: - id: agent-register name: Register Agent - cli: relay agent register + cli: relay agent register [--type ] [--persona ] description: Register a new agent and print its auth token location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-add name: Add Agent - cli: relay agent add - description: Add an agent to the workspace (interactive) + cli: relay agent add [--type ] + description: Register an additional agent in the workspace location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-list name: List Agents - cli: relay agent list + cli: relay agent list [--status ] description: Show all agents in the workspace with status location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-remove name: Remove Agent cli: relay agent remove description: Remove an agent from the workspace location: packages/cli/src/cli/commands/agent.ts - verify_tier: 2 + verify_tier: 3 - id: agent-capabilities name: List Capabilities - cli: relay capabilities + cli: relay capabilities list description: Show available agent capabilities location: packages/cli/src/cli/commands/capabilities.ts - verify_tier: 2 + verify_tier: 3 + + - id: capabilities-register + name: Register Capability + cli: relay capabilities register --description --handler + description: Register a named capability and the agent that handles it + location: packages/cli/src/cli/commands/capabilities.ts + verify_tier: 3 + + - id: capabilities-delete + name: Delete Capability + cli: relay capabilities delete + description: Delete a previously registered capability + location: packages/cli/src/cli/commands/capabilities.ts + verify_tier: 3 + + - id: system-status + name: Composite System Status + cli: relay status + description: Show the current project path, local broker state, and Cloud-login state + location: packages/cli/src/cli/commands/status.ts + verify_tier: 1 messaging-channels: name: Channel Messaging @@ -111,7 +166,7 @@ categories: features: - id: channel-create name: Create Channel - cli: relay channel create + cli: relay channel create [--topic ] mcp: create_channel description: Create a new messaging channel location: packages/cli/src/cli/commands/channel.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -119,7 +174,7 @@ categories: - id: channel-list name: List Channels - cli: relay channel list + cli: relay channel list [--archived] mcp: list_channels description: Show all channels in the workspace location: packages/cli/src/cli/commands/channel.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -172,7 +227,7 @@ categories: features: - id: message-post name: Post Message - cli: relay message post + cli: relay message post mcp: post_message description: Send a message to a channel as the current agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -180,7 +235,7 @@ categories: - id: message-list name: List Messages - cli: relay message list + cli: relay message list [--limit ] mcp: list_messages description: Retrieve message history from a channel location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -188,28 +243,35 @@ categories: - id: message-reply name: Reply to Message (Threads) - cli: relay message reply - mcp: reply_to_message + cli: relay message reply + mcp: reply_to_thread description: Reply to a message, creating or extending a thread location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 - id: message-get-thread name: Get Thread - cli: relay message get_thread - mcp: get_thread + cli: relay message get_thread + mcp: get_message_thread description: Retrieve all messages in a thread location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 - id: message-search name: Search Messages - cli: relay message search + cli: relay message search [--channel ] [--from ] [--limit ] mcp: search_messages description: Search for messages by content across the workspace location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 3 + - id: message-file-upload + name: Upload File Attachment + cli: relay message file upload --channel [--text ] + description: Send a channel message containing a local file attachment + location: packages/cli/src/cli/commands/message.ts + verify_tier: 3 + messaging-dm: name: Direct Messages description: Private messaging between agents @@ -217,23 +279,29 @@ categories: features: - id: dm-send name: Send DM - cli: relay message dm send + cli: relay message dm send mcp: send_dm description: Send a private direct message to another agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 - id: dm-list - name: List DMs - cli: relay message dm list - mcp: list_dms - description: List direct message conversations for the current agent + name: List Messages in a DM Conversation + cli: relay message dm list [--limit ] + description: List the messages in a known direct-message conversation location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 + - id: dm-list-conversations + name: List DM Conversations + mcp: list_dms + description: List direct-message conversations visible to the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 4 + - id: dm-send-group name: Send Group DM - cli: relay message dm send_group + cli: relay message dm send_group --to mcp: send_group_dm description: Create a group DM and send the first message to multiple agents location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts @@ -246,19 +314,19 @@ categories: features: - id: reaction-add name: Add Reaction - cli: relay message reaction add + cli: relay message reaction add mcp: add_reaction description: Add an emoji reaction to a message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: reaction-remove name: Remove Reaction - cli: relay message reaction remove + cli: relay message reaction remove mcp: remove_reaction description: Remove an emoji reaction from a message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 messaging-inbox: name: Inbox & Read Status @@ -267,24 +335,24 @@ categories: features: - id: inbox-check name: Check Inbox - cli: relay message inbox check - mcp: list_inbox + cli: relay message inbox check [--limit ] + mcp: check_inbox description: Show unread messages, mentions, and DMs directed to this agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: inbox-mark-read name: Mark Message Read - cli: relay message inbox mark_read + cli: relay message inbox mark_read mcp: mark_message_read description: Mark a message or thread as read for the current agent location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 4 - id: inbox-get-readers name: Get Readers - cli: relay message inbox get_readers - mcp: get_readers + cli: relay message inbox get_readers + mcp: get_message_readers description: Show which agents have read a specific message location: packages/cli/src/cli/commands/message.ts, packages/cli/src/cli/mcp/messaging-tools.ts verify_tier: 4 @@ -296,17 +364,17 @@ categories: features: - id: local-agent-spawn name: Spawn Agent - cli: relay node agent spawn --name + cli: relay node agent spawn [--name ] [--task ] [--spawn-mode task-exit] [--exit-after-task] description: Launch an AI agent process attached to the local broker location: packages/cli/src/cli/commands/local-agent.ts verify_tier: 2 - id: local-agent-new name: Spawn and Attach - cli: relay node agent new --name + cli: relay node agent new [--name ] description: Spawn an agent and immediately attach your terminal to it location: packages/cli/src/cli/commands/local-agent.ts - verify_tier: 2 + verify_tier: 6 - id: local-agent-release name: Release Agent @@ -325,7 +393,7 @@ categories: - id: local-agent-set-model name: Switch Agent Model cli: relay node agent set-model - description: Change the AI model a running agent uses (sends /model to TUI) + description: Request a best-effort /model switch from a running agent's provider TUI location: packages/cli/src/cli/commands/local-agent.ts verify_tier: 2 @@ -334,7 +402,7 @@ categories: cli: relay node agent attach description: Interactively connect terminal to a running agent (drive/view/passthrough) location: packages/cli/src/cli/commands/local-agent.ts - verify_tier: 2 + verify_tier: 6 - id: local-agent-flush name: Flush Agent Messages @@ -361,7 +429,7 @@ categories: name: Tail Broker Events cli: relay node tail [--agent ] description: Stream live broker events, optionally filtered to one agent - location: packages/cli/src/cli/commands/local-agent.ts + location: packages/cli/src/cli/commands/local-agent.ts, packages/cli/src/cli/commands/node.ts verify_tier: 2 local-workflows: @@ -371,10 +439,24 @@ categories: features: - id: local-workflow-run name: Run Local Workflow - cli: relay node workflow run - description: Execute a workflow YAML or TypeScript/JS file on the local broker + cli: relay node workflow run [--file-type ] [--json] + description: Run a local YAML, TypeScript, JavaScript, Python, or shell workflow and record its result locally location: packages/cli/src/cli/commands/local-workflow.ts - verify_tier: 2 + verify_tier: 1 + + - id: local-workflow-logs + name: Read Local Workflow Logs + cli: relay node workflow logs [--follow] [--poll-interval ] [--offset ] [--json] + description: Read or follow the locally recorded output of a workflow run + location: packages/cli/src/cli/commands/local-workflow.ts + verify_tier: 1 + + - id: local-workflow-sync + name: Sync Local Workflow Result + cli: relay node workflow sync [--dry-run] [--json] + description: Report the local synchronization state for a completed workflow run + location: packages/cli/src/cli/commands/local-workflow.ts + verify_tier: 1 cloud: name: Cloud Features @@ -400,74 +482,74 @@ categories: cli: relay cloud whoami description: Show current cloud authentication status (readable if already logged in) location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-session name: Cloud Session cli: relay cloud session description: Show canonical cloud session details (readable if already logged in) location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-connect name: Connect Provider - cli: relay cloud connect + cli: relay cloud connect [--language ] [--timeout ] description: Connect a provider via interactive SSH session location: packages/cli/src/cli/commands/cloud.ts - verify_tier: 5 + verify_tier: 6 - id: cloud-enroll name: Enroll Fleet Node - cli: relay cloud enroll + cli: relay cloud enroll (--token | --workspace ) [--name ] [--max-agents ] [--json] description: Register this machine as a cloud-managed fleet node location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-run name: Submit Cloud Workflow - cli: relay cloud run + cli: relay cloud run [--file-type ] [--no-sync-code] [--json] description: Submit a workflow for remote cloud execution location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-schedule name: Schedule Cloud Workflow - cli: relay cloud schedule + cli: relay cloud schedule (--cron | --at ) [--timezone ] [--json] description: Schedule a repeatable cloud workflow run with cron location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-schedules name: List Cloud Schedules - cli: relay cloud schedules + cli: relay cloud schedules [--json] description: List all scheduled workflow runs location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-status name: Cloud Run Status - cli: relay cloud status + cli: relay cloud status [--json] description: Fetch status of a workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-logs name: Cloud Run Logs - cli: relay cloud logs + cli: relay cloud logs [--follow] [--agent ] [--json] description: Read logs from a workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-sync name: Sync Cloud Results - cli: relay cloud sync + cli: relay cloud sync [--dir ] [--dry-run] description: Download and apply code changes from a completed cloud workflow run location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 - id: cloud-cancel name: Cancel Cloud Run - cli: relay cloud cancel + cli: relay cloud cancel [--json] description: Cancel a running cloud workflow location: packages/cli/src/cli/commands/cloud.ts verify_tier: 5 @@ -479,31 +561,31 @@ categories: features: - id: cloud-worker-register name: Register Cloud Worker - cli: relay cloud worker register + cli: relay cloud worker register --token --name [--base-url ] [--json] description: Register this machine as a cloud worker node location: packages/cli/src/cli/commands/cloud-worker.ts verify_tier: 5 - id: cloud-worker-start name: Start Cloud Worker - cli: relay cloud worker start + cli: relay cloud worker start [--worker-id | --name ] [--daemon] [--once] description: Start the cloud worker service on this machine location: packages/cli/src/cli/commands/cloud-worker.ts verify_tier: 5 - id: cloud-worker-status name: Cloud Worker Status - cli: relay cloud worker status - description: Show local cloud worker daemon status (reads local state, no cloud call) + cli: relay cloud worker status [--worker-id | --name ] [--json] + description: Show the state of a previously registered local Cloud worker location: packages/cli/src/cli/commands/cloud-worker.ts - verify_tier: 2 + verify_tier: 5 - id: cloud-worker-logs name: Cloud Worker Logs - cli: relay cloud worker logs + cli: relay cloud worker logs [--worker-id | --name ] [--follow] description: Read local cloud worker daemon logs location: packages/cli/src/cli/commands/cloud-worker.ts - verify_tier: 2 + verify_tier: 5 fleet: name: Fleet Management @@ -515,14 +597,14 @@ categories: cli: relay fleet nodes description: Show all fleet nodes registered in the workspace (returns empty list if none configured) location: packages/cli/src/cli/commands/fleet.ts - verify_tier: 2 + verify_tier: 5 - id: fleet-config name: Fleet Config cli: relay fleet config - description: Show workspace fleet node configuration (reads local config, no cloud needed) + description: Show the workspace fleet-node configuration location: packages/cli/src/cli/commands/fleet.ts - verify_tier: 2 + verify_tier: 5 - id: fleet-enable name: Enable Fleet @@ -560,16 +642,16 @@ categories: - id: workspace-active name: Show Active Workspace cli: relay workspace active - description: Show the currently active workspace + description: Resolve and show the canonical Cloud identifiers for the active workspace location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 5 - id: workspace-create name: Create Workspace - cli: relay workspace create + cli: relay workspace create [--base-url ] description: Create a new workspace and store its key location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 5 - id: workspace-list name: List Workspaces @@ -580,24 +662,24 @@ categories: - id: workspace-set-key name: Store Workspace Key - cli: relay workspace set_key + cli: relay workspace set_key description: Store a workspace key under a name location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 - id: workspace-join name: Join Workspace - cli: relay workspace join + cli: relay workspace join description: Join a workspace by key and make it active location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 - id: workspace-switch name: Switch Workspace cli: relay workspace switch description: Switch the active workspace to a stored one location: packages/cli/src/cli/commands/workspace.ts - verify_tier: 2 + verify_tier: 1 skills: name: Skills Management @@ -609,7 +691,7 @@ categories: cli: relay skills add description: Install the /orchestrate skill into one or more AI harnesses location: packages/cli/src/cli/commands/skills.ts - verify_tier: 1 + verify_tier: 5 integration: name: Integrations & Webhooks @@ -618,94 +700,101 @@ categories: features: - id: integration-subscribe name: Subscribe Integration - cli: relay integration subscribe [provider] + cli: relay integration subscribe --resource --to <@agent|#channel> [--no-input] description: Subscribe a relay recipient to a relayfile integration location: packages/cli/src/cli/commands/integration.ts - verify_tier: 2 + verify_tier: 6 - id: integration-unsubscribe name: Unsubscribe Integration - cli: relay integration unsubscribe + cli: relay integration unsubscribe --resource description: Remove a relayfile integration subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 2 + verify_tier: 6 + + - id: integration-list-bindings + name: List Relayfile Integration Bindings + cli: relay integration subscribe --list + description: List active Relayfile bindings and their Relay resources + location: packages/cli/src/cli/commands/integration.ts + verify_tier: 5 - id: webhook-create name: Create Webhook - cli: relay integration webhook create + cli: relay integration webhook create [--event ] description: Register an outbound webhook location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-list name: List Webhooks cli: relay integration webhook list description: Show all registered webhooks location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-delete name: Delete Webhook cli: relay integration webhook delete description: Remove a registered webhook location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-trigger name: Trigger Webhook - cli: relay integration webhook trigger + cli: relay integration webhook trigger [--payload ] description: Manually fire a webhook for testing location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-create name: Create Inbound Webhook - cli: relay integration webhook create-inbound + cli: relay integration webhook create-inbound [--name ] description: Create an endpoint external services POST to, delivering into a channel location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-list name: List Inbound Webhooks cli: relay integration webhook list-inbound description: Show all inbound webhook endpoints location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: webhook-inbound-delete name: Delete Inbound Webhook - cli: relay integration webhook delete-inbound + cli: relay integration webhook delete-inbound description: Remove an inbound webhook endpoint location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-create name: Create Event Subscription - cli: relay integration subscription create + cli: relay integration subscription create [--filter ] [--url ] [--secret ] description: Subscribe to workspace events location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-list name: List Subscriptions cli: relay integration subscription list description: Show all event subscriptions location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-get name: Get Subscription cli: relay integration subscription get description: Show details for a specific subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 - id: subscription-delete name: Delete Subscription cli: relay integration subscription delete description: Remove an event subscription location: packages/cli/src/cli/commands/integration.ts - verify_tier: 3 + verify_tier: 5 reflex: name: Reflex @@ -717,21 +806,21 @@ categories: cli: relay reflex on description: Enable reflex history sync so agents receive relay activity context location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 6 - id: reflex-off name: Reflex Off cli: relay reflex off description: Disable reflex history sync location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 1 - id: reflex-status name: Reflex Status cli: relay reflex status description: Show whether reflex history sync is enabled location: packages/cli/src/cli/commands/reflex.ts - verify_tier: 2 + verify_tier: 1 mcp: name: MCP Server Mode @@ -743,98 +832,231 @@ categories: cli: relay mcp description: Run agent-relay as an MCP server so harnesses can use relay tools location: packages/cli/src/cli/agent-relay-mcp.ts - verify_tier: 2 + verify_tier: 1 + + - id: mcp-create-workspace + name: MCP create_workspace + mcp: create_workspace + description: Create a workspace and retain its key in the MCP session + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-set-workspace-key + name: MCP set_workspace_key + mcp: set_workspace_key + description: Set the workspace key used by the MCP session + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 - id: mcp-register-agent name: MCP register_agent mcp: register_agent description: Register a new agent from within a harness - location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 2 + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 - id: mcp-list-agents name: MCP list_agents mcp: list_agents description: List workspace agents from within a harness - location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 2 + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-query-nodes + name: MCP query_nodes + mcp: query_nodes + description: Query fleet nodes in the current workspace + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 5 + + - id: mcp-add-agent + name: MCP add_agent + mcp: add_agent + description: Spawn a managed worker through the workspace API + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 + + - id: mcp-spawn + name: MCP spawn + mcp: spawn + description: Invoke the workspace spawn action for an agent identity + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 + + - id: mcp-remove-agent + name: MCP remove_agent + mcp: remove_agent + description: Release a managed worker from active duty + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 - id: mcp-create-channel name: MCP create_channel mcp: create_channel description: Create a channel from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-channels name: MCP list_channels mcp: list_channels description: List channels from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-join-channel + name: MCP join_channel + mcp: join_channel + description: Join a channel as the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-leave-channel + name: MCP leave_channel + mcp: leave_channel + description: Leave a channel as the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-invite-to-channel + name: MCP invite_to_channel + mcp: invite_to_channel + description: Invite an agent to a channel + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-set-channel-topic + name: MCP set_channel_topic + mcp: set_channel_topic + description: Update a channel topic + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-archive-channel + name: MCP archive_channel + mcp: archive_channel + description: Archive a channel + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-post-message name: MCP post_message mcp: post_message description: Post a message from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-messages name: MCP list_messages mcp: list_messages description: Retrieve channel history from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-reply-to-message - name: MCP reply_to_message - mcp: reply_to_message + name: MCP reply_to_thread + mcp: reply_to_thread description: Reply in a thread from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-get-message-thread + name: MCP get_message_thread + mcp: get_message_thread + description: Retrieve a message thread from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-send-dm name: MCP send_dm mcp: send_dm description: Send a DM from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 4 + verify_tier: 5 + + - id: mcp-list-dms + name: MCP list_dms + mcp: list_dms + description: List direct-message conversations for the current MCP identity + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-send-group-dm + name: MCP send_group_dm + mcp: send_group_dm + description: Create a group direct-message conversation and post its first message + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-add-reaction name: MCP add_reaction mcp: add_reaction description: Add a reaction from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-remove-reaction + name: MCP remove_reaction + mcp: remove_reaction + description: Remove a reaction from a message from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-search-messages name: MCP search_messages mcp: search_messages description: Search messages from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-inbox - name: MCP list_inbox - mcp: list_inbox + name: MCP check_inbox + mcp: check_inbox description: Check unread inbox from within a harness location: packages/cli/src/cli/mcp/messaging-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-mark-message-read + name: MCP mark_message_read + mcp: mark_message_read + description: Mark a message read from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 + + - id: mcp-get-message-readers + name: MCP get_message_readers + mcp: get_message_readers + description: List the agents that read a message from within a harness + location: packages/cli/src/cli/mcp/messaging-tools.ts + verify_tier: 5 - id: mcp-invoke-action name: MCP invoke_action mcp: invoke_action description: Fire an action from within a harness location: packages/cli/src/cli/mcp/action-tools.ts - verify_tier: 3 + verify_tier: 5 - id: mcp-list-actions name: MCP list_actions mcp: list_actions description: List available actions from within a harness location: packages/cli/src/cli/mcp/action-tools.ts - verify_tier: 3 + verify_tier: 5 + + - id: mcp-system-prompt + name: MCP system prompt + mcp_prompt: system + description: Retrieve Agent Relay's default collaboration instructions through the MCP prompts interface + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 1 + + - id: mcp-submit-result + name: MCP submit_result + mcp: submit_result + description: Submit a structured task result when AGENT_RELAY_RESULT_URL and AGENT_RELAY_RESULT_TOKEN configure the callback + location: packages/cli/src/cli/agent-relay-mcp.ts + verify_tier: 6 harnesses: name: AI Harness Integrations @@ -845,19 +1067,19 @@ categories: name: Claude Code Harness description: Run agents using Claude Code (claude CLI) with PTY injection location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-codex name: Codex Harness description: Run agents using OpenAI Codex CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-gemini name: Gemini Harness description: Run agents using Google Gemini CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-cursor name: Cursor Harness @@ -869,31 +1091,37 @@ categories: name: Aider Harness description: Run agents using the Aider code editing tool location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-goose name: Goose Harness description: Run agents using Block's Goose automation tool location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-grok name: Grok Harness description: Run agents using xAI Grok CLI location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 - id: harness-opencode name: OpenCode Harness description: Run agents using the OpenCode extension location: packages/harnesses/ - verify_tier: 2 + verify_tier: 6 + + - id: harness-droid + name: Droid Harness + description: Run agents using Factory's Droid CLI + location: packages/harnesses/src/index.ts + verify_tier: 6 - id: harness-custom name: Custom Harness description: Define a bespoke harness via the defineHarness contract location: packages/harnesses/ - verify_tier: 2 + verify_tier: 3 sdk: name: SDK (Programmatic API) @@ -920,10 +1148,197 @@ categories: - id: sdk-delivery name: SDK Delivery Control - description: Control message delivery hold/flush/auto programmatically + description: Process inbox work with DeliveryRunner acknowledgements, failures, and deferrals location: packages/sdk/src/ verify_tier: 3 + - id: sdk-workspace + name: SDK Workspace and Identity + description: Create or reconnect to workspaces and register agent or human participants + location: packages/sdk/src/agent-relay.ts, packages/sdk/src/facade.ts + verify_tier: 5 + + - id: sdk-realtime + name: SDK Realtime Listeners + description: Consume workspace events through listeners, event fan-in, and observer mode + location: packages/sdk/src/listeners.ts, packages/sdk/src/messaging/ + verify_tier: 5 + + - id: sdk-integrations + name: SDK Nodes and Integrations + description: Manage fleet nodes, triggers, event subscriptions, and inbound webhooks programmatically + location: packages/sdk/src/facade.ts, packages/sdk/src/messaging/ + verify_tier: 5 + + - id: sdk-harness-contract + name: SDK Harness and Session Contracts + description: Define custom harnesses and session capabilities for Relay-connected agents + location: packages/sdk/src/session/, packages/sdk/src/agent-relay.ts + verify_tier: 3 + + python-sdk: + name: Python SDK + description: Python orchestration, Relay communication, adapters, and workflow building + criticality: hot + features: + - id: python-sdk-orchestration + name: Python AgentRelay Orchestration + description: Spawn, message, release, and wait for managed Python AgentRelay agents + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + - id: python-sdk-communication + name: Python Relay Communication + description: Coordinate agents through the Relay communication interface + location: packages/sdk-py/src/agent_relay/ + verify_tier: 3 + + - id: python-sdk-adapters + name: Python Relay Adapters + description: Attach Relay behavior to supported Python agent frameworks with on_relay adapters + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + - id: python-sdk-workflows + name: Python Workflow Builder + description: Define Python workflow templates and multi-agent workflow steps + location: packages/sdk-py/src/agent_relay/ + verify_tier: 6 + + swift-sdk: + name: Swift SDKs + description: Swift clients for hosted Agent Relay and the local broker + criticality: standard + features: + - id: swift-sdk-hosted + name: AgentRelaySDK + description: Use the hosted Swift Agent Relay client for workspaces, messaging, and events + location: packages/sdk-swift/Sources/ + verify_tier: 5 + + - id: swift-sdk-broker + name: AgentRelayBrokerSDK + description: Use the local-broker Swift SDK for broker-backed agent coordination + location: packages/sdk-swift/Sources/ + verify_tier: 2 + + opencode-plugin: + name: OpenCode Relay Plugin + description: OpenCode-native Relaycast tools and session lifecycle hooks + criticality: hot + features: + - id: opencode-relay-connect + name: OpenCode relay_connect + description: Connect an OpenCode session to an Agent Relay workspace + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-send + name: OpenCode relay_send + description: Send a direct message from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-inbox + name: OpenCode relay_inbox + description: Check the Relay inbox from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-agents + name: OpenCode relay_agents + description: List Relay agents from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-post + name: OpenCode relay_post + description: Post a channel message from an OpenCode session + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 5 + + - id: opencode-relay-spawn + name: OpenCode relay_spawn + description: Spawn a Relay-connected OpenCode worker + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 6 + + - id: opencode-relay-dismiss + name: OpenCode relay_dismiss + description: Stop and release a spawned OpenCode worker + location: plugins/opencode-relay-plugin/src/tools.ts + verify_tier: 6 + + - id: opencode-relay-hooks + name: OpenCode Relay Lifecycle Hooks + description: Preserve Relay state across compaction, poll while idle, and clean up workers on session end + location: plugins/opencode-relay-plugin/src/hooks.ts + verify_tier: 6 + + codex-relay-skill: + name: Codex Relay Skill + description: Codex-native Relay coordination skill, MCP setup, hooks, and relay-worker template + criticality: hot + features: + - id: codex-relay-install + name: Install Codex Relay Skill + description: Install the project or user-scoped skill and idempotently configure Codex Relay MCP support + location: plugins/codex-relay-skill/scripts/setup.sh + verify_tier: 5 + + - id: codex-relay-worker + name: Codex relay-worker + description: Delegate a bounded task to the Relay-aware Codex worker template + location: plugins/codex-relay-skill/codex-config/relay-worker.toml + verify_tier: 6 + + - id: codex-relay-hooks + name: Codex Relay Hooks + description: Connect on session start, inject inbox state, and protect completion while messages remain unread + location: plugins/codex-relay-skill/hooks/ + verify_tier: 6 + + gemini-relay-extension: + name: Gemini CLI Relay Extension + description: Gemini-native Relay MCP integration, subagents, hooks, and orchestration commands + criticality: hot + features: + - id: gemini-relay-install + name: Install Gemini Relay Extension + description: Install the Relay extension into Gemini CLI and connect a Relay workspace + location: plugins/gemini-relay-extension/gemini-extension.json + verify_tier: 6 + + - id: gemini-relay-status + name: Gemini /relay:status + description: Show Relay connection and team status from Gemini CLI + location: plugins/gemini-relay-extension/commands/status/ + verify_tier: 6 + + - id: gemini-relay-team + name: Gemini /relay:team + description: Delegate a coordinated team task to Relay-aware Gemini workers + location: plugins/gemini-relay-extension/commands/team/ + verify_tier: 6 + + - id: gemini-relay-fanout + name: Gemini /relay:fanout + description: Fan out independent work to Relay-aware Gemini workers + location: plugins/gemini-relay-extension/commands/fanout/ + verify_tier: 6 + + - id: gemini-relay-workers + name: Gemini Relay Worker Roles + description: Run the relay-worker, relay-researcher, or relay-reviewer Gemini subagent roles + location: plugins/gemini-relay-extension/agents/ + verify_tier: 6 + + - id: gemini-relay-hooks + name: Gemini Relay Lifecycle Hooks + description: Auto-connect, poll/inject inbox content, guard stop, and clean up session workers + location: plugins/gemini-relay-extension/hooks/ + verify_tier: 6 + setup: name: Setup & Health description: Initial setup, diagnostics, and CLI lifecycle @@ -938,11 +1353,18 @@ categories: - id: cli-update name: Update CLI - cli: relay update + cli: relay update [--check] description: Check for and install CLI updates location: packages/cli/src/cli/commands/core.ts verify_tier: 1 + - id: cli-uninstall + name: Uninstall CLI + cli: relay uninstall [--keep-data] [--force] [--dry-run] + description: Remove Agent Relay runtime data, configuration, and global binaries + location: packages/cli/src/cli/commands/core.ts + verify_tier: 6 + telemetry: name: Telemetry description: Usage analytics opt-in/out @@ -952,21 +1374,21 @@ categories: name: Enable Telemetry cli: relay telemetry enable description: Opt in to usage telemetry reporting - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 - id: telemetry-disable name: Disable Telemetry cli: relay telemetry disable description: Opt out of usage telemetry reporting - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 - id: telemetry-status name: Telemetry Status cli: relay telemetry status description: Show current telemetry opt-in/out status - location: packages/cli/src/cli/telemetry/ + location: packages/cli/src/cli/commands/setup.ts, packages/cli/src/cli/telemetry/config.ts verify_tier: 1 node: @@ -976,14 +1398,14 @@ categories: features: - id: node-up name: Node Up - cli: relay node up - description: Start this node's local broker (use --background to run as daemon) + cli: relay node up [--no-spawn] [--config ] + description: Start this node in the foreground, optionally serving a node-definition file location: packages/cli/src/cli/commands/node.ts verify_tier: 1 - id: node-workflow - name: Node Workflow - cli: relay node workflow - description: Run and inspect workflows on this node + name: Workflow Command Discovery + cli: relay node workflow --help + description: Display the available local workflow commands; run, logs, and sync are cataloged separately location: packages/cli/src/cli/commands/node.ts - verify_tier: 2 + verify_tier: 1 diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 260004fb6..4a4b710ee 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -1,402 +1,385 @@ # Feature Verification Procedures -How an agent verifies that each feature works from a user perspective. Organized by tier (what's required to run). Always run lower tiers first — they establish prerequisites for higher ones. +This document is the executable companion to `../manifest.yaml`. Its `verification.categories` map assigns every feature in a category to one procedure below. The tier orders work, but never replaces the stated prerequisites, assertions, cleanup, or automation limitation. ---- +Use `relay` (or the equivalent `agent-relay` binary). Run all mutations in a disposable project, workspace, account, configuration directory, or provider fixture. Never use global `update`, non-dry-run `sync`/`uninstall`, `node down --all`, or production credentials in an unattended test. -## Tier 1 — No dependencies +## Shared fixtures -Features that can be verified with just the CLI installed. Run these first. +For a local broker, use a disposable project and only stop the broker it created: -### CLI Health +```bash +RUN="feature-$(date +%s)"; TMP="$(mktemp -d)"; cd "$TMP" +relay node up --background --no-spawn +trap 'relay node down --force 2>/dev/null || true; rm -rf "$TMP"' EXIT +``` + +For hosted APIs, use explicit disposable credentials rather than an operator's active workspace: ```bash -relay version -# → prints version string like "10.x.x" +export RELAY_WORKSPACE_KEY='' +export RELAY_BASE_URL='' +RUN="feature-$(date +%s)" +A_JSON="$(relay agent register "audit-a-$RUN")"; TOKEN_A="$(jq -er .token <<<"$A_JSON")" +B_JSON="$(relay agent register "audit-b-$RUN")"; TOKEN_B="$(jq -er .token <<<"$B_JSON")" +cleanup_agents() { relay agent remove "audit-a-$RUN" 2>/dev/null || true; relay agent remove "audit-b-$RUN" 2>/dev/null || true; } +``` -relay --help -# → shows command list without error +SDK-backed CLI commands emit JSON already; do not add an unsupported `--json` flag. -relay telemetry status -# → prints "enabled" or "disabled" +## broker-lifecycle -relay workspace list -# → prints list (may be empty) without error -``` +**Features:** `broker-up`, `broker-down`, `broker-status`, `broker-metrics`, `broker-deadletters`, `broker-redeliver`. -### Setup and Doctor +**Prerequisites:** local fixture and broker binary. ```bash -relay version -# → confirms CLI is installed (relay setup does not exist) - -relay node --help -# → shows node subcommand options without error +relay node status | grep -Eiq 'running|stopped' +relay node metrics | jq -e 'type == "object"' +relay node deadletters --json | jq -e 'type == "object" or type == "array"' +relay status | grep -Eiq 'Local broker:' +relay node down; relay node status | grep -Eiq 'stopped|not running' ``` -Pass criteria: all commands exit 0, output is non-empty and sensibly formatted. +Assert daemon state and structured metrics/deadletters, then restart only when a later local test needs it. A successful redelivery needs a real dead-letter addressed to a live worker; the public CLI cannot seed one. Use the broker integration fixture, run exactly one of `relay node redeliver ` or `relay node redeliver --all`, and assert the delivery leaves the dead-letter queue. + +## workspace-agents-and-capabilities ---- +**Features:** `agent-register`, `agent-add`, `agent-list`, `agent-remove`, `agent-capabilities`, `capabilities-register`, `capabilities-delete`, `system-status`. -## Tier 2 — Broker running +**Prerequisites:** hosted fixture; a local broker is not the dependency. -Start the broker first: `relay node up --background` +```bash +CAP="audit-cap-$RUN" +relay agent list | jq -e --arg n "audit-a-$RUN" '.[] | select(.name == $n)' +relay agent add "audit-extra-$RUN" | jq -e '.token' +relay capabilities register "$CAP" --description 'audit capability' --handler "audit-a-$RUN" +relay capabilities list | jq -e --arg c "$CAP" '.[] | select(.command == $c)' +relay capabilities delete "$CAP"; relay agent remove "audit-extra-$RUN" +relay status | grep -Eiq 'Workspace:|Local broker:|Cloud:' +``` -Confirm it started: `relay status` should show "running". +Assert list visibility after every create and absence after every delete; finish with `cleanup_agents`. -### Broker Lifecycle +## channel-messaging + +**Features:** `channel-create`, `channel-list`, `channel-join`, `channel-leave`, `channel-invite`, `channel-set-topic`, `channel-archive`. + +**Prerequisites:** hosted fixture with two identities. ```bash -relay node up --background -relay status -# → shows "running", agent count, queue stats +CH="audit-channel-$RUN" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" --topic 'first topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel set_topic "$CH" 'second topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite "$CH" "audit-b-$RUN" +RELAY_AGENT_TOKEN="$TOKEN_B" relay channel list | jq -e --arg n "$CH" '.[] | select(.name == $n)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel leave "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list --archived | jq -e --arg n "$CH" '.[] | select(.name == $n)' +``` + +Assert the new topic and invited-agent membership, not just success output. Archive the test channel and remove test identities. -relay metrics -# → shows memory, CPU, message throughput +## message-round-trip -relay deadletters -# → shows empty list or existing dead letters (not an error) +**Features:** `message-post`, `message-list`, `message-reply`, `message-get-thread`, `message-search`, `message-file-upload`. -relay node down -relay status -# → shows "stopped" or "not running" +**Prerequisites:** hosted fixture and a channel owned by A. -relay node up --background # restart for subsequent tests +```bash +CH="audit-messages-$RUN"; TEXT="audit message $RUN" +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post "$CH" "$TEXT")"; MSG_ID="$(jq -er '.id // .messageId' <<<"$POST")" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message list "$CH" --limit 10 | jq -e --arg t "$TEXT" '.[] | select(.text == $t)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message reply "$MSG_ID" 'audit reply' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message get_thread "$MSG_ID" | jq -e 'tostring | contains("audit reply")' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message search "$RUN" --channel "$CH" --from "audit-a-$RUN" | jq -e 'tostring | contains("audit message")' +echo "attachment $RUN" > "$TMP/attachment.txt" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message file upload "$TMP/attachment.txt" --channel "$CH" --text 'attachment audit' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message list "$CH" --limit 20 | jq -e 'tostring | contains("attachment audit")' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" ``` -### Agent Management +Assert that the returned parent ID links the reply, search filters return the unique text, and the attachment is listed. Remove the temp file, archive, and call `cleanup_agents`. -```bash -# Register agents -relay agent register verify-agent-1 -# → prints token: RELAY_AGENT_TOKEN= +## direct-messages -relay agent list -# → shows verify-agent-1 in list +**Features:** `dm-send`, `dm-list`, `dm-list-conversations`, `dm-send-group`. -relay agent remove verify-agent-1 -relay agent list -# → verify-agent-1 no longer appears +**Prerequisites:** hosted fixture; group DM needs third identity C. + +```bash +C_JSON="$(relay agent register "audit-c-$RUN")"; TOKEN_C="$(jq -er .token <<<"$C_JSON")" +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "dm $RUN")"; CONV="$(jq -er '.conversationId // .conversation_id' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV" | jq -e --arg t "dm $RUN" 'tostring | contains($t)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group $RUN" --to "audit-b-$RUN" "audit-c-$RUN" +relay agent remove "audit-c-$RUN" ``` -### Local Agent Orchestration +Use `list_dms` through the MCP client and assert the same conversation appears. Assert both group recipients see it before cleanup. + +## reactions-and-read-status + +**Features:** `reaction-add`, `reaction-remove`, `inbox-check`, `inbox-mark-read`, `inbox-get-readers`. + +**Prerequisites:** hosted fixture with two identities. ```bash -relay node agent list -# → shows empty list or running agents +CH="audit-read-$RUN"; RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" +POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post "$CH" "read $RUN")"; MSG_ID="$(jq -er '.id // .messageId' <<<"$POST")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction add "$MSG_ID" thumbsup +RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction remove "$MSG_ID" thumbsup +DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "inbox $RUN")"; DM_ID="$(jq -er '.id // .messageId' <<<"$DM")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox check | jq -e --arg t "inbox $RUN" 'tostring | contains($t)' +RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox mark_read "$DM_ID" +RELAY_AGENT_TOKEN="$TOKEN_A" relay message inbox get_readers "$DM_ID" | jq -e --arg n "audit-b-$RUN" 'tostring | contains($n)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" +``` -# If claude harness is available: -relay node agent spawn claude --name test-worker -relay node agent list -# → shows test-worker with status active +Assert the reaction is observable after add and absent after remove, and B's unread message becomes A-visible read receipt. Archive and remove identities. -relay node agent message hold test-worker -# → message delivery paused +## local-agent-lifecycle -relay node agent message auto test-worker -# → message delivery resumed +**Features:** `local-agent-spawn`, `local-agent-new`, `local-agent-release`, `local-agent-list`, `local-agent-set-model`, `local-agent-attach`, `local-agent-flush`, `local-agent-hold`, `local-agent-auto`, `local-agent-tail`. -relay node agent release test-worker -relay node agent list -# → test-worker no longer appears +**Prerequisites:** local broker; spawn/new/set-model need an installed authenticated provider CLI and can incur cost. + +```bash +AGENT="audit-worker-$RUN" +relay node agent list | jq -e 'type == "array"' +relay node agent spawn "$PROVIDER" --name "$AGENT" --task 'Reply relay-e2e-ok' --spawn-mode task-exit --exit-after-task +relay node agent list | jq -e --arg n "$AGENT" '.[] | select(.name == $n)' +relay node agent message hold "$AGENT" | jq -e 'tostring | test("manual|hold"; "i")' +relay node agent message auto "$AGENT" | jq -e 'tostring | test("auto"; "i")' +relay node agent release "$AGENT" || true ``` -### Local Workflow +`new` and `attach` are attended PTY checks; verify `drive`, `view`, and `passthrough`. `tail` streams until interrupted: filter by agent, create output, assert it, then stop its exact child process. A model switch proves broker acceptance only, not provider-side model change. + +## local-workflow-lifecycle + +**Features:** `local-workflow-run`, `local-workflow-logs`, `local-workflow-sync`. + +**Prerequisites:** disposable local project and selected workflow runtime; no broker. ```bash -# Requires a minimal workflow file; check examples/ or create one: -cat > /tmp/test-workflow.yaml << 'EOF' -version: "1" -swarm: - agents: - - name: test-agent - harness: claude - workflows: - - name: health-check - steps: - - agent: test-agent - prompt: "Reply with just: OK" -EOF - -relay node workflow run /tmp/test-workflow.yaml -# → executes without crashing, agent responds +echo 'console.log("relay-workflow-e2e")' > "$TMP/workflow.js" +RUN_JSON="$(relay node workflow run "$TMP/workflow.js" --json)"; RUN_ID="$(jq -er .runId <<<"$RUN_JSON")" +relay node workflow logs "$RUN_ID" --follow --json | jq -e 'tostring | contains("relay-workflow-e2e")' +relay node workflow sync "$RUN_ID" --dry-run --json | jq -e 'type == "object"' ``` -### Workspace +Assert completed status and log content. Local run records have no delete command; remove only the disposable state/project directory. -```bash -relay workspace active -# → prints active workspace name/id +## cloud-workflows -relay workspace list -# → lists stored workspaces -``` +**Features:** `cloud-login`, `cloud-logout`, `cloud-whoami`, `cloud-session`, `cloud-connect`, `cloud-enroll`, `cloud-run`, `cloud-schedule`, `cloud-schedules`, `cloud-status`, `cloud-logs`, `cloud-sync`, `cloud-cancel`. -### Fleet Status +**Prerequisites:** dedicated Cloud account/workspace. Login and connect are browser/SSH interactive; enrollment needs a one-time test token or test-workspace mint. ```bash -relay fleet status -# → shows local broker status and provider attachment (even if no fleet configured) +relay cloud whoami +relay cloud session --json | jq -e '.apiUrl' +RUN_JSON="$(relay cloud run "$TMP/noop.yaml" --no-sync-code --json)"; CLOUD_RUN="$(jq -er '.runId // .id' <<<"$RUN_JSON")" +relay cloud status "$CLOUD_RUN" --json | jq -e 'type == "object"' +relay cloud logs "$CLOUD_RUN" --follow --json | jq -e 'type == "object"' +relay cloud sync "$CLOUD_RUN" --dry-run ``` -Pass criteria: each command exits 0, output matches expected description. +Use a no-op workflow, long disposable workflow for cancel, and only `sync --dry-run`. Schedules lack a CLI delete, so create them only in an isolated workspace and remove through Cloud control plane. Logout only an isolated account. ---- +## cloud-workers -## Tier 3 — Broker running + agent token +**Features:** `cloud-worker-register`, `cloud-worker-start`, `cloud-worker-status`, `cloud-worker-logs`. -Register at least one agent (`relay agent register `) and export its token: +**Prerequisites:** dedicated enrollment token and isolated worker state. ```bash -export RELAY_AGENT_TOKEN=$(relay agent register verify-agent | jq -r '.token') +relay cloud worker register --token "$WORKER_TOKEN" --name "audit-worker-$RUN" --json | jq -e '.workerId' +relay cloud worker start --name "audit-worker-$RUN" --daemon +relay cloud worker status --name "audit-worker-$RUN" --json | jq -e '.localDaemon' +relay cloud worker logs --name "audit-worker-$RUN" ``` -Or pass `--token ` to commands that support it. +Record the daemon PID/log path and terminate only that daemon; no worker-stop command exists. Use `--once` when a bounded assignment fixture is available. -### Channel Operations +## fleet-management + +**Features:** `fleet-nodes`, `fleet-config`, `fleet-enable`, `fleet-disable`, `fleet-inherit`, `fleet-status`. + +**Prerequisites:** disposable workspace; `fleet-status` additionally benefits from a local broker. ```bash -# Create -relay channel create verify-test-channel -# → success message - -# List -relay channel list -# → shows verify-test-channel - -# Join -relay channel join verify-test-channel -# → success message - -# Set topic -relay channel set_topic verify-test-channel "verification test" -# → success message - -# Leave -relay channel leave verify-test-channel -# → success message - -# Archive -relay channel archive verify-test-channel -# → success message -relay channel list -# → verify-test-channel absent (or present with archived flag if --archived passed) +relay fleet nodes | jq -e '.nodes' +BEFORE="$(relay fleet config)" +relay fleet enable; relay fleet config | jq -e 'type == "object"' +relay fleet disable; relay fleet inherit; relay fleet status | jq -e '.broker' ``` -### Message Operations +Snapshot and restore configuration or discard the workspace. For full two-node dispatch/enrollment coverage, run `npm run test:e2e` with `tests/e2e/fleet/README.md` prerequisites. + +## workspace-management + +**Features:** `workspace-active`, `workspace-create`, `workspace-list`, `workspace-set-key`, `workspace-join`, `workspace-switch`. + +**Prerequisites:** isolated `AGENT_RELAY_HOME`; active/create also require Cloud API/auth. ```bash -# Post -relay channel create verify-msgs -relay channel join verify-msgs -relay message post verify-msgs "verification message $(date +%s)" -# → success - -# List and confirm delivery -relay message list verify-msgs --limit 1 -# → shows the message just posted with correct text - -# Reply (create thread) -MSG_ID=$(relay message list verify-msgs --limit 1 --json | jq -r '.[0].id') -relay message reply "$MSG_ID" "thread reply" -# → success - -# Get thread -relay message get_thread --message-id "$MSG_ID" -# → shows original message + the reply - -# Search -relay message search --query "verification" -# → returns at least one result containing the posted message - -# Inbox -relay message inbox check -# → shows unread count (may be 0 if no messages directed to this agent) - -# Mark read -relay message inbox mark_read --message-id "$MSG_ID" -# → success +export AGENT_RELAY_HOME="$(mktemp -d)" +relay workspace set_key audit-one rk_live_example +relay workspace join audit-two rk_live_example_two +relay workspace switch audit-one +relay workspace list | jq -e '.active == "audit-one"' +rm -rf "$AGENT_RELAY_HOME" ``` -### Reactions +Create remote workspaces only in a test tenant, assert `workspace active --json` returns canonical IDs, and delete them through the control plane. -```bash -relay message reaction add --message-id "$MSG_ID" --emoji thumbsup -# → success +## skills-installation -relay message reaction remove --message-id "$MSG_ID" --emoji thumbsup -# → success -``` +**Features:** `skills-add`. -### Webhooks (requires auth token) +**Prerequisites:** network and disposable project/configuration. ```bash -relay integration webhook list -# → success (empty or populated) +mkdir -p "$TMP/skills-project"; cd "$TMP/skills-project" +relay skills add --local --harness codex +test -e .agents/skills/orchestrate/SKILL.md +``` -HOOK_ID=$(relay integration webhook create https://example.com/hook --event message.created | jq -r '.id') -# → prints webhook id +Assert the downloaded skill and delete only this disposable project. Do not run `--global` in automation. -relay integration webhook delete "$HOOK_ID" -# → success -``` +## integrations-and-webhooks -### Subscriptions +**Features:** `integration-subscribe`, `integration-unsubscribe`, `integration-list-bindings`, all `webhook-*`, and all `subscription-*` entries. + +**Prerequisites:** disposable workspace, identity, externally reachable controlled receiver. Relayfile also needs compatible authenticated daemon, connected provider, real provider resource, and provider observation API; first connection can require browser OAuth. ```bash -relay integration subscription list -# → success (empty or populated) +HOOK="$(relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" +relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' +relay integration webhook list | jq -e --arg id "$HOOK_ID" 'tostring | contains($id)' +relay integration webhook delete "$HOOK_ID" ``` -Pass criteria: all channel/message round-trips show data that matches what was written. +Assert exact receiver payload/signature, then delete it. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. ---- +## reflex-history -## Tier 4 — Broker running + two agents +**Features:** `reflex-on`, `reflex-off`, `reflex-status`. -Register two agents and test cross-agent features: +**Prerequisites:** isolated home; `on` is interactive and can use Cloud login. ```bash -export TOKEN_A=$(relay agent register verify-alice | jq -r '.token') -export TOKEN_B=$(relay agent register verify-bob | jq -r '.token') +echo y | relay reflex on; relay reflex status | grep -Eiq 'on' +relay reflex off; relay reflex status | grep -Eiq 'off' ``` -### Channel Invite +Run only against a disposable home/configuration and assert cleanup leaves Reflex off. -```bash -RELAY_AGENT_TOKEN=$TOKEN_A relay channel create private-verify -RELAY_AGENT_TOKEN=$TOKEN_A relay channel join private-verify -RELAY_AGENT_TOKEN=$TOKEN_A relay channel invite private-verify verify-bob -# → success +## mcp-stdio -RELAY_AGENT_TOKEN=$TOKEN_B relay channel list -# → shows private-verify as a channel bob is in -``` +**Features:** `mcp-server-start` and all `mcp-*` tool/prompt entries. -### Direct Messages +**Prerequisites:** Node MCP SDK client, disposable hosted workspace, registered identities. `add_agent`, `spawn`, and `submit_result` need real provider/callback fixtures. -```bash -# Send DM and capture conversationId from JSON output: -DM_RESPONSE=$(RELAY_AGENT_TOKEN=$TOKEN_A relay message dm send verify-bob "hello bob") -CONV_ID=$(echo "$DM_RESPONSE" | jq -r '.conversationId') -# → success; conversationId captured +Launch `relay mcp` as stdio child. Initialize it, call `tools/list` and `prompts/list`, and assert all 31 static tool names in the manifest plus prompt `system`. Call `set_workspace_key`/`register_agent` for A and B; repeat channel/message/thread/DM/reaction/read assertions through MCP fields (`message_id`, `include_archived`, `participants`, `as`). Register an action before `list_actions`/`invoke_action`. Spawn only a disposable worker then `remove_agent`. Configure result callback environment before testing conditional `submit_result`, assert receiver payload, and close the child cleanly. -# Verify recipient can list the conversation: -RELAY_AGENT_TOKEN=$TOKEN_B relay message dm list "$CONV_ID" -# → shows the DM from verify-alice +## harnesses -RELAY_AGENT_TOKEN=$TOKEN_A relay message dm send_group "group hello" --to verify-bob -# → success -``` +**Features:** all `harness-*` entries. -### Read Receipts +**Prerequisites:** named provider CLI installed/authenticated, disposable project, and budget; custom harness only needs the SDK contract. -```bash -MSG_ID=$(RELAY_AGENT_TOKEN=$TOKEN_A relay message list private-verify --limit 1 --json | jq -r '.[0].id') -RELAY_AGENT_TOKEN=$TOKEN_B relay message inbox mark_read --message-id "$MSG_ID" +Run the bounded local-agent spawn test for each installed provider with `--task`, `--spawn-mode task-exit`, and `--exit-after-task`; assert list, sentinel response, and release. PTY providers need attended verification. For a custom harness, run `npm test --workspace @agent-relay/harnesses` plus a `defineHarness` create/send/release fixture. -RELAY_AGENT_TOKEN=$TOKEN_A relay message inbox get_readers --message-id "$MSG_ID" -# → shows verify-bob has read the message -``` +## typescript-sdk -### Cleanup +**Features:** all `sdk-*` entries. + +**Prerequisites:** local engine or disposable hosted workspace. ```bash -relay agent remove verify-alice -relay agent remove verify-bob +node tests/integration/sdk/v8-api-smoke.mjs ``` -Pass criteria: messages posted by agent A appear in agent B's list, DMs route correctly. +Assert bootstrap/reconnect, participants, channel/thread/reaction/DM/group-DM/listener/action/DeliveryRunner/webhook/node behavior. Release all identities and clean up remote workspace through its control plane; the smoke script does not delete it. + +## python-sdk ---- +**Features:** all `python-sdk-*` entries. -## Tier 5 — Cloud auth required +```bash +cd packages/sdk-py && python -m pytest +``` -Requires `relay cloud login` to have been completed. +For true E2E, create a disposable Relay, send/receive unique message, then release agents and shut down. Adapter/workflow tests require their provider CLI and must clean up spawned work. -### Auth Checks +## swift-sdk -```bash -relay cloud whoami -# → shows authenticated user/org +**Features:** `swift-sdk-hosted`, `swift-sdk-broker`. -relay cloud session -# → shows session details (workspace, expiry, etc.) +```bash +cd packages/sdk-swift && swift test ``` -### Cloud Workflow Run +Run hosted and local-broker round trips, assert receipt/listener behavior, then release identities and clean up the disposable workspace. -```bash -relay cloud run examples/basic-workflow.yaml -# → prints run ID +## opencode-plugin -RUN_ID= -relay cloud status "$RUN_ID" -# → shows run status (queued, running, completed) +**Features:** all `opencode-relay-*` entries. -relay cloud logs "$RUN_ID" -# → shows log output from the run +```bash +npm --prefix plugins/opencode-relay-plugin test ``` -### Schedules +Connect two disposable OpenCode sessions, prove each native tool, spawn one worker, use `relay_dismiss`, and assert idle/compaction/end hooks preserve then clean state. -```bash -relay cloud schedule examples/basic-workflow.yaml --cron "0 * * * *" -# → prints schedule ID +## codex-relay-skill -relay cloud schedules -# → shows the schedule just created -``` +**Features:** all `codex-relay-*` entries. + +Copy the skill into a temporary `.agents/skills/agent-relay`, run setup twice, and diff resulting `.codex/config.toml`, hooks, and worker template to prove idempotence. In disposable Codex, require a relay-worker ACK/STATUS/DONE via MCP and assert hooks connect, surface inbox, and protect completion with unread work. + +## gemini-relay-extension + +**Features:** all `gemini-relay-*` entries. -Pass criteria: cloud commands return data consistent with the authenticated account. +Install with `gemini extensions install AgentWorkforce/relay`; with experimental agents and disposable workspace run `/relay:status`, `/relay:team `, `/relay:fanout `. Assert worker ACK/DONE and session-start/after-tool/before-model/stop/session-end hook behavior, including cleanup. ---- +## cli-maintenance -## Tier 6 — Manual / Browser +**Features:** `cli-version`, `cli-update`, `cli-uninstall`. -These cannot be automated from CLI. A human must verify them. +```bash +relay version | grep -E '.' +relay update --check +relay uninstall --dry-run --keep-data +``` -| Feature | How to Verify | -| ------------------------------------ | --------------------------------------------------------------------------- | -| `relay cloud login` | Open browser OAuth flow, complete auth, check `relay cloud whoami` succeeds | -| Cursor harness | Open Cursor, run cursor-agent, confirm PTY injection works | -| Web dashboard (relay-dashboard repo) | Navigate to dashboard, confirm agents/channels visible | -| `relay cloud connect` | SSH session must complete provider auth interactively | -| `relay cloud enroll` | Machine enrollment requires cloud account and browser confirmation | +Assert version/update/removal-plan output. Mutating update/uninstall requires isolated OS/container and attended approval. ---- +## telemetry -## Verification Checklist by Change Type +**Features:** `telemetry-enable`, `telemetry-disable`, `telemetry-status`. -Use this to determine which tiers to run: +```bash +export AGENT_RELAY_DATA_DIR="$(mktemp -d)" +relay telemetry disable; relay telemetry status | grep -Eiq 'Enabled: No' +relay telemetry enable; relay telemetry status | grep -Eiq 'Enabled: Yes' +rm -rf "$AGENT_RELAY_DATA_DIR" +``` -| Change area | Tiers to run | -| ------------------------------------------------------------------ | ------------------------------------ | -| Broker Rust code (`crates/broker/`) | 1, 2, 3, 4 | -| PTY/harness code (`crates/relay-pty/`, `packages/harness-driver/`) | 2 (spawn tests) | -| CLI commands (`packages/cli/src/cli/commands/`) | 1 + tier matching the command | -| MCP tools (`packages/cli/src/cli/mcp/`) | 2, 3 (MCP server + tool calls) | -| SDK (`packages/sdk/`) | 3, 4 | -| Cloud client (`packages/cloud/`) | 5 | -| Any auth/token change | 2, 3, 4 | -| Message ordering/delivery | 3, 4 (post then list, confirm order) | -| Harness definitions (`packages/harnesses/`) | 2 (spawn that harness) | +Assert both persisted states; unset telemetry opt-out environment variables because they override stored preference. ---- +## node-command-discovery -## Quick Sanity Check (Run After Any Change) +**Features:** `node-up`, `node-workflow`. ```bash -relay version && relay status || relay node up --background && relay status -AGENT_NAME="quick-check-$(date +%s)" -export RELAY_AGENT_TOKEN=$(relay agent register "$AGENT_NAME" | jq -r '.token') -relay agent list -relay channel create quick-check-ch -relay channel join quick-check-ch -relay message post quick-check-ch "sanity $(date)" -relay message list quick-check-ch --limit 1 -relay channel archive quick-check-ch -relay agent remove "$AGENT_NAME" +relay node up --help | grep -Eiq 'config|no-spawn' +relay node workflow --help | grep -Eiq 'run|logs|sync' ``` -All steps should succeed. Total runtime: under 10 seconds. +Use `broker-lifecycle` for start behavior and `local-workflow-lifecycle` for the three workflow leaves. This group procedure is discoverability only, not workflow execution. diff --git a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json index ec034c701..4cfc6e9ef 100644 --- a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json @@ -35,6 +35,39 @@ "reasoning": "Panic may occur on a tokio worker thread and the process may abort before the async sender loop drains; a dedicated std::thread with its own runtime avoids nested-runtime panic, and the reqwest timeout bounds process teardown" }, "significance": "high" + }, + { + "ts": 1784578305956, + "type": "decision", + "content": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose: Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "raw": { + "question": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "chosen": "Audit exact Commander registrations and SDK/MCP exports rather than relying on existing manifest prose", + "alternatives": [], + "reasoning": "The manifest already contains stale aliases and incomplete argument syntax, so repository source is the authoritative contract." + }, + "significance": "high" + }, + { + "ts": 1784579530918, + "type": "reflection", + "content": "Parallel audits reconciled the manifest with Commander registrations, MCP tools, SDKs, harnesses, and plugins. The catalog now uses explicit category-to-procedure mappings with contract tests so CLI and MCP drift fails locally before review.", + "raw": { + "focalPoints": [ + "surface-accuracy", + "e2e-verification", + "drift-prevention" + ], + "adjustments": "Added executable procedures and a manifest contract test; retained explicit external/interactive limits rather than claiming unsupported automation.", + "confidence": 0.9 + }, + "significance": "high", + "tags": [ + "focal:surface-accuracy", + "focal:e2e-verification", + "focal:drift-prevention", + "confidence:0.9" + ] } ] } @@ -47,4 +80,4 @@ "startRef": "765e42535063b9d61262a537b1fd364d3e965000", "endRef": "765e42535063b9d61262a537b1fd364d3e965000" } -} +} \ No newline at end of file diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index 3a5b1dad6..774a11543 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -23,7 +23,7 @@ Use when you need to verify that a specific feature or set of features works cor ```bash # View all features in a category -grep -A 10 "category: messaging-messages" .agentworkforce/features/manifest.yaml +grep -A 60 "^ messaging-messages:" .agentworkforce/features/manifest.yaml # Find a feature by id grep -A 8 "id: message-post" .agentworkforce/features/manifest.yaml @@ -34,37 +34,26 @@ The manifest tells you: - `criticality` — how important is this feature - `verify_tier` — what's required to verify it (1=nothing, 2=broker, 3=agent token, 4=two agents, 5=cloud, 6=manual) - `location` — which source files implement it +- `verification.categories` — the named procedure in `verify/procedures.md` that supplies prerequisites, commands, assertions, cleanup, and automation limits ### 2. Check verify tier requirements -| Tier | Requires | Setup command | -| ---- | -------------------- | -------------------------------------------------------------------------------- | -| 1 | Nothing | (none) | -| 2 | Broker running | `relay node up --background` | -| 3 | Broker + agent token | `relay node up --background && relay agent register ` | -| 4 | Broker + 2 agents | `relay node up --background && relay agent register a && relay agent register b` | -| 5 | Cloud auth | `relay cloud whoami` (must already be logged in) | -| 6 | Manual only | Human must verify in browser or interactive session | +| Tier | Primary environment | Important qualification | +| ---- | ----------------------------------------- | ------------------------------------------------------------------------------ | +| 1 | Isolated local CLI/filesystem | May still mutate local config; use a temp directory. | +| 2 | Local broker | Start with `relay node up --background --no-spawn`. | +| 3 | Hosted workspace + one agent | Requires explicit workspace key/API, not merely a broker. | +| 4 | Hosted workspace + two agents | Use separate actor tokens for cross-agent assertions. | +| 5 | Authenticated disposable external service | Use a test workspace, receiver, or account and clean it up. | +| 6 | Interactive/pre-provisioned integration | Browser, SSH, PTY, provider credentials, or external callback may be required. | ### 3. Run the verification -Follow `.agentworkforce/features/verify/procedures.md` for the relevant tier. Always start from the lowest tier that applies and work up. +Resolve the category through `verification.categories`, then follow the matching procedure in `.agentworkforce/features/verify/procedures.md`. Always run lower prerequisites first. **Quick sanity check after any change:** -```bash -relay version && \ -relay status || relay node up --background && \ -relay status && \ -export RELAY_AGENT_TOKEN=$(relay agent register quick-check | jq -r '.token') && \ -relay agent list && \ -relay channel create quick-check-ch && \ -relay channel join quick-check-ch && \ -relay message post quick-check-ch "sanity $(date)" && \ -relay message list quick-check-ch --limit 1 && \ -relay channel archive quick-check-ch && \ -relay agent remove quick-check -``` +Start with the `Fast Health Triage` sequence in `critical-paths.md`; use the hosted-workspace fixture in `procedures.md` before attempting channel or message checks. ### 4. Determine which features to verify for a given change @@ -80,13 +69,14 @@ relay agent remove quick-check ### 5. Check critical paths last -Always run the 5 critical paths from `critical-paths.md` before declaring a change verified: +Always run the applicable critical paths from `critical-paths.md` before declaring a change verified: -1. `broker-up` → `agent-register` → `agent-list` → `status` -2. Channel create → join → post → list (two agents) -3. `local agent spawn` → `hold` → `flush` → `release` -4. `relay mcp` → `list_channels` → `post_message` → `list_messages` -5. `local workflow run` (basic workflow) +1. Local broker lifecycle +2. Cross-agent channel message +3. Managed local-agent lifecycle (when a provider is available) +4. MCP stdio round trip +5. Local workflow run/logs/sync +6. Direct message and read receipt ## When to update the manifest @@ -105,9 +95,9 @@ Update `critical-paths.md` when: ## Example: verifying after a messaging change ```text -1. Read the manifest: grep "messaging-messages" manifest.yaml → verify_tier: 3 -2. Setup: relay node up --background && export RELAY_AGENT_TOKEN=$(relay agent register test-agent | jq -r '.token') -3. Follow tier 3 procedures: post → list → reply → get_thread → search -4. Run critical path 2 (channel messaging) with two agents -5. Clean up: relay agent remove test-agent +1. Read the manifest category and resolve its verification procedure. +2. Create the exact disposable fixture stated by that procedure. +3. Run every listed command and assert values written by the test are read back. +4. Run the applicable critical path with separate agents where required. +5. Perform the procedure's cleanup and prove the test resources are gone. ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e0af4e2..0db0e2a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Changed + +- Feature verification catalog now records the exact CLI and MCP surfaces, adds previously unlisted SDK and plugin integrations, and maps every category to an end-to-end procedure with prerequisites, assertions, cleanup, and automation limits. ## [10.6.6] - 2026-07-19 From 4d1ae3a94c07eddbe9989a4987f091140108dc7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 20 Jul 2026 20:34:39 +0000 Subject: [PATCH 2/6] style: auto-format with Prettier --- .../trajectories/active/traj_rtyte8r4g07t/trajectory.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json index 4cfc6e9ef..440d01472 100644 --- a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json @@ -53,11 +53,7 @@ "type": "reflection", "content": "Parallel audits reconciled the manifest with Commander registrations, MCP tools, SDKs, harnesses, and plugins. The catalog now uses explicit category-to-procedure mappings with contract tests so CLI and MCP drift fails locally before review.", "raw": { - "focalPoints": [ - "surface-accuracy", - "e2e-verification", - "drift-prevention" - ], + "focalPoints": ["surface-accuracy", "e2e-verification", "drift-prevention"], "adjustments": "Added executable procedures and a manifest contract test; retained explicit external/interactive limits rather than claiming unsupported automation.", "confidence": 0.9 }, @@ -80,4 +76,4 @@ "startRef": "765e42535063b9d61262a537b1fd364d3e965000", "endRef": "765e42535063b9d61262a537b1fd364d3e965000" } -} \ No newline at end of file +} From 229ce4724839892e29d46782323b5b4464d1affd Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 20 Jul 2026 22:36:25 +0200 Subject: [PATCH 3/6] test: harden feature manifest contract --- .../agents/relay-feature-guardian/manifest-contract.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts index 22b675301..1375741b9 100644 --- a/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts +++ b/.agentworkforce/agents/relay-feature-guardian/manifest-contract.test.ts @@ -4,7 +4,7 @@ import { parse } from 'yaml'; import { describe, expect, it } from 'vitest'; type Feature = { id: string; cli?: string; mcp?: string; mcp_prompt?: string; location?: string }; -type Category = { features: Feature[] }; +type Category = { features?: Feature[] }; type Manifest = { version: string; verification: { document: string; categories: Record }; @@ -15,7 +15,7 @@ const manifestPath = new URL('../../features/manifest.yaml', import.meta.url); const proceduresPath = new URL('../../features/verify/procedures.md', import.meta.url); const manifest = parse(readFileSync(manifestPath, 'utf8')) as Manifest; const procedures = readFileSync(proceduresPath, 'utf8'); -const features = Object.values(manifest.categories).flatMap((category) => category.features); +const features = Object.values(manifest.categories).flatMap((category) => category.features ?? []); describe('feature manifest contract', () => { it('maps every category to a detailed verification procedure', () => { From 86ac7875cd3d999c91f6a30b03af11e06cfd39d1 Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 20 Jul 2026 22:48:17 +0200 Subject: [PATCH 4/6] docs: harden feature verification procedures --- .agentworkforce/features/critical-paths.md | 7 +- .agentworkforce/features/verify/procedures.md | 64 ++++++++++++++++--- .claude/skills/verify-features.md | 24 +++++-- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/.agentworkforce/features/critical-paths.md b/.agentworkforce/features/critical-paths.md index 992ea03cc..41c5fe421 100644 --- a/.agentworkforce/features/critical-paths.md +++ b/.agentworkforce/features/critical-paths.md @@ -17,7 +17,8 @@ The first status must report running and the last must report stopped. Use an is ## Path 2: Cross-Agent Channel Message -**Prerequisite:** disposable hosted workspace with `TOKEN_A` and `TOKEN_B` from two registrations. +**Prerequisite:** disposable hosted workspace; `TOKEN_A` must be the token returned for +`critical-a`, and `TOKEN_B` must be the token returned for `critical-b`. ```bash RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create critical-path @@ -34,6 +35,7 @@ Agent B must see the exact text from A. Remove both test identities after archiv **Prerequisite:** a locally installed and authenticated provider CLI. ```bash +PROVIDER="${PROVIDER:-claude}" # installed and authenticated fixture provider relay node up --background --no-spawn relay node agent spawn "$PROVIDER" --name critical-worker --task 'Reply critical-ok' --spawn-mode task-exit --exit-after-task relay node agent list @@ -74,7 +76,8 @@ Assert completed status and the sentinel output. Run it in a disposable project ## Path 6: Direct Message and Read Receipt -**Prerequisite:** disposable hosted workspace with two identities. +**Prerequisite:** disposable hosted workspace; `TOKEN_A` must be the token returned for +`critical-a`, and `TOKEN_B` must be the token returned for `critical-b`. ```bash DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send critical-b 'critical-dm')" diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index 4a4b710ee..edd48923e 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -27,6 +27,28 @@ cleanup_agents() { relay agent remove "audit-a-$RUN" 2>/dev/null || true; relay SDK-backed CLI commands emit JSON already; do not add an unsupported `--json` flag. +## Externally provisioned fixtures + +Two procedures need fixtures that the Relay CLI cannot create. Create only the +fixture required by the procedure you are about to run; its command block +performs the corresponding fail-fast check. + +```bash +# A Cloud *worker* enrollment token minted for the disposable Cloud workspace. +# This is not the node-enrollment token used by `relay cloud enroll`. +export WORKER_TOKEN='' + +# A unique, internet-reachable receiver owned by the test harness. It must +# accept the webhook POST, retain request headers and body for assertion, and +# return a 2xx response. Localhost is suitable only when the tested Relay API +# is local; hosted APIs require a tunneled or CI-provisioned receiver. +export CAPTURE_URL='https://controlled-test-receiver.example/unique-run-path' +``` + +Do not substitute a public request-bin or a shared endpoint: webhook payloads +and signatures are test data that must remain isolated. Each run gets its own +receiver path and destroys it after the webhook/subscription cleanup below. + ## broker-lifecycle **Features:** `broker-up`, `broker-down`, `broker-status`, `broker-metrics`, `broker-deadletters`, `broker-redeliver`. @@ -55,7 +77,10 @@ relay agent list | jq -e --arg n "audit-a-$RUN" '.[] | select(.name == $n)' relay agent add "audit-extra-$RUN" | jq -e '.token' relay capabilities register "$CAP" --description 'audit capability' --handler "audit-a-$RUN" relay capabilities list | jq -e --arg c "$CAP" '.[] | select(.command == $c)' -relay capabilities delete "$CAP"; relay agent remove "audit-extra-$RUN" +relay capabilities delete "$CAP" +relay capabilities list | jq -e --arg c "$CAP" 'all(.[]; .command != $c)' +relay agent remove "audit-extra-$RUN" +relay agent list | jq -e --arg n "audit-extra-$RUN" 'all(.[]; .name != $n)' relay status | grep -Eiq 'Workspace:|Local broker:|Cloud:' ``` @@ -70,10 +95,18 @@ Assert list visibility after every create and absence after every delete; finish ```bash CH="audit-channel-$RUN" RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" --topic 'first topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list | jq -e --arg n "$CH" \ + '.[] | select(.name == $n)' RELAY_AGENT_TOKEN="$TOKEN_A" relay channel join "$CH" RELAY_AGENT_TOKEN="$TOKEN_A" relay channel set_topic "$CH" 'second topic' +RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list | jq -e --arg n "$CH" --arg topic 'second topic' \ + '.[] | select(.name == $n and .topic == $topic)' RELAY_AGENT_TOKEN="$TOKEN_A" relay channel invite "$CH" "audit-b-$RUN" -RELAY_AGENT_TOKEN="$TOKEN_B" relay channel list | jq -e --arg n "$CH" '.[] | select(.name == $n)' +RELAY_AGENT_TOKEN="$TOKEN_B" relay channel list | jq -e --arg n "$CH" --arg agent "audit-b-$RUN" \ + '.[] | select( + .name == $n and + ((.members // []) | map(if type == "string" then . else (.agentName // .name) end) | index($agent)) + )' RELAY_AGENT_TOKEN="$TOKEN_A" relay channel leave "$CH" RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" RELAY_AGENT_TOKEN="$TOKEN_A" relay channel list --archived | jq -e --arg n "$CH" '.[] | select(.name == $n)' @@ -113,7 +146,9 @@ Assert that the returned parent ID links the reply, search filters return the un C_JSON="$(relay agent register "audit-c-$RUN")"; TOKEN_C="$(jq -er .token <<<"$C_JSON")" DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "dm $RUN")"; CONV="$(jq -er '.conversationId // .conversation_id' <<<"$DM")" RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$CONV" | jq -e --arg t "dm $RUN" 'tostring | contains($t)' -RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group $RUN" --to "audit-b-$RUN" "audit-c-$RUN" +GROUP="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send_group "group $RUN" --to "audit-b-$RUN" "audit-c-$RUN")"; GROUP_CONV="$(jq -er '.conversationId // .conversation_id' <<<"$GROUP")" +RELAY_AGENT_TOKEN="$TOKEN_B" relay message dm list "$GROUP_CONV" | jq -e --arg t "group $RUN" 'tostring | contains($t)' +RELAY_AGENT_TOKEN="$TOKEN_C" relay message dm list "$GROUP_CONV" | jq -e --arg t "group $RUN" 'tostring | contains($t)' relay agent remove "audit-c-$RUN" ``` @@ -127,9 +162,19 @@ Use `list_dms` through the MCP client and assert the same conversation appears. ```bash CH="audit-read-$RUN"; RELAY_AGENT_TOKEN="$TOKEN_A" relay channel create "$CH" +RELAY_AGENT_TOKEN="$TOKEN_B" relay channel join "$CH" POST="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message post "$CH" "read $RUN")"; MSG_ID="$(jq -er '.id // .messageId' <<<"$POST")" RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction add "$MSG_ID" thumbsup +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list "$CH" --limit 10 | jq -e --arg id "$MSG_ID" ' + .[] | select((.id == $id) or (.messageId == $id)) + | .reactions[]? | select(.emoji == "thumbsup" and (.count // 0) >= 1) +' RELAY_AGENT_TOKEN="$TOKEN_B" relay message reaction remove "$MSG_ID" thumbsup +RELAY_AGENT_TOKEN="$TOKEN_B" relay message list "$CH" --limit 10 | jq -e --arg id "$MSG_ID" ' + [ .[] | select((.id == $id) or (.messageId == $id)) + | .reactions[]? | select(.emoji == "thumbsup" and (.count // 0) > 0) ] + | length == 0 +' DM="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay message dm send "audit-b-$RUN" "inbox $RUN")"; DM_ID="$(jq -er '.id // .messageId' <<<"$DM")" RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox check | jq -e --arg t "inbox $RUN" 'tostring | contains($t)' RELAY_AGENT_TOKEN="$TOKEN_B" relay message inbox mark_read "$DM_ID" @@ -147,12 +192,13 @@ Assert the reaction is observable after add and absent after remove, and B's unr ```bash AGENT="audit-worker-$RUN" +PROVIDER="${PROVIDER:-claude}" # installed and authenticated fixture provider relay node agent list | jq -e 'type == "array"' relay node agent spawn "$PROVIDER" --name "$AGENT" --task 'Reply relay-e2e-ok' --spawn-mode task-exit --exit-after-task relay node agent list | jq -e --arg n "$AGENT" '.[] | select(.name == $n)' relay node agent message hold "$AGENT" | jq -e 'tostring | test("manual|hold"; "i")' relay node agent message auto "$AGENT" | jq -e 'tostring | test("auto"; "i")' -relay node agent release "$AGENT" || true +relay node agent release "$AGENT" ``` `new` and `attach` are attended PTY checks; verify `drive`, `view`, and `passthrough`. `tail` streams until interrupted: filter by agent, create output, assert it, then stop its exact child process. A model switch proves broker acceptance only, not provider-side model change. @@ -196,6 +242,7 @@ Use a no-op workflow, long disposable workflow for cancel, and only `sync --dry- **Prerequisites:** dedicated enrollment token and isolated worker state. ```bash +: "${WORKER_TOKEN:?See Externally provisioned fixtures}" relay cloud worker register --token "$WORKER_TOKEN" --name "audit-worker-$RUN" --json | jq -e '.workerId' relay cloud worker start --name "audit-worker-$RUN" --daemon relay cloud worker status --name "audit-worker-$RUN" --json | jq -e '.localDaemon' @@ -257,10 +304,11 @@ Assert the downloaded skill and delete only this disposable project. Do not run **Prerequisites:** disposable workspace, identity, externally reachable controlled receiver. Relayfile also needs compatible authenticated daemon, connected provider, real provider resource, and provider observation API; first connection can require browser OAuth. ```bash -HOOK="$(relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" -relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' -relay integration webhook list | jq -e --arg id "$HOOK_ID" 'tostring | contains($id)' -relay integration webhook delete "$HOOK_ID" +: "${CAPTURE_URL:?See Externally provisioned fixtures}" +HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook list | jq -e --arg id "$HOOK_ID" 'tostring | contains($id)' +RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook delete "$HOOK_ID" ``` Assert exact receiver payload/signature, then delete it. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. diff --git a/.claude/skills/verify-features.md b/.claude/skills/verify-features.md index 774a11543..f93c12d71 100644 --- a/.claude/skills/verify-features.md +++ b/.claude/skills/verify-features.md @@ -22,11 +22,25 @@ Use when you need to verify that a specific feature or set of features works cor ### 1. Read the manifest to find the feature ```bash -# View all features in a category -grep -A 60 "^ messaging-messages:" .agentworkforce/features/manifest.yaml - -# Find a feature by id -grep -A 8 "id: message-post" .agentworkforce/features/manifest.yaml +# Prefer a structural query: category/feature length and indentation can change. +if command -v yq >/dev/null 2>&1; then + yq '.categories."messaging-messages"' .agentworkforce/features/manifest.yaml + yq '.. | select(type == "!!map" and .id == "message-post")' .agentworkforce/features/manifest.yaml +else + # Fallback for environments without mikefarah/yq: stop at the next sibling + # category/feature instead of assuming a fixed number of following lines. + awk ' + $0 == " messaging-messages:" { in_category = 1 } + in_category && $0 != " messaging-messages:" && /^ [[:alnum:]][[:alnum:]-]*:$/ { exit } + in_category { print } + ' .agentworkforce/features/manifest.yaml + awk ' + $0 == " - id: message-post" { in_feature = 1 } + in_feature && $0 != " - id: message-post" && /^ - id: / { exit } + in_feature && /^ [[:alnum:]][[:alnum:]-]*:$/ { exit } + in_feature { print } + ' .agentworkforce/features/manifest.yaml +fi ``` The manifest tells you: From a9e356da72df29f23b02366e33ec2dedd8b52c9e Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 20 Jul 2026 23:01:10 +0200 Subject: [PATCH 5/6] docs: make feature verification executable --- .../agents/relay-feature-guardian/agent.ts | 16 ++- .agentworkforce/features/verify/procedures.md | 130 +++++++++++++++--- .../active/traj_rtyte8r4g07t/trajectory.json | 12 ++ 3 files changed, 135 insertions(+), 23 deletions(-) diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.ts b/.agentworkforce/agents/relay-feature-guardian/agent.ts index abbe2c586..b83319f1f 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.ts @@ -546,9 +546,13 @@ function pickNextFeature(features: Feature[], checkedIds: Set): Feature // ── quiz generation ─────────────────────────────────────────────────────────── async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise { - const mcpNote = feature.mcp ? `\nMCP tool: \`${feature.mcp}\`` : ''; - const mcpPromptNote = feature.mcpPrompt ? `\nMCP prompt: \`${feature.mcpPrompt}\`` : ''; - const surface = feature.cli ? `CLI: ${feature.cli}` : 'CLI: (MCP-only surface)'; + const surface = [ + feature.cli ? `CLI command: ${feature.cli}` : null, + feature.mcp ? `MCP tool: ${feature.mcp}` : null, + feature.mcpPrompt ? `MCP prompt: ${feature.mcpPrompt}` : null, + ] + .filter((entry): entry is string => entry !== null) + .join('\n'); const tierLabel = { 1: 'isolated local CLI/filesystem', @@ -561,13 +565,13 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise const prompt = [ 'You are the Relay Feature Guardian, a proactive Slack bot for the Agent Relay team.', - 'Write a brief, conversational Slack message (3-5 sentences, no markdown headers) asking the team to confirm whether a specific CLI feature is working as intended.', - 'Be specific: name the feature, describe what it should do, show the CLI command, and ask if it behaves this way or if anything has drifted.', + 'Write a brief, conversational Slack message (3-5 sentences, no markdown headers) asking the team to confirm whether a specific feature is working as intended. The feature can be a CLI command or an MCP tool/prompt.', + 'Be specific: name the feature, describe what it should do, show the relevant CLI command or MCP tool/prompt, and ask if it behaves this way or if anything has drifted.', 'End with: "React ✅ if working as expected, 🔧 if something is off, or ❓ if untested."', 'Keep it casual and direct — this is an internal team check.', '', `Feature: ${feature.name}`, - `${surface}${mcpNote}${mcpPromptNote}`, + surface, `What it should do: ${feature.desc}`, `Verify tier: ${feature.tier} (${tierLabel})`, `Criticality: ${feature.criticality}`, diff --git a/.agentworkforce/features/verify/procedures.md b/.agentworkforce/features/verify/procedures.md index edd48923e..5d4cfbf70 100644 --- a/.agentworkforce/features/verify/procedures.md +++ b/.agentworkforce/features/verify/procedures.md @@ -6,12 +6,68 @@ Use `relay` (or the equivalent `agent-relay` binary). Run all mutations in a dis ## Shared fixtures -For a local broker, use a disposable project and only stop the broker it created: +Start every procedure in a Bash shell with an isolated run identifier, strict +failure handling, and cleanup. The trap safely cleans only resources named for +this run, whether the procedure uses a broker, hosted APIs, or both: ```bash -RUN="feature-$(date +%s)"; TMP="$(mktemp -d)"; cd "$TMP" +set -Eeuo pipefail +RUN_ID="${CI_RUN_ID:-${GITHUB_RUN_ID:-local}}" +RUN_RANDOM="$(od -An -N6 -tx1 /dev/urandom | tr -d '[:space:]')" +RUN="feature-${RUN_ID}-$(date +%s)-$RUN_RANDOM" +TMP="$(mktemp -d)" +TAIL_PID="" +CLOUD_WORKER_PID="" +CLOUD_WORKER_ID="" +cleanup_cloud_worker_daemon() { + [[ -n "$CLOUD_WORKER_PID" && -n "$CLOUD_WORKER_ID" ]] || return 0 + kill -0 "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + + local command + command="$(ps -p "$CLOUD_WORKER_PID" -o command= 2>/dev/null || true)" + if ! printf '%s\n' "$command" | grep -F -- "cloud worker start --worker-id $CLOUD_WORKER_ID" >/dev/null || \ + ! printf '%s\n' "$command" | grep -F -- '--foreground-child' >/dev/null; then + printf 'Refusing to terminate unexpected cloud-worker process %s\n' "$CLOUD_WORKER_PID" >&2 + return 1 + fi + + kill -TERM "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + for _ in 1 2 3 4 5; do + kill -0 "$CLOUD_WORKER_PID" 2>/dev/null || return 0 + sleep 1 + done + + # Recheck the command before escalating in case the PID was reused. + command="$(ps -p "$CLOUD_WORKER_PID" -o command= 2>/dev/null || true)" + if printf '%s\n' "$command" | grep -F -- "cloud worker start --worker-id $CLOUD_WORKER_ID" >/dev/null && \ + printf '%s\n' "$command" | grep -F -- '--foreground-child' >/dev/null; then + kill -KILL "$CLOUD_WORKER_PID" 2>/dev/null || true + fi +} +cleanup() { + local status=$? + if [[ -n "$TAIL_PID" ]]; then + kill "$TAIL_PID" 2>/dev/null || true + wait "$TAIL_PID" 2>/dev/null || true + fi + cleanup_cloud_worker_daemon || true + relay agent remove "audit-extra-$RUN" 2>/dev/null || true + relay agent remove "audit-c-$RUN" 2>/dev/null || true + relay agent remove "audit-b-$RUN" 2>/dev/null || true + relay agent remove "audit-a-$RUN" 2>/dev/null || true + relay node down --force 2>/dev/null || true + rm -rf "$TMP" || true + exit "$status" +} +trap cleanup EXIT +``` + +For a local broker, use the shared temporary project and only stop the broker it +created: + +```bash +cd "$TMP" relay node up --background --no-spawn -trap 'relay node down --force 2>/dev/null || true; rm -rf "$TMP"' EXIT ``` For hosted APIs, use explicit disposable credentials rather than an operator's active workspace: @@ -19,10 +75,8 @@ For hosted APIs, use explicit disposable credentials rather than an operator's a ```bash export RELAY_WORKSPACE_KEY='' export RELAY_BASE_URL='' -RUN="feature-$(date +%s)" A_JSON="$(relay agent register "audit-a-$RUN")"; TOKEN_A="$(jq -er .token <<<"$A_JSON")" B_JSON="$(relay agent register "audit-b-$RUN")"; TOKEN_B="$(jq -er .token <<<"$B_JSON")" -cleanup_agents() { relay agent remove "audit-a-$RUN" 2>/dev/null || true; relay agent remove "audit-b-$RUN" 2>/dev/null || true; } ``` SDK-backed CLI commands emit JSON already; do not add an unsupported `--json` flag. @@ -43,6 +97,8 @@ export WORKER_TOKEN='' # return a 2xx response. Localhost is suitable only when the tested Relay API # is local; hosted APIs require a tunneled or CI-provisioned receiver. export CAPTURE_URL='https://controlled-test-receiver.example/unique-run-path' +# Fetch only this run's captured requests from the same controlled receiver. +export CAPTURE_FETCH_URL='https://controlled-test-receiver.example/unique-run-path/requests' ``` Do not substitute a public request-bin or a shared endpoint: webhook payloads @@ -84,7 +140,7 @@ relay agent list | jq -e --arg n "audit-extra-$RUN" 'all(.[]; .name != $n)' relay status | grep -Eiq 'Workspace:|Local broker:|Cloud:' ``` -Assert list visibility after every create and absence after every delete; finish with `cleanup_agents`. +Assert list visibility after every create and absence after every delete; the shared exit trap removes all disposable identities. ## channel-messaging @@ -134,7 +190,7 @@ RELAY_AGENT_TOKEN="$TOKEN_A" relay message list "$CH" --limit 20 | jq -e 'tostri RELAY_AGENT_TOKEN="$TOKEN_A" relay channel archive "$CH" ``` -Assert that the returned parent ID links the reply, search filters return the unique text, and the attachment is listed. Remove the temp file, archive, and call `cleanup_agents`. +Assert that the returned parent ID links the reply, search filters return the unique text, and the attachment is listed. Remove the temp file, archive, and let the shared exit trap remove identities. ## direct-messages @@ -191,17 +247,27 @@ Assert the reaction is observable after add and absent after remove, and B's unr **Prerequisites:** local broker; spawn/new/set-model need an installed authenticated provider CLI and can incur cost. ```bash -AGENT="audit-worker-$RUN" PROVIDER="${PROVIDER:-claude}" # installed and authenticated fixture provider +TASK_AGENT="audit-task-$RUN" +TASK_TAIL="$TMP/$TASK_AGENT.tail" relay node agent list | jq -e 'type == "array"' -relay node agent spawn "$PROVIDER" --name "$AGENT" --task 'Reply relay-e2e-ok' --spawn-mode task-exit --exit-after-task +relay node tail --agent "$TASK_AGENT" >"$TASK_TAIL" 2>&1 & TAIL_PID=$! +relay node agent spawn "$PROVIDER" --name "$TASK_AGENT" --task 'Reply with exactly relay-e2e-ok, then exit.' --spawn-mode task-exit --exit-after-task +for _ in $(seq 1 60); do + grep -Fq 'relay-e2e-ok' "$TASK_TAIL" && break + sleep 1 +done +grep -Fq 'relay-e2e-ok' "$TASK_TAIL" +kill "$TAIL_PID" 2>/dev/null || true; wait "$TAIL_PID" 2>/dev/null || true; TAIL_PID="" +AGENT="audit-worker-$RUN" +relay node agent spawn "$PROVIDER" --name "$AGENT" relay node agent list | jq -e --arg n "$AGENT" '.[] | select(.name == $n)' relay node agent message hold "$AGENT" | jq -e 'tostring | test("manual|hold"; "i")' relay node agent message auto "$AGENT" | jq -e 'tostring | test("auto"; "i")' relay node agent release "$AGENT" ``` -`new` and `attach` are attended PTY checks; verify `drive`, `view`, and `passthrough`. `tail` streams until interrupted: filter by agent, create output, assert it, then stop its exact child process. A model switch proves broker acceptance only, not provider-side model change. +The bounded task-exit worker proves task output through `relay node tail` before it exits; the separate interactive worker proves hold, auto, and release. `new` and `attach` are attended PTY checks; verify `drive`, `view`, and `passthrough`. A model switch proves broker acceptance only, not provider-side model change. ## local-workflow-lifecycle @@ -243,13 +309,23 @@ Use a no-op workflow, long disposable workflow for cancel, and only `sync --dry- ```bash : "${WORKER_TOKEN:?See Externally provisioned fixtures}" -relay cloud worker register --token "$WORKER_TOKEN" --name "audit-worker-$RUN" --json | jq -e '.workerId' -relay cloud worker start --name "audit-worker-$RUN" --daemon -relay cloud worker status --name "audit-worker-$RUN" --json | jq -e '.localDaemon' -relay cloud worker logs --name "audit-worker-$RUN" +export AGENT_RELAY_HOME="$TMP/cloud-worker-state" +WORKER_NAME="audit-worker-$RUN" +REGISTERED="$(relay cloud worker register --token "$WORKER_TOKEN" --name "$WORKER_NAME" --json)" +WORKER_ID="$(jq -er '.workerId | strings | select(length > 0)' <<<"$REGISTERED")" +relay cloud worker start --worker-id "$WORKER_ID" --daemon +WORKER_STATUS="$(relay cloud worker status --worker-id "$WORKER_ID" --json)" +CLOUD_WORKER_PID="$(jq -er '.localDaemon.pid | numbers | select(. > 0)' <<<"$WORKER_STATUS")" +CLOUD_WORKER_ID="$WORKER_ID" +CLOUD_WORKER_LOG_PATH="$(jq -er '.localDaemon.logPath | strings | select(length > 0)' <<<"$WORKER_STATUS")" +jq -e '.localDaemon.running == true' <<<"$WORKER_STATUS" +test -r "$CLOUD_WORKER_LOG_PATH" +relay cloud worker logs --worker-id "$WORKER_ID" >/dev/null +cleanup_cloud_worker_daemon +CLOUD_WORKER_PID=""; CLOUD_WORKER_ID="" ``` -Record the daemon PID/log path and terminate only that daemon; no worker-stop command exists. Use `--once` when a bounded assignment fixture is available. +The CLI persists the daemon PID and log path in its isolated local worker state, so the procedure derives both through `status --json` rather than parsing command output. Cleanup checks that the PID is still the `--foreground-child` process for this worker ID before sending `TERM` (then `KILL` only after the same check); it never kills a PID merely because it was recorded. There is no worker stop or deregister CLI command: this terminates only the local daemon, while its local record and remote registration remain for the disposable test workspace/Cloud control plane to clean up. For a fixture with one bounded assignment, add `--once` to `start`; it exits after that assignment, but retain the cleanup check in case it is still running. ## fleet-management @@ -301,17 +377,37 @@ Assert the downloaded skill and delete only this disposable project. Do not run **Features:** `integration-subscribe`, `integration-unsubscribe`, `integration-list-bindings`, all `webhook-*`, and all `subscription-*` entries. -**Prerequisites:** disposable workspace, identity, externally reachable controlled receiver. Relayfile also needs compatible authenticated daemon, connected provider, real provider resource, and provider observation API; first connection can require browser OAuth. +**Prerequisites:** disposable workspace, identity, externally reachable controlled receiver. Relayfile also needs compatible authenticated daemon, connected provider, real provider resource, and provider observation API; first connection can require browser OAuth. The test-owned receiver must expose `CAPTURE_URL` for delivery and a run-scoped `CAPTURE_FETCH_URL` that returns `{"requests":[{"body":,"headers":{"lowercase-header-name":"string value"}}]}`. It must retain requests until this test reads them. ```bash : "${CAPTURE_URL:?See Externally provisioned fixtures}" +: "${CAPTURE_FETCH_URL:?GET endpoint returning the current run captured requests}" HOOK="$(RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook create "$CAPTURE_URL" --event message.created)"; HOOK_ID="$(jq -er '.id // .webhookId' <<<"$HOOK")" RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook trigger "$HOOK_ID" --payload '{"audit":true}' +DELIVERED=false +for _ in $(seq 1 15); do + CAPTURE="$(curl --fail --silent --show-error "$CAPTURE_FETCH_URL")" + if jq -e ' + .requests[] + | select(.body == {"audit": true}) + | select( + [.headers | to_entries[] + | select(.key | contains("signature")) + | select(.value | (type == "string" and length > 0))] + | length > 0 + ) + ' <<<"$CAPTURE" >/dev/null; then + DELIVERED=true + break + fi + sleep 1 +done +test "$DELIVERED" = true RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook list | jq -e --arg id "$HOOK_ID" 'tostring | contains($id)' RELAY_AGENT_TOKEN="$TOKEN_A" relay integration webhook delete "$HOOK_ID" ``` -Assert exact receiver payload/signature, then delete it. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. +The capture assertion requires the exact parsed payload `{"audit":true}` and a nonempty header whose lowercased name contains `signature`; it runs before deletion. For inbound, create channel/hook, POST the returned URL with token and documented payload, assert message, delete hook. Create/list/get/delete a unique subscription. For Relayfile, `subscribe --no-input`, assert `subscribe --list`, cause provider event and Relay reply, `unsubscribe` with same provider/resource, assert absent. Localhost cannot receive hosted webhooks. ## reflex-history diff --git a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json index 440d01472..51283b464 100644 --- a/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json +++ b/.agentworkforce/trajectories/active/traj_rtyte8r4g07t/trajectory.json @@ -64,6 +64,18 @@ "focal:drift-prevention", "confidence:0.9" ] + }, + { + "ts": 1784581060376, + "type": "decision", + "content": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child: Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "raw": { + "question": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "chosen": "Cloud worker teardown verifies the stored PID still belongs to the worker's foreground child", + "alternatives": [], + "reasoning": "The daemon is detached and persisted locally; checking worker ID plus --foreground-child before TERM/KILL prevents a stale or reused PID from targeting an unrelated process." + }, + "significance": "high" } ] } From b40c3f436a357b253ab068413468bb66ff313ee7 Mon Sep 17 00:00:00 2001 From: Miya Date: Mon, 20 Jul 2026 23:07:51 +0200 Subject: [PATCH 6/6] fix: preserve guardian fallback surfaces --- .../relay-feature-guardian/agent.test.ts | 19 +++++++++++++++++++ .../agents/relay-feature-guardian/agent.ts | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.test.ts b/.agentworkforce/agents/relay-feature-guardian/agent.test.ts index d08570397..7d7d7a249 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.test.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.test.ts @@ -330,6 +330,25 @@ describe('relay-feature-guardian runtime paths', () => { expect(persona.inputs.SLACK_CHANNEL.default).toBe('C0AEKNLDNKW'); }); + it('falls back with every declared MCP surface when quiz generation fails', async () => { + const transport = new IdempotentSlackTransport(); + const restore = bindPreviewTransport(transport); + const mcpOnlyManifest = manifest.replace(' cli: relay node up', ' mcp: create_workspace'); + const { ctx } = exactStateContext(JSON.stringify(progressState(0)), mcpOnlyManifest); + ctx.llm.complete = vi.fn(async () => { + throw new Error('simulated quiz model failure'); + }); + + try { + await guardian.handler(ctx, { type: 'cron.tick' } as never); + const text = (transport.attempts[0]?.body as { text: string }).text; + expect(text).toContain('MCP tool: create_workspace'); + expect(text).not.toContain('CLI command:'); + } finally { + restore(); + } + }); + it('uses a dedicated low-reasoning model path instead of shared subscription quota', () => { expect(persona).toMatchObject({ harness: 'opencode', diff --git a/.agentworkforce/agents/relay-feature-guardian/agent.ts b/.agentworkforce/agents/relay-feature-guardian/agent.ts index b83319f1f..fce38661a 100644 --- a/.agentworkforce/agents/relay-feature-guardian/agent.ts +++ b/.agentworkforce/agents/relay-feature-guardian/agent.ts @@ -584,7 +584,7 @@ async function generateQuizMessage(ctx: WorkforceCtx, feature: Feature): Promise return [ `🔍 *Relay Feature Check: ${feature.name}*`, ``, - `\`${feature.cli ?? feature.mcp ?? feature.mcpPrompt ?? feature.name}\`${mcpNote}${mcpPromptNote}`, + surface, ``, `This should: ${feature.desc}`, ``,