diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f800e129..b722dce3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `agent-relay fleet serve|nodes|status` runs a fleet node sidecar and inspects registered nodes, and the broker MCP surface adds `query_nodes` and `spawn` tools. - `@agent-relay/config` `CLI_AUTH_CONFIG` adds an `xai` provider (Grok CLI): `grok login --device-auth` device-code connect, `~/.grok/auth.json` credential capture, and the official x.ai installer as the sandbox fallback — so cloud sandboxes can authenticate the `grok` harness from a connected account instead of an API key. - `@agent-relay/sdk` wires the durable delivery surface to the Relaycast backend: `inbox.list`, `inbox.subscribe`, `inbox.ack/fail/defer`, and `deliveries.ack/fail/defer` now use the hosted delivery ledger, agent-scoped capabilities report `serverDeliveryState: true`, and `DeliveryRunner` works against Relaycast-backed inbox items. +- `@agent-relay/sdk` adds `placement.spawn({ capability, node?, repo? })` — node-targeted/`self`/least-eligible placement that gates on advertised capability and repo-key map, queues with a bounded TTL until an eligible live node appears, and surfaces queue/fail visibility through `onReconcile` events. A `spawn:` capability pins the broker harness — a mismatched `input.cli` is rejected — and the exported `RelayPlacementError` reports `capability_mismatch` / `placement_queue_full` / `placement_ttl_expired` / `unmapped_repo`. - Two-node fleet E2E (`tests/e2e/fleet`, `npm run test:e2e`, `Fleet E2E` CI workflow): boots a real relaycast engine plus two `agent-relay fleet serve` nodes (real Rust broker + sidecar each) and asserts the live control wire — boot/register (real broker `Authorization: Bearer` node auth), negative auth, capability-filtered roster, cross-node action dispatch + ack, declarative trigger fire-once with loop guard, end-to-end spawn completion (token mint+inject), capability-routed + least-loaded + resume placement, `capability_mismatch` failure, in-flight reschedule on node death + restart reconcile, and bounded-mailbox TTL dead-letter. ### Changed diff --git a/packages/sdk/src/messaging/index.ts b/packages/sdk/src/messaging/index.ts index 346d1de85..85a6d0b0e 100644 --- a/packages/sdk/src/messaging/index.ts +++ b/packages/sdk/src/messaging/index.ts @@ -1,3 +1,7 @@ export * from './types.js'; export * from './normalize.js'; -export { RelaycastMessagingClient, type RelaycastMessagingOptions } from './relaycast.js'; +export { + RelayPlacementError, + RelaycastMessagingClient, + type RelaycastMessagingOptions, +} from './relaycast.js'; diff --git a/packages/sdk/src/messaging/placement.test.mts b/packages/sdk/src/messaging/placement.test.mts new file mode 100644 index 000000000..f4553204f --- /dev/null +++ b/packages/sdk/src/messaging/placement.test.mts @@ -0,0 +1,523 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { RelayPlacementError, RelaycastMessagingClient } from './index.js'; + +type RawNode = { + id: string; + name: string; + status: string; + live?: boolean; + capabilities: Array<{ name: string; kind?: string }>; + repo_keys?: string[]; +}; + +function createClient( + nodes: RawNode[], + options: { + placementLog?: (message: string) => void; + selfNodeName?: string; + maxQueuedPlacements?: number; + } = {} +) { + const invoke = vi.fn(async (name: string, input?: Record) => ({ + invocation_id: `inv-${invoke.mock.calls.length}`, + action_name: name, + handler_node_id: input?.target_node === 'node-b' ? 'node_b' : 'node_a', + dispatched_node_id: input?.target_node === 'node-b' ? 'node_b' : 'node_a', + input, + status: 'invoked', + })); + const relaycast = { + agents: { + list: vi.fn(async () => []), + get: vi.fn(), + register: vi.fn(), + update: vi.fn(), + delete: vi.fn(), + presence: vi.fn(async () => []), + }, + channels: { list: vi.fn(async () => []), get: vi.fn() }, + messages: { list: vi.fn(async () => []), get: vi.fn(), thread: vi.fn(), reactions: vi.fn() }, + nodes: { + list: vi.fn(async (query?: { capability?: string; name?: string }) => + nodes.filter( + (node) => + (!query?.name || node.name === query.name) && + (!query?.capability || + node.capabilities.some((capability) => capability.name === query.capability)) + ) + ), + get: vi.fn(async (name: string) => nodes.find((node) => node.name === name) ?? null), + }, + }; + const agentClient = { + actions: { + invoke, + getInvocation: vi.fn(), + completeInvocation: vi.fn(), + }, + }; + const client = new RelaycastMessagingClient({ + relaycast: relaycast as never, + agentClient: agentClient as never, + placementTtlMs: 60, + ...options, + }); + return { client, invoke, nodes }; +} + +describe('RelaycastMessagingClient placement', () => { + it('places a targeted spawn on the named live eligible node', async () => { + const { client, invoke } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + const ack = await client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-1', task: 'ship' }, + }); + + expect(ack.placement).toMatchObject({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + attempts: 1, + queued: false, + }); + expect(ack.handlerNodeId).toBe('node_a'); + expect(invoke).toHaveBeenCalledWith('spawn', { + name: 'worker-1', + task: 'ship', + capability: 'spawn:claude', + node: 'node-a', + target_node: 'node-a', + repo: 'relay', + ttl_override_ms: 60, + cli: 'claude', + }); + }); + + it('rejects a spawn whose input cli does not match the spawn: capability', async () => { + const { client, invoke } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + await expect( + client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-mismatch', cli: 'codex' }, + }) + ).rejects.toMatchObject({ + name: 'RelayPlacementError', + code: 'capability_mismatch', + capability: 'spawn:claude', + }); + // The broker is never invoked with the wrong harness. + expect(invoke).not.toHaveBeenCalled(); + }); + + it('overwrites cli from the spawn: capability when the input cli already matches', async () => { + const { client, invoke } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + await client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-match', cli: 'claude' }, + }); + + expect(invoke).toHaveBeenCalledWith('spawn', expect.objectContaining({ cli: 'claude' })); + }); + + it('hard-fails a named node that does not advertise the requested capability', async () => { + const { client, invoke } = createClient([ + { + id: 'node_b', + name: 'node-b', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:codex', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + await expect( + client.placement.spawn({ capability: 'spawn:claude', node: 'node-b', repo: 'relay' }) + ).rejects.toMatchObject({ + name: 'RelayPlacementError', + code: 'capability_mismatch', + capability: 'spawn:claude', + node: 'node-b', + }); + expect(invoke).not.toHaveBeenCalled(); + }); + + it('resolves node self through the client self node name', async () => { + const { client, invoke } = createClient( + [ + { + id: 'node_self', + name: 'laptop', + status: 'online', + live: true, + capabilities: [{ name: 'workflow:run', kind: 'action' }], + repo_keys: ['relay'], + }, + ], + { selfNodeName: 'laptop' } + ); + + const ack = await client.placement.spawn({ + capability: 'workflow:run', + node: 'self', + repo: 'relay', + input: { workflow: 'factory.yml' }, + }); + + expect(ack.placement.node).toBe('laptop'); + expect(invoke).toHaveBeenCalledWith( + 'workflow:run', + expect.objectContaining({ workflow: 'factory.yml', node: 'laptop', target_node: 'laptop' }) + ); + }); + + it('places exactly once when two nodes are simultaneously eligible (no bleed)', async () => { + const { client, invoke } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + { + id: 'node_b', + name: 'node-b', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + const ack = await client.placement.spawn({ + capability: 'spawn:claude', + repo: 'relay', + input: { name: 'worker-2nodes' }, + }); + + // A single placement is dispatched — no cross-node double-dispatch. + expect(invoke).toHaveBeenCalledTimes(1); + expect(['node-a', 'node-b']).toContain(ack.placement.node); + expect(ack.placement).toMatchObject({ queued: false, attempts: 1 }); + }); + + it('rejects with placement_queue_full and reconciles a failed event when the queue is full', async () => { + const reconciled: unknown[] = []; + const logs: string[] = []; + const { client, invoke } = createClient([], { + maxQueuedPlacements: 0, + placementLog: (line) => logs.push(line), + }); + + await expect( + client.placement.spawn({ + capability: 'spawn:claude', + repo: 'relay', + input: { name: 'worker-overflow' }, + ttlMs: 1_000, + pollIntervalMs: 25, + onReconcile: (event) => { + reconciled.push(event); + }, + }) + ).rejects.toMatchObject({ + name: 'RelayPlacementError', + code: 'placement_queue_full', + attempts: 1, + }); + + expect(invoke).not.toHaveBeenCalled(); + expect(reconciled).toContainEqual( + expect.objectContaining({ action: 'failed', reason: 'no_eligible_node' }) + ); + expect(logs.join('\n')).toContain('placement queue full'); + }); + + it('fails fast with no eligible node after a single attempt and reconciles failed', async () => { + const reconciled: unknown[] = []; + const { client, invoke } = createClient([]); + + await expect( + client.placement.spawn({ + capability: 'workflow:run', + failFast: true, + onReconcile: (event) => { + reconciled.push(event); + }, + }) + ).rejects.toMatchObject({ + name: 'RelayPlacementError', + code: 'placement_ttl_expired', + attempts: 1, + }); + + expect(invoke).not.toHaveBeenCalled(); + expect(reconciled).toEqual([expect.objectContaining({ action: 'failed', reason: 'no_eligible_node' })]); + }); + + it('fails fast with code unmapped_repo when a live capable node never maps the repo', async () => { + const reconciled: unknown[] = []; + const { client, invoke } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['cloud'], + }, + ]); + + await expect( + client.placement.spawn({ + capability: 'spawn:claude', + repo: 'relay', + failFast: true, + onReconcile: (event) => { + reconciled.push(event); + }, + }) + ).rejects.toMatchObject({ + name: 'RelayPlacementError', + code: 'unmapped_repo', + capability: 'spawn:claude', + repo: 'relay', + }); + + expect(invoke).not.toHaveBeenCalled(); + expect(reconciled).toEqual([ + expect.objectContaining({ action: 'failed', reason: 'unmapped_repo', repo: 'relay' }), + ]); + }); + + it('isolates a throwing onReconcile hook so placement still drains', async () => { + const { client, invoke, nodes } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'offline', + live: false, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + const placement = client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-throwing-hook' }, + pollIntervalMs: 25, + onReconcile: () => { + throw new Error('observability sink down'); + }, + }); + await new Promise((resolve) => setTimeout(resolve, 35)); + nodes[0] = { ...nodes[0], status: 'online', live: true }; + + const ack = await placement; + expect(ack.placement).toMatchObject({ node: 'node-a', queued: true }); + expect(invoke).toHaveBeenCalledTimes(1); + }); + + it('queues a targeted offline node with reason target_offline and drains once it is live', async () => { + const reconciled: unknown[] = []; + const { client, invoke, nodes } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'offline', + live: false, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + const placement = client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-offline' }, + pollIntervalMs: 25, + onReconcile: (event) => { + reconciled.push(event); + }, + }); + await new Promise((resolve) => setTimeout(resolve, 35)); + nodes[0] = { ...nodes[0], status: 'online', live: true }; + + const ack = await placement; + + expect(ack.placement).toMatchObject({ node: 'node-a', queued: true }); + expect(invoke).toHaveBeenCalledTimes(1); + expect(reconciled).toContainEqual( + expect.objectContaining({ action: 'queued', reason: 'target_offline', node: 'node-a' }) + ); + }); + + it('queues a targeted node that does not map the repo and drains once the repo map updates', async () => { + const reconciled: unknown[] = []; + const logs: string[] = []; + const { client, invoke, nodes } = createClient( + [ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['cloud'], + }, + ], + { placementLog: (line) => logs.push(line) } + ); + + const placement = client.placement.spawn({ + capability: 'spawn:claude', + node: 'node-a', + repo: 'relay', + input: { name: 'worker-targeted-unmapped' }, + pollIntervalMs: 25, + onReconcile: (event) => { + reconciled.push(event); + }, + }); + await new Promise((resolve) => setTimeout(resolve, 35)); + nodes[0] = { ...nodes[0], repo_keys: ['cloud', 'relay'] }; + + const ack = await placement; + + expect(ack.placement).toMatchObject({ node: 'node-a', repo: 'relay', queued: true }); + expect(invoke).toHaveBeenCalledTimes(1); + expect(reconciled).toContainEqual( + expect.objectContaining({ action: 'queued', reason: 'unmapped_repo', node: 'node-a' }) + ); + expect(logs.join('\n')).toContain('does not map repo "relay"'); + }); + + it('reconciles an unmapped repo by queueing until a mapped eligible node appears', async () => { + const logs: string[] = []; + const reconciled: unknown[] = []; + const { client, invoke, nodes } = createClient( + [ + { + id: 'node_a', + name: 'node-a', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['cloud'], + }, + ], + { placementLog: (line) => logs.push(line) } + ); + + const placement = client.placement.spawn({ + capability: 'spawn:claude', + repo: 'relay', + input: { name: 'worker-2' }, + pollIntervalMs: 25, + onReconcile: (event) => { + reconciled.push(event); + }, + }); + await new Promise((resolve) => setTimeout(resolve, 35)); + nodes.push({ + id: 'node_b', + name: 'node-b', + status: 'online', + live: true, + capabilities: [{ name: 'spawn:claude', kind: 'spawn' }], + repo_keys: ['relay'], + }); + + const ack = await placement; + + expect(ack.placement).toMatchObject({ node: 'node-b', repo: 'relay', queued: true }); + expect(invoke).toHaveBeenCalledWith( + 'spawn', + expect.objectContaining({ target_node: 'node-b', repo: 'relay', cli: 'claude' }) + ); + expect(logs.join('\n')).toContain('maps repo "relay"'); + expect(reconciled).toContainEqual( + expect.objectContaining({ action: 'queued', reason: 'unmapped_repo', repo: 'relay' }) + ); + }); + + it('queues when no eligible node is live and drains before TTL', async () => { + const { client, nodes } = createClient([ + { + id: 'node_a', + name: 'node-a', + status: 'offline', + live: false, + capabilities: [{ name: 'spawn:codex', kind: 'spawn' }], + repo_keys: ['relay'], + }, + ]); + + const placement = client.placement.spawn({ + capability: 'spawn:codex', + repo: 'relay', + input: { name: 'worker-3' }, + pollIntervalMs: 25, + }); + await new Promise((resolve) => setTimeout(resolve, 35)); + nodes[0] = { ...nodes[0], status: 'online', live: true }; + + await expect(placement).resolves.toMatchObject({ + placement: { node: 'node-a', queued: true }, + }); + }); + + it('fails after placement TTL instead of silently dropping the spawn', async () => { + const logs: string[] = []; + const { client, invoke } = createClient([], { placementLog: (line) => logs.push(line) }); + + await expect( + client.placement.spawn({ capability: 'workflow:run', ttlMs: 30, pollIntervalMs: 25 }) + ).rejects.toBeInstanceOf(RelayPlacementError); + + expect(invoke).not.toHaveBeenCalled(); + expect(logs.join('\n')).toContain('placement TTL expired'); + }); +}); diff --git a/packages/sdk/src/messaging/relaycast.ts b/packages/sdk/src/messaging/relaycast.ts index 37499ebb1..d82bf3748 100644 --- a/packages/sdk/src/messaging/relaycast.ts +++ b/packages/sdk/src/messaging/relaycast.ts @@ -71,6 +71,7 @@ import type { RelayMessagingEventMap, RelayNode, RelayNodeCapability, + RelayPlacementReconcileEvent, RelayReadReceipt, RelayRegisterAgentInput, RelayReplyMessageInput, @@ -78,6 +79,8 @@ import type { RelaySendChannelMessageInput, RelaySendDirectMessageInput, RelaySendGroupDirectMessageInput, + RelaySpawnPlacementAck, + RelaySpawnPlacementInput, RelayThread, RelayTrigger, RelayTriggerInput, @@ -130,6 +133,7 @@ function toRelayNode(raw: unknown): RelayNode { status: rawStatus === 'online' || rawStatus === 'offline' ? rawStatus : 'unknown', live: readBoolean(node, 'live'), capabilities: Array.isArray(node.capabilities) ? node.capabilities.map(toRelayNodeCapability) : [], + repoKeys: readRepoKeys(node), maxAgents: readNumber(node, 'maxAgents', 'max_agents'), activeAgents: readNumber(node, 'activeAgents', 'active_agents'), handlersLive: readBoolean(node, 'handlersLive', 'handlers_live'), @@ -141,6 +145,13 @@ function toRelayNode(raw: unknown): RelayNode { }; } +function readRepoKeys(node: Record): string[] | undefined { + const direct = readStringArray(node, 'repoKeys') ?? readStringArray(node, 'repo_keys'); + if (direct) return direct; + const repoPaths = readRecord(node, 'repoPaths', 'repo_paths'); + return repoPaths ? Object.keys(repoPaths).filter(Boolean) : undefined; +} + function toRelayNodeCapability(raw: unknown): RelayNodeCapability { const capability = (raw ?? {}) as Record; return { @@ -232,6 +243,93 @@ function readRecord(record: Record, ...keys: string[]): Record< return undefined; } +type PlacementReconcileReason = 'no_eligible_node' | 'target_offline' | 'unmapped_repo'; + +type PlacementSelection = + | { node: RelayNode; message?: never; hardFail?: never; reason?: never; reconcileReason?: never } + | { + // Hard failure — thrown before any side effect; `reason` is the error code. + node?: never; + message: string; + hardFail: true; + reason: 'capability_mismatch'; + reconcileReason: PlacementReconcileReason; + } + | { + // Retryable — queued and reconciled; only `reconcileReason` is consumed. + node?: never; + message: string; + hardFail?: false; + reason?: never; + reconcileReason: PlacementReconcileReason; + }; + +export class RelayPlacementError extends Error { + readonly code: 'capability_mismatch' | 'placement_queue_full' | 'placement_ttl_expired' | 'unmapped_repo'; + readonly capability: string; + readonly node?: string; + readonly repo?: string; + readonly attempts: number; + + constructor( + code: RelayPlacementError['code'], + message: string, + context: { capability: string; node?: string; repo?: string; attempts: number } + ) { + super(message); + this.name = 'RelayPlacementError'; + this.code = code; + this.capability = context.capability; + this.node = context.node; + this.repo = context.repo; + this.attempts = context.attempts; + } +} + +function nonEmptyPlacement(value: string, label: string): string { + const trimmed = value.trim(); + if (!trimmed) throw new Error(`${label} is required.`); + return trimmed; +} + +function placementActionName(capability: string): string { + return capability.startsWith('spawn:') ? 'spawn' : capability; +} + +function placementActionInput( + input: Record | undefined, + placement: { capability: string; node: string; repo?: string; ttlMs: number } +): Record { + const payload = { ...(input ?? {}) }; + payload.capability = placement.capability; + payload.node = placement.node; + payload.target_node = placement.node; + if (placement.repo) payload.repo = placement.repo; + if (placement.ttlMs > 0) { + payload.ttl_override_ms = placement.ttlMs; + } + if (placement.capability.startsWith('spawn:')) { + // The broker picks the harness from `cli`, but node eligibility was gated on + // the `spawn:` capability. An explicit, mismatched `cli` would select a + // harness the chosen node never advertised — reject it instead of silently + // dispatching the wrong harness. + const capabilityCli = placement.capability.slice('spawn:'.length); + if (typeof payload.cli === 'string' && payload.cli !== capabilityCli) { + throw new RelayPlacementError( + 'capability_mismatch', + `Placement rejected: input cli "${payload.cli}" does not match capability "${placement.capability}"`, + { capability: placement.capability, node: placement.node, repo: placement.repo, attempts: 0 } + ); + } + payload.cli = capabilityCli; + } + return payload; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + /** Normalize a relaycast invoke ack (camelized) into the relay `RelayActionInvocationAck`. */ function normalizeActionInvocationAck(raw: unknown): RelayActionInvocationAck { const record = asRecord(raw); @@ -241,6 +339,12 @@ function normalizeActionInvocationAck(raw: unknown): RelayActionInvocationAck { ...(readStr(record, 'handlerAgentId', 'handler_agent_id') ? { handlerAgentId: readStr(record, 'handlerAgentId', 'handler_agent_id') } : {}), + ...(readStr(record, 'handlerNodeId', 'handler_node_id') + ? { handlerNodeId: readStr(record, 'handlerNodeId', 'handler_node_id') } + : {}), + ...(readStr(record, 'dispatchedNodeId', 'dispatched_node_id') + ? { dispatchedNodeId: readStr(record, 'dispatchedNodeId', 'dispatched_node_id') } + : {}), ...(readRecord(record, 'input') ? { input: readRecord(record, 'input') } : {}), ...(readStr(record, 'status') ? { status: readStr(record, 'status') } : {}), ...(readStr(record, 'createdAt', 'created_at') @@ -480,6 +584,14 @@ export interface RelaycastMessagingOptions extends RelaycastTelemetryOptions { agentToken?: string; agentClient?: RelaycastAgentLike; agentClientOptions?: AgentClientOptions; + /** Local node name used to resolve placement requests with `node: "self"`. */ + selfNodeName?: string; + /** Default bounded placement queue TTL. RFC placeholder default is one hour. */ + placementTtlMs?: number; + /** Max in-process placement requests allowed to wait for an eligible node. */ + maxQueuedPlacements?: number; + /** Receives placement queue/reject/fail log lines. */ + placementLog?: (message: string) => void; } function definedOptions>(options: T): Partial { @@ -527,6 +639,11 @@ export class RelaycastMessagingClient implements RelayMessagingClient { private readonly relaycast: RelaycastWorkspaceLike; private readonly agentClient?: RelaycastAgentLike; + private readonly selfNodeName?: string; + private readonly placementTtlMs: number; + private readonly maxQueuedPlacements: number; + private readonly placementLog?: (message: string) => void; + private queuedPlacements = 0; private readonly eventHandlers = new Map< keyof RelayMessagingEventMap, Set<(event: RelayMessagingEvent) => void | Promise> @@ -538,6 +655,10 @@ export class RelaycastMessagingClient implements RelayMessagingClient { this.agentClient = options.agentClient ?? (options.agentToken ? this.relaycast.as?.(options.agentToken, options.agentClientOptions) : undefined); + this.selfNodeName = options.selfNodeName; + this.placementTtlMs = options.placementTtlMs ?? 60 * 60 * 1000; + this.maxQueuedPlacements = options.maxQueuedPlacements ?? 100; + this.placementLog = options.placementLog; // Durable delivery state is agent-scoped: it requires an agent client that // exposes the relaycast delivery ledger (deliveries list + transitions). const durable = this.deliverySurface() !== undefined; @@ -996,6 +1117,121 @@ export class RelaycastMessagingClient implements RelayMessagingClient { }, }; + readonly placement = { + spawn: async (input: RelaySpawnPlacementInput): Promise => { + const capability = nonEmptyPlacement(input.capability, 'placement capability'); + const repo = input.repo?.trim() || undefined; + const targetNode = this.resolvePlacementNode(input.node, input.selfNodeName); + const ttlMs = Math.max(0, input.ttlMs ?? input.ttlOverrideMs ?? this.placementTtlMs); + const pollIntervalMs = Math.max(25, input.pollIntervalMs ?? 1_000); + const startedAt = Date.now(); + let queued = false; + let attempts = 0; + + try { + while (true) { + attempts += 1; + const decision = await this.selectPlacementNode({ capability, repo, targetNode }); + if (decision.node) { + const actionName = input.actionName ?? placementActionName(capability); + const actionInput = placementActionInput(input.input, { + capability, + node: decision.node.name, + repo, + ttlMs, + }); + const ack = await this.commands.invoke(actionName, actionInput); + return { + ...ack, + node: decision.node, + placement: { + capability, + node: decision.node.name, + ...(repo ? { repo } : {}), + attempts, + queued, + }, + }; + } + + if (decision.hardFail) { + this.logPlacement(input, decision.message); + throw new RelayPlacementError(decision.reason, decision.message, { + capability, + node: targetNode, + repo, + attempts, + }); + } + + if (input.failFast || Date.now() - startedAt >= ttlMs) { + // A repo that no live, capable node maps will never drain by waiting, + // so report it as `unmapped_repo` rather than a generic TTL expiry. + const code: RelayPlacementError['code'] = + decision.reconcileReason === 'unmapped_repo' ? 'unmapped_repo' : 'placement_ttl_expired'; + const message = + code === 'unmapped_repo' + ? `${decision.message}; no node maps the requested repo` + : `${decision.message}; placement TTL expired`; + await this.reconcilePlacement(input, { + action: 'failed', + reason: decision.reconcileReason, + capability, + ...(targetNode ? { node: targetNode } : {}), + ...(repo ? { repo } : {}), + attempts, + message, + }); + throw new RelayPlacementError(code, message, { + capability, + node: targetNode, + repo, + attempts, + }); + } + + if (!queued) { + if (this.queuedPlacements >= this.maxQueuedPlacements) { + const message = `${decision.message}; placement queue full`; + await this.reconcilePlacement(input, { + action: 'failed', + reason: decision.reconcileReason, + capability, + ...(targetNode ? { node: targetNode } : {}), + ...(repo ? { repo } : {}), + attempts, + message, + }); + throw new RelayPlacementError('placement_queue_full', message, { + capability, + node: targetNode, + repo, + attempts, + }); + } + this.queuedPlacements += 1; + queued = true; + await this.reconcilePlacement(input, { + action: 'queued', + reason: decision.reconcileReason, + capability, + ...(targetNode ? { node: targetNode } : {}), + ...(repo ? { repo } : {}), + attempts, + message: decision.message, + }); + } + + // Floor the queued delay at a small minimum so a near-zero remaining + // TTL cannot busy-spin the poll loop before the next expiry check. + await delay(Math.max(5, Math.min(pollIntervalMs, ttlMs - (Date.now() - startedAt)))); + } + } finally { + if (queued) this.queuedPlacements = Math.max(0, this.queuedPlacements - 1); + } + }, + }; + readonly triggers = { list: async (): Promise => (await this.requireTriggers().list()).map(toRelayTrigger), create: async (input: RelayTriggerInput): Promise => @@ -1016,6 +1252,114 @@ export class RelaycastMessagingClient implements RelayMessagingClient { }, }; + private resolvePlacementNode(node: string | 'self' | undefined, selfNodeName?: string): string | undefined { + if (!node) return undefined; + if (node !== 'self') return nonEmptyPlacement(node, 'placement node'); + const resolved = selfNodeName ?? this.selfNodeName; + if (!resolved) { + throw new Error('placement node "self" requires selfNodeName on the request or client.'); + } + return nonEmptyPlacement(resolved, 'placement self node'); + } + + private async selectPlacementNode(input: { + capability: string; + repo?: string; + targetNode?: string; + }): Promise { + if (input.targetNode) { + const node = await this.nodes.get(input.targetNode); + if (!node) { + return { + message: `Placement queued: target node "${input.targetNode}" is not registered`, + reconcileReason: 'target_offline', + }; + } + if (!this.nodeHasCapability(node, input.capability)) { + return { + message: `Placement rejected: node "${node.name}" does not advertise capability "${input.capability}"`, + hardFail: true, + reason: 'capability_mismatch', + reconcileReason: 'no_eligible_node', + }; + } + if (!node.live) { + return { + message: `Placement queued: target node "${node.name}" is offline`, + reconcileReason: 'target_offline', + }; + } + if (!this.nodeMapsRepo(node, input.repo)) { + return { + message: `Placement queued: node "${node.name}" does not map repo "${input.repo}"`, + reconcileReason: 'unmapped_repo', + }; + } + return { node }; + } + + const nodes = await this.nodes.list({ capability: input.capability }); + const capable = nodes.filter((node) => this.nodeHasCapability(node, input.capability)); + const live = capable.filter((node) => node.live); + const eligible = live.filter((node) => this.nodeMapsRepo(node, input.repo)); + if (eligible[0]) return { node: eligible[0] }; + + if (input.repo && live.length > 0) { + return { + message: `Placement queued: no live node advertising "${input.capability}" maps repo "${input.repo}"`, + reconcileReason: 'unmapped_repo', + }; + } + return { + message: `Placement queued: no live node advertises capability "${input.capability}"`, + reconcileReason: 'no_eligible_node', + }; + } + + private nodeHasCapability(node: RelayNode, capability: string): boolean { + return node.capabilities.some((item) => item.name === capability); + } + + private nodeMapsRepo(node: RelayNode, repo: string | undefined): boolean { + if (!repo) return true; + return Boolean(node.repoKeys?.includes(repo)); + } + + private async reconcilePlacement( + input: RelaySpawnPlacementInput, + event: RelayPlacementReconcileEvent + ): Promise { + this.logPlacement(input, event.message); + // A throwing/rejecting reconcile hook (e.g. a Slack/log sink outage) must not + // break an otherwise valid placement — isolate it and log the failure. + try { + await input.onReconcile?.(event); + } catch (error) { + this.placementLog?.( + `[agent-relay] placement reconcile hook threw: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + private logPlacement(input: RelaySpawnPlacementInput, message: string): void { + const line = `[agent-relay] ${message}`; + // Observability log sinks are caller-provided; never let them break placement. + try { + input.log?.(line); + } catch (error) { + this.placementLog?.( + `[agent-relay] placement log hook threw: ${error instanceof Error ? error.message : String(error)}` + ); + } + if (input.log !== this.placementLog) { + try { + this.placementLog?.(line); + } catch { + // Intentionally swallow the client log-sink failure; nothing else to report to. + } + } + } + private requireWebhooks(): NonNullable { if (!this.relaycast.webhooks) { throw new Error('RelaycastMessagingClient.integrations.webhooks requires the relaycast webhooks API.'); diff --git a/packages/sdk/src/messaging/types.ts b/packages/sdk/src/messaging/types.ts index a58b21b7c..516183691 100644 --- a/packages/sdk/src/messaging/types.ts +++ b/packages/sdk/src/messaging/types.ts @@ -46,6 +46,8 @@ export interface RelayNode { status: RelayNodeStatus; live?: boolean; capabilities: RelayNodeCapability[]; + /** Repository keys from NodeConfig.repoPaths that this node can service. */ + repoKeys?: string[]; maxAgents?: number; activeAgents?: number; handlersLive?: boolean; @@ -495,6 +497,8 @@ export interface RelayActionInvocationAck { invocationId: string; actionName: string; handlerAgentId?: string; + handlerNodeId?: string | null; + dispatchedNodeId?: string | null; input?: Record; status?: string; createdAt?: string; @@ -522,6 +526,65 @@ export interface RelayCompleteInvocationInput { durationMs?: number; } +export type RelayPlacementRejectReason = + | 'capability_mismatch' + | 'placement_queue_full' + | 'placement_ttl_expired' + | 'unmapped_repo'; + +export type RelayPlacementReconcileReason = 'no_eligible_node' | 'target_offline' | 'unmapped_repo'; + +export interface RelayPlacementReconcileEvent { + action: 'queued' | 'failed'; + reason: RelayPlacementReconcileReason; + capability: string; + node?: string; + repo?: string; + attempts: number; + message: string; +} + +export interface RelaySpawnPlacementInput { + /** Node capability to dispatch, e.g. `spawn:claude` or `workflow:run`. */ + capability: string; + /** + * Optional exact node target. `self` resolves through `selfNodeName` on this + * input, then the messaging client default self node name. + */ + node?: string | 'self'; + /** Explicit self-node name used when `node: "self"` is requested. */ + selfNodeName?: string; + /** Repo label/key that must be present in the selected node's repo map. */ + repo?: string; + /** Action name to invoke once placement is resolved. Defaults to the capability. */ + actionName?: string; + /** Action payload passed to the node after placement metadata is added. */ + input?: Record; + /** Per-placement queue TTL. Defaults to the client placement TTL. */ + ttlMs?: number; + /** RFC-compatible alias for `ttlMs`. */ + ttlOverrideMs?: number; + /** Poll cadence while a placement is queued. */ + pollIntervalMs?: number; + /** Fail immediately instead of queueing when no currently eligible node exists. */ + failFast?: boolean; + /** Placement log sink. Defaults to the client placement logger. */ + log?: (message: string) => void; + /** Reconcile hook for queue/fail visibility, e.g. Slack surfacing by callers. */ + onReconcile?: (event: RelayPlacementReconcileEvent) => void | Promise; +} + +export interface RelaySpawnPlacementAck extends RelayActionInvocationAck { + node: RelayNode; + placement: { + capability: string; + node: string; + repo?: string; + attempts: number; + queued: boolean; + }; +} + // ── Workspace ─────────────────────────────────────────────────────────────── export interface RelayWorkspaceInfo { @@ -843,6 +906,9 @@ export interface RelayMessagingClient { list(options?: RelayListNodesOptions): Promise; get(name: string): Promise; }; + readonly placement: { + spawn(input: RelaySpawnPlacementInput): Promise; + }; readonly triggers: { list(): Promise; create(input: RelayTriggerInput): Promise; diff --git a/packages/sdk/src/messaging/vitest.placement.config.mts b/packages/sdk/src/messaging/vitest.placement.config.mts new file mode 100644 index 000000000..c81940b2b --- /dev/null +++ b/packages/sdk/src/messaging/vitest.placement.config.mts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['packages/sdk/src/messaging/placement.test.mts'], + }, +});