diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b8a04b66..3ea13aa6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `.agentworkforce/features/critical-paths.md` — documents the 4 product-critical sequences that must work end-to-end (broker+registration, channel messaging, local agent lifecycle, MCP server). - `@agent-relay/sdk` `./workflows` subpath export, enabling local workflow files to import the workflow builder without a build step. +### Fixed + +- `agent-relay node up --background` now preserves persisted Cloud enrollment credentials and node identity through detached startup, and fails instead of reporting healthy when the enrolled node cannot connect. + ## [10.6.1] - 2026-07-16 ### Fixed diff --git a/packages/cli/src/cli/commands/core.test.ts b/packages/cli/src/cli/commands/core.test.ts index 7152a56f4..d5871388b 100644 --- a/packages/cli/src/cli/commands/core.test.ts +++ b/packages/cli/src/cli/commands/core.test.ts @@ -8,7 +8,14 @@ import { readProjectWorkspaceKey } from '../lib/project-workspace-key.js'; const sdkStatusClient = { getStatus: vi.fn(async () => ({ agent_count: 0, pending_delivery_count: 0 })), - getSession: vi.fn(async () => ({ workspace_key: '' }) as { workspace_key?: string }), + getSession: vi.fn( + async () => + ({ workspace_key: '' }) as { + workspace_key?: string; + node_id?: string; + node_name?: string; + } + ), disconnect: vi.fn(() => undefined), }; @@ -534,7 +541,7 @@ describe('registerCoreCommands', () => { } }); - it('up --background re-execs a Bun standalone binary without adding its virtual entrypoint', async () => { + it('up --background preserves an enrolled identity through a Bun standalone re-exec', async () => { const spawnedProcess = createSpawnedProcessMock(); let now = 0; const fs = createFsMock(); @@ -546,25 +553,237 @@ describe('registerCoreCommands', () => { if ((pid === 9001 || pid === 4242) && signal === 0) return; throw new Error('unexpected kill check'); }); + sdkStatusClient.getStatus.mockResolvedValue({ + agent_count: 0, + pending_delivery_count: 0, + node_connected: true, + node_delivery: { token_present: true, connected: true }, + }); + sdkStatusClient.getSession.mockResolvedValue({ + workspace_key: 'rk_enrolled', + node_id: 'node_enrolled', + node_name: 'sf-mini', + }); const { program, deps } = createHarness({ fs, + env: { + RELAY_NODE_ID: 'node_enrolled', + RELAY_NODE_TOKEN: 'nt_enrolled', + }, spawnedProcess, killImpl, nowImpl: vi.fn(() => now), sleepImpl, execPath: '/tmp/agent-relay-darwin-arm64', cliScript: '/$bunfs/root/agent-relay-darwin-arm64', - argv: ['bun', '/$bunfs/root/agent-relay-darwin-arm64', 'up', '--background'], + argv: [ + 'bun', + '/$bunfs/root/agent-relay-darwin-arm64', + 'node', + 'up', + '--background', + '--config', + 'agent-relay.mjs', + ], }); - const exitCode = await runCommand(program, ['up', '--background']); + const exitCode = await runCommand(program, ['up', '--background', '--broker-name', 'sf-mini']); expect(exitCode).toBe(0); - expect(deps.spawnProcess).toHaveBeenCalledWith('/tmp/agent-relay-darwin-arm64', ['up'], { - detached: true, - stdio: 'ignore', - env: deps.env, + expect(deps.spawnProcess).toHaveBeenCalledWith( + '/tmp/agent-relay-darwin-arm64', + ['node', 'up', '--config', 'agent-relay.mjs', '--broker-name', 'sf-mini'], + { + detached: true, + stdio: 'ignore', + env: expect.objectContaining({ + RELAY_NODE_ID: 'node_enrolled', + RELAY_NODE_TOKEN: 'nt_enrolled', + }), + } + ); + }); + + it('up --background fails loudly when the broker reports the wrong enrolled identity', async () => { + const spawnedProcess = createSpawnedProcessMock(); + let now = 0; + const fs = createFsMock(); + const sleepImpl = vi.fn(async (ms: number) => { + now += ms; + fs.writeFileSync('/tmp/project/.agentworkforce/relay/connection.json', connectionFile(4242)); + }); + const stopped = new Set(); + const killImpl = vi.fn((pid: number, signal?: NodeJS.Signals | number) => { + if ((pid === 9001 || pid === 4242) && signal === 0 && !stopped.has(pid)) return; + if ((pid === 9001 || pid === 4242) && (signal === 'SIGTERM' || signal === 'SIGKILL')) { + stopped.add(pid); + return; + } + throw new Error('unexpected kill check'); + }); + sdkStatusClient.getStatus.mockResolvedValue({ + node_connected: true, + node_delivery: { token_present: true, connected: true }, + }); + sdkStatusClient.getSession.mockResolvedValue({ + workspace_key: 'rk_enrolled', + node_id: 'node_enrolled', + node_name: 'project', + }); + const { program, deps } = createHarness({ + fs, + env: { + RELAY_NODE_ID: 'node_enrolled', + RELAY_NODE_TOKEN: 'nt_enrolled', + }, + spawnedProcess, + killImpl, + nowImpl: vi.fn(() => now), + sleepImpl, + }); + + const exitCode = await runCommand(program, ['up', '--background', '--broker-name', 'sf-mini']); + + expect(exitCode).toBe(1); + expect(deps.error).toHaveBeenCalledWith( + 'Cloud enrollment identity mismatch: expected node name "sf-mini", got "project".' + ); + expect(killImpl).toHaveBeenCalledWith(4242, 'SIGTERM'); + expect(deps.log).not.toHaveBeenCalledWith('Broker started.'); + }); + + it('up --background rejects an enrolled node token without a node id', async () => { + const spawnedProcess = createSpawnedProcessMock(); + let now = 0; + const fs = createFsMock(); + const sleepImpl = vi.fn(async (ms: number) => { + now += ms; + fs.writeFileSync('/tmp/project/.agentworkforce/relay/connection.json', connectionFile(4242)); + }); + const stopped = new Set(); + const killImpl = vi.fn((pid: number, signal?: NodeJS.Signals | number) => { + if ((pid === 9001 || pid === 4242) && signal === 0 && !stopped.has(pid)) return; + if ((pid === 9001 || pid === 4242) && (signal === 'SIGTERM' || signal === 'SIGKILL')) { + stopped.add(pid); + return; + } + throw new Error('unexpected kill check'); + }); + const { program, deps } = createHarness({ + fs, + env: { RELAY_NODE_ID: ' ', RELAY_NODE_TOKEN: 'nt_incomplete' }, + spawnedProcess, + killImpl, + nowImpl: vi.fn(() => now), + sleepImpl, + }); + + const exitCode = await runCommand(program, ['up', '--background', '--broker-name', 'sf-mini']); + + expect(exitCode).toBe(1); + expect(deps.error).toHaveBeenCalledWith( + 'Cloud enrollment credentials are incomplete: RELAY_NODE_ID is required when RELAY_NODE_TOKEN is set.' + ); + expect(killImpl).toHaveBeenCalledWith(4242, 'SIGTERM'); + expect(deps.log).not.toHaveBeenCalledWith('Broker started.'); + }); + + it('up --background retains broker state when failed enrollment cleanup cannot stop the broker', async () => { + const spawnedProcess = createSpawnedProcessMock(); + let now = 0; + const fs = createFsMock(); + const connectionPath = '/tmp/project/.agentworkforce/relay/connection.json'; + const sleepImpl = vi.fn(async (ms: number) => { + now += ms; + fs.writeFileSync(connectionPath, connectionFile(4242)); + }); + const runningPids = new Set([9001, 4242]); + const killImpl = vi.fn((pid: number, signal?: NodeJS.Signals | number) => { + if (signal === 0) { + if (runningPids.has(pid)) return; + throw new Error('not running'); + } + if (pid === 9001 && (signal === 'SIGTERM' || signal === 'SIGKILL')) { + runningPids.delete(pid); + } + }); + sdkStatusClient.getStatus.mockResolvedValue({ + node_connected: true, + node_delivery: { token_present: true, connected: true }, + }); + sdkStatusClient.getSession.mockResolvedValue({ + workspace_key: 'rk_enrolled', + node_id: 'node_enrolled', + node_name: 'project', + }); + const { program, deps } = createHarness({ + fs, + env: { + RELAY_NODE_ID: 'node_enrolled', + RELAY_NODE_TOKEN: 'nt_enrolled', + }, + spawnedProcess, + killImpl, + nowImpl: vi.fn(() => now), + sleepImpl, + }); + + const exitCode = await runCommand(program, ['up', '--background', '--broker-name', 'sf-mini']); + + expect(exitCode).toBe(1); + expect(deps.error).toHaveBeenCalledWith( + 'Failed to stop broker process after Cloud enrollment startup failed (pid: 4242). ' + + 'Run `agent-relay down --force` to retry cleanup.' + ); + expect(fs.existsSync(connectionPath)).toBe(true); + expect(deps.log).not.toHaveBeenCalledWith('Broker started.'); + }); + + it('up --background fails when enrolled node delivery never connects', async () => { + const spawnedProcess = createSpawnedProcessMock(); + let now = 0; + const fs = createFsMock(); + const sleepImpl = vi.fn(async (ms: number) => { + now += ms; + fs.writeFileSync('/tmp/project/.agentworkforce/relay/connection.json', connectionFile(4242)); + }); + const stopped = new Set(); + const killImpl = vi.fn((pid: number, signal?: NodeJS.Signals | number) => { + if ((pid === 9001 || pid === 4242) && signal === 0 && !stopped.has(pid)) return; + if ((pid === 9001 || pid === 4242) && (signal === 'SIGTERM' || signal === 'SIGKILL')) { + stopped.add(pid); + return; + } + throw new Error('unexpected kill check'); }); + sdkStatusClient.getStatus.mockResolvedValue({ + node_connected: false, + node_delivery: { token_present: true, connected: false }, + }); + sdkStatusClient.getSession.mockResolvedValue({ + workspace_key: 'rk_enrolled', + node_id: 'node_enrolled', + node_name: 'sf-mini', + }); + const { program, deps } = createHarness({ + fs, + env: { + RELAY_NODE_ID: 'node_enrolled', + RELAY_NODE_TOKEN: 'nt_expired', + }, + spawnedProcess, + killImpl, + nowImpl: vi.fn(() => now), + sleepImpl, + }); + + const exitCode = await runCommand(program, ['up', '--background', '--broker-name', 'sf-mini']); + + expect(exitCode).toBe(1); + expect(deps.error).toHaveBeenCalledWith( + 'Cloud enrollment for node "sf-mini" did not become ready. Node delivery: DOWN (node websocket disconnected)' + ); + expect(deps.log).not.toHaveBeenCalledWith('Broker started.'); }); it('up --background exits non-zero when the detached broker never becomes ready', async () => { @@ -896,7 +1115,11 @@ describe('registerCoreCommands', () => { const connectionPath = '/tmp/project/.agentworkforce/relay/connection.json'; const fs = createFsMock({ [connectionPath]: connectionFile(4242) }); sdkStatusClient.getStatus.mockResolvedValueOnce({ agent_count: 4, pending_delivery_count: 2 }); - sdkStatusClient.getSession.mockResolvedValueOnce({ workspace_key: 'rk_live_test123' }); + sdkStatusClient.getSession.mockResolvedValueOnce({ + workspace_key: 'rk_live_test123', + node_id: 'node_enrolled', + node_name: 'sf-mini', + }); const { program, deps } = createHarness({ fs }); @@ -906,6 +1129,7 @@ describe('registerCoreCommands', () => { expect(deps.log).toHaveBeenCalledWith('Status: RUNNING'); expect(deps.log).toHaveBeenCalledWith('Agents: 4'); expect(deps.log).toHaveBeenCalledWith('Pending deliveries: 2'); + expect(deps.log).toHaveBeenCalledWith('Node: sf-mini (node_enrolled)'); expect(deps.log).toHaveBeenCalledWith('Workspace Key: rk_live_test123'); expect(deps.log).toHaveBeenCalledWith('Observer: https://agentrelay.com/observer?key=rk_live_test123'); expect(sdkStatusClient.disconnect).toHaveBeenCalled(); diff --git a/packages/cli/src/cli/commands/node.test.ts b/packages/cli/src/cli/commands/node.test.ts index c97940a49..186cf9eae 100644 --- a/packages/cli/src/cli/commands/node.test.ts +++ b/packages/cli/src/cli/commands/node.test.ts @@ -107,6 +107,28 @@ describe('registerNodeCommands', () => { expect(log.mock.calls.flat().join('\n')).toContain('rw_123'); }); + it('preserves the enrolled identity when background startup re-execs the CLI', async () => { + const resolveEnrollment = vi.fn( + () => enrollmentRecord + ) as unknown as NodeCommandDependencies['resolveEnrollment']; + const { program, env } = createNodeHarness({ env: {}, resolveEnrollment }); + + await program.parseAsync(['node', 'up', '--background'], { from: 'user' }); + + expect(env).toMatchObject({ + RELAY_NODE_ID: 'node_abc', + RELAY_NODE_TOKEN: 'nt_secret', + }); + expect(brokerMocks.runUpCommand).toHaveBeenCalledWith( + expect.objectContaining({ + background: true, + brokerName: 'kjglaptop', + nodeName: 'kjglaptop', + }), + expect.anything() + ); + }); + it('lets --broker-name beat the enrolled node name', async () => { const resolveEnrollment = vi.fn( () => enrollmentRecord @@ -166,6 +188,24 @@ describe('registerNodeCommands', () => { expect(brokerMocks.runUpCommand).toHaveBeenCalledTimes(1); }); + it('reuses the forwarded enrolled name when the detached child already has credentials', async () => { + const resolveEnrollment = vi.fn( + () => enrollmentRecord + ) as unknown as NodeCommandDependencies['resolveEnrollment']; + const { program } = createNodeHarness({ + env: { RELAY_NODE_ID: 'node_abc', RELAY_NODE_TOKEN: 'nt_secret' }, + resolveEnrollment, + }); + + await program.parseAsync(['node', 'up', '--broker-name', 'kjglaptop'], { from: 'user' }); + + expect(resolveEnrollment).not.toHaveBeenCalled(); + expect(brokerMocks.runUpCommand).toHaveBeenCalledWith( + expect.objectContaining({ brokerName: 'kjglaptop', nodeName: 'kjglaptop' }), + expect.anything() + ); + }); + it('keeps an existing RELAY_BASE_URL when applying enrollment creds', async () => { const resolveEnrollment = vi.fn( () => enrollmentRecord diff --git a/packages/cli/src/cli/commands/node.ts b/packages/cli/src/cli/commands/node.ts index 3556e61b6..abd29f3e9 100644 --- a/packages/cli/src/cli/commands/node.ts +++ b/packages/cli/src/cli/commands/node.ts @@ -111,13 +111,15 @@ async function runNodeUp(options: UpCommandOptions, deps: NodeCommandDependencie } } + const nodeName = options.brokerName ?? enrolledNodeName; await runUpCommand( { ...options, discoverConfig: true, - ...((options.brokerName ?? enrolledNodeName) - ? { nodeName: options.brokerName ?? enrolledNodeName } - : {}), + // The broker name is also its registered fleet-node name. Keeping both + // fields aligned makes foreground startup use the enrolled identity and + // lets detached startup preserve it via the existing --broker-name arg. + ...(nodeName ? { brokerName: nodeName, nodeName } : {}), }, deps.core ); diff --git a/packages/cli/src/cli/lib/broker-lifecycle.ts b/packages/cli/src/cli/lib/broker-lifecycle.ts index a9c3f684c..b02ddd7b0 100644 --- a/packages/cli/src/cli/lib/broker-lifecycle.ts +++ b/packages/cli/src/cli/lib/broker-lifecycle.ts @@ -82,6 +82,11 @@ type BrokerStatusDetails = { session: Awaited> | null; }; +type EnrolledNodeExpectation = { + nodeId: string; + nodeName?: string; +}; + type NodeDeliveryStatus = { tokenPresent: boolean; connected: boolean; @@ -1032,6 +1037,51 @@ async function waitForBrokerReadiness( return latest; } +async function waitForEnrolledNodeReadiness( + conn: BrokerConnection, + deps: CoreDependencies, + expected: EnrolledNodeExpectation, + initialDetails?: BrokerStatusDetails | null +): Promise<{ ready: boolean; reason?: string }> { + const deadline = deps.now() + DETACHED_START_READY_TIMEOUT_MS; + let details = initialDetails ?? null; + + for (;;) { + details ??= await readBrokerStatusDetails(conn); + const actualNodeId = details?.session?.node_id?.trim(); + const actualNodeName = details?.session?.node_name?.trim(); + if (actualNodeId && actualNodeId !== expected.nodeId) { + return { + ready: false, + reason: `Cloud enrollment identity mismatch: expected node id "${expected.nodeId}", got "${actualNodeId}".`, + }; + } + if (expected.nodeName && actualNodeName && actualNodeName !== expected.nodeName) { + return { + ready: false, + reason: `Cloud enrollment identity mismatch: expected node name "${expected.nodeName}", got "${actualNodeName}".`, + }; + } + if ( + actualNodeId === expected.nodeId && + (!expected.nodeName || actualNodeName === expected.nodeName) && + nodeDeliveryReady(details?.status) + ) { + return { ready: true }; + } + if (deps.now() >= deadline) { + return { + ready: false, + reason: + `Cloud enrollment for node "${expected.nodeName ?? expected.nodeId}" did not become ready. ` + + `Node delivery: ${formatNodeDeliveryStatus(details?.status)}`, + }; + } + await deps.sleep(Math.min(STATUS_POLL_INTERVAL_MS, Math.max(0, deadline - deps.now()))); + details = null; + } +} + export async function waitForNodeDelivery( relay: CoreRelay, deps: CoreDependencies, @@ -1243,6 +1293,51 @@ export async function runUpCommand(options: UpOptions, deps: CoreDependencies): deps.exit(1); return; } + const enrolledNodeToken = deps.env.RELAY_NODE_TOKEN?.trim(); + const enrolledNodeId = enrolledNodeToken ? deps.env.RELAY_NODE_ID?.trim() : undefined; + let enrollmentFailureReason: string | undefined; + if (enrolledNodeToken && !enrolledNodeId) { + enrollmentFailureReason = + 'Cloud enrollment credentials are incomplete: RELAY_NODE_ID is required when RELAY_NODE_TOKEN is set.'; + } else if (enrolledNodeId) { + const enrolledReadiness = await waitForEnrolledNodeReadiness( + readiness.conn, + deps, + { + nodeId: enrolledNodeId, + ...(options.brokerName?.trim() ? { nodeName: options.brokerName.trim() } : {}), + }, + readiness.statusDetails + ); + if (!enrolledReadiness.ready) { + enrollmentFailureReason = enrolledReadiness.reason ?? 'Cloud enrollment did not become ready.'; + } + } + if (enrollmentFailureReason) { + deps.error(enrollmentFailureReason); + const cleanupPids = new Set(); + if (typeof child.pid === 'number' && child.pid > 0) { + cleanupPids.add(child.pid); + } + cleanupPids.add(readiness.conn.pid); + let allStopped = true; + for (const cleanupPid of cleanupPids) { + deps.warn(`Cleaning up failed broker start (pid: ${cleanupPid})`); + const stopped = await terminateProcess(cleanupPid, deps, true); + if (!stopped) { + allStopped = false; + deps.error( + `Failed to stop broker process after Cloud enrollment startup failed (pid: ${cleanupPid}). ` + + 'Run `agent-relay down --force` to retry cleanup.' + ); + } + } + if (allStopped) { + cleanupBrokerFiles(paths, deps); + } + deps.exit(1); + return; + } deps.log('Broker started.'); deps.log(`Broker PID: ${readiness.conn.pid}`); deps.log('Stop with: agent-relay down'); @@ -1598,6 +1693,9 @@ export async function runStatusCommand( deps.log(`Pending deliveries: ${status.pending_delivery_count}`); } deps.log(`Node delivery: ${formatNodeDeliveryStatus(status)}`); + if (session?.node_id) { + deps.log(`Node: ${session.node_name?.trim() || session.node_id} (${session.node_id})`); + } if (session?.workspace_key) { deps.log(`Workspace Key: ${session.workspace_key}`); deps.log(`Observer: https://agentrelay.com/observer?key=${session.workspace_key}`);