Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"id": "traj_lbmykojlyhtm",
"version": 1,
"task": {
"title": "Implement workspace-scoped node delivery fix"
},
"status": "active",
"startedAt": "2026-06-30T15:37:17.052Z",
"agents": [
{
"name": "default",
"role": "lead",
"joinedAt": "2026-06-30T15:39:47.866Z"
}
],
"chapters": [
{
"id": "chap_w24sy92u02nm",
"title": "Work",
"agentName": "default",
"startedAt": "2026-06-30T15:39:47.866Z",
"events": [
{
"ts": 1782833987867,
"type": "decision",
"content": "Workspace-scope auto-derived node IDs only: Workspace-scope auto-derived node IDs only",
"raw": {
"question": "Workspace-scope auto-derived node IDs only",
"chosen": "Workspace-scope auto-derived node IDs only",
"alternatives": [],
"reasoning": "The pinned RELAY_NODE_TOKEN path must keep using the enrolled machine seed verbatim, while create_node auto-mint needs node IDs unique across workspaces for the same cwd."
},
"significance": "high"
}
]
}
],
"commits": [],
"filesChanged": [],
"projectId": "AgentWorkforce/relay",
"tags": [],
"_trace": {
"startRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e",
"endRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Trajectory: Probe Relay create_node HTTP status for local-up node conflict

> **Status:** ✅ Completed
> **Confidence:** 90%
> **Started:** June 30, 2026 at 08:35 AM
> **Completed:** June 30, 2026 at 08:37 AM

---

## Summary

Captured create_node probe: current restarted local-up workspace returned HTTP 500 internal_error with failed nodes insert for reused node_id; reported status/body to lead.

**Approach:** Standard approach
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"id": "traj_eowv9c937zq9",
"version": 1,
"task": {
"title": "Probe Relay create_node HTTP status for local-up node conflict"
},
"status": "completed",
"startedAt": "2026-06-30T15:35:59.618Z",
"completedAt": "2026-06-30T15:37:06.842Z",
"agents": [],
"chapters": [],
"retrospective": {
"summary": "Captured create_node probe: current restarted local-up workspace returned HTTP 500 internal_error with failed nodes insert for reused node_id; reported status/body to lead.",
"approach": "Standard approach",
"confidence": 0.9
},
"commits": [],
"filesChanged": [],
"projectId": "AgentWorkforce/relay",
"tags": [],
"_trace": {
"startRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e",
"endRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Trajectory: Investigate local up injection path

> **Status:** ✅ Completed
> **Confidence:** 90%
> **Started:** June 30, 2026 at 08:26 AM
> **Completed:** June 30, 2026 at 08:28 AM

---

## Summary

Investigated local-up runtime config and PTY broker logs. Found local-up bound to workspace 197732795721383936, home cloud/workspace config lacks explicit persisted coverage for that workspace, and parent broker logs show node_token_missing followed by node_not_found binding failures for claude and codex.

**Approach:** Standard approach

---

## Key Decisions

### Treat parent relay-hyperagent log as authoritative for local-up injection failure

- **Chose:** Treat parent relay-hyperagent log as authoritative for local-up injection failure
- **Reasoning:** Child PTY logs were empty or only had readiness warnings, while the parent broker log records node token mint failure and node binding failures for claude and codex.

---

## Chapters

### 1. Work

_Agent: default_

- Treat parent relay-hyperagent log as authoritative for local-up injection failure: Treat parent relay-hyperagent log as authoritative for local-up injection failure
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"id": "traj_wwmbn0x18dso",
"version": 1,
"task": {
"title": "Investigate local up injection path"
},
"status": "completed",
"startedAt": "2026-06-30T15:26:27.723Z",
"completedAt": "2026-06-30T15:28:57.709Z",
"agents": [
{
"name": "default",
"role": "lead",
"joinedAt": "2026-06-30T15:28:57.629Z"
}
],
"chapters": [
{
"id": "chap_iaz4nysvb3oe",
"title": "Work",
"agentName": "default",
"startedAt": "2026-06-30T15:28:57.629Z",
"endedAt": "2026-06-30T15:28:57.709Z",
"events": [
{
"ts": 1782833337630,
"type": "decision",
"content": "Treat parent relay-hyperagent log as authoritative for local-up injection failure: Treat parent relay-hyperagent log as authoritative for local-up injection failure",
"raw": {
"question": "Treat parent relay-hyperagent log as authoritative for local-up injection failure",
"chosen": "Treat parent relay-hyperagent log as authoritative for local-up injection failure",
"alternatives": [],
"reasoning": "Child PTY logs were empty or only had readiness warnings, while the parent broker log records node token mint failure and node binding failures for claude and codex."
},
"significance": "high"
}
]
}
],
"retrospective": {
"summary": "Investigated local-up runtime config and PTY broker logs. Found local-up bound to workspace 197732795721383936, home cloud/workspace config lacks explicit persisted coverage for that workspace, and parent broker logs show node_token_missing followed by node_not_found binding failures for claude and codex.",
"approach": "Standard approach",
"confidence": 0.9
},
"commits": [],
"filesChanged": [],
"projectId": "AgentWorkforce/relay",
"tags": [],
"_trace": {
"startRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e",
"endRef": "05d586b789623067f35a1fd2e9c0a4c1cdc1a96e"
}
}
5 changes: 4 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `agent-relay integration subscribe` now resolves provider-native `--resource` values through relayfile before binding, so Slack channel names, GitHub repos, Linear team keys, and Telegram chats bind to matching relayfile VFS globs while explicit `/`-prefixed globs still work.
- `agent-relay integration subscribe` is now idempotent and supports multiple resources/channels per provider. Each inbound webhook is scoped to its `(provider, resource)` binding (not one-per-provider), so subscribing a second Slack channel — or two sources into the same relay channel — no longer collides on the unique `(workspace, webhook name)` index or clobbers the other binding's webhook. Re-subscribing creates the replacement webhook/subscription before retiring the old one, so a transient failure can't leave you with no working binding; a failed cleanup now warns instead of being silently swallowed. The relay channel id is normalized (`#general` → `general`) consistently across the webhook, subscription filter, relayfile bind, and writeback-secret lookup, and `listBindings` now maps relayfile's `pathGlob` field so unsubscribe/replace match correctly.
- `agent-relay-broker` bootstrap `node.register` no longer advertises a generic `"spawn"` capability. Because the engine does not treat bare `"spawn"` as a placement capability (only `spawn:*`), it materialized a `spawn` action pinned to whichever node bootstrapped first, which then hijacked capability-based spawn placement for the whole workspace — every `spawn` invoke was dispatched to that node, ignoring `cli`/`target_node`/least-loaded routing. The pre-sidecar descriptor now carries no capabilities; real `spawn:*`/action capabilities arrive on the sidecar's `node.register`.
- `agent-relay-broker` node id, when the broker auto-mints its node, is derived from the machine-id seed plus a hash of the working directory, so multiple brokers / workspaces on one host no longer collide on `create_node` (stable across restarts in the same directory, distinct across directories). When an explicit `RELAY_NODE_TOKEN` is supplied (operator-enrolled / fleet nodes), the pinned node id is used verbatim so `node.register` matches the token's node instead of being rejected with `node_id_mismatch`.
- `agent-relay-broker` node id, when the broker auto-mints its node, is derived from the machine-id seed plus a hash of the working directory **and the workspace id**, so the same project directory re-pointed at a different workspace (e.g. `agent-relay up` minting a fresh workspace) mints a distinct node instead of reusing a node id already owned by the old workspace. Previously that reuse made `create_node` fail the mint and silently disabled all realtime injection. Stable across restarts in the same directory+workspace; distinct across directories or workspaces. When an explicit `RELAY_NODE_TOKEN` is supplied (operator-enrolled / fleet nodes), the pinned node id is used verbatim so `node.register` matches the token's node instead of being rejected with `node_id_mismatch`.
- `agent-relay-broker` `create_node` mint failures now log the real HTTP status and response body instead of collapsing to `Max retries exceeded`, and non-retryable `4xx` responses are no longer retried — so a node-token mint failure is diagnosable from the broker log.
- `agent-relay up` refuses to auto-spawn agents when broker node delivery (`/v1/node/ws`) is not connected, exiting non-zero with guidance, instead of spawning agents that can never receive realtime injection.
- `agent-relay status` and `agent-relay doctor` report node-delivery health (node token present + `/v1/node/ws` connected); the broker `/health` and `/api/status` expose `nodeConnected`/`nodeDelivery`, so a broker that is relaycast-connected but cannot inject is no longer indistinguishable from a healthy one.
- `agent-relay-broker` node-only delivery: agents spawned via the HTTP-register fallback (when node-control `agent.register` is unavailable) are now bound to the broker's node so the engine delivers to them; a failed bind surfaces a loud `registration_warning` instead of silently producing an undeliverable agent. A missing node token is now logged as a hard fault (realtime delivery disabled) rather than a quiet warning.
- `agent-relay-broker` node-only delivery: the persisted node token is now scoped to the workspace (and engine base URL) it was minted for, so a token cached for one workspace/engine is no longer reused against another and rejected with HTTP 401. A node-control `/v1/node/ws` 401 now discards the stale token and re-mints a fresh one (bounded re-minting) instead of looping forever on the rejected token.
- `agent-relay-broker` node token cache is now scoped per `node_id` (`node-tokens/{node_id}.json`) instead of one host-wide file, so two brokers in different working directories on one host no longer overwrite each other's cached token and re-mint on every restart.
Expand Down
65 changes: 61 additions & 4 deletions crates/broker/src/listen_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use uuid::Uuid;
use crate::worker_request::{RequestWorkerError, DEFAULT_REQUEST_TIMEOUT};

const LISTEN_API_SEND_TIMEOUT: Duration = Duration::from_secs(30);
const HEALTH_STATUS_TIMEOUT: Duration = Duration::from_millis(100);

type PtyInputSerializers = Arc<tokio::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>;

Expand Down Expand Up @@ -499,16 +500,72 @@ pub(crate) fn listen_api_health_payload(
"wsConnections": 0,
"memoryMb": 0,
"relaycastConnected": startup_error_code.is_none(),
"nodeConnected": false,
"nodeDelivery": {
"tokenPresent": false,
"connected": false,
},
})
}

async fn listen_api_health(
axum::extract::State(state): axum::extract::State<ListenApiState>,
) -> axum::Json<Value> {
axum::Json(listen_api_health_payload(
state.default_workspace_id,
state.memberships,
))
let mut payload = listen_api_health_payload(state.default_workspace_id, state.memberships);
if let Some(status) = fetch_status_for_health(&state.tx).await {
merge_status_into_health_payload(&mut payload, &status);
}
axum::Json(payload)
}

async fn fetch_status_for_health(tx: &mpsc::Sender<ListenApiRequest>) -> Option<Value> {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
tx.try_send(ListenApiRequest::GetStatus { reply: reply_tx })
.ok()?;
timeout(HEALTH_STATUS_TIMEOUT, reply_rx)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
.await
.ok()?
.ok()?
.ok()
}
Comment on lines +521 to +530

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The /health endpoint can hang indefinitely if the broker's event loop is deadlocked or overloaded, because tx.send(...).await blocks until there is space in the bounded channel. Using tx.try_send(...) instead of tx.send(...).await prevents blocking and ensures the health check fails fast under load or deadlock.

async fn fetch_status_for_health(tx: &mpsc::Sender<ListenApiRequest>) -> Option<Value> {
    let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
    tx.try_send(ListenApiRequest::GetStatus { reply: reply_tx })
        .ok()?;
    timeout(HEALTH_STATUS_TIMEOUT, reply_rx)
        .await
        .ok()?
        .ok()?
        .ok()
}


fn merge_status_into_health_payload(payload: &mut Value, status: &Value) {
let Some(object) = payload.as_object_mut() else {
return;
};
if let Some(agent_count) = status.get("agent_count").and_then(Value::as_u64) {
object.insert("agentCount".to_string(), json!(agent_count));
}
if let Some(pending_count) = status.get("pending_delivery_count").and_then(Value::as_u64) {
object.insert("pendingDeliveryCount".to_string(), json!(pending_count));
}
let token_present = status
.get("node_delivery")
.and_then(|value| value.get("token_present"))
.and_then(Value::as_bool)
.unwrap_or(false);
let connected = status
.get("node_connected")
.and_then(Value::as_bool)
.or_else(|| {
status
.get("node_delivery")
.and_then(|value| value.get("connected"))
.and_then(Value::as_bool)
})
.unwrap_or(false);
object.insert("nodeConnected".to_string(), json!(connected));
object.insert(
"nodeDelivery".to_string(),
json!({
"tokenPresent": token_present,
"connected": connected,
}),
);
object.insert(
"wsConnections".to_string(),
json!(if connected { 1 } else { 0 }),
);
}

/// Authenticated endpoint that returns broker configuration, including the
Expand Down
Loading
Loading