From 2b2e0d6ee8c11c192605f1ccd5fa635b2d4d87e9 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Thu, 21 May 2026 12:16:09 +1000 Subject: [PATCH 1/3] fix: adapt realtime relay RPCs to talk sessions --- src/lib/gateway-client-realtime.test.ts | 79 +++++++++++++++++++++++++ src/lib/gateway-client.ts | 68 +++++++++++++++++++-- 2 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 src/lib/gateway-client-realtime.test.ts diff --git a/src/lib/gateway-client-realtime.test.ts b/src/lib/gateway-client-realtime.test.ts new file mode 100644 index 00000000..e39ea9d5 --- /dev/null +++ b/src/lib/gateway-client-realtime.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from "vitest"; +import { GatewayClient, type DeviceIdentity } from "./gateway-client"; + +const device: DeviceIdentity = { + deviceId: "device_1", + publicKeyRawBase64Url: "pub", + privateKeyPem: "private", + source: "configured", +}; + +describe("GatewayClient realtime Talk compatibility", () => { + it("creates gateway relay sessions through the unified Talk session API", async () => { + const client = new GatewayClient("ws://localhost:18789", null, device); + const rpc = vi.spyOn(client, "rpc").mockResolvedValueOnce({ + transport: "gateway-relay", + relaySessionId: "relay_1", + }); + + await expect(client.realtimeTalkSession({ + sessionKey: "main", + provider: "openai", + agentId: "NEO", + })).resolves.toMatchObject({ + transport: "gateway-relay", + relaySessionId: "relay_1", + }); + + expect(rpc).toHaveBeenCalledWith("talk.session.create", { + sessionKey: "main", + provider: "openai", + mode: "realtime", + transport: "gateway-relay", + brain: "agent-consult", + }); + }); + + it("maps relay audio, tool results, and stop onto unified session methods", async () => { + const client = new GatewayClient("ws://localhost:18789", null, device); + const rpc = vi.spyOn(client, "rpc").mockResolvedValue({ ok: true }); + + await client.realtimeRelayAudio({ + relaySessionId: "relay_1", + audioBase64: "AAAA", + timestamp: 123, + }); + await client.realtimeRelayToolResult({ + relaySessionId: "relay_1", + callId: "call_1", + result: { ok: true }, + }); + await client.realtimeRelayStop("relay_1"); + + expect(rpc).toHaveBeenNthCalledWith(1, "talk.session.appendAudio", { + sessionId: "relay_1", + audioBase64: "AAAA", + timestamp: 123, + }); + expect(rpc).toHaveBeenNthCalledWith(2, "talk.session.submitToolResult", { + sessionId: "relay_1", + callId: "call_1", + result: { ok: true }, + }); + expect(rpc).toHaveBeenNthCalledWith(3, "talk.session.close", { + sessionId: "relay_1", + }); + }); + + it("keeps relay mark acknowledgements local for the current OpenClaw API", async () => { + const client = new GatewayClient("ws://localhost:18789", null, device); + const rpc = vi.spyOn(client, "rpc"); + + await expect(client.realtimeRelayMark({ + relaySessionId: "relay_1", + markName: "done", + })).resolves.toEqual({ ok: true }); + + expect(rpc).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/gateway-client.ts b/src/lib/gateway-client.ts index 3d76bd65..c3c896d7 100644 --- a/src/lib/gateway-client.ts +++ b/src/lib/gateway-client.ts @@ -165,6 +165,9 @@ export interface GatewayRealtimeTalkSessionParams extends Record>(record: T): Record { + return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)); +} + +function isLikelyMissingGatewayMethod(err: unknown): boolean { + const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase(); + return ( + message.includes("unknown method") + || message.includes("method not found") + || message.includes("unsupported method") + || message.includes("no handler") + ); +} + /** * Build v3 device auth payload string for signing. */ @@ -714,23 +735,60 @@ export class GatewayClient { async realtimeTalkSession( params: GatewayRealtimeTalkSessionParams = {}, ): Promise { - return this.rpc("talk.realtime.session", params); + try { + const sessionParams = withoutUndefined(params); + delete sessionParams.agentId; + return await this.rpc("talk.session.create", { + ...sessionParams, + mode: params.mode ?? "realtime", + transport: params.transport ?? "gateway-relay", + brain: params.brain ?? "agent-consult", + }); + } catch (err) { + if (!isLikelyMissingGatewayMethod(err)) throw err; + return this.rpc("talk.realtime.session", params); + } } async realtimeRelayAudio(params: GatewayRealtimeRelayAudioParams): Promise<{ ok?: boolean }> { - return this.rpc<{ ok?: boolean }>("talk.realtime.relayAudio", params); + try { + return await this.rpc<{ ok?: boolean }>("talk.session.appendAudio", withoutUndefined({ + sessionId: params.relaySessionId, + audioBase64: params.audioBase64, + timestamp: params.timestamp, + })); + } catch (err) { + if (!isLikelyMissingGatewayMethod(err)) throw err; + return this.rpc<{ ok?: boolean }>("talk.realtime.relayAudio", params); + } } async realtimeRelayMark(params: GatewayRealtimeRelayMarkParams): Promise<{ ok?: boolean }> { - return this.rpc<{ ok?: boolean }>("talk.realtime.relayMark", params); + void params; + return { ok: true }; } async realtimeRelayToolResult(params: GatewayRealtimeRelayToolResultParams): Promise<{ ok?: boolean }> { - return this.rpc<{ ok?: boolean }>("talk.realtime.relayToolResult", params); + try { + return await this.rpc<{ ok?: boolean }>("talk.session.submitToolResult", withoutUndefined({ + sessionId: params.relaySessionId, + callId: params.callId, + result: params.result, + options: params.options, + })); + } catch (err) { + if (!isLikelyMissingGatewayMethod(err)) throw err; + return this.rpc<{ ok?: boolean }>("talk.realtime.relayToolResult", params); + } } async realtimeRelayStop(relaySessionId: string): Promise<{ ok?: boolean }> { - return this.rpc<{ ok?: boolean }>("talk.realtime.relayStop", { relaySessionId }); + try { + return await this.rpc<{ ok?: boolean }>("talk.session.close", { sessionId: relaySessionId }); + } catch (err) { + if (!isLikelyMissingGatewayMethod(err)) throw err; + return this.rpc<{ ok?: boolean }>("talk.realtime.relayStop", { relaySessionId }); + } } async cronList(): Promise { From 799574dd1aee740129d5a604d6810eb7a2a997b3 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Thu, 21 May 2026 12:16:17 +1000 Subject: [PATCH 2/3] fix: subscribe realtime relay to talk event channel --- src/app/api/runtimes/[id]/talk/realtime/events/route.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app/api/runtimes/[id]/talk/realtime/events/route.ts b/src/app/api/runtimes/[id]/talk/realtime/events/route.ts index 728f5285..15fae8cd 100644 --- a/src/app/api/runtimes/[id]/talk/realtime/events/route.ts +++ b/src/app/api/runtimes/[id]/talk/realtime/events/route.ts @@ -47,12 +47,14 @@ export async function GET( }; cleanup = () => { if (!cleanup) return; + client.off("talk.event", onRelay); client.off("talk.realtime.relay", onRelay); request.signal.removeEventListener("abort", cleanup); releaseClient(client); cleanup = null; }; + client.on("talk.event", onRelay); client.on("talk.realtime.relay", onRelay); request.signal.addEventListener("abort", cleanup, { once: true }); send("realtime_ready", { relaySessionId }); From 70c0d900ab451e61b0aa9f50513969b7f7c9bec2 Mon Sep 17 00:00:00 2001 From: Roger Chappel Date: Thu, 21 May 2026 12:16:27 +1000 Subject: [PATCH 3/3] fix: advertise unified realtime talk capabilities --- src/lib/runtime-capabilities.test.ts | 3 ++- src/lib/runtime-capabilities.ts | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lib/runtime-capabilities.test.ts b/src/lib/runtime-capabilities.test.ts index 7942079a..a74f26a7 100644 --- a/src/lib/runtime-capabilities.test.ts +++ b/src/lib/runtime-capabilities.test.ts @@ -21,7 +21,8 @@ describe("deriveRuntimeCapabilitySnapshot realtime voice", () => { configuredProviders: [], transports: ["webrtc-sdp", "json-pcm-websocket", "gateway-relay"], }); - expect(snapshot.realtimeVoice?.gatewayMethods).toContain("talk.realtime.session"); + expect(snapshot.realtimeVoice?.gatewayMethods).toContain("talk.session.create"); + expect(snapshot.realtimeVoice?.gatewayMethods).toContain("talk.event"); }); it("reads explicit realtime talk provider config when present", () => { diff --git a/src/lib/runtime-capabilities.ts b/src/lib/runtime-capabilities.ts index 8076afd6..19228774 100644 --- a/src/lib/runtime-capabilities.ts +++ b/src/lib/runtime-capabilities.ts @@ -149,11 +149,11 @@ function deriveRealtimeVoiceSummary(params: { configuredProviders, transports: ["webrtc-sdp", "json-pcm-websocket", "gateway-relay"], gatewayMethods: [ - "talk.realtime.session", - "talk.realtime.relayAudio", - "talk.realtime.relayMark", - "talk.realtime.relayToolResult", - "talk.realtime.relayStop", + "talk.session.create", + "talk.session.appendAudio", + "talk.session.submitToolResult", + "talk.session.close", + "talk.event", ], notes: [ "Capability is config-derived only; route-level probing still determines whether the selected runtime accepts realtime talk sessions.",