diff --git a/lib/public/js/components/gateway.js b/lib/public/js/components/gateway.js index 99dda239..2d0724be 100644 --- a/lib/public/js/components/gateway.js +++ b/lib/public/js/components/gateway.js @@ -38,19 +38,26 @@ export const Gateway = ({ : watchdogStatus?.health; const hasConfigurationError = watchdogStatus?.lifecycle === "configuration_error"; + // Safe mode: gateway healthy but channel autostart suppressed by its + // crash-loop breaker — a green "healthy" badge would be misleading. + const isSafeMode = !!watchdogStatus?.safeMode; const watchdogDotClass = - watchdogHealth === "healthy" - ? "ac-status-dot ac-status-dot--healthy ac-status-dot--healthy-offset" - : watchdogHealth === "degraded" - ? "bg-yellow-500" - : watchdogHealth === "unhealthy" || watchdogHealth === "crash_loop" - ? "bg-red-500" - : "bg-gray-500"; + isSafeMode && watchdogHealth === "healthy" + ? "bg-yellow-500" + : watchdogHealth === "healthy" + ? "ac-status-dot ac-status-dot--healthy ac-status-dot--healthy-offset" + : watchdogHealth === "degraded" + ? "bg-yellow-500" + : watchdogHealth === "unhealthy" || watchdogHealth === "crash_loop" + ? "bg-red-500" + : "bg-gray-500"; const watchdogLabel = hasConfigurationError ? "configuration error" - : watchdogHealth === "unknown" - ? "initializing" - : watchdogHealth || "unknown"; + : isSafeMode + ? "safe mode" + : watchdogHealth === "unknown" + ? "initializing" + : watchdogHealth || "unknown"; const isRepairInProgress = repairing || !!watchdogStatus?.operationInProgress; const showInspectButton = watchdogHealth === "degraded" && !!onOpenWatchdog; const showRepairButton = diff --git a/lib/public/js/components/watchdog-tab/helpers.js b/lib/public/js/components/watchdog-tab/helpers.js index baaa244e..ff4264d7 100644 --- a/lib/public/js/components/watchdog-tab/helpers.js +++ b/lib/public/js/components/watchdog-tab/helpers.js @@ -115,6 +115,26 @@ export const getIncidentStatusTone = (event) => { }; }; +// OpenClaw 2026.7.1+ can boot into control-plane-safe mode after its +// crash-loop breaker trips: the gateway reports healthy while channel +// autostart stays suppressed. Returns null when no banner should render. +export const buildSafeModeBannerModel = (watchdogStatus = null) => { + if (!watchdogStatus?.safeMode) return null; + const channels = Array.isArray(watchdogStatus.suppressedChannels) + ? watchdogStatus.suppressedChannels + .map((entry) => String(entry || "").trim()) + .filter(Boolean) + : []; + return { + title: "Gateway is in safe mode", + body: + channels.length > 0 + ? `Channel autostart was suppressed by the gateway's crash-loop breaker. Suppressed: ${channels.join(", ")}. These channels are not delivering messages.` + : "Channel autostart was suppressed by the gateway's crash-loop breaker.", + channels, + }; +}; + export const formatWatchdogCopyAllText = ({ logs = "", generatedAt = null, diff --git a/lib/public/js/components/watchdog-tab/index.js b/lib/public/js/components/watchdog-tab/index.js index b56936ca..ba9e67f3 100644 --- a/lib/public/js/components/watchdog-tab/index.js +++ b/lib/public/js/components/watchdog-tab/index.js @@ -2,6 +2,7 @@ import { h } from "preact"; import htm from "htm"; import { Gateway } from "../gateway.js"; import { useWatchdogTab } from "./use-watchdog-tab.js"; +import { WatchdogSafeModeBanner } from "./safe-mode-banner.js"; import { WatchdogResourcesCard } from "./resources/index.js"; import { WatchdogSettingsCard } from "./settings/index.js"; import { WatchdogConsoleCard } from "./console/index.js"; @@ -26,6 +27,12 @@ export const WatchdogTab = ({ return html`
+ <${WatchdogSafeModeBanner} + watchdogStatus=${state.currentWatchdogStatus} + onResumeChannels=${state.onResumeChannels} + resuming=${state.resumingChannels} + /> + <${Gateway} status=${gatewayStatus} openclawVersion=${openclawVersion} diff --git a/lib/public/js/components/watchdog-tab/safe-mode-banner.js b/lib/public/js/components/watchdog-tab/safe-mode-banner.js new file mode 100644 index 00000000..c32215ce --- /dev/null +++ b/lib/public/js/components/watchdog-tab/safe-mode-banner.js @@ -0,0 +1,37 @@ +import { h } from "preact"; +import htm from "htm"; +import { UpdateActionButton } from "../update-action-button.js"; +import { buildSafeModeBannerModel } from "./helpers.js"; + +const html = htm.bind(h); + +export const WatchdogSafeModeBanner = ({ + watchdogStatus = null, + onResumeChannels = () => {}, + resuming = false, +}) => { + const model = buildSafeModeBannerModel(watchdogStatus); + if (!model) return null; + + return html` +
+
+
+
+ + ${model.title} +
+

${model.body}

+
+ <${UpdateActionButton} + onClick=${onResumeChannels} + loading=${resuming} + disabled=${resuming} + warning=${true} + idleLabel="Resume channels" + loadingLabel="Resuming..." + /> +
+
+ `; +}; diff --git a/lib/public/js/components/watchdog-tab/settings/use-settings.js b/lib/public/js/components/watchdog-tab/settings/use-settings.js index 228e36d3..c87c9804 100644 --- a/lib/public/js/components/watchdog-tab/settings/use-settings.js +++ b/lib/public/js/components/watchdog-tab/settings/use-settings.js @@ -1,6 +1,7 @@ import { useEffect, useState } from "preact/hooks"; import { fetchWatchdogSettings, + resumeWatchdogChannels, triggerWatchdogRepair, updateWatchdogSettings, } from "../../../lib/api.js"; @@ -17,6 +18,7 @@ export const useWatchdogSettings = ({ }); const [savingSettings, setSavingSettings] = useState(false); const [repairing, setRepairing] = useState(false); + const [resumingChannels, setResumingChannels] = useState(false); const isRepairInProgress = repairing || !!(watchdogStatus || {})?.operationInProgress; @@ -106,6 +108,24 @@ export const useWatchdogSettings = ({ } }; + const onResumeChannels = async () => { + if (resumingChannels) return; + setResumingChannels(true); + try { + const data = await resumeWatchdogChannels(); + if (!data.ok) throw new Error(data.error || "Resume failed"); + showToast("Channels resuming", "success"); + setTimeout(() => { + onRefreshStatuses(); + onRefreshIncidents(); + }, 800); + } catch (error) { + showToast(error.message || "Could not resume channels", "error"); + } finally { + setResumingChannels(false); + } + }; + return { settings, savingSettings, @@ -113,5 +133,7 @@ export const useWatchdogSettings = ({ onToggleAutoRepair, onToggleNotifications, onRepair, + onResumeChannels, + resumingChannels, }; }; diff --git a/lib/public/js/components/watchdog-tab/use-watchdog-tab.js b/lib/public/js/components/watchdog-tab/use-watchdog-tab.js index 9abd45f9..f2277567 100644 --- a/lib/public/js/components/watchdog-tab/use-watchdog-tab.js +++ b/lib/public/js/components/watchdog-tab/use-watchdog-tab.js @@ -34,6 +34,8 @@ export const useWatchdogTab = ({ onToggleNotifications: settings.onToggleNotifications, onRepair: settings.onRepair, isRepairInProgress: settings.isRepairInProgress, + onResumeChannels: settings.onResumeChannels, + resumingChannels: settings.resumingChannels, logs: consoleState.logs, loadingLogs: consoleState.loadingLogs, copyingAll: consoleState.copyingAll, diff --git a/lib/public/js/lib/api.js b/lib/public/js/lib/api.js index 3f582997..3b2f9658 100644 --- a/lib/public/js/lib/api.js +++ b/lib/public/js/lib/api.js @@ -445,6 +445,13 @@ export async function triggerWatchdogRepair() { return parseJsonOrThrow(res, "Could not trigger watchdog repair"); } +export async function resumeWatchdogChannels() { + const res = await authFetch("/api/watchdog/resume-channels", { + method: "POST", + }); + return parseJsonOrThrow(res, "Could not resume channels"); +} + export async function fetchWatchdogResources() { const res = await authFetch("/api/watchdog/resources"); return parseJsonOrThrow(res, "Could not load system resources"); diff --git a/lib/server.js b/lib/server.js index e0ae22e4..de78afdd 100644 --- a/lib/server.js +++ b/lib/server.js @@ -291,6 +291,7 @@ const watchdog = createWatchdog({ reloadEnv, resolveSetupUrl, resolveGatewayHealthUrl: () => `${getGatewayUrl()}/health`, + resolveGatewayReadyzUrl: () => `${getGatewayUrl()}/readyz`, }); const watchdogTerminal = createWatchdogTerminalService({ cwd: constants.OPENCLAW_DIR, diff --git a/lib/server/routes/watchdog.js b/lib/server/routes/watchdog.js index 45b1f078..23ddfef1 100644 --- a/lib/server/routes/watchdog.js +++ b/lib/server/routes/watchdog.js @@ -51,6 +51,19 @@ const registerWatchdogRoutes = ({ } }); + app.post("/api/watchdog/resume-channels", requireAuth, async (req, res) => { + try { + const result = await watchdog.resumeChannels(); + if (result?.skipped) { + res.status(409).json({ ok: false, error: result.reason, result }); + return; + } + res.json({ ok: !!result?.ok, result }); + } catch (err) { + res.status(500).json({ ok: false, error: err.message }); + } + }); + app.get("/api/watchdog/settings", requireAuth, (req, res) => { try { res.json({ ok: true, settings: watchdog.getSettings() }); diff --git a/lib/server/watchdog.js b/lib/server/watchdog.js index 856e7fca..e9788ca0 100644 --- a/lib/server/watchdog.js +++ b/lib/server/watchdog.js @@ -11,6 +11,16 @@ const kHealthStartupGraceMs = 30 * 1000; const kBootstrapHealthCheckMs = 5 * 1000; const kExpectedRestartWindowMs = 15 * 1000; const kGatewayHealthTimeoutMs = 5 * 1000; +// OpenClaw 2026.7.1+ exits with EX_CONFIG (78, sysexits.h) on fatal +// configuration errors. The contract is "do not restart until the config is +// fixed" — restarting blindly recreates the restart storm the gateway is +// trying to prevent. +const kOpenclawConfigErrorExitCode = 78; + +const shellEscapeArg = (value) => { + const safeValue = String(value || ""); + return `'${safeValue.replace(/'/g, `'\\''`)}'`; +}; const isTruthy = (value) => ["1", "true", "yes", "on"].includes( @@ -42,6 +52,7 @@ const createWatchdog = ({ reloadEnv, resolveSetupUrl, resolveGatewayHealthUrl = () => "", + resolveGatewayReadyzUrl = () => "", }) => { const state = { lifecycle: "stopped", @@ -64,6 +75,9 @@ const createWatchdog = ({ awaitingAutoRepairRecovery: false, startupConsecutiveHealthFailures: 0, configurationErrorActive: false, + safeMode: false, + suppressedChannels: [], + safeModeNotifiedKey: "", }; let healthTimer = null; let bootstrapHealthTimer = null; @@ -338,6 +352,138 @@ const createWatchdog = ({ } }; + // OpenClaw 2026.7.1+ can boot into control-plane-safe mode after its own + // crash-loop breaker trips: /health stays green while channel autostart is + // suppressed. /readyz reports the suppressed channels. + const probeGatewayReadiness = async () => { + const readyzUrl = String(resolveGatewayReadyzUrl() || "").trim(); + if (!readyzUrl) { + return { ok: false, reason: "gateway readyz URL unavailable" }; + } + const controller = new AbortController(); + const timeoutId = setTimeout( + () => controller.abort(), + kGatewayHealthTimeoutMs, + ); + try { + const response = await fetch(readyzUrl, { + method: "GET", + headers: { Accept: "application/json" }, + signal: controller.signal, + }); + const rawBody = await response.text(); + let parsedBody = null; + try { + parsedBody = rawBody ? JSON.parse(rawBody) : null; + } catch {} + if (!response.ok || !parsedBody || typeof parsedBody !== "object") { + return { + ok: false, + reason: `gateway readyz returned HTTP ${response.status}`, + }; + } + return { + ok: true, + ready: parsedBody.ready !== false, + failing: Array.isArray(parsedBody.failing) ? parsedBody.failing : [], + suppressed: Array.isArray(parsedBody.suppressed) + ? parsedBody.suppressed.map((entry) => String(entry || "")).filter(Boolean) + : [], + }; + } catch (error) { + const message = + error?.name === "AbortError" + ? `gateway readyz timed out after ${kGatewayHealthTimeoutMs}ms` + : error?.message || "gateway readyz request failed"; + return { ok: false, reason: message }; + } finally { + clearTimeout(timeoutId); + } + }; + + const clearSafeModeState = () => { + state.safeMode = false; + state.suppressedChannels = []; + state.safeModeNotifiedKey = ""; + }; + + const evaluateChannelSuppression = async (source, correlationId) => { + const readiness = await probeGatewayReadiness(); + if (!readiness.ok) return; + const suppressed = readiness.suppressed; + if (suppressed.length > 0) { + const notifiedKey = suppressed.slice().sort().join(","); + const changed = state.safeModeNotifiedKey !== notifiedKey; + state.safeMode = true; + state.suppressedChannels = suppressed; + if (!changed) return; + state.safeModeNotifiedKey = notifiedKey; + logEvent( + "safe_mode", + source, + "failed", + { suppressed, failing: readiness.failing }, + correlationId, + ); + await notify( + [ + "🐺 *AlphaClaw Watchdog*", + withViewLogsSuffix( + "🟡 Gateway is in safe mode — channel autostart suppressed by its crash-loop breaker", + ), + `Suppressed channels: ${suppressed.join(", ")}`, + "The gateway is up but these channels are not delivering messages. Resume them from the Watchdog tab once the crash cause is fixed.", + ].join("\n"), + correlationId, + "crash", + ); + return; + } + if (state.safeMode) { + clearSafeModeState(); + logEvent("safe_mode", source, "ok", { recovered: true }, correlationId); + await notify( + [ + "🐺 *AlphaClaw Watchdog*", + withViewLogsSuffix("🟢 Gateway safe mode cleared — channels resumed"), + ].join("\n"), + correlationId, + "recovery", + ); + } + }; + + const resumeChannels = async () => { + const correlationId = createCorrelationId(); + const channels = [...state.suppressedChannels]; + if (channels.length === 0) { + return { ok: false, skipped: true, reason: "no_suppressed_channels" }; + } + const results = []; + for (const channel of channels) { + const params = JSON.stringify({ channel }); + const result = await clawCmd( + `gateway call channels.start --params ${shellEscapeArg(params)}`, + { quiet: true }, + ); + const ok = !!result?.ok; + results.push({ channel, ok, stderr: ok ? undefined : result?.stderr }); + logEvent( + "safe_mode_resume", + "manual", + ok ? "ok" : "failed", + { channel, stderr: result?.stderr || null }, + correlationId, + ); + } + await runHealthCheck({ + source: "resume_channels", + allowAutoRepair: false, + allowDuringOperation: true, + }); + return { ok: results.every((entry) => entry.ok), results }; + }; + const updateSettings = ({ autoRepair, notificationsEnabled } = {}) => { const hasAutoRepair = typeof autoRepair === "boolean"; const hasNotificationsEnabled = typeof notificationsEnabled === "boolean"; @@ -573,6 +719,7 @@ const createWatchdog = ({ parsed.details || { ok: true }, correlationId, ); + await evaluateChannelSuppression(source, correlationId); return true; } if (restartWindowActive) { @@ -702,6 +849,7 @@ const createWatchdog = ({ } = {}) => { const correlationId = createCorrelationId(); clearDegradedHealthCheckTimer(); + clearSafeModeState(); if (expectedExit && (code == null || code === 0)) { state.lifecycle = "restarting"; state.health = "unknown"; @@ -742,7 +890,7 @@ const createWatchdog = ({ return; } - if (code === 78) { + if (code === kOpenclawConfigErrorExitCode) { state.configurationErrorActive = true; state.lifecycle = "configuration_error"; state.health = "unhealthy"; @@ -843,6 +991,7 @@ const createWatchdog = ({ const onExpectedRestart = () => { clearDegradedHealthCheckTimer(); + clearSafeModeState(); state.lifecycle = "restarting"; state.health = "unknown"; state.uptimeStartedAt = null; @@ -886,6 +1035,7 @@ const createWatchdog = ({ state.startupConsecutiveHealthFailures = 0; state.awaitingAutoRepairRecovery = false; state.pendingRecoveryNoticeSource = ""; + clearSafeModeState(); closeIncident(); }; @@ -906,6 +1056,8 @@ const createWatchdog = ({ crashLoopWindowMs: kWatchdogCrashLoopWindowMs, operationInProgress: state.operationInProgress, gatewayPid: state.gatewayPid, + safeMode: state.safeMode, + suppressedChannels: [...state.suppressedChannels], }; }; @@ -914,6 +1066,7 @@ const createWatchdog = ({ getSettings, updateSettings, triggerRepair, + resumeChannels, onExpectedRestart, onGatewayExit, onGatewayLaunch, diff --git a/package-lock.json b/package-lock.json index db86e1e6..46d69c14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "express": "^4.21.0", "http-proxy": "^1.18.1", - "openclaw": "2026.7.1", + "openclaw": "2026.7.1-2", "ws": "^8.19.0" }, "bin": { @@ -1039,18 +1039,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/node": { - "version": "25.6.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", - "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "undici-types": "~7.19.0" - } - }, "node_modules/@vitest/coverage-v8": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", @@ -2252,18 +2240,6 @@ "node": ">=8" } }, - "node_modules/jiti": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", - "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", @@ -2549,9 +2525,9 @@ } }, "node_modules/openclaw": { - "version": "2026.7.1", - "resolved": "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1.tgz", - "integrity": "sha512-ge/Xss99CHAjPL/ikmH/UFoiOrjcxDB4sW3y9mhyCD+dYW3wzV7TKbAVdkrXFgAG2d2BjpJofP97zUZ+umxo8g==", + "version": "2026.7.1-2", + "resolved": "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1-2.tgz", + "integrity": "sha512-ycF3yPcbjN6bUPeaUx6Mh6vze1hQWoD3CT/wWcmD7a8xaHHHRUaAlaq+lFxMHf1ssEgODVAwjlzYqp2twkYZ7g==", "hasInstallScript": true, "hasShrinkwrap": true, "license": "MIT", @@ -2569,7 +2545,7 @@ "@mistralai/mistralai": "2.4.0", "@modelcontextprotocol/sdk": "1.29.0", "@mozilla/readability": "0.6.0", - "@openclaw/ai": "2026.7.1", + "@openclaw/ai": "2026.7.1-2", "@openclaw/fs-safe": "0.4.1", "@openclaw/proxyline": "0.3.3", "@silvia-odwyer/photon-node": "0.3.4", @@ -2974,9 +2950,9 @@ } }, "node_modules/openclaw/node_modules/@openclaw/ai": { - "version": "2026.7.1", - "resolved": "https://registry.npmjs.org/@openclaw/ai/-/ai-2026.7.1.tgz", - "integrity": "sha512-FsKy5DXSHf4qyN8Huoz/10HZRgoEwLF4uk8UWaCafaIler+q5Fsl51HcrIqIrEe0S38OT7LOaxnR++MOshAlmw==", + "version": "2026.7.1-2", + "resolved": "https://registry.npmjs.org/@openclaw/ai/-/ai-2026.7.1-2.tgz", + "integrity": "sha512-st+NH0cxlQqdbEur//yYqM7WlYBjeEBnop3cztJTSCONKjv6LNoGguI9cH65asZG94FdM/39z857isRGPnZvEw==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.109.1", @@ -7224,15 +7200,6 @@ "node": ">= 0.6" } }, - "node_modules/undici-types": { - "version": "7.19.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", - "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -7962,24 +7929,6 @@ "optional": true } } - }, - "node_modules/yaml": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.4.tgz", - "integrity": "sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } } } } diff --git a/package.json b/package.json index fa5f739a..36ebaea4 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "dependencies": { "express": "^4.21.0", "http-proxy": "^1.18.1", - "openclaw": "2026.7.1", + "openclaw": "2026.7.1-2", "ws": "^8.19.0" }, "devDependencies": { diff --git a/tests/frontend/api.test.js b/tests/frontend/api.test.js index 5f982b06..78f90d66 100644 --- a/tests/frontend/api.test.js +++ b/tests/frontend/api.test.js @@ -655,4 +655,31 @@ describe("frontend/api", () => { expectLastFetchHeaders("application/json"); expect(result).toEqual({ ok: true }); }); + + it("resumeWatchdogChannels posts to the resume endpoint and returns the result", async () => { + const payload = { + ok: true, + result: { ok: true, results: [{ channel: "telegram", ok: true }] }, + }; + global.fetch.mockResolvedValue(mockJsonResponse(200, payload)); + const api = await loadApiModule(); + + const result = await api.resumeWatchdogChannels(); + + const [url, options] = global.fetch.mock.calls.at(-1); + expect(url).toBe("/api/watchdog/resume-channels"); + expect(options.method).toBe("POST"); + expect(result).toEqual(payload); + }); + + it("resumeWatchdogChannels surfaces server error messages", async () => { + global.fetch.mockResolvedValue( + mockJsonResponse(409, { ok: false, error: "no_suppressed_channels" }), + ); + const api = await loadApiModule(); + + await expect(api.resumeWatchdogChannels()).rejects.toThrow( + "no_suppressed_channels", + ); + }); }); diff --git a/tests/frontend/watchdog-helpers.test.js b/tests/frontend/watchdog-helpers.test.js index 56771377..6ff594eb 100644 --- a/tests/frontend/watchdog-helpers.test.js +++ b/tests/frontend/watchdog-helpers.test.js @@ -27,4 +27,39 @@ describe("frontend/watchdog-helpers", () => { expect(text).toContain("## Gateway Logs"); expect(text).toContain("No logs yet."); }); + + it("returns no safe-mode banner model when the gateway is not in safe mode", async () => { + const { buildSafeModeBannerModel } = await loadWatchdogHelpers(); + + expect(buildSafeModeBannerModel(null)).toBeNull(); + expect(buildSafeModeBannerModel({})).toBeNull(); + expect(buildSafeModeBannerModel({ safeMode: false })).toBeNull(); + }); + + it("builds a safe-mode banner model listing suppressed channels", async () => { + const { buildSafeModeBannerModel } = await loadWatchdogHelpers(); + + const model = buildSafeModeBannerModel({ + safeMode: true, + suppressedChannels: ["telegram", " discord ", "", null], + }); + + expect(model.title).toBe("Gateway is in safe mode"); + expect(model.channels).toEqual(["telegram", "discord"]); + expect(model.body).toContain("Suppressed: telegram, discord"); + expect(model.body).toContain("not delivering messages"); + }); + + it("builds a safe-mode banner model without a channel list when none reported", async () => { + const { buildSafeModeBannerModel } = await loadWatchdogHelpers(); + + const model = buildSafeModeBannerModel({ + safeMode: true, + suppressedChannels: "not-an-array", + }); + + expect(model.channels).toEqual([]); + expect(model.body).toContain("crash-loop breaker"); + expect(model.body).not.toContain("Suppressed:"); + }); }); diff --git a/tests/server/routes-watchdog.test.js b/tests/server/routes-watchdog.test.js index b718089b..29461224 100644 --- a/tests/server/routes-watchdog.test.js +++ b/tests/server/routes-watchdog.test.js @@ -8,6 +8,10 @@ const createDeps = () => { const watchdog = { getStatus: vi.fn(() => ({ lifecycle: "running", health: "healthy" })), triggerRepair: vi.fn(async () => ({ ok: true })), + resumeChannels: vi.fn(async () => ({ + ok: true, + results: [{ channel: "telegram", ok: true }], + })), getSettings: vi.fn(() => ({ autoRepair: true, notificationsEnabled: true })), updateSettings: vi.fn(({ autoRepair }) => ({ autoRepair, notificationsEnabled: true })), }; @@ -98,6 +102,39 @@ describe("server/routes/watchdog", () => { }); }); + it("resumes suppressed channels on POST /api/watchdog/resume-channels", async () => { + const deps = createDeps(); + const app = createApp(deps); + + const res = await request(app).post("/api/watchdog/resume-channels"); + + expect(res.status).toBe(200); + expect(deps.watchdog.resumeChannels).toHaveBeenCalledTimes(1); + expect(res.body).toEqual({ + ok: true, + result: { + ok: true, + results: [{ channel: "telegram", ok: true }], + }, + }); + }); + + it("returns 409 when resume-channels has nothing to resume", async () => { + const deps = createDeps(); + deps.watchdog.resumeChannels.mockResolvedValue({ + ok: false, + skipped: true, + reason: "no_suppressed_channels", + }); + const app = createApp(deps); + + const res = await request(app).post("/api/watchdog/resume-channels"); + + expect(res.status).toBe(409); + expect(res.body.ok).toBe(false); + expect(res.body.error).toBe("no_suppressed_channels"); + }); + it("returns 400 when updateSettings throws", async () => { const deps = createDeps(); deps.watchdog.updateSettings.mockImplementation(() => { diff --git a/tests/server/watchdog-gateway-hardening.e2e.test.js b/tests/server/watchdog-gateway-hardening.e2e.test.js new file mode 100644 index 00000000..47087e59 --- /dev/null +++ b/tests/server/watchdog-gateway-hardening.e2e.test.js @@ -0,0 +1,225 @@ +const express = require("express"); +const request = require("supertest"); + +const { createWatchdog } = require("../../lib/server/watchdog"); +const { registerWatchdogRoutes } = require("../../lib/server/routes/watchdog"); + +// End-to-end coverage for the OpenClaw 2026.7.1+ gateway-lifecycle contract: +// exit code 78 (EX_CONFIG) fatal config errors, and control-plane-safe mode +// where /health stays green while /readyz reports suppressed channels. Uses +// the real watchdog wired into the real routes against a stateful fake +// gateway. + +const flushMicrotasks = async () => + new Promise((resolve) => { + setImmediate(resolve); + }); + +const kOriginalAutoRepair = process.env.WATCHDOG_AUTO_REPAIR; +const kOriginalNotificationsDisabled = + process.env.WATCHDOG_NOTIFICATIONS_DISABLED; +const kOriginalFetch = global.fetch; + +const createFakeGateway = () => ({ + healthy: true, + suppressed: [], +}); + +const createStack = ({ autoRepair = false, fakeGateway } = {}) => { + process.env.WATCHDOG_AUTO_REPAIR = autoRepair ? "true" : "false"; + process.env.WATCHDOG_NOTIFICATIONS_DISABLED = "false"; + + const gateway = fakeGateway || createFakeGateway(); + + global.fetch = vi.fn(async (url) => { + if (!gateway.healthy) throw new Error("gateway unavailable"); + if (String(url).includes("/readyz")) { + return { + ok: true, + status: 200, + text: async () => + JSON.stringify({ + ready: true, + failing: [], + ...(gateway.suppressed.length > 0 + ? { suppressed: gateway.suppressed } + : {}), + }), + }; + } + return { + ok: true, + status: 200, + text: async () => JSON.stringify({ ok: true, status: "live" }), + }; + }); + + const clawCmd = vi.fn(async (command) => { + if (command.startsWith("gateway call channels.start")) { + const paramsJson = command.replace( + /^gateway call channels\.start --params '(.*)'$/, + "$1", + ); + const params = JSON.parse(paramsJson); + gateway.suppressed = gateway.suppressed.filter( + (channel) => channel !== params.channel, + ); + return { ok: true, stdout: "{}" }; + } + if (command === "doctor --fix --yes") { + gateway.healthy = true; + return { ok: true, stdout: "fixed" }; + } + return { ok: true, stdout: "" }; + }); + + const launchGatewayProcess = vi.fn(() => ({ pid: 4242 })); + const insertWatchdogEvent = vi.fn(); + const notifier = { notify: vi.fn(async () => ({ ok: true })) }; + + const watchdog = createWatchdog({ + clawCmd, + launchGatewayProcess, + insertWatchdogEvent, + notifier, + readEnvFile: vi.fn(() => []), + writeEnvFile: vi.fn(), + reloadEnv: vi.fn(), + resolveSetupUrl: () => "https://setup.example.com", + resolveGatewayHealthUrl: () => "http://127.0.0.1:18789/health", + resolveGatewayReadyzUrl: () => "http://127.0.0.1:18789/readyz", + }); + + const app = express(); + app.use(express.json()); + registerWatchdogRoutes({ + app, + requireAuth: (req, res, next) => next(), + watchdog, + watchdogNotifier: notifier, + getRecentEvents: vi.fn(() => []), + readLogTail: vi.fn(() => ""), + watchdogTerminal: {}, + }); + + return { + app, + watchdog, + gateway, + clawCmd, + launchGatewayProcess, + insertWatchdogEvent, + notifier, + }; +}; + +describe("server/watchdog gateway hardening (e2e)", () => { + afterEach(() => { + if (kOriginalAutoRepair == null) { + delete process.env.WATCHDOG_AUTO_REPAIR; + } else { + process.env.WATCHDOG_AUTO_REPAIR = kOriginalAutoRepair; + } + if (kOriginalNotificationsDisabled == null) { + delete process.env.WATCHDOG_NOTIFICATIONS_DISABLED; + } else { + process.env.WATCHDOG_NOTIFICATIONS_DISABLED = + kOriginalNotificationsDisabled; + } + if (kOriginalFetch == null) { + delete global.fetch; + } else { + global.fetch = kOriginalFetch; + } + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("surfaces safe mode in the status API and recovers via resume-channels", async () => { + const { app, watchdog, gateway, clawCmd } = createStack(); + gateway.suppressed = ["telegram", "discord"]; + + watchdog.start(); + await flushMicrotasks(); + await flushMicrotasks(); + + const statusRes = await request(app).get("/api/watchdog/status"); + expect(statusRes.status).toBe(200); + expect(statusRes.body.status).toEqual( + expect.objectContaining({ + lifecycle: "running", + health: "healthy", + safeMode: true, + suppressedChannels: ["telegram", "discord"], + }), + ); + + const resumeRes = await request(app).post("/api/watchdog/resume-channels"); + expect(resumeRes.status).toBe(200); + expect(resumeRes.body.ok).toBe(true); + expect(resumeRes.body.result.results).toEqual([ + { channel: "telegram", ok: true }, + { channel: "discord", ok: true }, + ]); + expect(clawCmd).toHaveBeenCalledWith( + `gateway call channels.start --params '{"channel":"telegram"}'`, + { quiet: true }, + ); + expect(clawCmd).toHaveBeenCalledWith( + `gateway call channels.start --params '{"channel":"discord"}'`, + { quiet: true }, + ); + expect(gateway.suppressed).toEqual([]); + + const clearedRes = await request(app).get("/api/watchdog/status"); + expect(clearedRes.body.status).toEqual( + expect.objectContaining({ safeMode: false, suppressedChannels: [] }), + ); + + const idempotentRes = await request(app).post( + "/api/watchdog/resume-channels", + ); + expect(idempotentRes.status).toBe(409); + expect(idempotentRes.body.error).toBe("no_suppressed_channels"); + watchdog.stop(); + }); + + it("reports configuration_error on exit 78 and recovers through manual repair", async () => { + const { app, watchdog, gateway, clawCmd, launchGatewayProcess } = + createStack({ autoRepair: false }); + gateway.healthy = false; + + watchdog.onGatewayLaunch({ startedAt: Date.now(), pid: 1234 }); + watchdog.onGatewayExit({ + code: 78, + expectedExit: false, + stderrTail: ["fatal configuration error"], + }); + await flushMicrotasks(); + + const statusRes = await request(app).get("/api/watchdog/status"); + expect(statusRes.body.status).toEqual( + expect.objectContaining({ + lifecycle: "configuration_error", + health: "unhealthy", + crashCountInWindow: 0, + }), + ); + // The EX_CONFIG contract: no automatic relaunch without repair. + expect(launchGatewayProcess).not.toHaveBeenCalled(); + + const repairRes = await request(app).post("/api/watchdog/repair"); + expect(repairRes.status).toBe(200); + expect(repairRes.body.ok).toBe(true); + expect(clawCmd).toHaveBeenCalledWith("doctor --fix --yes", { + quiet: true, + }); + expect(launchGatewayProcess).toHaveBeenCalledTimes(1); + + const recoveredRes = await request(app).get("/api/watchdog/status"); + expect(recoveredRes.body.status).toEqual( + expect.objectContaining({ lifecycle: "running", health: "healthy" }), + ); + watchdog.stop(); + }); +}); diff --git a/tests/server/watchdog.test.js b/tests/server/watchdog.test.js index 06b6a3c5..8c5c8f75 100644 --- a/tests/server/watchdog.test.js +++ b/tests/server/watchdog.test.js @@ -15,6 +15,7 @@ const createHarness = ({ clawCmdImpl, resolveSetupUrl = () => "https://setup.example.com", resolveGatewayHealthUrl = () => "http://127.0.0.1:18789/health", + resolveGatewayReadyzUrl = () => "", fetchImpl = async () => ({ ok: true, status: 200, @@ -49,6 +50,7 @@ const createHarness = ({ reloadEnv, resolveSetupUrl, resolveGatewayHealthUrl, + resolveGatewayReadyzUrl, }); return { @@ -714,4 +716,172 @@ describe("server/watchdog", () => { notificationsEnabled: false, }); }); + + const buildSafeModeFetch = (gatewayState) => async (url) => { + if (String(url).includes("/readyz")) { + return { + ok: true, + status: 200, + text: async () => + JSON.stringify({ + ready: true, + failing: [], + ...(gatewayState.suppressed.length > 0 + ? { suppressed: gatewayState.suppressed } + : {}), + }), + }; + } + return { + ok: true, + status: 200, + text: async () => JSON.stringify({ ok: true, status: "live" }), + }; + }; + + it("detects gateway safe mode from readyz and notifies once", async () => { + vi.useFakeTimers(); + const gatewayState = { suppressed: ["telegram", "discord"] }; + const { watchdog, insertWatchdogEvent, notifier } = createHarness({ + autoRepair: false, + resolveGatewayReadyzUrl: () => "http://127.0.0.1:18789/readyz", + fetchImpl: buildSafeModeFetch(gatewayState), + }); + + watchdog.start(); + await vi.advanceTimersByTimeAsync(10); + + expect(watchdog.getStatus()).toEqual( + expect.objectContaining({ + lifecycle: "running", + health: "healthy", + safeMode: true, + suppressedChannels: ["telegram", "discord"], + }), + ); + expect(insertWatchdogEvent).toHaveBeenCalledWith( + expect.objectContaining({ + eventType: "safe_mode", + status: "failed", + details: expect.objectContaining({ + suppressed: ["telegram", "discord"], + }), + }), + ); + const safeModeNotices = () => + notifier.notify.mock.calls.filter((call) => + String(call?.[0] || "").includes("safe mode"), + ); + expect(safeModeNotices()).toHaveLength(1); + + // Subsequent checks with unchanged suppression must not re-notify. + await vi.advanceTimersByTimeAsync(120_000); + expect(safeModeNotices()).toHaveLength(1); + watchdog.stop(); + }); + + it("clears safe mode and notifies recovery when suppression ends", async () => { + vi.useFakeTimers(); + const gatewayState = { suppressed: ["telegram"] }; + const { watchdog, insertWatchdogEvent, notifier } = createHarness({ + autoRepair: false, + resolveGatewayReadyzUrl: () => "http://127.0.0.1:18789/readyz", + fetchImpl: buildSafeModeFetch(gatewayState), + }); + + watchdog.start(); + await vi.advanceTimersByTimeAsync(10); + expect(watchdog.getStatus().safeMode).toBe(true); + + gatewayState.suppressed = []; + await vi.advanceTimersByTimeAsync(120_000); + + expect(watchdog.getStatus()).toEqual( + expect.objectContaining({ safeMode: false, suppressedChannels: [] }), + ); + expect(insertWatchdogEvent).toHaveBeenCalledWith( + expect.objectContaining({ + eventType: "safe_mode", + status: "ok", + details: expect.objectContaining({ recovered: true }), + }), + ); + expect( + notifier.notify.mock.calls.some((call) => + String(call?.[0] || "").includes("safe mode cleared"), + ), + ).toBe(true); + watchdog.stop(); + }); + + it("resumeChannels issues channels.start for each suppressed channel", async () => { + vi.useFakeTimers(); + const gatewayState = { suppressed: ["telegram", "discord"] }; + const startCalls = []; + const { watchdog } = createHarness({ + autoRepair: false, + resolveGatewayReadyzUrl: () => "http://127.0.0.1:18789/readyz", + fetchImpl: buildSafeModeFetch(gatewayState), + clawCmdImpl: async (command) => { + if (command.startsWith("gateway call channels.start")) { + startCalls.push(command); + return { ok: true, stdout: "{}" }; + } + return { ok: true, stdout: "" }; + }, + }); + + watchdog.start(); + await vi.advanceTimersByTimeAsync(10); + expect(watchdog.getStatus().safeMode).toBe(true); + + gatewayState.suppressed = []; + const resultPromise = watchdog.resumeChannels(); + await vi.advanceTimersByTimeAsync(10); + const result = await resultPromise; + + expect(result.ok).toBe(true); + expect(startCalls).toEqual([ + `gateway call channels.start --params '{"channel":"telegram"}'`, + `gateway call channels.start --params '{"channel":"discord"}'`, + ]); + expect(watchdog.getStatus()).toEqual( + expect.objectContaining({ safeMode: false, suppressedChannels: [] }), + ); + watchdog.stop(); + }); + + it("resumeChannels skips when no channels are suppressed", async () => { + const { watchdog, clawCmd } = createHarness({ autoRepair: false }); + + const result = await watchdog.resumeChannels(); + + expect(result).toEqual({ + ok: false, + skipped: true, + reason: "no_suppressed_channels", + }); + expect(clawCmd).not.toHaveBeenCalled(); + }); + + it("clears safe-mode status when the gateway exits", async () => { + vi.useFakeTimers(); + const gatewayState = { suppressed: ["telegram"] }; + const { watchdog } = createHarness({ + autoRepair: false, + resolveGatewayReadyzUrl: () => "http://127.0.0.1:18789/readyz", + fetchImpl: buildSafeModeFetch(gatewayState), + }); + + watchdog.start(); + await vi.advanceTimersByTimeAsync(10); + expect(watchdog.getStatus().safeMode).toBe(true); + + watchdog.onGatewayExit({ code: 1, expectedExit: false }); + + expect(watchdog.getStatus()).toEqual( + expect.objectContaining({ safeMode: false, suppressedChannels: [] }), + ); + watchdog.stop(); + }); }); diff --git a/vitest.config.js b/vitest.config.js index bd171f4b..7a320d1e 100644 --- a/vitest.config.js +++ b/vitest.config.js @@ -7,5 +7,8 @@ export default defineConfig({ include: ["tests/**/*.test.js"], restoreMocks: true, clearMocks: true, + // Tests that touch the openclaw plugin-sdk pay a >5s dynamic-import cost + // on first load per worker, which flakes under parallel machine load. + testTimeout: 30000, }, });