From 6b6af042ef76f67b549c4bf2efd804ede2a48e90 Mon Sep 17 00:00:00 2001 From: Tyler Dane Date: Mon, 7 Sep 2026 11:07:18 -0600 Subject: [PATCH] feat(web): show a reconnecting badge and reload prompt when the sse stream degrades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live-update EventSource already reported sse_connection_degraded to PostHog after 15s non-OPEN, but the only UI was the sidebar status text, which loses to saving/sync status and hides with the sidebar. Users saw a normal calendar that had silently stopped updating. The SSE client now records when degradation began instead of a boolean. CalendarHeader renders a pulsing "Reconnecting…" badge while degraded and, after 30s, a Refresh control that reloads the page, since native EventSource never retries once the browser marks it CLOSED. Like UpdateAvailableButton, the tooltip and blocked-click hint name Mod+R because the calendar is keyboard-only. Also documents the three live PostHog alerts, including the new hourly "SSE connection degraded burst (production)" alert (more than 3 events in a trailing 60 minutes). Co-Authored-By: Claude Fable 5.1 --- docs/development/launch-ops-checklist.md | 20 +++- .../CalendarHeader/CalendarHeader.tsx | 2 + .../CalendarHeader/LiveUpdatesStatus.test.tsx | 104 ++++++++++++++++++ .../CalendarHeader/LiveUpdatesStatus.tsx | 76 +++++++++++++ .../web/src/sse/client/sse.client.test.ts | 39 +++++++ packages/web/src/sse/client/sse.client.ts | 33 ++++-- packages/web/src/sse/hooks/useSseDegraded.ts | 10 ++ 7 files changed, 272 insertions(+), 12 deletions(-) create mode 100644 packages/web/src/components/CalendarHeader/LiveUpdatesStatus.test.tsx create mode 100644 packages/web/src/components/CalendarHeader/LiveUpdatesStatus.tsx diff --git a/docs/development/launch-ops-checklist.md b/docs/development/launch-ops-checklist.md index 4a62241910..af24347d63 100644 --- a/docs/development/launch-ops-checklist.md +++ b/docs/development/launch-ops-checklist.md @@ -28,7 +28,25 @@ Alert on `sync_health_snapshot` properties (low cardinality — safe to alert): Also watch: - Web `$exception` rate (Error Tracking) -- Client event `sse_connection_degraded` (prolonged EventSource non-OPEN) + +### Alerts that already exist in PostHog + +Created by hand in the PostHog UI or via the PostHog MCP; all evaluate hourly +and email the founder's PostHog account. Check the +[alerts page](https://us.posthog.com/project/165441/alerts) before creating +another one. + +| Alert | Insight | Fires when | +| --- | --- | --- | +| Sync job terminal failure | hourly count of `sync_job_terminal_failure` | count above 0 in the current hour | +| Sync reconcile sweep starved (production) | `sync_reconcile_sweep` completions, trailing 45 minutes, production | count below 1 | +| SSE connection degraded burst (production) | `sse_connection_degraded`, trailing 60 minutes, production | count above 3 | + +The SSE alert samples a trailing 60-minute window once an hour, so a burst +that straddles two checks can be under-counted. Widen the insight's window to +90 minutes if that bites; 15-minute evaluation needs a PostHog add-on. The +web app shows its own "Reconnecting…" header badge for the same condition +(`LiveUpdatesStatus`), with a reload prompt after 30 seconds. ## During launch diff --git a/packages/web/src/components/CalendarHeader/CalendarHeader.tsx b/packages/web/src/components/CalendarHeader/CalendarHeader.tsx index 88c809e22d..943adc6133 100644 --- a/packages/web/src/components/CalendarHeader/CalendarHeader.tsx +++ b/packages/web/src/components/CalendarHeader/CalendarHeader.tsx @@ -1,5 +1,6 @@ import { type FC } from "react"; import { ArrowButton } from "@web/components/Button/ArrowButton"; +import { LiveUpdatesStatus } from "@web/components/CalendarHeader/LiveUpdatesStatus"; import { UpdateAvailableButton } from "@web/components/CalendarHeader/UpdateAvailableButton"; import { SelectView } from "@web/components/SelectView/SelectView"; import { useVersionCheck } from "@web/components/Sidebar/SidebarActions/useVersionCheck"; @@ -63,6 +64,7 @@ export const CalendarHeader: FC = ({ )} + {isUpdateAvailable ? : null} diff --git a/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.test.tsx b/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.test.tsx new file mode 100644 index 0000000000..c6191b28b5 --- /dev/null +++ b/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.test.tsx @@ -0,0 +1,104 @@ +import { act, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterAll, afterEach, describe, expect, it, mock } from "bun:test"; +import "@testing-library/jest-dom"; +import { mockModuleForFile } from "@web/__tests__/utils/mock-module.test.util"; +import * as realBrowserNavigation from "@web/common/utils/browser/browser-navigation.util"; +import * as realUseSseDegraded from "@web/sse/hooks/useSseDegraded"; + +let degradedSinceMs: number | null = null; +let isDegradedMocked = true; +const actualUseSseDegradedSince = realUseSseDegraded.useSseDegradedSince; +mockModuleForFile("@web/sse/hooks/useSseDegraded", realUseSseDegraded, { + useSseDegradedSince: () => + isDegradedMocked ? degradedSinceMs : actualUseSseDegradedSince(), +}); + +const reloadLocation = mock(); +mockModuleForFile( + "@web/common/utils/browser/browser-navigation.util", + realBrowserNavigation, + { reloadLocation }, +); + +afterAll(() => { + isDegradedMocked = false; +}); + +// Cache-bust so this file's mocks apply even when another suite already +// loaded the component with the real hooks. +const moduleUrl = new URL( + `./LiveUpdatesStatus.tsx?test=${Math.random().toString(36).slice(2)}`, + import.meta.url, +); +const { LiveUpdatesStatus, REFRESH_AFTER_DEGRADED_MS } = (await import( + moduleUrl.href +)) as typeof import("./LiveUpdatesStatus"); + +describe("LiveUpdatesStatus", () => { + afterEach(() => { + degradedSinceMs = null; + reloadLocation.mockClear(); + }); + + it("renders nothing while the stream is healthy", () => { + render(); + + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + }); + + it("shows the reconnecting badge without a refresh control at first", () => { + degradedSinceMs = Date.now(); + render(); + + expect(screen.getByRole("status")).toHaveTextContent("Reconnecting…"); + expect( + screen.queryByRole("button", { name: "Refresh" }), + ).not.toBeInTheDocument(); + }); + + it("offers a reload once the outage has outlived the refresh window", async () => { + const user = userEvent.setup(); + degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS - 1_000; + render(); + + const refresh = screen.getByRole("button", { name: "Refresh" }); + await user.hover(refresh); + const tooltip = await screen.findByRole("tooltip"); + expect( + within(tooltip).getByText("Reload for the latest calendar"), + ).toBeInTheDocument(); + expect(within(tooltip).getByText("R")).toBeInTheDocument(); + + // Bare focus() opens the tooltip outside React's act; wrap it. + act(() => refresh.focus()); + await user.keyboard("{Enter}"); + expect(reloadLocation).toHaveBeenCalledTimes(1); + }); + + it("adds the refresh control when the remaining window elapses", async () => { + degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS + 50; + render(); + + expect( + screen.queryByRole("button", { name: "Refresh" }), + ).not.toBeInTheDocument(); + expect( + await screen.findByRole("button", { name: "Refresh" }), + ).toBeInTheDocument(); + }); + + it("clears the badge when the stream reopens", () => { + degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS - 1_000; + const { rerender } = render(); + expect(screen.getByRole("status")).toBeInTheDocument(); + + degradedSinceMs = null; + rerender(); + + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Refresh" }), + ).not.toBeInTheDocument(); + }); +}); diff --git a/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.tsx b/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.tsx new file mode 100644 index 0000000000..fc8a8b856b --- /dev/null +++ b/packages/web/src/components/CalendarHeader/LiveUpdatesStatus.tsx @@ -0,0 +1,76 @@ +import { ArrowClockwiseIcon } from "@phosphor-icons/react"; +import { type FC, useEffect, useState } from "react"; +import { reloadLocation } from "@web/common/utils/browser/browser-navigation.util"; +import { TooltipWrapper } from "@web/components/Tooltip/TooltipWrapper"; +import { useSseDegradedSince } from "@web/sse/hooks/useSseDegraded"; + +// Native EventSource retries on its own for transient drops but gives up for +// good once the browser marks it CLOSED. Past this age the badge stops +// implying recovery is imminent and offers the reload that always works. +export const REFRESH_AFTER_DEGRADED_MS = 30_000; + +const useRefreshDue = (degradedSinceMs: number | null): boolean => { + const [due, setDue] = useState(false); + + useEffect(() => { + if (degradedSinceMs === null) { + setDue(false); + return; + } + const remainingMs = + degradedSinceMs + REFRESH_AFTER_DEGRADED_MS - Date.now(); + if (remainingMs <= 0) { + setDue(true); + return; + } + setDue(false); + const timer = setTimeout(() => setDue(true), remainingMs); + return () => clearTimeout(timer); + }, [degradedSinceMs]); + + return due; +}; + +/** + * Header badge for a live-update stream that has been down for 15s+ (see + * sse.client's degraded window). Clears itself when the stream reopens. After + * 30s it adds a Refresh control that reloads the page; like + * UpdateAvailableButton, the tooltip and blocked-click hint name Mod+R + * because pointer clicks are suppressed in the keyboard-only calendar. + */ +export const LiveUpdatesStatus: FC = () => { + const degradedSinceMs = useSseDegradedSince(); + const refreshDue = useRefreshDue(degradedSinceMs); + + if (degradedSinceMs === null) return null; + + return ( +
+
+ ); +}; diff --git a/packages/web/src/sse/client/sse.client.test.ts b/packages/web/src/sse/client/sse.client.test.ts index d68e4c2b52..a5cc03b155 100644 --- a/packages/web/src/sse/client/sse.client.test.ts +++ b/packages/web/src/sse/client/sse.client.test.ts @@ -1,6 +1,7 @@ import * as posthogBootstrap from "@web/auth/posthog/posthog.bootstrap"; import { closeStream, + getSseDegradedSinceMs, isSseDegraded, openStream, subscribeSseDegraded, @@ -191,6 +192,44 @@ describe("sse.client degraded state", () => { expect(isSseDegraded()).toBe(false); }); + it("records when degradation began and keeps the first timestamp", () => { + expect(getSseDegradedSinceMs()).toBeNull(); + + openStream(); + fakeEs.dispatch("error"); + runDegradedTimer(); + expect(getSseDegradedSinceMs()).toBe(nowMs); + + // A later error re-arms the timer; a second fire must not restart the + // header's reload countdown. + const firstDegradedAt = nowMs; + nowMs += 20_000; + fakeEs.dispatch("error"); + for (const timer of timerCallbacks.filter((t) => t.delayMs === 15_000)) { + timer.callback(); + } + expect(getSseDegradedSinceMs()).toBe(firstDegradedAt); + }); + + it("clears the degraded timestamp on reopen and on close", () => { + openStream(); + fakeEs.dispatch("error"); + runDegradedTimer(); + fakeEs.readyState = FakeEventSource.OPEN; + fakeEs.dispatch("open"); + expect(getSseDegradedSinceMs()).toBeNull(); + + fakeEs.readyState = FakeEventSource.CONNECTING; + fakeEs.dispatch("error"); + for (const timer of timerCallbacks.filter((t) => t.delayMs === 15_000)) { + timer.callback(); + } + expect(getSseDegradedSinceMs()).toBe(nowMs); + + closeStream(); + expect(getSseDegradedSinceMs()).toBeNull(); + }); + it("captures diagnostic properties on the first degraded report", () => { window.history.replaceState(null, "", "/week"); openStream(); diff --git a/packages/web/src/sse/client/sse.client.ts b/packages/web/src/sse/client/sse.client.ts index 9f826c1d81..dbd56049ea 100644 --- a/packages/web/src/sse/client/sse.client.ts +++ b/packages/web/src/sse/client/sse.client.ts @@ -73,19 +73,26 @@ function resetConnectionDiagnostics() { lastErrorType = "timeout"; } -// Whether the live stream has been down long enough that displayed data can -// no longer be trusted as fresh. Previously this was analytics-only -// (sse_connection_degraded, PostHog) with no UI representation at all: a tab -// with a dead stream kept showing "Calendar connected" and a "Updated N -// minutes ago" timestamp that both silently stopped being true. -const sseDegradedStore = createExternalStore(false); +// Epoch ms at which the live stream had been down long enough that displayed +// data can no longer be trusted as fresh, or null while healthy. Previously +// this was analytics-only (sse_connection_degraded, PostHog) with no UI +// representation at all: a tab with a dead stream kept showing "Calendar +// connected" and a "Updated N minutes ago" timestamp that both silently +// stopped being true. The timestamp (not a boolean) lets the header offer a +// reload once an outage has outlived native reconnect's usefulness, and it +// survives header remounts on view switches. +const sseDegradedSinceStore = createExternalStore(null); export function isSseDegraded(): boolean { - return sseDegradedStore.get(); + return sseDegradedSinceStore.get() !== null; +} + +export function getSseDegradedSinceMs(): number | null { + return sseDegradedSinceStore.get(); } export function subscribeSseDegraded(onChange: () => void): () => void { - return sseDegradedStore.subscribe(onChange); + return sseDegradedSinceStore.subscribe(onChange); } function clearDegradedTimer() { @@ -96,7 +103,11 @@ function clearDegradedTimer() { } function reportSseDegraded() { - sseDegradedStore.set(true); + // The timer re-arms on every error, so it can fire more than once per + // outage; the badge keeps the first timestamp. + if (sseDegradedSinceStore.get() === null) { + sseDegradedSinceStore.set(Date.now()); + } if (hasReportedDegraded) return; hasReportedDegraded = true; try { @@ -158,7 +169,7 @@ export const openStream = (): EventSource => { connectionOpenedAtMs = Date.now(); userEventCount = 0; lastErrorType = "timeout"; - sseDegradedStore.set(false); + sseDegradedSinceStore.set(null); for (const listener of reopenListeners) { listener(); } @@ -183,7 +194,7 @@ export const openStream = (): EventSource => { export const closeStream = (): void => { clearDegradedTimer(); - sseDegradedStore.set(false); + sseDegradedSinceStore.set(null); if (es && forwardingHandler) { es.removeEventListener(SSE_MESSAGE_EVENT, forwardingHandler); } diff --git a/packages/web/src/sse/hooks/useSseDegraded.ts b/packages/web/src/sse/hooks/useSseDegraded.ts index 43eb7c2df8..9665ce7f9f 100644 --- a/packages/web/src/sse/hooks/useSseDegraded.ts +++ b/packages/web/src/sse/hooks/useSseDegraded.ts @@ -1,5 +1,6 @@ import { useSyncExternalStore } from "react"; import { + getSseDegradedSinceMs, isSseDegraded, subscribeSseDegraded, } from "@web/sse/client/sse.client"; @@ -13,3 +14,12 @@ import { export function useSseDegraded(): boolean { return useSyncExternalStore(subscribeSseDegraded, isSseDegraded); } + +/** + * Epoch ms at which the stream became degraded, or null while healthy. The + * header uses it to offer a reload once an outage has run long enough that + * native EventSource reconnect is unlikely to recover on its own. + */ +export function useSseDegradedSince(): number | null { + return useSyncExternalStore(subscribeSseDegraded, getSseDegradedSinceMs); +}