feat(web): reconnecting badge and reload prompt for degraded sse stream

docs/development/launch-ops-checklist.md

@@ -28,7 +28,25 @@ Alert on `sync_health_snapshot` properties (low cardinality — safe to alert):
 Also watch:
 
 - Web `$exception` rate (Error Tracking)
-- Client event `sse_connection_degraded` (prolonged EventSource non-OPEN)
+
+### Alerts that already exist in PostHog
+
+Created by hand in the PostHog UI or via the PostHog MCP; all evaluate hourly
+and email the founder's PostHog account. Check the
+[alerts page](https://us.posthog.com/project/165441/alerts) before creating
+another one.
+
+| Alert | Insight | Fires when |
+| --- | --- | --- |
+| Sync job terminal failure | hourly count of `sync_job_terminal_failure` | count above 0 in the current hour |
+| Sync reconcile sweep starved (production) | `sync_reconcile_sweep` completions, trailing 45 minutes, production | count below 1 |
+| SSE connection degraded burst (production) | `sse_connection_degraded`, trailing 60 minutes, production | count above 3 |
+
+The SSE alert samples a trailing 60-minute window once an hour, so a burst
+that straddles two checks can be under-counted. Widen the insight's window to
+90 minutes if that bites; 15-minute evaluation needs a PostHog add-on. The
+web app shows its own "Reconnecting…" header badge for the same condition
+(`LiveUpdatesStatus`), with a reload prompt after 30 seconds.
 
 ## During launch
 

packages/web/src/components/CalendarHeader/CalendarHeader.tsx

@@ -1,5 +1,6 @@
 import { type FC } from "react";
 import { ArrowButton } from "@web/components/Button/ArrowButton";
+import { LiveUpdatesStatus } from "@web/components/CalendarHeader/LiveUpdatesStatus";
 import { UpdateAvailableButton } from "@web/components/CalendarHeader/UpdateAvailableButton";
 import { SelectView } from "@web/components/SelectView/SelectView";
 import { useVersionCheck } from "@web/components/Sidebar/SidebarActions/useVersionCheck";
@@ -63,6 +64,7 @@ export const CalendarHeader: FC<Props> = ({
           </div>
         )}
         <SelectView label={label} onToday={onToday} />
+        <LiveUpdatesStatus />
         {isUpdateAvailable ? <UpdateAvailableButton /> : null}
       </div>
 

packages/web/src/components/CalendarHeader/LiveUpdatesStatus.test.tsx

@@ -0,0 +1,104 @@
+import { act, render, screen, within } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterAll, afterEach, describe, expect, it, mock } from "bun:test";
+import "@testing-library/jest-dom";
+import { mockModuleForFile } from "@web/__tests__/utils/mock-module.test.util";
+import * as realBrowserNavigation from "@web/common/utils/browser/browser-navigation.util";
+import * as realUseSseDegraded from "@web/sse/hooks/useSseDegraded";
+
+let degradedSinceMs: number | null = null;
+let isDegradedMocked = true;
+const actualUseSseDegradedSince = realUseSseDegraded.useSseDegradedSince;
+mockModuleForFile("@web/sse/hooks/useSseDegraded", realUseSseDegraded, {
+  useSseDegradedSince: () =>
+    isDegradedMocked ? degradedSinceMs : actualUseSseDegradedSince(),
+});
+
+const reloadLocation = mock();
+mockModuleForFile(
+  "@web/common/utils/browser/browser-navigation.util",
+  realBrowserNavigation,
+  { reloadLocation },
+);
+
+afterAll(() => {
+  isDegradedMocked = false;
+});
+
+// Cache-bust so this file's mocks apply even when another suite already
+// loaded the component with the real hooks.
+const moduleUrl = new URL(
+  `./LiveUpdatesStatus.tsx?test=${Math.random().toString(36).slice(2)}`,
+  import.meta.url,
+);
+const { LiveUpdatesStatus, REFRESH_AFTER_DEGRADED_MS } = (await import(
+  moduleUrl.href
+)) as typeof import("./LiveUpdatesStatus");
+
+describe("LiveUpdatesStatus", () => {
+  afterEach(() => {
+    degradedSinceMs = null;
+    reloadLocation.mockClear();
+  });
+
+  it("renders nothing while the stream is healthy", () => {
+    render(<LiveUpdatesStatus />);
+
+    expect(screen.queryByRole("status")).not.toBeInTheDocument();
+  });
+
+  it("shows the reconnecting badge without a refresh control at first", () => {
+    degradedSinceMs = Date.now();
+    render(<LiveUpdatesStatus />);
+
+    expect(screen.getByRole("status")).toHaveTextContent("Reconnecting…");
+    expect(
+      screen.queryByRole("button", { name: "Refresh" }),
+    ).not.toBeInTheDocument();
+  });
+
+  it("offers a reload once the outage has outlived the refresh window", async () => {
+    const user = userEvent.setup();
+    degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS - 1_000;
+    render(<LiveUpdatesStatus />);
+
+    const refresh = screen.getByRole("button", { name: "Refresh" });
+    await user.hover(refresh);
+    const tooltip = await screen.findByRole("tooltip");
+    expect(
+      within(tooltip).getByText("Reload for the latest calendar"),
+    ).toBeInTheDocument();
+    expect(within(tooltip).getByText("R")).toBeInTheDocument();
+
+    // Bare focus() opens the tooltip outside React's act; wrap it.
+    act(() => refresh.focus());
+    await user.keyboard("{Enter}");
+    expect(reloadLocation).toHaveBeenCalledTimes(1);
+  });
+
+  it("adds the refresh control when the remaining window elapses", async () => {
+    degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS + 50;
+    render(<LiveUpdatesStatus />);
+
+    expect(
+      screen.queryByRole("button", { name: "Refresh" }),
+    ).not.toBeInTheDocument();
+    expect(
+      await screen.findByRole("button", { name: "Refresh" }),
+    ).toBeInTheDocument();
+  });
+
+  it("clears the badge when the stream reopens", () => {
+    degradedSinceMs = Date.now() - REFRESH_AFTER_DEGRADED_MS - 1_000;
+    const { rerender } = render(<LiveUpdatesStatus />);
+    expect(screen.getByRole("status")).toBeInTheDocument();
+
+    degradedSinceMs = null;
+    rerender(<LiveUpdatesStatus />);
+
+    expect(screen.queryByRole("status")).not.toBeInTheDocument();
+    expect(
+      screen.queryByRole("button", { name: "Refresh" }),
+    ).not.toBeInTheDocument();
+  });
+});

packages/web/src/components/CalendarHeader/LiveUpdatesStatus.tsx

@@ -0,0 +1,76 @@
+import { ArrowClockwiseIcon } from "@phosphor-icons/react";
+import { type FC, useEffect, useState } from "react";
+import { reloadLocation } from "@web/common/utils/browser/browser-navigation.util";
+import { TooltipWrapper } from "@web/components/Tooltip/TooltipWrapper";
+import { useSseDegradedSince } from "@web/sse/hooks/useSseDegraded";
+
+// Native EventSource retries on its own for transient drops but gives up for
+// good once the browser marks it CLOSED. Past this age the badge stops
+// implying recovery is imminent and offers the reload that always works.
+export const REFRESH_AFTER_DEGRADED_MS = 30_000;
+
+const useRefreshDue = (degradedSinceMs: number | null): boolean => {
+  const [due, setDue] = useState(false);
+
+  useEffect(() => {
+    if (degradedSinceMs === null) {
+      setDue(false);
+      return;
+    }
+    const remainingMs =
+      degradedSinceMs + REFRESH_AFTER_DEGRADED_MS - Date.now();
+    if (remainingMs <= 0) {
+      setDue(true);
+      return;
+    }
+    setDue(false);
+    const timer = setTimeout(() => setDue(true), remainingMs);
+    return () => clearTimeout(timer);
+  }, [degradedSinceMs]);
+
+  return due;
+};
+
+/**
+ * Header badge for a live-update stream that has been down for 15s+ (see
+ * sse.client's degraded window). Clears itself when the stream reopens. After
+ * 30s it adds a Refresh control that reloads the page; like
+ * UpdateAvailableButton, the tooltip and blocked-click hint name Mod+R
+ * because pointer clicks are suppressed in the keyboard-only calendar.
+ */
+export const LiveUpdatesStatus: FC = () => {
+  const degradedSinceMs = useSseDegradedSince();
+  const refreshDue = useRefreshDue(degradedSinceMs);
+
+  if (degradedSinceMs === null) return null;
+
+  return (
+    <div
+      aria-live="polite"
+      className="flex shrink-0 items-center gap-2 text-warning text-xs"
+      role="status"
+    >
+      <span
+        aria-hidden="true"
+        className="size-1.5 shrink-0 rounded-full bg-warning motion-safe:animate-pulse"
+      />
+      <span>Reconnecting…</span>
+      {refreshDue ? (
+        <TooltipWrapper
+          description="Reload for the latest calendar"
+          onClick={reloadLocation}
+          shortcut={["Mod", "R"]}
+        >
+          <button
+            aria-label="Refresh"
+            className="c-focus-ring inline-flex items-center gap-1 rounded-xs px-1.5 py-0.5 font-medium text-text hover:bg-surface-overlay"
+            type="button"
+          >
+            <ArrowClockwiseIcon aria-hidden="true" size={14} />
+            Refresh
+          </button>
+        </TooltipWrapper>
+      ) : null}
+    </div>
+  );
+};

packages/web/src/sse/client/sse.client.test.ts

@@ -1,6 +1,7 @@
 import * as posthogBootstrap from "@web/auth/posthog/posthog.bootstrap";
 import {
   closeStream,
+  getSseDegradedSinceMs,
   isSseDegraded,
   openStream,
   subscribeSseDegraded,
@@ -191,6 +192,44 @@ describe("sse.client degraded state", () => {
     expect(isSseDegraded()).toBe(false);
   });
 
+  it("records when degradation began and keeps the first timestamp", () => {
+    expect(getSseDegradedSinceMs()).toBeNull();
+
+    openStream();
+    fakeEs.dispatch("error");
+    runDegradedTimer();
+    expect(getSseDegradedSinceMs()).toBe(nowMs);
+
+    // A later error re-arms the timer; a second fire must not restart the
+    // header's reload countdown.
+    const firstDegradedAt = nowMs;
+    nowMs += 20_000;
+    fakeEs.dispatch("error");
+    for (const timer of timerCallbacks.filter((t) => t.delayMs === 15_000)) {
+      timer.callback();
+    }
+    expect(getSseDegradedSinceMs()).toBe(firstDegradedAt);
+  });
+
+  it("clears the degraded timestamp on reopen and on close", () => {
+    openStream();
+    fakeEs.dispatch("error");
+    runDegradedTimer();
+    fakeEs.readyState = FakeEventSource.OPEN;
+    fakeEs.dispatch("open");
+    expect(getSseDegradedSinceMs()).toBeNull();
+
+    fakeEs.readyState = FakeEventSource.CONNECTING;
+    fakeEs.dispatch("error");
+    for (const timer of timerCallbacks.filter((t) => t.delayMs === 15_000)) {
+      timer.callback();
+    }
+    expect(getSseDegradedSinceMs()).toBe(nowMs);
+
+    closeStream();
+    expect(getSseDegradedSinceMs()).toBeNull();
+  });
+
   it("captures diagnostic properties on the first degraded report", () => {
     window.history.replaceState(null, "", "/week");
     openStream();

packages/web/src/sse/client/sse.client.ts

@@ -73,19 +73,26 @@ function resetConnectionDiagnostics() {
   lastErrorType = "timeout";
 }
 
-// Whether the live stream has been down long enough that displayed data can
-// no longer be trusted as fresh. Previously this was analytics-only
-// (sse_connection_degraded, PostHog) with no UI representation at all: a tab
-// with a dead stream kept showing "Calendar connected" and a "Updated N
-// minutes ago" timestamp that both silently stopped being true.
-const sseDegradedStore = createExternalStore(false);
+// Epoch ms at which the live stream had been down long enough that displayed
+// data can no longer be trusted as fresh, or null while healthy. Previously
+// this was analytics-only (sse_connection_degraded, PostHog) with no UI
+// representation at all: a tab with a dead stream kept showing "Calendar
+// connected" and a "Updated N minutes ago" timestamp that both silently
+// stopped being true. The timestamp (not a boolean) lets the header offer a
+// reload once an outage has outlived native reconnect's usefulness, and it
+// survives header remounts on view switches.
+const sseDegradedSinceStore = createExternalStore<number | null>(null);
 
 export function isSseDegraded(): boolean {
-  return sseDegradedStore.get();
+  return sseDegradedSinceStore.get() !== null;
+}
+
+export function getSseDegradedSinceMs(): number | null {
+  return sseDegradedSinceStore.get();
 }
 
 export function subscribeSseDegraded(onChange: () => void): () => void {
-  return sseDegradedStore.subscribe(onChange);
+  return sseDegradedSinceStore.subscribe(onChange);
 }
 
 function clearDegradedTimer() {
@@ -96,7 +103,11 @@ function clearDegradedTimer() {
 }
 
 function reportSseDegraded() {
-  sseDegradedStore.set(true);
+  // The timer re-arms on every error, so it can fire more than once per
+  // outage; the badge keeps the first timestamp.
+  if (sseDegradedSinceStore.get() === null) {
+    sseDegradedSinceStore.set(Date.now());
+  }
   if (hasReportedDegraded) return;
   hasReportedDegraded = true;
   try {
@@ -158,7 +169,7 @@ export const openStream = (): EventSource => {
     connectionOpenedAtMs = Date.now();
     userEventCount = 0;
     lastErrorType = "timeout";
-    sseDegradedStore.set(false);
+    sseDegradedSinceStore.set(null);
     for (const listener of reopenListeners) {
       listener();
     }
@@ -183,7 +194,7 @@ export const openStream = (): EventSource => {
 
 export const closeStream = (): void => {
   clearDegradedTimer();
-  sseDegradedStore.set(false);
+  sseDegradedSinceStore.set(null);
   if (es && forwardingHandler) {
     es.removeEventListener(SSE_MESSAGE_EVENT, forwardingHandler);
   }

packages/web/src/sse/hooks/useSseDegraded.ts

@@ -1,5 +1,6 @@
 import { useSyncExternalStore } from "react";
 import {
+  getSseDegradedSinceMs,
   isSseDegraded,
   subscribeSseDegraded,
 } from "@web/sse/client/sse.client";
@@ -13,3 +14,12 @@ import {
 export function useSseDegraded(): boolean {
   return useSyncExternalStore(subscribeSseDegraded, isSseDegraded);
 }
+
+/**
+ * Epoch ms at which the stream became degraded, or null while healthy. The
+ * header uses it to offer a reload once an outage has run long enough that
+ * native EventSource reconnect is unlikely to recover on its own.
+ */
+export function useSseDegradedSince(): number | null {
+  return useSyncExternalStore(subscribeSseDegraded, getSseDegradedSinceMs);
+}