feat(studio-desktop): signed background helper — db/serve survive app close/crash (#599)

apps/axctl/src/cli/commands/serve.ts

@@ -24,8 +24,29 @@ const serveStopCommand = Command.make("stop", {}, () =>
 
 export const serveCommand = Command.make(
     "serve",
-    { port: Flag.integer("port").pipe(Flag.withDefault(DEFAULT_DASHBOARD_PORT)) },
-    ({ port }) => Effect.promise(() => serveDashboard([`--port=${port}`])),
+    {
+        port: Flag.integer("port").pipe(Flag.withDefault(DEFAULT_DASHBOARD_PORT)),
+        managedDb: Flag.boolean("managed-db").pipe(
+            Flag.withDefault(false),
+            Flag.withDescription(
+                "Spawn and supervise the bundled surreal binary as a child process before serving. " +
+                "Resolves surreal as a sibling of the bun execPath (used by the macOS background helper).",
+            ),
+        ),
+        ingestEvery: Flag.string("ingest-every").pipe(
+            Flag.optional,
+            Flag.withDescription(
+                "Run the ingest pipeline on this interval (e.g. '2m', '30s'). " +
+                "Requires a running DB. Off by default.",
+            ),
+        ),
+    },
+    ({ port, managedDb, ingestEvery }) => {
+        const args: string[] = [`--port=${port}`];
+        if (managedDb) args.push("--managed-db");
+        if (ingestEvery._tag === "Some") args.push(`--ingest-every=${ingestEvery.value}`);
+        return Effect.promise(() => serveDashboard(args));
+    },
 ).pipe(
     Command.withDescription("Serve the live web dashboard locally (status/stop manage a running daemon)"),
     Command.withSubcommands([serveStatusCommand, serveStopCommand]),

apps/axctl/src/cli/install.test.ts

@@ -6,6 +6,7 @@ import {
     formatDaemonStatus,
     formatDoctorReport,
     parseDaemonCommand,
+    probeDbQueryPure,
     resolveDaemonHostPort,
     staleRunningIngestRuns,
     watcherProfilePublishDoctorCheck,
@@ -118,6 +119,7 @@ describe("cli install operations", () => {
             dataDir: "/tmp/ax",
             logDir: "/tmp/ax/logs",
             dbListening: true,
+            dbQueryOk: true,
             endpoint: {
                 host: "127.0.0.1",
                 port: 8521,
@@ -156,6 +158,7 @@ describe("cli install operations", () => {
             dataDir: "/tmp/ax",
             logDir: "/tmp/ax/logs",
             dbListening: true,
+            dbQueryOk: true,
             endpoint: {
                 host: "127.0.0.1",
                 port: 8521,
@@ -177,6 +180,7 @@ describe("cli install operations", () => {
             dataDir: "/tmp/ax",
             logDir: "/tmp/ax/logs",
             dbListening: true,
+            dbQueryOk: true,
             endpoint: {
                 host: "127.0.0.1",
                 port: 8521,
@@ -195,6 +199,33 @@ describe("cli install operations", () => {
         expect(text).not.toContain("conflict: port");
     });
 
+    // --- TDD: wedge detector (Step 1 - written failing before formatter branch exists) ---
+    test("wedged db: port listening but query timed out → shows 'wedged' and restart hint", () => {
+        const status: DaemonStatus = {
+            platform: "darwin",
+            macosLaunchd: true,
+            dataDir: "/tmp/ax",
+            logDir: "/tmp/ax/logs",
+            dbListening: true,
+            dbQueryOk: false,
+            endpoint: {
+                host: "127.0.0.1",
+                port: 8521,
+                url: "ws://127.0.0.1:8521",
+                listening: true,
+                conflict: null,
+                runtimeStatePath: "/tmp/ax/runtime.json",
+            },
+            ideModel: false,
+            agents: [],
+        };
+        const text = formatDaemonStatus(status);
+        expect(text).toContain("wedged");
+        expect(text).toContain("ax daemon restart");
+        // Must NOT claim it's healthy
+        expect(text).not.toMatch(/database: listening on 127\.0\.0\.1:8521\s*$/m);
+    });
+
     test("formats doctor checks", () => {
         const report: DoctorReport = {
             platform: "darwin",
@@ -238,6 +269,19 @@ describe("cli install operations", () => {
     });
 });
 
+describe("probeDbQueryPure (wedge probe: fail-closed on dead port)", () => {
+    // This test exercises the fail-closed contract on a guaranteed-dead port (1)
+    // WITHOUT touching the live db on :8521. The 1.5s timeout must not hang.
+    test("returns false quickly for a port nobody listens on", async () => {
+        const start = Date.now();
+        const result = await probeDbQueryPure("127.0.0.1", 1); // port 1 = unreachable
+        const elapsed = Date.now() - start;
+        expect(result).toBe(false);
+        // Must resolve within 3s (well under the 1.5s probe timeout + connection refuse)
+        expect(elapsed).toBeLessThan(3000);
+    });
+});
+
 describe("doctor stale ingest_run detection", () => {
     const NOW = Date.parse("2026-06-11T12:00:00.000Z");
     const STALE_AFTER_MS = 960_000; // 900s timeout + 60s grace

apps/axctl/src/cli/install.ts

@@ -606,6 +606,14 @@ export interface DaemonStatus {
     readonly dataDir: string;
     readonly logDir: string;
     readonly dbListening: boolean;
+    /**
+     * Result of a real `SELECT 1` round-trip against the DB endpoint.
+     * - `true`  → port listening AND query answered (healthy)
+     * - `false` → port listening BUT query timed out / errored (wedged - the
+     *             production incident: socket accepts but queries hang forever)
+     * - `null`  → port not listening (probe skipped)
+     */
+    readonly dbQueryOk: boolean | null;
     readonly endpoint: DaemonEndpoint;
     readonly agents: readonly AgentRuntimeStatus[];
     /**
@@ -804,12 +812,18 @@ function collectDaemonStatus(): Effect.Effect<DaemonStatus, never, FileSystem.Fi
     return Effect.gen(function* () {
         const ideModel = yield* detectIdeModel();
         const endpoint = yield* collectDaemonEndpoint(ideModel);
+        // Probe a real query round-trip ONLY when the socket is open, with a
+        // hard timeout so a wedged db can't hang this command.
+        const dbQueryOk: boolean | null = endpoint.listening
+            ? yield* Effect.promise(() => probeDbQueryPure(endpoint.host, endpoint.port))
+            : null;
         return {
             platform: process.platform,
             macosLaunchd: isMacos(),
             dataDir: DATA_DIR,
             logDir: LOG_DIR,
             dbListening: endpoint.listening,
+            dbQueryOk,
             endpoint,
             ideModel,
             agents: [
@@ -827,9 +841,11 @@ export function formatDaemonStatus(status: DaemonStatus, json = false): string {
     if (json) return prettyPrint(status);
     const ep = status.endpoint;
     const endpointDesc = `${ep.host}:${ep.port}`;
-    const dbLine = status.dbListening
-        ? `listening on ${endpointDesc}`
-        : `not listening on ${endpointDesc}`;
+    const dbLine = !status.dbListening
+        ? `not listening on ${endpointDesc}`
+        : status.dbQueryOk === false
+            ? `listening on ${endpointDesc} but NOT answering queries (wedged) - restart with \`ax daemon restart\``
+            : `listening on ${endpointDesc}`;
     const lines = [
         "axctl daemon",
         `  platform: ${status.platform}${status.macosLaunchd ? "" : " (launchd unavailable)"}`,
@@ -856,6 +872,38 @@ export function formatDaemonStatus(status: DaemonStatus, json = false): string {
     return lines.join("\n");
 }
 
+/**
+ * Execute a real `SELECT 1` round-trip against SurrealDB's HTTP `/sql`
+ * endpoint to detect a WEDGED database - one whose socket is open but whose
+ * query engine is hung and never replies (the production incident: status
+ * reported "listening" the whole time the db was stuck).
+ *
+ * Fail-CLOSED: any error or timeout → `false`. Hard 1.5s timeout so this
+ * NEVER hangs `ax daemon status` even when the db is fully wedged.
+ *
+ * Exported as `probeDbQueryPure` for unit tests (verifies the dead-port
+ * fail-closed path without touching the live :8521 db).
+ */
+export async function probeDbQueryPure(host: string, port: number): Promise<boolean> {
+    try {
+        const db = readDbEnvConfig();
+        const res = await fetch(`http://${host}:${port}/sql`, {
+            method: "POST",
+            headers: {
+                Accept: "application/json",
+                Authorization: `Basic ${Buffer.from(`${db.user}:${db.pass}`).toString("base64")}`,
+                "surreal-ns": db.ns,
+                "surreal-db": db.db,
+            },
+            body: "SELECT 1;",
+            signal: AbortSignal.timeout(1500),
+        });
+        return true;
+    } catch {
+        return false;
+    }
+}
+
 /**
  * Buckets schema.surql defines. A missing one means the schema import rolled
  * back partway - historically a bucket BACKEND path outside the daemon's
@@ -986,7 +1034,7 @@ export function collectDoctorReport(): Effect.Effect<
         const watcherPlistText = yield* fs
             .readFileString(WATCH_PLIST)
             .pipe(orAbsent<string | null>(null));
-        const dbReachable = daemon.dbListening && daemon.endpoint.conflict === null;
+        const dbReachable = daemon.dbListening && daemon.dbQueryOk !== false && daemon.endpoint.conflict === null;
         const missingBuckets = dbReachable
             ? yield* Effect.promise(() => probeMissingBuckets(daemon.endpoint))
             : null;

apps/axctl/src/dashboard/SurrealWatchdog.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Tests for the SurrealDB wedge watchdog.
+ *
+ * Uses TestClock so that time-dependent Schedule behaviour is deterministic
+ * and millisecond-fast rather than waiting for real wall-clock intervals.
+ */
+import { describe, it, expect } from "bun:test";
+import { Duration, Effect, Fiber, Ref } from "effect";
+import { TestClock } from "effect/testing";
+import { makeSurrealWatchdog } from "./SurrealWatchdog.ts";
+
+// Helper: run an Effect with a TestClock so time is deterministic.
+const runTest = <A>(effect: Effect.Effect<A, unknown, never>): Promise<A> =>
+    Effect.runPromise(effect.pipe(Effect.provide(TestClock.layer())));
+
+describe("makeSurrealWatchdog", () => {
+    it("trips onWedged after failuresToTrip consecutive failures", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe: Effect.succeed(false), // always wedged
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                // Advance 15 s → sleep(5s) fires 3 times → counter reaches 3 → 1 trip
+                yield* TestClock.adjust(Duration.seconds(15));
+                expect(yield* Ref.get(trips)).toBe(1);
+            }),
+        ));
+
+    it("does NOT trip when failures are fewer than failuresToTrip", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe: Effect.succeed(false),
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                // 2 ticks → counter=2, not yet tripped
+                yield* TestClock.adjust(Duration.seconds(10));
+                expect(yield* Ref.get(trips)).toBe(0);
+            }),
+        ));
+
+    it("resets counter and re-arms after a trip", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe: Effect.succeed(false),
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                // 6 ticks → trips at 3 (reset), trips again at 6 → 2 trips total
+                yield* TestClock.adjust(Duration.seconds(30));
+                expect(yield* Ref.get(trips)).toBe(2);
+            }),
+        ));
+
+    it("resets counter to 0 on a successful probe", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                let callCount = 0;
+                // 2 failures, then 1 success, then 2 more failures → no trip
+                const probe = Effect.sync(() => {
+                    callCount++;
+                    if (callCount === 3) return true; // success resets counter
+                    return false;
+                });
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe,
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                // 4 ticks: fail, fail, success(reset), fail → counter=1, no trip
+                yield* TestClock.adjust(Duration.seconds(20));
+                expect(yield* Ref.get(trips)).toBe(0);
+            }),
+        ));
+
+    it("trips exactly once per failuresToTrip failures even with mixed results", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                let callCount = 0;
+                // pattern: F F F S F F F → trip at 3, reset, 3 more failures → 2 trips
+                const probe = Effect.sync(() => {
+                    callCount++;
+                    // success only at tick 4
+                    return callCount === 4;
+                });
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe,
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                // 7 ticks: F F F(trip) S F F F(trip)
+                yield* TestClock.adjust(Duration.seconds(35));
+                expect(yield* Ref.get(trips)).toBe(2);
+            }),
+        ));
+
+    it("probe failure (Effect failure, not false return) is treated as not-ok", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe: Effect.fail(new Error("connection refused")),
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                yield* TestClock.adjust(Duration.seconds(15));
+                expect(yield* Ref.get(trips)).toBe(1);
+            }),
+        ));
+
+    it("interruption of the forked fiber stops the watchdog cleanly", () =>
+        runTest(
+            Effect.gen(function* () {
+                const trips = yield* Ref.make(0);
+                const fiber = yield* Effect.forkChild(
+                    makeSurrealWatchdog({
+                        probe: Effect.succeed(false),
+                        onWedged: Ref.update(trips, (n) => n + 1),
+                        interval: Duration.seconds(5),
+                        failuresToTrip: 3,
+                    }),
+                );
+                yield* TestClock.adjust(Duration.seconds(10));
+                yield* Fiber.interrupt(fiber);
+                // No trip should have occurred (only 2 ticks)
+                expect(yield* Ref.get(trips)).toBe(0);
+            }),
+        ));
+});