diff --git a/apps/axctl/src/cli/commands/serve.ts b/apps/axctl/src/cli/commands/serve.ts index d13d19df..70d8604f 100644 --- a/apps/axctl/src/cli/commands/serve.ts +++ b/apps/axctl/src/cli/commands/serve.ts @@ -24,8 +24,29 @@ const serveStopCommand = Command.make("stop", {}, () => export const serveCommand = Command.make( "serve", - { port: Flag.integer("port").pipe(Flag.withDefault(DEFAULT_DASHBOARD_PORT)) }, - ({ port }) => Effect.promise(() => serveDashboard([`--port=${port}`])), + { + port: Flag.integer("port").pipe(Flag.withDefault(DEFAULT_DASHBOARD_PORT)), + managedDb: Flag.boolean("managed-db").pipe( + Flag.withDefault(false), + Flag.withDescription( + "Spawn and supervise the bundled surreal binary as a child process before serving. " + + "Resolves surreal as a sibling of the bun execPath (used by the macOS background helper).", + ), + ), + ingestEvery: Flag.string("ingest-every").pipe( + Flag.optional, + Flag.withDescription( + "Run the ingest pipeline on this interval (e.g. '2m', '30s'). " + + "Requires a running DB. Off by default.", + ), + ), + }, + ({ port, managedDb, ingestEvery }) => { + const args: string[] = [`--port=${port}`]; + if (managedDb) args.push("--managed-db"); + if (ingestEvery._tag === "Some") args.push(`--ingest-every=${ingestEvery.value}`); + return Effect.promise(() => serveDashboard(args)); + }, ).pipe( Command.withDescription("Serve the live web dashboard locally (status/stop manage a running daemon)"), Command.withSubcommands([serveStatusCommand, serveStopCommand]), diff --git a/apps/axctl/src/cli/install.test.ts b/apps/axctl/src/cli/install.test.ts index 4c677c45..26fd3262 100644 --- a/apps/axctl/src/cli/install.test.ts +++ b/apps/axctl/src/cli/install.test.ts @@ -6,6 +6,7 @@ import { formatDaemonStatus, formatDoctorReport, parseDaemonCommand, + probeDbQueryPure, resolveDaemonHostPort, staleRunningIngestRuns, watcherProfilePublishDoctorCheck, @@ -118,6 +119,7 @@ describe("cli install operations", () => { dataDir: "/tmp/ax", logDir: "/tmp/ax/logs", dbListening: true, + dbQueryOk: true, endpoint: { host: "127.0.0.1", port: 8521, @@ -156,6 +158,7 @@ describe("cli install operations", () => { dataDir: "/tmp/ax", logDir: "/tmp/ax/logs", dbListening: true, + dbQueryOk: true, endpoint: { host: "127.0.0.1", port: 8521, @@ -177,6 +180,7 @@ describe("cli install operations", () => { dataDir: "/tmp/ax", logDir: "/tmp/ax/logs", dbListening: true, + dbQueryOk: true, endpoint: { host: "127.0.0.1", port: 8521, @@ -195,6 +199,33 @@ describe("cli install operations", () => { expect(text).not.toContain("conflict: port"); }); + // --- TDD: wedge detector (Step 1 - written failing before formatter branch exists) --- + test("wedged db: port listening but query timed out → shows 'wedged' and restart hint", () => { + const status: DaemonStatus = { + platform: "darwin", + macosLaunchd: true, + dataDir: "/tmp/ax", + logDir: "/tmp/ax/logs", + dbListening: true, + dbQueryOk: false, + endpoint: { + host: "127.0.0.1", + port: 8521, + url: "ws://127.0.0.1:8521", + listening: true, + conflict: null, + runtimeStatePath: "/tmp/ax/runtime.json", + }, + ideModel: false, + agents: [], + }; + const text = formatDaemonStatus(status); + expect(text).toContain("wedged"); + expect(text).toContain("ax daemon restart"); + // Must NOT claim it's healthy + expect(text).not.toMatch(/database: listening on 127\.0\.0\.1:8521\s*$/m); + }); + test("formats doctor checks", () => { const report: DoctorReport = { platform: "darwin", @@ -238,6 +269,19 @@ describe("cli install operations", () => { }); }); +describe("probeDbQueryPure (wedge probe: fail-closed on dead port)", () => { + // This test exercises the fail-closed contract on a guaranteed-dead port (1) + // WITHOUT touching the live db on :8521. The 1.5s timeout must not hang. + test("returns false quickly for a port nobody listens on", async () => { + const start = Date.now(); + const result = await probeDbQueryPure("127.0.0.1", 1); // port 1 = unreachable + const elapsed = Date.now() - start; + expect(result).toBe(false); + // Must resolve within 3s (well under the 1.5s probe timeout + connection refuse) + expect(elapsed).toBeLessThan(3000); + }); +}); + describe("doctor stale ingest_run detection", () => { const NOW = Date.parse("2026-06-11T12:00:00.000Z"); const STALE_AFTER_MS = 960_000; // 900s timeout + 60s grace diff --git a/apps/axctl/src/cli/install.ts b/apps/axctl/src/cli/install.ts index 64774cd2..fb2ad8c8 100644 --- a/apps/axctl/src/cli/install.ts +++ b/apps/axctl/src/cli/install.ts @@ -606,6 +606,14 @@ export interface DaemonStatus { readonly dataDir: string; readonly logDir: string; readonly dbListening: boolean; + /** + * Result of a real `SELECT 1` round-trip against the DB endpoint. + * - `true` → port listening AND query answered (healthy) + * - `false` → port listening BUT query timed out / errored (wedged - the + * production incident: socket accepts but queries hang forever) + * - `null` → port not listening (probe skipped) + */ + readonly dbQueryOk: boolean | null; readonly endpoint: DaemonEndpoint; readonly agents: readonly AgentRuntimeStatus[]; /** @@ -804,12 +812,18 @@ function collectDaemonStatus(): Effect.Effect probeDbQueryPure(endpoint.host, endpoint.port)) + : null; return { platform: process.platform, macosLaunchd: isMacos(), dataDir: DATA_DIR, logDir: LOG_DIR, dbListening: endpoint.listening, + dbQueryOk, endpoint, ideModel, agents: [ @@ -827,9 +841,11 @@ export function formatDaemonStatus(status: DaemonStatus, json = false): string { if (json) return prettyPrint(status); const ep = status.endpoint; const endpointDesc = `${ep.host}:${ep.port}`; - const dbLine = status.dbListening - ? `listening on ${endpointDesc}` - : `not listening on ${endpointDesc}`; + const dbLine = !status.dbListening + ? `not listening on ${endpointDesc}` + : status.dbQueryOk === false + ? `listening on ${endpointDesc} but NOT answering queries (wedged) - restart with \`ax daemon restart\`` + : `listening on ${endpointDesc}`; const lines = [ "axctl daemon", ` platform: ${status.platform}${status.macosLaunchd ? "" : " (launchd unavailable)"}`, @@ -856,6 +872,38 @@ export function formatDaemonStatus(status: DaemonStatus, json = false): string { return lines.join("\n"); } +/** + * Execute a real `SELECT 1` round-trip against SurrealDB's HTTP `/sql` + * endpoint to detect a WEDGED database - one whose socket is open but whose + * query engine is hung and never replies (the production incident: status + * reported "listening" the whole time the db was stuck). + * + * Fail-CLOSED: any error or timeout → `false`. Hard 1.5s timeout so this + * NEVER hangs `ax daemon status` even when the db is fully wedged. + * + * Exported as `probeDbQueryPure` for unit tests (verifies the dead-port + * fail-closed path without touching the live :8521 db). + */ +export async function probeDbQueryPure(host: string, port: number): Promise { + try { + const db = readDbEnvConfig(); + const res = await fetch(`http://${host}:${port}/sql`, { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Basic ${Buffer.from(`${db.user}:${db.pass}`).toString("base64")}`, + "surreal-ns": db.ns, + "surreal-db": db.db, + }, + body: "SELECT 1;", + signal: AbortSignal.timeout(1500), + }); + return true; + } catch { + return false; + } +} + /** * Buckets schema.surql defines. A missing one means the schema import rolled * back partway - historically a bucket BACKEND path outside the daemon's @@ -986,7 +1034,7 @@ export function collectDoctorReport(): Effect.Effect< const watcherPlistText = yield* fs .readFileString(WATCH_PLIST) .pipe(orAbsent(null)); - const dbReachable = daemon.dbListening && daemon.endpoint.conflict === null; + const dbReachable = daemon.dbListening && daemon.dbQueryOk !== false && daemon.endpoint.conflict === null; const missingBuckets = dbReachable ? yield* Effect.promise(() => probeMissingBuckets(daemon.endpoint)) : null; diff --git a/apps/axctl/src/dashboard/SurrealWatchdog.test.ts b/apps/axctl/src/dashboard/SurrealWatchdog.test.ts new file mode 100644 index 00000000..06df2bf9 --- /dev/null +++ b/apps/axctl/src/dashboard/SurrealWatchdog.test.ts @@ -0,0 +1,156 @@ +/** + * Tests for the SurrealDB wedge watchdog. + * + * Uses TestClock so that time-dependent Schedule behaviour is deterministic + * and millisecond-fast rather than waiting for real wall-clock intervals. + */ +import { describe, it, expect } from "bun:test"; +import { Duration, Effect, Fiber, Ref } from "effect"; +import { TestClock } from "effect/testing"; +import { makeSurrealWatchdog } from "./SurrealWatchdog.ts"; + +// Helper: run an Effect with a TestClock so time is deterministic. +const runTest = (effect: Effect.Effect): Promise => + Effect.runPromise(effect.pipe(Effect.provide(TestClock.layer()))); + +describe("makeSurrealWatchdog", () => { + it("trips onWedged after failuresToTrip consecutive failures", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe: Effect.succeed(false), // always wedged + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + // Advance 15 s → sleep(5s) fires 3 times → counter reaches 3 → 1 trip + yield* TestClock.adjust(Duration.seconds(15)); + expect(yield* Ref.get(trips)).toBe(1); + }), + )); + + it("does NOT trip when failures are fewer than failuresToTrip", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe: Effect.succeed(false), + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + // 2 ticks → counter=2, not yet tripped + yield* TestClock.adjust(Duration.seconds(10)); + expect(yield* Ref.get(trips)).toBe(0); + }), + )); + + it("resets counter and re-arms after a trip", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe: Effect.succeed(false), + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + // 6 ticks → trips at 3 (reset), trips again at 6 → 2 trips total + yield* TestClock.adjust(Duration.seconds(30)); + expect(yield* Ref.get(trips)).toBe(2); + }), + )); + + it("resets counter to 0 on a successful probe", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + let callCount = 0; + // 2 failures, then 1 success, then 2 more failures → no trip + const probe = Effect.sync(() => { + callCount++; + if (callCount === 3) return true; // success resets counter + return false; + }); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe, + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + // 4 ticks: fail, fail, success(reset), fail → counter=1, no trip + yield* TestClock.adjust(Duration.seconds(20)); + expect(yield* Ref.get(trips)).toBe(0); + }), + )); + + it("trips exactly once per failuresToTrip failures even with mixed results", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + let callCount = 0; + // pattern: F F F S F F F → trip at 3, reset, 3 more failures → 2 trips + const probe = Effect.sync(() => { + callCount++; + // success only at tick 4 + return callCount === 4; + }); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe, + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + // 7 ticks: F F F(trip) S F F F(trip) + yield* TestClock.adjust(Duration.seconds(35)); + expect(yield* Ref.get(trips)).toBe(2); + }), + )); + + it("probe failure (Effect failure, not false return) is treated as not-ok", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + yield* Effect.forkChild( + makeSurrealWatchdog({ + probe: Effect.fail(new Error("connection refused")), + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + yield* TestClock.adjust(Duration.seconds(15)); + expect(yield* Ref.get(trips)).toBe(1); + }), + )); + + it("interruption of the forked fiber stops the watchdog cleanly", () => + runTest( + Effect.gen(function* () { + const trips = yield* Ref.make(0); + const fiber = yield* Effect.forkChild( + makeSurrealWatchdog({ + probe: Effect.succeed(false), + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + }), + ); + yield* TestClock.adjust(Duration.seconds(10)); + yield* Fiber.interrupt(fiber); + // No trip should have occurred (only 2 ticks) + expect(yield* Ref.get(trips)).toBe(0); + }), + )); +}); diff --git a/apps/axctl/src/dashboard/SurrealWatchdog.ts b/apps/axctl/src/dashboard/SurrealWatchdog.ts new file mode 100644 index 00000000..fdcad96e --- /dev/null +++ b/apps/axctl/src/dashboard/SurrealWatchdog.ts @@ -0,0 +1,114 @@ +/** + * Real-query watchdog for the managed SurrealDB process. + * + * The 4-day production incident: SurrealDB held its LISTEN socket while + * refusing to answer queries. KeepAlive only restarts on process *exit* - a + * wedge never exits, so nothing recovered. This watchdog detects the wedge + * via a real query round-trip (not `/health`, which passes on a wedge) and + * force-restarts via SIGKILL. + * + * Design: + * - Pure / TestClock-drivable: no wall-clock imports, only `Effect.sleep` + * and `Ref` for state. + * - The loop body: sleep(interval) → probe → update counter → maybe trip. + * - Sleep-first so the first probe fires at t=interval, not t=0 (lets the + * managed process settle before the first check). + * - `probe` returning false OR failing (e.g. timeout, connection refused) + * both count as "not ok". + * - After a trip, the counter resets to 0 so the watchdog re-arms and will + * detect a subsequent wedge after the restart. + */ +import { Duration, Effect, Ref } from "effect"; + +// --------------------------------------------------------------------------- +// Public interface +// --------------------------------------------------------------------------- + +export interface SurrealWatchdogOpts { + /** + * Effect that probes the database. + * + * - Returns `true` → DB is responsive; success resets the failure counter. + * - Returns `false` → DB did not answer satisfactorily; increments counter. + * - Fails with any error → treated the same as returning `false`. + * + * The error channel is `unknown` so callers can pass any probe effect + * without needing to coerce errors first; the watchdog handles all + * failures via `Effect.orElseSucceed(() => false)`. + * + * The caller is responsible for supplying a 1-second hard timeout around + * the probe so that a wedged DB does not stall the loop indefinitely. + */ + readonly probe: Effect.Effect; + + /** + * Effect executed when `failuresToTrip` consecutive failures are detected. + * + * In production this SIGKILLs the wedged surreal pid and re-spawns it. + * Must not throw / fail in ways that stop the watchdog (the outer loop + * wraps it with `Effect.ignore`). + */ + readonly onWedged: Effect.Effect; + + /** Wait this long between probes. */ + readonly interval: Duration.Duration; + + /** + * Number of consecutive probe failures needed to fire `onWedged`. + * Reset to 0 after each trip and after each successful probe. + */ + readonly failuresToTrip: number; +} + +/** + * Returns an infinite Effect that implements the watchdog loop. + * + * Callers should `Effect.fork` or `Effect.forkScoped` the result so it runs + * as a background fiber. Interrupting the fiber stops the watchdog cleanly. + * + * The returned Effect has no service requirements: all dependencies (probe, + * onWedged) are provided by the caller via the options record. + * + * @example + * ```typescript + * yield* Effect.forkScoped(makeSurrealWatchdog({ + * probe: probeSelect1, + * onWedged: killAndRespawn, + * interval: Duration.seconds(15), + * failuresToTrip: 3, + * })); + * ``` + */ +export const makeSurrealWatchdog = ( + opts: SurrealWatchdogOpts, +): Effect.Effect => + Ref.make(0).pipe( + Effect.flatMap((counter) => + Effect.forever( + Effect.gen(function* () { + // Sleep first so the first probe fires after one interval, + // giving the managed process time to settle. + yield* Effect.sleep(opts.interval); + + // Treat both probe failure and probe returning false as "not ok". + const ok = yield* opts.probe.pipe( + Effect.map(Boolean), + Effect.orElseSucceed(() => false), + ); + + if (ok) { + // Healthy response - reset the failure streak. + yield* Ref.set(counter, 0); + } else { + const failures = yield* Ref.updateAndGet(counter, (n) => n + 1); + if (failures >= opts.failuresToTrip) { + // Trip the watchdog. + yield* opts.onWedged.pipe(Effect.ignore); + // Re-arm: reset counter so we detect the *next* wedge too. + yield* Ref.set(counter, 0); + } + } + }), + ), + ), + ); diff --git a/apps/axctl/src/dashboard/managed-db.test.ts b/apps/axctl/src/dashboard/managed-db.test.ts new file mode 100644 index 00000000..96bf43e5 --- /dev/null +++ b/apps/axctl/src/dashboard/managed-db.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect } from "bun:test"; +import { Duration, Effect, Layer, Ref } from "effect"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { resolveManagedSurrealPath, parseDurationString, makeManagedDb } from "./managed-db.ts"; + +describe("resolveManagedSurrealPath", () => { + it("resolves surreal as a sibling of the bun execPath", () => { + expect(resolveManagedSurrealPath("/Applications/ax studio.app/Contents/Resources/bin/arm64/bun")) + .toBe("/Applications/ax studio.app/Contents/Resources/bin/arm64/surreal"); + }); + + it("works for x64 arch", () => { + expect(resolveManagedSurrealPath("/Applications/ax studio.app/Contents/Resources/bin/x64/bun")) + .toBe("/Applications/ax studio.app/Contents/Resources/bin/x64/surreal"); + }); +}); + +describe("parseDurationString", () => { + it("parses '2m' as 2 minutes", () => { + const d = parseDurationString("2m"); + expect(d).not.toBeNull(); + expect(Duration.toMillis(d!)).toBe(2 * 60 * 1000); + }); + + it("parses '30s' as 30 seconds", () => { + const d = parseDurationString("30s"); + expect(d).not.toBeNull(); + expect(Duration.toMillis(d!)).toBe(30 * 1000); + }); + + it("parses '1h' as 1 hour", () => { + const d = parseDurationString("1h"); + expect(d).not.toBeNull(); + expect(Duration.toMillis(d!)).toBe(60 * 60 * 1000); + }); + + it("parses '500ms' as 500 milliseconds", () => { + const d = parseDurationString("500ms"); + expect(d).not.toBeNull(); + expect(Duration.toMillis(d!)).toBe(500); + }); + + it("returns null for unrecognised format", () => { + expect(parseDurationString("bad")).toBeNull(); + expect(parseDurationString("2 minutes")).toBeNull(); + expect(parseDurationString("")).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Pre-spawn idempotency check (restart-storm guard) +// --------------------------------------------------------------------------- +// These tests verify the owned-vs-attached distinction: +// attached (healthy probe) → NO spawn, NO finalizer, NO watchdog +// owned (unhealthy probe) → spawn IS invoked; watchdog + finalizer registered +// +// Stubbing approach: provide mock Layer implementations for +// ChildProcessSpawner (tracks spawn call count) and HttpClient (controls +// the probe response). Scope is satisfied by Effect.scoped. +// --------------------------------------------------------------------------- + +const TEST_OPTS = { + surrealPath: "/fake/surreal", + host: "127.0.0.1", + port: 19991, + dataDir: "/fake/data", +} as const; + +describe("makeManagedDb - pre-spawn idempotency (owned-vs-attached)", () => { + it("does NOT spawn when pre-spawn probe reports healthy (attach path)", async () => { + const spawnCount = await Effect.runPromise( + Effect.gen(function* () { + const countRef = yield* Ref.make(0); + + // Spawner that would fail loudly if ever called. + const mockSpawner = ChildProcessSpawner.make((_command) => + Ref.update(countRef, (n) => n + 1).pipe( + Effect.flatMap(() => + Effect.die(new Error("managed-db test: spawn must NOT be called in attach path")), + ), + ), + ); + + // HttpClient that always responds 200 → probe sees "healthy". + const mockHttp = HttpClient.make((req, _url, _signal, _fiber) => + Effect.succeed(HttpClientResponse.fromWeb(req, new Response(null, { status: 200 }))), + ); + + // Effect must succeed (early return, no spawn). + yield* makeManagedDb(TEST_OPTS).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, mockSpawner), + Layer.succeed(HttpClient.HttpClient, mockHttp), + ), + ), + ); + + return yield* Ref.get(countRef); + }), + ); + + // Spawner must never have been invoked. + expect(spawnCount).toBe(0); + }); + + it("DOES spawn when pre-spawn probe reports unhealthy (owned path)", async () => { + const spawnCount = await Effect.runPromise( + Effect.gen(function* () { + const countRef = yield* Ref.make(0); + + // Spawner: records the call, then fails fast (we only need to + // verify it was invoked; completing the full spawn flow would + // require a real process or a complex fake handle). + const mockSpawner = ChildProcessSpawner.make((_command) => + Ref.update(countRef, (n) => n + 1).pipe( + Effect.flatMap(() => + Effect.die(new Error("managed-db test: spawn stub - intentional fast fail")), + ), + ), + ); + + // HttpClient that always responds 503 → pre-spawn probe sees + // "not healthy" → proceeds to spawn; post-spawn readiness probe + // would also fail but spawn dies first so it's never reached. + const mockHttp = HttpClient.make((req, _url, _signal, _fiber) => + Effect.succeed(HttpClientResponse.fromWeb(req, new Response(null, { status: 503 }))), + ); + + // Capture the exit so the defect from the mock spawner doesn't + // escape to the test runner. We only care about spawnCount. + yield* Effect.exit( + makeManagedDb(TEST_OPTS).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, mockSpawner), + Layer.succeed(HttpClient.HttpClient, mockHttp), + ), + ), + ), + ); + + return yield* Ref.get(countRef); + }), + ); + + // Spawner must have been invoked exactly once. + expect(spawnCount).toBe(1); + }); +}); diff --git a/apps/axctl/src/dashboard/managed-db.ts b/apps/axctl/src/dashboard/managed-db.ts new file mode 100644 index 00000000..72262971 --- /dev/null +++ b/apps/axctl/src/dashboard/managed-db.ts @@ -0,0 +1,323 @@ +/** + * Managed SurrealDB child-process supervisor for `ax serve --managed-db`. + * + * When the macOS background helper (SMAppService plist) invokes: + * bun /apps/axctl/src/cli/index.ts serve --managed-db --port=1738 --ingest-every=2m + * this module spawns the bundled `surreal` binary as a supervised child, + * waits for its HTTP readiness probe, and registers a Scope finalizer that + * terminates it on shutdown. + * + * Bundle-location independence: `resolveManagedSurrealPath(process.execPath)` + * resolves `surreal` as a sibling of the bun binary - both live in + * `Contents/Resources/bin//` inside the app bundle. + */ +import { Data, Duration, Effect, Ref, Schedule, Scope, Stream } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { posixPath } from "@ax/lib/shared/path"; +import { makeSurrealWatchdog } from "./SurrealWatchdog.ts"; + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +export class ManagedDbError extends Data.TaggedError("ManagedDbError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +// --------------------------------------------------------------------------- +// Path resolution +// --------------------------------------------------------------------------- + +/** + * Resolve the surreal binary path as a sibling of the bun execPath. + * + * In the app bundle both bun and surreal live in the same directory: + * `Contents/Resources/bin//bun` + * `Contents/Resources/bin//surreal` + * + * @example + * resolveManagedSurrealPath("/App.app/Contents/Resources/bin/arm64/bun") + * // => "/App.app/Contents/Resources/bin/arm64/surreal" + */ +export const resolveManagedSurrealPath = (execPath: string): string => + posixPath.join(posixPath.dirname(execPath), "surreal"); + +// --------------------------------------------------------------------------- +// Duration string parsing ("2m", "30s", "1h") +// --------------------------------------------------------------------------- + +/** + * Parse compact duration strings used in the `--ingest-every` flag. + * Returns null for unrecognised formats. + * Supported units: ms, s, m, h, d. + */ +export const parseDurationString = (s: string): Duration.Duration | null => { + const match = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/.exec(s.trim()); + if (!match) return null; + const n = Number(match[1]); + switch (match[2]) { + case "ms": return Duration.millis(n); + case "s": return Duration.seconds(n); + case "m": return Duration.minutes(n); + case "h": return Duration.hours(n); + case "d": return Duration.days(n); + default: return null; + } +}; + +// --------------------------------------------------------------------------- +// Readiness probe constants +// --------------------------------------------------------------------------- + +const READINESS_TIMEOUT = Duration.seconds(30); +const READINESS_INTERVAL = Duration.millis(250); +const READINESS_REQUEST_TIMEOUT = Duration.seconds(2); +const TERMINATE_GRACE = Duration.seconds(5); + +/** + * Short timeout for the pre-spawn idempotency probe. + * Fail-closed: if nothing answers in 1 s we assume "not listening" and spawn. + */ +const PRE_SPAWN_PROBE_TIMEOUT = Duration.seconds(1); + +// --------------------------------------------------------------------------- +// Watchdog constants +// --------------------------------------------------------------------------- + +/** + * How long to wait between each SQL probe round-trip. + * 15 s is conservative but fast enough to detect a 4-day stall in <1 min. + */ +const WATCHDOG_INTERVAL = Duration.seconds(15); + +/** + * How many consecutive probe failures before we declare a wedge and + * force-restart. 3 × 15 s = 45 s before triggering a restart. + */ +const WATCHDOG_FAILURES_TO_TRIP = 3; + +// --------------------------------------------------------------------------- +// makeManagedDb +// --------------------------------------------------------------------------- + +/** + * Spawn a bundled `surreal` process, wait for its HTTP health probe, and + * register a Scope finalizer that terminates it. + * + * Requirements: + * - `Scope.Scope` - finalizer is registered here; the caller owns + * the scope lifetime (close it to kill surreal). + * - `ChildProcessSpawner` - platform spawn implementation (BunChildProcessSpawner). + * - `HttpClient` - for the `/health` readiness probe. + * + * The effect resolves as `void` once surreal is ready to accept connections. + * Shutting down the enclosing scope sends SIGTERM then (after grace) SIGKILL. + */ +export const makeManagedDb = (opts: { + readonly surrealPath: string; + readonly host: string; + readonly port: number; + readonly dataDir: string; +}): Effect.Effect< + void, + ManagedDbError, + Scope.Scope | ChildProcessSpawner.ChildProcessSpawner | HttpClient.HttpClient +> => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const httpClient = yield* HttpClient.HttpClient; + + // Move healthUrl up here so the pre-spawn probe can use it. + const healthUrl = new URL(`http://${opts.host}:${opts.port}/health`); + + // --------------------------------------------------------------------------- + // Pre-spawn idempotency check (restart-storm guard) + // --------------------------------------------------------------------------- + // If a healthy surreal is already listening on host:port, attach to it + // instead of spawning a new one. + // + // Failure mode prevented: helper crash → SIGKILL → launchd restarts + // helper → new surreal collides with orphan (rocksdb-locked) → + // 30-s readiness timeout → exit → restart → storm. + // + // Fail-closed: timeout / connection error → false → proceed to spawn. + // In the attach case we do NOT register a finalizer or start the + // watchdog - we didn't spawn the process so we must not kill or + // monitor it. + const alreadyHealthy = yield* httpClient.pipe(HttpClient.filterStatusOk) + .get(healthUrl) + .pipe( + Effect.timeout(PRE_SPAWN_PROBE_TIMEOUT), + Effect.map(() => true as boolean), + Effect.orElseSucceed(() => false as boolean), + ); + + if (alreadyHealthy) { + yield* Effect.logInfo( + `[managed-db] surreal already healthy on ${opts.host}:${opts.port} - attaching, not spawning`, + ); + return; // no spawn, no finalizer, no watchdog - we don't own this process + } + + const args = [ + "start", + "--user", "root", + "--pass", "root", + "--bind", `${opts.host}:${opts.port}`, + "--log", "info", + "--allow-experimental=files", + `rocksdb://${opts.dataDir}/db`, + ]; + + yield* Effect.logInfo(`[managed-db] spawning surreal`, { + path: opts.surrealPath, + bind: `${opts.host}:${opts.port}`, + dataDir: opts.dataDir, + }); + + const command = ChildProcess.make(opts.surrealPath, args, { + cwd: opts.dataDir, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + + // HTTP readiness probe: poll /health until surreal is ready. + const readyClient = httpClient.pipe( + HttpClient.filterStatusOk, + HttpClient.transformResponse(Effect.timeout(READINESS_REQUEST_TIMEOUT)), + HttpClient.retry(Schedule.spaced(READINESS_INTERVAL)), + ); + + const probeHealth = readyClient.get(healthUrl).pipe( + Effect.asVoid, + Effect.timeout(READINESS_TIMEOUT), + Effect.mapError(() => + new ManagedDbError({ + message: `Timed out waiting for surreal at ${healthUrl.href} (${Duration.toSeconds(READINESS_TIMEOUT)}s)`, + }), + ), + ); + + // Capture the managed-db scope so that spawnAndReady can register the + // spawner's scope finalizer in it even when called from onWedged (which + // has no Scope in its own R type). + const managedDbScope = yield* Effect.scope; + + // --------------------------------------------------------------------------- + // Spawn helper: spawn surreal, drain output streams, wait for readiness. + // Returns a handle; registers the spawner's SIGTERM finalizer in the + // managed-db scope so that ALL spawned processes are cleaned up when the + // overall scope closes. + // + // Scope is provided via `provideService` so `spawnAndReady` has no Scope + // in its own R type, allowing it to be called from `onWedged` (which must + // have R = never to satisfy SurrealWatchdogOpts). + // --------------------------------------------------------------------------- + const spawnAndReady: Effect.Effect< + ChildProcessSpawner.ChildProcessHandle, + ManagedDbError + > = Effect.gen(function* () { + // Spawn the child. The ChildProcessSpawner registers a scope finalizer + // (SIGTERM) in managedDbScope when the enclosing scope closes. + const handle = yield* spawner.spawn(command).pipe( + Effect.mapError((cause) => + new ManagedDbError({ + message: `Failed to spawn surreal: ${String(cause.message ?? cause)}`, + cause, + }), + ), + ); + + yield* Effect.logInfo(`[managed-db] surreal spawned`, { pid: handle.pid }); + + // Drain stdout/stderr in background (prevent pipe-buffer stall). + // Effect.ignore swallows failures from the stream (e.g. if surreal + // exits before we drain), so these don't surface as errors. + // forkDetach so drain fibers have no Scope requirement; they terminate + // naturally when the process exits (pipe closes). + yield* handle.stdout.pipe(Stream.runDrain, Effect.ignore, Effect.forkDetach); + yield* handle.stderr.pipe(Stream.runDrain, Effect.ignore, Effect.forkDetach); + + yield* probeHealth; + + return handle; + }).pipe( + // Provide the managed-db scope so spawner.spawn can register its + // finalizer there without Scope appearing in spawnAndReady's own R type. + Effect.provideService(Scope.Scope, managedDbScope), + ); + + // Initial spawn. + const initialHandle = yield* spawnAndReady; + + // Track the *current* handle in a Ref so the finalizer and watchdog + // always operate on the live process, not the originally-spawned one. + const handleRef = yield* Ref.make(initialHandle); + + // Explicit finalizer: graceful SIGTERM → wait → SIGKILL. + // Reads from handleRef so it always targets the most-recently spawned pid. + // The ChildProcessSpawner's own scope finalizers also fire, but those may + // target stale pids (already killed by the watchdog); Effect.ignore makes + // those no-ops. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* Effect.logInfo("[managed-db] shutting down surreal (SIGTERM)"); + const handle = yield* Ref.get(handleRef); + yield* handle.kill({ killSignal: "SIGTERM" }).pipe(Effect.ignore); + yield* Effect.sleep(TERMINATE_GRACE); + yield* handle.kill({ killSignal: "SIGKILL" }).pipe(Effect.ignore); + }), + ); + + // --------------------------------------------------------------------------- + // Watchdog: detect a wedged SurrealDB via a real SELECT 1 round-trip. + // + // `/health` passes on a wedge (socket open, process alive). An actual SQL + // query is the only reliable signal that the DB is serving queries. + // + // On trip: SIGKILL (not SIGTERM - SIGTERM was ignored in the incident), + // then respawn + re-probe health. Logged as a structured warning. + // --------------------------------------------------------------------------- + const sqlUrl = new URL(`http://${opts.host}:${opts.port}/sql`); + const watchdogProbe: Effect.Effect = httpClient.execute( + HttpClientRequest.post(sqlUrl).pipe( + HttpClientRequest.basicAuth("root", "root"), + HttpClientRequest.bodyText("SELECT 1", "text/plain"), + ), + ).pipe( + Effect.timeout(Duration.seconds(1)), + Effect.map(() => true as boolean), + Effect.orElseSucceed(() => false as boolean), + ); + + const onWedged: Effect.Effect = Effect.gen(function* () { + yield* Effect.logWarning( + "[managed-db] watchdog: surreal wedge detected - SIGKILLing and respawning", + ); + const staleHandle = yield* Ref.get(handleRef); + // Go straight to SIGKILL - the incident showed SIGTERM was ignored. + yield* staleHandle.kill({ killSignal: "SIGKILL" }).pipe(Effect.ignore); + // Respawn and wait for readiness before re-arming the watchdog counter. + const newHandle = yield* spawnAndReady; + yield* Ref.set(handleRef, newHandle); + yield* Effect.logInfo("[managed-db] watchdog: surreal restarted and ready"); + }).pipe( + // Don't let restart errors propagate to the watchdog loop - the loop + // re-arms and will try again after the next trip. + Effect.ignore, + ); + + yield* Effect.forkScoped( + makeSurrealWatchdog({ + probe: watchdogProbe, + onWedged, + interval: WATCHDOG_INTERVAL, + failuresToTrip: WATCHDOG_FAILURES_TO_TRIP, + }), + ); + + yield* Effect.logInfo("[managed-db] surreal is ready"); + }); diff --git a/apps/axctl/src/dashboard/serve-ingest-loop.test.ts b/apps/axctl/src/dashboard/serve-ingest-loop.test.ts new file mode 100644 index 00000000..fecc4653 --- /dev/null +++ b/apps/axctl/src/dashboard/serve-ingest-loop.test.ts @@ -0,0 +1,87 @@ +/** + * Tests for serve-ingest-loop.ts. + * + * These tests do NOT require a live SurrealDB. They verify structural + * properties of `runIngestLoop`: + * - The loop keeps running after an iteration failure (fail-soft). + * - `runIngestLoop` exports a valid function with the expected signature. + */ +import { describe, it, expect } from "bun:test"; +import { Cause, Duration, Effect, Layer, Schedule } from "effect"; +import { AxConfig } from "@ax/lib/config"; +import { SurrealClient } from "@ax/lib/db"; +import { ProcessService } from "@ax/lib/process"; +import { TraceSink } from "@ax/lib/live-traces/Sink"; +import { StageRegistry } from "../ingest/stage/registry.ts"; +import { runIngestLoop } from "./serve-ingest-loop.ts"; + +// Stub layer that satisfies IngestBaseServices with empty mocks. +const stubLayer = Layer.mergeAll( + Layer.succeed(SurrealClient, {} as SurrealClient["Service"]), + Layer.succeed(AxConfig, {} as AxConfig["Service"]), + Layer.succeed(ProcessService, {} as ProcessService["Service"]), + Layer.succeed(StageRegistry, { + all: () => [] as unknown[], + get: () => undefined, + has: () => false, + } as unknown as StageRegistry["Service"]), + Layer.succeed(TraceSink, {} as TraceSink["Service"]), +); + +describe("runIngestLoop", () => { + it("is exported and accepts opts with every + sinceDays", () => { + const loopEffect = runIngestLoop({ every: Duration.minutes(2), sinceDays: 2 }, stubLayer); + expect(loopEffect).toBeDefined(); + expect(typeof loopEffect).toBe("object"); + }); + + it("fails-soft: loop continues to run after iteration errors", async () => { + // Build a fail-soft loop directly (mirrors the internal pattern) + // to verify the catchCause + repeat pattern works as specified. + let iterationCount = 0; + + const alwaysFailingIteration = Effect.gen(function* () { + iterationCount++; + return yield* Effect.fail(new Error("mock iteration failure")); + }); + + const loop = alwaysFailingIteration.pipe( + // Same fail-soft pattern as in runIngestLoop + Effect.catchCause((cause) => + Effect.logWarning("[test] iteration failed", { cause: Cause.pretty(cause) }), + ), + Effect.repeat(Schedule.spaced(Duration.millis(40))), + Effect.asVoid, + Effect.flatMap(() => Effect.never), + ); + + await Effect.runPromise( + loop.pipe( + Effect.race(Effect.sleep(Duration.millis(250))), + Effect.asVoid, + Effect.catchCause(() => Effect.void), + ), + ); + + // At 40ms intervals over 250ms window, expect at least 4 iterations. + expect(iterationCount).toBeGreaterThan(3); + }); + + it("runIngestLoop with stubLayer races against a timeout without hanging", async () => { + // The loop itself fails-soft (registry.all is not a function on stub), + // but the important thing is it doesn't die completely. + const loop = runIngestLoop({ every: Duration.millis(50), sinceDays: 1 }, stubLayer); + + // Race: only the sleep wins; confirms the loop never resolves on its own. + await Effect.runPromise( + (loop as Effect.Effect).pipe( + Effect.race(Effect.sleep(Duration.millis(200))), + Effect.asVoid, + Effect.catchCause(() => Effect.void), + ), + ); + + // If we get here, the loop didn't hang or throw uncaught exceptions. + expect(true).toBe(true); + }); +}); diff --git a/apps/axctl/src/dashboard/serve-ingest-loop.ts b/apps/axctl/src/dashboard/serve-ingest-loop.ts new file mode 100644 index 00000000..622c7466 --- /dev/null +++ b/apps/axctl/src/dashboard/serve-ingest-loop.ts @@ -0,0 +1,88 @@ +/** + * Background ingest loop for `ax serve --ingest-every=`. + * + * When the macOS background helper starts `ax serve --managed-db --ingest-every=2m`, + * this module provides `runIngestLoop` which calls the same `runIngest` + * pipeline that `POST /api/ingest` uses, on a recurring schedule. Each + * iteration fails-soft: a crash in one ingest run is logged and the loop + * continues rather than dying. + * + * The loop is forked as a detached daemon fiber (via `Effect.forkDetach`) in + * the serve runtime's managed scope so it outlives individual HTTP requests + * but is interrupted when the server shuts down. + */ +import { Cause, Duration, Effect, Layer, Schedule } from "effect"; +import { TraceSink } from "@ax/lib/live-traces/Sink"; +import { runIngest, type RunIngestOptions } from "../ingest/run.ts"; +import type { IngestBaseServices } from "./ingest-workflow.ts"; + +/** A no-op TraceSink for background ingest runs (no live-trace bus needed). */ +const noopTraceSinkLayer: Layer.Layer = Layer.succeed(TraceSink, { + emit: () => undefined, +}); + +// --------------------------------------------------------------------------- +// runIngestLoop +// --------------------------------------------------------------------------- + +/** + * Run the ingest pipeline on a fixed interval. + * + * Each iteration calls `runIngest` in-process (same pipeline as CLI / + * POST /api/ingest) and fail-soft swallows failures so one bad run never + * kills the loop. + * + * `baseLayer` must satisfy `runIngest`'s service requirements + * (SurrealClient | AxConfig | ProcessService | StageRegistry | TraceSink). + * Using the `IngestBaseLayer` from `ingest-workflow.ts` is recommended. + * + * @param opts.every - Time between ingest runs (e.g. `Duration.minutes(2)`). + * @param opts.sinceDays - `--since=N` window passed to each ingest run. + * Defaults to 2 (pick up the last 2 days of changes). + * + * Returns an effect that never resolves (loops forever). Fork it with + * `Effect.forkDetach` or `Effect.forkScoped` from inside the serve runtime. + */ +export const runIngestLoop = ( + opts: { + readonly every: Duration.Duration; + readonly sinceDays?: number; + }, + baseLayer: Layer.Layer, +): Effect.Effect => { + const sinceDays = opts.sinceDays ?? 2; + const everyMs = Duration.toMillis(opts.every); + + const ingestOpts: RunIngestOptions = { + command: "ingest", + args: [`--since=${sinceDays}`], + cwd: process.cwd(), + }; + + const oneRun: Effect.Effect = runIngest(ingestOpts).pipe( + Effect.asVoid, + // Provide the no-op TraceSink first (lower priority), then the + // caller's baseLayer on top so it can override if needed. + Effect.provide(baseLayer), + Effect.provide(noopTraceSinkLayer), + Effect.scoped, + Effect.catchCause((cause) => + Effect.logWarning( + "[ingest-loop] ingest iteration failed, will retry next interval", + { cause: Cause.pretty(cause) }, + ), + ), + ); + + // repeat(Schedule.spaced) waits `every` AFTER each run completes. + return oneRun.pipe( + Effect.repeat(Schedule.spaced(Duration.millis(everyMs))), + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logError("[ingest-loop] unexpected loop error", { + cause: Cause.pretty(cause), + }), + ), + Effect.flatMap(() => Effect.never), + ); +}; diff --git a/apps/axctl/src/dashboard/server.test.ts b/apps/axctl/src/dashboard/server.test.ts index 00b9ebd2..f9a86214 100644 --- a/apps/axctl/src/dashboard/server.test.ts +++ b/apps/axctl/src/dashboard/server.test.ts @@ -90,6 +90,29 @@ describe("dashboard server", () => { expect(parseDashboardServeArgs(["--port=1800"]).port).toBe(1800); }); + test("parseDashboardServeArgs: --managed-db defaults to false", () => { + expect(parseDashboardServeArgs([]).managedDb).toBe(false); + }); + + test("parseDashboardServeArgs: --managed-db sets managedDb to true", () => { + expect(parseDashboardServeArgs(["--managed-db"]).managedDb).toBe(true); + }); + + test("parseDashboardServeArgs: --ingest-every defaults to null", () => { + expect(parseDashboardServeArgs([]).ingestEvery).toBeNull(); + }); + + test("parseDashboardServeArgs: --ingest-every=2m parses to 2 minutes", () => { + const { Duration } = require("effect"); + const result = parseDashboardServeArgs(["--ingest-every=2m"]); + expect(result.ingestEvery).not.toBeNull(); + expect(Duration.toMillis(result.ingestEvery!)).toBe(2 * 60 * 1000); + }); + + test("parseDashboardServeArgs: invalid --ingest-every throws", () => { + expect(() => parseDashboardServeArgs(["--ingest-every=bad"])).toThrow(); + }); + test("graph explorer is disabled unless explicitly enabled", () => { expect(isGraphExplorerEnabled({})).toBe(false); expect(isGraphExplorerEnabled({ AX_ENABLE_GRAPH_EXPLORER: "0" })).toBe(false); diff --git a/apps/axctl/src/dashboard/server.ts b/apps/axctl/src/dashboard/server.ts index 67611238..fb4c8547 100644 --- a/apps/axctl/src/dashboard/server.ts +++ b/apps/axctl/src/dashboard/server.ts @@ -1,5 +1,12 @@ import { DEFAULT_DASHBOARD_PORT } from "@ax/lib/dashboard-port"; -import { Layer } from "effect"; +import { DEFAULT_DB_HOST, DEFAULT_DB_PORT } from "@ax/lib/runtime-state"; +import { Duration, Effect, Exit, Layer, Scope } from "effect"; +import * as BunChildProcessSpawner from "@effect/platform-bun/BunChildProcessSpawner"; +import { BunFileSystem, BunPath } from "@effect/platform-bun"; +import * as BunHttpClient from "@effect/platform-bun/BunHttpClient"; +import { makeManagedDb, parseDurationString, resolveManagedSurrealPath } from "./managed-db.ts"; +import { runIngestLoop } from "./serve-ingest-loop.ts"; +import { IngestRuntimeLayer } from "../ingest/stage/runtime.ts"; import { isContractRequest, makeContractWebHandler, @@ -20,11 +27,29 @@ import { import { defaultRuntimeFactory, makeServeRuntime } from "./serve-runtime.ts"; import { serveStudioAsset } from "./studio-assets.ts"; -export function parseDashboardServeArgs(args: string[]): { port: number } { - const raw = args.find((arg) => arg.startsWith("--port="))?.split("=")[1]; - const port = raw === undefined ? DEFAULT_DASHBOARD_PORT : Number(raw); - if (!Number.isInteger(port) || port <= 0) throw new Error(`--port must be a positive integer (got ${raw})`); - return { port }; +export interface DashboardServeArgs { + readonly port: number; + readonly managedDb: boolean; + readonly ingestEvery: Duration.Duration | null; +} + +export function parseDashboardServeArgs(args: string[]): DashboardServeArgs { + const rawPort = args.find((arg) => arg.startsWith("--port="))?.split("=")[1]; + const port = rawPort === undefined ? DEFAULT_DASHBOARD_PORT : Number(rawPort); + if (!Number.isInteger(port) || port <= 0) throw new Error(`--port must be a positive integer (got ${rawPort})`); + + const managedDb = args.includes("--managed-db"); + + const rawEvery = args.find((a) => a.startsWith("--ingest-every="))?.split("=")[1]; + let ingestEvery: Duration.Duration | null = null; + if (rawEvery !== undefined) { + ingestEvery = parseDurationString(rawEvery); + if (ingestEvery === null) { + throw new Error(`--ingest-every: unrecognised duration '${rawEvery}' (use e.g. '2m', '30s', '1h')`); + } + } + + return { port, managedDb, ingestEvery }; } /** @@ -179,8 +204,19 @@ export async function handleDashboardRequestWithCors( return response; } +/** + * Layer providing ChildProcessSpawner + HttpClient for `--managed-db`. + * Built once at module level so it's not rebuilt per invocation. + */ +const managedDbLayer = Layer.mergeAll( + BunChildProcessSpawner.layer.pipe( + Layer.provide(Layer.mergeAll(BunFileSystem.layer, BunPath.layer)), + ), + BunHttpClient.layer, +); + export async function serveDashboard(args: string[]): Promise { - const { port } = parseDashboardServeArgs(args); + const { port, managedDb, ingestEvery } = parseDashboardServeArgs(args); // Pre-flight: ask whoever already holds the port to identify itself, // BEFORE any sidecar/runtime noise. The common failure here is "I forgot @@ -203,6 +239,37 @@ export async function serveDashboard(args: string[]): Promise { return; } + // --managed-db: spawn and supervise the bundled surreal binary. + // Must happen BEFORE the serve runtime warmup so the DB is ready when + // SurrealClient tries to connect. + let managedDbScope: Scope.Closeable | null = null; + if (managedDb) { + const dbHost = DEFAULT_DB_HOST; + const dbPort = DEFAULT_DB_PORT; + const dataDir = + process.env.AX_DATA_DIR ?? + `${process.env.HOME ?? "~"}/.local/share/ax`; + const surrealPath = resolveManagedSurrealPath(process.execPath); + + try { + managedDbScope = await Effect.runPromise(Scope.make("sequential")); + await Effect.runPromise( + makeManagedDb({ surrealPath, host: dbHost, port: dbPort, dataDir }).pipe( + Effect.provide(managedDbLayer), + Effect.provideService(Scope.Scope, managedDbScope), + ), + ); + } catch (err) { + if (managedDbScope) { + await Effect.runPromise(Scope.close(managedDbScope, Exit.void)).catch(() => undefined); + managedDbScope = null; + } + console.error(`[ax] --managed-db: failed to start surreal: ${err instanceof Error ? err.message : String(err)}`); + process.exitCode = 1; + return; + } + } + // Start the Durable Streams sidecar ONCE, before we accept requests, so // POST /api/ingest can publish progress the browser subscribes to directly // (the sidecar has permissive CORS). Graceful degradation: the sidecar @@ -285,6 +352,16 @@ export async function serveDashboard(args: string[]): Promise { const { prewarmDashboardCaches } = await import("./prewarm.ts"); void handle.runner(prewarmDashboardCaches()).catch(() => undefined); + // --ingest-every: fork a background ingest loop via Effect.runFork. + // runIngestLoop has no R requirements (it closes over IngestRuntimeLayer), + // so it can be forked without going through handle.runner. The fiber is + // NOT attached to any scope intentionally - it runs until the process exits + // or is interrupted by the signal handler below (which kills the process). + if (ingestEvery !== null) { + Effect.runFork(runIngestLoop({ every: ingestEvery }, IngestRuntimeLayer)); + console.log(`[ax] ingest loop started (every ${Duration.toSeconds(ingestEvery)}s)`); + } + // Tear down the sidecar + runtime on shutdown. Bun's serve never resolves, // so the process exits via a signal; close the open run streams and dispose // the runtime (which interrupts any in-flight ingest daemon) first. @@ -300,6 +377,11 @@ export async function serveDashboard(args: string[]): Promise { if (stream) await stream.stop().catch(() => undefined); await contract.dispose().catch(() => undefined); await handle.dispose().catch(() => undefined); + // Close the managed-db scope AFTER the serve runtime disposes so + // in-flight ingest runs finish before surreal shuts down. + if (managedDbScope) { + await Effect.runPromise(Scope.close(managedDbScope, Exit.void)).catch(() => undefined); + } process.removeListener("SIGINT", onSigint); process.removeListener("SIGTERM", onSigterm); process.kill(process.pid, signal); diff --git a/apps/studio-desktop/build/LaunchAgents/com.necmttn.ax-studio.helper.plist b/apps/studio-desktop/build/LaunchAgents/com.necmttn.ax-studio.helper.plist new file mode 100644 index 00000000..29e06667 --- /dev/null +++ b/apps/studio-desktop/build/LaunchAgents/com.necmttn.ax-studio.helper.plist @@ -0,0 +1,145 @@ + + + + + + Label + com.necmttn.ax-studio.helper + + + BundleProgram + Contents/Resources/bin/arm64/bun + + + ProgramArguments + + Contents/Resources/bin/arm64/bun + Contents/Resources/ax-src/apps/axctl/src/cli/index.ts + serve + --managed-db + --port=1738 + --ingest-every=2m + + + + KeepAlive + + + + RunAtLoad + + + + ProcessType + Background + + + AssociatedBundleIdentifiers + + com.necmttn.ax-studio + + + + EnvironmentVariables + + AX_DB_URL + ws://127.0.0.1:8521 + AX_DB_NS + ax + AX_DB_DB + main + + + + ThrottleInterval + 5 + + + SoftResourceLimits + + NumberOfFiles + 65536 + + + + + diff --git a/apps/studio-desktop/electron-builder.config.cjs b/apps/studio-desktop/electron-builder.config.cjs index 1aacfea3..517839c3 100644 --- a/apps/studio-desktop/electron-builder.config.cjs +++ b/apps/studio-desktop/electron-builder.config.cjs @@ -66,6 +66,20 @@ module.exports = { { from: "resources/studio", to: "studio" }, { from: "build/icons", to: "icons" }, ], + // extraFiles: placed at Contents/ (mac) / (win/linux). Used for the + // LaunchAgent plist so SMAppService can find it at the required location: + // Contents/Library/LaunchAgents/com.necmttn.ax-studio.helper.plist + // + // electron-builder auto-signs every Mach-O it finds under the bundle when + // hardenedRuntime=true + a real Developer ID cert is present. The plist is + // not a Mach-O (no signing needed); the bun binary it references is already + // signed as part of extraResources (resources/bin//bun). + extraFiles: [ + { + from: "build/LaunchAgents/com.necmttn.ax-studio.helper.plist", + to: "Library/LaunchAgents/com.necmttn.ax-studio.helper.plist", + }, + ], mac: { // arm64-first. A split arm64/x64 build produces two clobbering // `latest-mac.yml` feed manifests; until a merge step exists, the update diff --git a/apps/studio-desktop/scripts/verify-helper-bundle.test.ts b/apps/studio-desktop/scripts/verify-helper-bundle.test.ts new file mode 100644 index 00000000..8a3f919d --- /dev/null +++ b/apps/studio-desktop/scripts/verify-helper-bundle.test.ts @@ -0,0 +1,113 @@ +/** + * verify-helper-bundle.test.ts - TDD gate for the ax studio helper plist. + * + * Asserts that build/LaunchAgents/com.necmttn.ax-studio.helper.plist: + * - Has the correct Label / RunAtLoad / KeepAlive / AssociatedBundleIdentifiers + * - BundleProgram points at the arch-specific bundled bun binary + * - ProgramArguments includes the ax serve flags (serve, --managed-db) + * - DOES NOT have StandardOutPath or StandardErrorPath + * (macOS 14.4+ rejects SMAppService jobs that set them) + * + * Run from apps/studio-desktop/: bun test scripts/verify-helper-bundle.test.ts + */ +import { describe, expect, it } from "bun:test"; +import { parseHelperPlist } from "./verify-helper-bundle.ts"; + +describe("com.necmttn.ax-studio.helper.plist", () => { + // Parse once; every assertion below reads from this shared object. + const plist = parseHelperPlist(); + + // ── Identity ──────────────────────────────────────────────────────────── + + it("Label == com.necmttn.ax-studio.helper", () => { + expect(plist["Label"]).toBe("com.necmttn.ax-studio.helper"); + }); + + // ── Executable ────────────────────────────────────────────────────────── + + it("BundleProgram is set", () => { + expect(typeof plist["BundleProgram"]).toBe("string"); + }); + + it("BundleProgram ends with /bun", () => { + expect((plist["BundleProgram"] as string).endsWith("/bun")).toBe(true); + }); + + it("BundleProgram uses full bundle-relative prefix Contents/Resources/bin/(arm64|x64)/bun", () => { + expect(plist["BundleProgram"] as string).toMatch(/^Contents\/Resources\/bin\/(arm64|x64)\/bun$/); + }); + + // ── ProgramArguments ──────────────────────────────────────────────────── + + it("ProgramArguments is an array", () => { + expect(Array.isArray(plist["ProgramArguments"])).toBe(true); + }); + + it("ProgramArguments contains 'serve'", () => { + const args = plist["ProgramArguments"] as string[]; + expect(args).toContain("serve"); + }); + + it("ProgramArguments contains '--managed-db'", () => { + const args = plist["ProgramArguments"] as string[]; + expect(args).toContain("--managed-db"); + }); + + it("ProgramArguments contains '--port=1738'", () => { + const args = plist["ProgramArguments"] as string[]; + expect(args).toContain("--port=1738"); + }); + + it("ProgramArguments contains '--ingest-every=2m'", () => { + const args = plist["ProgramArguments"] as string[]; + expect(args).toContain("--ingest-every=2m"); + }); + + // ── Lifecycle ─────────────────────────────────────────────────────────── + + it("RunAtLoad == true", () => { + expect(plist["RunAtLoad"]).toBe(true); + }); + + it("KeepAlive is truthy", () => { + expect(plist["KeepAlive"]).toBeTruthy(); + }); + + it("ProcessType == Background", () => { + expect(plist["ProcessType"]).toBe("Background"); + }); + + // ── Headroom ──────────────────────────────────────────────────────────── + + it("ThrottleInterval is a positive number", () => { + const interval = plist["ThrottleInterval"]; + expect(typeof interval).toBe("number"); + expect((interval as number) > 0).toBe(true); + }); + + it("SoftResourceLimits.NumberOfFiles == 65536", () => { + const limits = plist["SoftResourceLimits"] as Record; + expect(limits).toBeTruthy(); + expect(limits["NumberOfFiles"]).toBe(65536); + }); + + // ── Association ───────────────────────────────────────────────────────── + + it("AssociatedBundleIdentifiers contains com.necmttn.ax-studio", () => { + const ids = plist["AssociatedBundleIdentifiers"] as string[]; + expect(Array.isArray(ids)).toBe(true); + expect(ids).toContain("com.necmttn.ax-studio"); + }); + + // ── CRITICAL: forbidden keys ───────────────────────────────────────────── + // macOS 14.4+ rejects SMAppService jobs that set StandardOutPath or + // StandardErrorPath (SMAppServiceErrorDomain code 22 / status .notFound). + + it("does NOT have StandardOutPath (macOS 14.4+ rejects it on SMAppService jobs)", () => { + expect(plist["StandardOutPath"]).toBeUndefined(); + }); + + it("does NOT have StandardErrorPath (macOS 14.4+ rejects it on SMAppService jobs)", () => { + expect(plist["StandardErrorPath"]).toBeUndefined(); + }); +}); diff --git a/apps/studio-desktop/scripts/verify-helper-bundle.ts b/apps/studio-desktop/scripts/verify-helper-bundle.ts new file mode 100644 index 00000000..cc7a1cc3 --- /dev/null +++ b/apps/studio-desktop/scripts/verify-helper-bundle.ts @@ -0,0 +1,42 @@ +/** + * verify-helper-bundle.ts - parse the LaunchAgent plist for the ax studio helper + * and return it as a plain JSON object for assertion. + * + * Uses `plutil -convert json` (macOS built-in) so no extra deps are needed. + * This lives in scripts/ (not apps/ runtime), so node:* imports are allowed. + */ +import { spawnSync } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const APP_ROOT = dirname(HERE); // apps/studio-desktop + +/** + * Canonical path to the helper plist source file (committed, not staged). + * electron-builder copies it to Contents/Library/LaunchAgents/ at package time. + */ +export const PLIST_PATH = join( + APP_ROOT, + "build", + "LaunchAgents", + "com.necmttn.ax-studio.helper.plist", +); + +export type PlistJson = Record; + +/** + * Parse a plist file into a JSON object using `plutil -convert json`. + * Throws if plutil fails (file missing, malformed XML, etc.). + */ +export function parseHelperPlist(plistPath: string = PLIST_PATH): PlistJson { + const r = spawnSync("plutil", ["-convert", "json", "-o", "-", plistPath], { + encoding: "utf8", + }); + if (r.status !== 0 || r.error) { + throw new Error( + `plutil failed (exit ${r.status ?? "error"}) for ${plistPath}: ${r.stderr || r.stdout || String(r.error)}`, + ); + } + return JSON.parse(r.stdout) as PlistJson; +} diff --git a/apps/studio-desktop/src/app/DesktopApp.ts b/apps/studio-desktop/src/app/DesktopApp.ts index 3a9ec1d3..9f496d76 100644 --- a/apps/studio-desktop/src/app/DesktopApp.ts +++ b/apps/studio-desktop/src/app/DesktopApp.ts @@ -80,10 +80,21 @@ const startup = Effect.gen(function* () { yield* Effect.forkDetach(updates.checkForUpdates); } - // 7. IDE daemon-model continuity: register the app to launch at login as ONE - // Developer-ID Login Item (mainAppService), so ingest/serve resume without - // a separate background agent. Prod only (dev isn't in /Applications, and - // SMAppService registration there errors). Fail-soft: never block boot. + // 7. IDE daemon-model continuity (prod-only; dev isn't in /Applications so + // SMAppService registration errors there). Fail-soft on both items so a + // registration failure NEVER blocks boot. + // + // 7a. Register the app itself to launch at login (mainAppService). This + // ensures the Electron shell re-opens after a reboot, which in turn + // re-starts the backend supervisor in spawn mode. + // + // 7b. Register the background helper LaunchAgent (agentService) so launchd + // keeps surreal + ax serve alive even while the app window is closed. + // The helper and the app login item are INDEPENDENT: both are registered. + // Requires macOS 13+ and the plist bundled at + // Contents/Library/LaunchAgents/com.necmttn.ax-studio.helper.plist. + // On first registration macOS may return "requires-approval" - the user + // must allow it in System Settings → General → Login Items. if (!environment.isDevelopment) { yield* electronApp.setOpenAtLogin(true).pipe( Effect.tap(() => logStartupInfo("registered launch-at-login (mainAppService)")), @@ -93,6 +104,30 @@ const startup = Effect.gen(function* () { }), ), ); + + yield* electronApp.registerBackgroundHelper.pipe( + Effect.tap(() => + logStartupInfo("registered background helper (agentService)", { + serviceName: "com.necmttn.ax-studio.helper", + }), + ), + Effect.flatMap(() => electronApp.helperStatus), + Effect.tap((status) => { + if (status === "requires-approval") { + return logStartupInfo( + "background helper requires user approval: " + + "open System Settings → General → Login Items to allow it", + { status }, + ); + } + return logStartupInfo("background helper status", { status }); + }), + Effect.catchCause((cause) => + logStartupInfo("could not register background helper", { + cause: String(cause), + }), + ), + ); } // 8. Menubar tray: Open ax studio / Start at Login / Quit. Scoped so the diff --git a/apps/studio-desktop/src/backend/AxBackendManager.test.ts b/apps/studio-desktop/src/backend/AxBackendManager.test.ts index 3ef2a011..5429fdc1 100644 --- a/apps/studio-desktop/src/backend/AxBackendManager.test.ts +++ b/apps/studio-desktop/src/backend/AxBackendManager.test.ts @@ -266,6 +266,53 @@ test("attach mode opens window without spawning processes", async () => { expect(out.backendReady).toBe(true); }); +// --------------------------------------------------------------------------- +// (b2) attach mode: explicit stop() does not kill any processes +// (helper backend lifecycle not owned by the app) +// --------------------------------------------------------------------------- + +test("attach mode: stop() is a no-op for processes (does not kill helper backend)", async () => { + // When the launchd helper owns surreal + ax serve and the app attached to + // them, the app never spawned any supervised process. Calling stop() on + // app quit must NOT send SIGTERM/SIGKILL to the helper's processes - + // it must be a pure no-op for process lifecycle. + const program = Effect.gen(function* () { + const fakeProc = yield* makeFakeProcessFactory; + const fakeWindow = yield* makeFakeWindow; + + yield* Effect.scoped( + Effect.gen(function* () { + const manager = yield* AxBackendManager.AxBackendManager; + yield* manager.start; + // Explicitly call stop(), as DesktopApp.ts does on before-quit. + // In attach mode the manager never populated its `procs` Ref + // with surreal/ax-serve handles, so stop() must not emit any + // stop events (it guards on null-checks before killing). + yield* manager.stop(); + }).pipe( + Effect.provide( + AxBackendManager.layer(fakeProc.factory).pipe( + Layer.provide(arbitrationLayer({ mode: "attach" })), + Layer.provide(fakeWindow.layer), + Layer.provide(DesktopState.layer), + Layer.provide(envLayer), + Layer.provide(platformStubLayer), + ), + ), + ), + ); + + // After explicit stop() + scope-close finalizer (which also calls stop(), + // idempotently), zero process events: nothing was started, nothing is + // stopped. The helper's surreal/ax-serve remain alive. + return { events: yield* fakeProc.events }; + }); + + const out = await Effect.runPromise(program); + + expect(out.events).toEqual([]); +}); + // --------------------------------------------------------------------------- // (c) stop tears down in reverse order (ax serve before surreal) // --------------------------------------------------------------------------- diff --git a/apps/studio-desktop/src/backend/AxDaemonArbitration.test.ts b/apps/studio-desktop/src/backend/AxDaemonArbitration.test.ts index 111e7dc6..56af658f 100644 --- a/apps/studio-desktop/src/backend/AxDaemonArbitration.test.ts +++ b/apps/studio-desktop/src/backend/AxDaemonArbitration.test.ts @@ -42,6 +42,39 @@ test("partial (surreal up, daemon down) but ports occupied -> spawn-ax-only", () .toEqual({ mode: "spawn-ax-only" }); }); +// --------------------------------------------------------------------------- +// Launchd-helper invariant: daemonHealthy short-circuits to "attach" +// +// When the background helper owns surreal + ax serve, `daemonHealthy` will be +// true. The app must ATTACH to the existing pair regardless of what +// `surrealHealthy` or `portsFree` say - the daemon's /api/version being healthy +// is the decisive signal that the full backend is up and owned by the helper. +// These tests pin that short-circuit so a refactor can never accidentally add +// branching that ignores the daemon probe. +// --------------------------------------------------------------------------- + +test("launchd-helper running: daemonHealthy=true, surrealHealthy=true, portsFree=false → attach (no double-spawn)", () => { + // The canonical helper-running scenario: both probes healthy, ports occupied. + // Identical to the "both healthy" case above but explicitly named for the + // helper invariant so a future refactor knows what it would break. + expect(decideArbitration({ daemonHealthy: true, surrealHealthy: true, portsFree: false })) + .toEqual({ mode: "attach" }); +}); + +test("daemonHealthy=true short-circuits to attach even when surrealHealthy=false", () => { + // ax serve answered /api/version - the backend is up. The surreal /health + // probe result is irrelevant: if the daemon is healthy, surreal must be too. + expect(decideArbitration({ daemonHealthy: true, surrealHealthy: false, portsFree: false })) + .toEqual({ mode: "attach" }); +}); + +test("daemonHealthy=true short-circuits to attach even when portsFree=true", () => { + // Edge case: daemon probe wins even if the port-bind probe races and sees + // the ports as free (transient race at startup is theoretically possible). + expect(decideArbitration({ daemonHealthy: true, surrealHealthy: false, portsFree: true })) + .toEqual({ mode: "attach" }); +}); + // --------------------------------------------------------------------------- // Effect probes: HTTP failures collapse to `false`, success to `true`. // --------------------------------------------------------------------------- diff --git a/apps/studio-desktop/src/electron/ElectronApp.test.ts b/apps/studio-desktop/src/electron/ElectronApp.test.ts new file mode 100644 index 00000000..492fa27c --- /dev/null +++ b/apps/studio-desktop/src/electron/ElectronApp.test.ts @@ -0,0 +1,213 @@ +/** + * TDD: Task 5 - register background helper via Electron SMAppService agentService API. + * + * Tests the three new ElectronApp methods: + * - registerBackgroundHelper + * - unregisterBackgroundHelper + * - helperStatus + * + * Uses `makeFrom(stub)` to inject a test double instead of the real Electron app, + * keeping tests electron-free and runnable outside an Electron process. + */ +import { beforeEach, expect, test } from "bun:test"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { + BACKGROUND_HELPER_SERVICE_NAME, + ElectronApp, + type AgentServiceStatus, + type ElectronAppLike, + makeFrom, +} from "./ElectronApp.ts"; + +// --------------------------------------------------------------------------- +// Stub electron app +// --------------------------------------------------------------------------- + +type LoginItemCall = { + type?: string; + serviceName?: string; + openAtLogin?: boolean; +}; + +let setLoginItemSettingsCalls: LoginItemCall[] = []; +let stubStatus: AgentServiceStatus = "not-registered"; + +const makeStubApp = (): ElectronAppLike => ({ + getName: () => "ax studio", + getVersion: () => "0.0.0-test", + on: () => stubApp, + removeListener: () => stubApp, + quit: () => {}, + exit: () => {}, + relaunch: () => {}, + whenReady: () => Promise.resolve(), + requestSingleInstanceLock: () => true, + setLoginItemSettings: (opts) => { + setLoginItemSettingsCalls.push(opts as LoginItemCall); + }, + getLoginItemSettings: (opts) => ({ + openAtLogin: false, + status: stubStatus, + type: opts?.type, + serviceName: opts?.serviceName, + }), +}); + +// Re-create the stub before each test so call log is clean. +let stubApp: ElectronAppLike; +let testLayer: Layer.Layer; + +beforeEach(() => { + setLoginItemSettingsCalls = []; + stubStatus = "not-registered"; + stubApp = makeStubApp(); + testLayer = Layer.succeed(ElectronApp, makeFrom(stubApp)); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const run = (effect: Effect.Effect) => + Effect.runPromise(effect.pipe(Effect.provide(testLayer))); + +// --------------------------------------------------------------------------- +// Tests (TDD: these were written BEFORE the methods were added to ElectronApp) +// --------------------------------------------------------------------------- + +test("[RED→GREEN] registerBackgroundHelper calls setLoginItemSettings with agentService + openAtLogin:true", async () => { + await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + yield* ea.registerBackgroundHelper; + }), + ); + + // Only called on darwin; in CI (linux/macOS) the platform guard runs + if (process.platform === "darwin") { + expect(setLoginItemSettingsCalls).toHaveLength(1); + expect(setLoginItemSettingsCalls[0]).toEqual({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + openAtLogin: true, + }); + } else { + // non-darwin: no-op, no calls + expect(setLoginItemSettingsCalls).toHaveLength(0); + } +}); + +test("[RED→GREEN] unregisterBackgroundHelper calls setLoginItemSettings with openAtLogin:false", async () => { + await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + yield* ea.unregisterBackgroundHelper; + }), + ); + + if (process.platform === "darwin") { + expect(setLoginItemSettingsCalls).toHaveLength(1); + expect(setLoginItemSettingsCalls[0]).toEqual({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + openAtLogin: false, + }); + } else { + expect(setLoginItemSettingsCalls).toHaveLength(0); + } +}); + +test("[RED→GREEN] helperStatus maps status from getLoginItemSettings", async () => { + stubStatus = "requires-approval"; + stubApp = makeStubApp(); + testLayer = Layer.succeed(ElectronApp, makeFrom(stubApp)); + + const status = await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + return yield* ea.helperStatus; + }), + ); + + if (process.platform === "darwin") { + expect(status).toBe("requires-approval"); + } else { + expect(status).toBe("not-registered"); + } +}); + +test("helperStatus maps 'enabled' status", async () => { + stubStatus = "enabled"; + stubApp = makeStubApp(); + testLayer = Layer.succeed(ElectronApp, makeFrom(stubApp)); + + const status = await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + return yield* ea.helperStatus; + }), + ); + + if (process.platform === "darwin") { + expect(status).toBe("enabled"); + } else { + expect(status).toBe("not-registered"); + } +}); + +test("helperStatus maps 'not-found' status (plist missing from bundle)", async () => { + stubStatus = "not-found"; + stubApp = makeStubApp(); + testLayer = Layer.succeed(ElectronApp, makeFrom(stubApp)); + + const status = await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + return yield* ea.helperStatus; + }), + ); + + if (process.platform === "darwin") { + expect(status).toBe("not-found"); + } else { + expect(status).toBe("not-registered"); + } +}); + +test("existing setOpenAtLogin still works (regression guard)", async () => { + await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + yield* ea.setOpenAtLogin(true); + }), + ); + + // setOpenAtLogin always calls setLoginItemSettings (no platform guard) + expect(setLoginItemSettingsCalls).toHaveLength(1); + expect(setLoginItemSettingsCalls[0]).toMatchObject({ + type: "mainAppService", + openAtLogin: true, + }); +}); + +test("registerBackgroundHelper and setOpenAtLogin are independent (both work together)", async () => { + if (process.platform !== "darwin") return; // agentService only on darwin + + await run( + Effect.gen(function* () { + const ea = yield* ElectronApp; + yield* ea.setOpenAtLogin(true); + yield* ea.registerBackgroundHelper; + }), + ); + + expect(setLoginItemSettingsCalls).toHaveLength(2); + expect(setLoginItemSettingsCalls[0]).toMatchObject({ type: "mainAppService", openAtLogin: true }); + expect(setLoginItemSettingsCalls[1]).toMatchObject({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + openAtLogin: true, + }); +}); diff --git a/apps/studio-desktop/src/electron/ElectronApp.ts b/apps/studio-desktop/src/electron/ElectronApp.ts index b5aef043..d7ccb7db 100644 --- a/apps/studio-desktop/src/electron/ElectronApp.ts +++ b/apps/studio-desktop/src/electron/ElectronApp.ts @@ -10,6 +10,12 @@ export interface ElectronAppMetadata { readonly appVersion: string; } +/** Status values returned by SMAppService for an agentService (macOS 13+). */ +export type AgentServiceStatus = "not-registered" | "enabled" | "requires-approval" | "not-found"; + +/** Identifier for the background helper LaunchAgent (must match plist Label + filename). */ +export const BACKGROUND_HELPER_SERVICE_NAME = "com.necmttn.ax-studio.helper" as const; + export interface ElectronAppShape { readonly metadata: Effect.Effect; readonly on: >( @@ -29,52 +35,145 @@ export interface ElectronAppShape { */ readonly setOpenAtLogin: (enabled: boolean) => Effect.Effect; readonly getOpenAtLogin: Effect.Effect; + /** + * Register the background helper LaunchAgent (`com.necmttn.ax-studio.helper`) + * with launchd via SMAppService `agentService` (macOS 13+). The plist must be + * bundled at `Contents/Library/LaunchAgents/com.necmttn.ax-studio.helper.plist` + * (done by Task 4). No-op on non-darwin platforms. + * + * Runs alongside `setOpenAtLogin` (mainAppService) - the helper agent service + * and the app's own login item are independent: the helper owns the backend + * (surreal + ax serve), the login item ensures the app itself auto-launches. + */ + readonly registerBackgroundHelper: Effect.Effect; + /** + * Unregister the background helper from launchd. No-op on non-darwin. + */ + readonly unregisterBackgroundHelper: Effect.Effect; + /** + * Query the SMAppService status of the background helper. Returns + * `"not-registered"` on non-darwin platforms. `"not-found"` means the plist + * is absent from the bundle (app was not packaged correctly). + * `"requires-approval"` means the user must allow it in System Settings → + * General → Login Items (first-time registration on macOS 13+). + */ + readonly helperStatus: Effect.Effect; } export class ElectronApp extends Context.Service()( "@ax/studio-desktop/electron/ElectronApp", ) {} -const addScopedAppListener = >( - eventName: string, - listener: (...args: Args) => void, -): Effect.Effect => - Effect.acquireRelease( - Effect.sync(() => { - Electron.app.on(eventName as any, listener as any); - }), - () => +// --------------------------------------------------------------------------- +// Minimal interface satisfied by both Electron.app and test stubs. +// Only the methods actually used by ElectronApp are listed here. +// --------------------------------------------------------------------------- + +export interface ElectronAppLike { + getName(): string; + getVersion(): string; + on(event: string, listener: (...args: any[]) => void): this; + removeListener(event: string, listener: (...args: any[]) => void): this; + quit(): void; + exit(exitCode?: number): void; + relaunch(options?: { args?: string[]; execPath?: string }): void; + whenReady(): Promise; + requestSingleInstanceLock(additionalData?: Record): boolean; + setLoginItemSettings(settings: { + openAtLogin?: boolean; + type?: string; + serviceName?: string; + [key: string]: unknown; + }): void; + getLoginItemSettings(options?: { + type?: string; + serviceName?: string; + [key: string]: unknown; + }): { openAtLogin: boolean; status?: string; [key: string]: unknown }; +} + +// --------------------------------------------------------------------------- +// Factory (accepts a test stub or the real Electron.app) +// --------------------------------------------------------------------------- + +const addScopedListener = + (app: ElectronAppLike) => + >( + eventName: string, + listener: (...args: Args) => void, + ): Effect.Effect => + Effect.acquireRelease( Effect.sync(() => { - Electron.app.removeListener(eventName as any, listener as any); + app.on(eventName, listener as any); }), - ).pipe(Effect.asVoid); + () => + Effect.sync(() => { + app.removeListener(eventName, listener as any); + }), + ).pipe(Effect.asVoid); -const make = ElectronApp.of({ - metadata: Effect.sync(() => ({ - appName: Electron.app.getName(), - appVersion: Electron.app.getVersion(), - })), - on: addScopedAppListener, - quit: Effect.sync(() => { - Electron.app.quit(); - }), - exit: (code) => - Effect.sync(() => { - Electron.app.exit(code); - }), - relaunch: (options) => - Effect.sync(() => { - Electron.app.relaunch(options); - }), - whenReady: Effect.promise(() => Electron.app.whenReady()).pipe(Effect.asVoid), - requestSingleInstanceLock: Effect.sync(() => Electron.app.requestSingleInstanceLock()), - setOpenAtLogin: (enabled) => - Effect.sync(() => { - Electron.app.setLoginItemSettings({ openAtLogin: enabled, type: "mainAppService" }); +/** + * Build an `ElectronAppShape` from any object that satisfies `ElectronAppLike`. + * Used directly in tests (pass a stub); the production `layer` calls this with + * the real `Electron.app`. + */ +export const makeFrom = (app: ElectronAppLike): ElectronAppShape => { + const isDarwin = process.platform === "darwin"; + + return ElectronApp.of({ + metadata: Effect.sync(() => ({ + appName: app.getName(), + appVersion: app.getVersion(), + })), + on: addScopedListener(app), + quit: Effect.sync(() => { + app.quit(); }), - getOpenAtLogin: Effect.sync( - () => Electron.app.getLoginItemSettings({ type: "mainAppService" }).openAtLogin, - ), -}); + exit: (code) => + Effect.sync(() => { + app.exit(code); + }), + relaunch: (options) => + Effect.sync(() => { + app.relaunch(options); + }), + whenReady: Effect.promise(() => app.whenReady()).pipe(Effect.asVoid), + requestSingleInstanceLock: Effect.sync(() => app.requestSingleInstanceLock()), + setOpenAtLogin: (enabled) => + Effect.sync(() => { + app.setLoginItemSettings({ openAtLogin: enabled, type: "mainAppService" }); + }), + getOpenAtLogin: Effect.sync( + () => app.getLoginItemSettings({ type: "mainAppService" }).openAtLogin, + ), + registerBackgroundHelper: isDarwin + ? Effect.sync(() => { + app.setLoginItemSettings({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + openAtLogin: true, + }); + }) + : Effect.void, + unregisterBackgroundHelper: isDarwin + ? Effect.sync(() => { + app.setLoginItemSettings({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + openAtLogin: false, + }); + }) + : Effect.void, + helperStatus: isDarwin + ? Effect.sync(() => { + const { status } = app.getLoginItemSettings({ + type: "agentService", + serviceName: BACKGROUND_HELPER_SERVICE_NAME, + }) as { status: AgentServiceStatus }; + return status ?? "not-registered"; + }) + : Effect.succeed("not-registered" as AgentServiceStatus), + }); +}; -export const layer = Layer.succeed(ElectronApp, make); +export const layer = Layer.succeed(ElectronApp, makeFrom(Electron.app as unknown as ElectronAppLike)); diff --git a/docs/superpowers/notes/2026-06-24-agentservice-contract.md b/docs/superpowers/notes/2026-06-24-agentservice-contract.md new file mode 100644 index 00000000..8aba3a88 --- /dev/null +++ b/docs/superpowers/notes/2026-06-24-agentservice-contract.md @@ -0,0 +1,325 @@ +# Electron `agentService` Contract - Spike Findings + +**Date:** 2026-06-24 +**Issue:** #599 - studio app registers db + serve via launchd +**Status:** DONE - strategy resolved, no gate blocked + +--- + +## 1. Sources consulted + +- Electron 41.5.0 docs: `app.setLoginItemSettings` / `app.getLoginItemSettings` +- Electron source: `shell/common/platform_util_mac.mm` (v41.5.0, read via GitHub) +- Electron source: `shell/common/gin_converters/login_item_settings_converter.*` +- Apple: `SMAppService.agentServiceWithPlistName:` (ServiceManagement.framework) +- Real-world plist samples: `supaku/kith`, `JP1222/Mac-Monitor`, `haasonsaas/ambient-agent`, + `antimatter-studios/diskjockey`, `parleq/parleq-speech` +- `apps/studio-desktop/electron-builder.config.cjs` - `appId`, signing config +- `apps/studio-desktop/build/entitlements.mac.plist` - hardenedRuntime entitlements +- `apps/studio-desktop/src/electron/ElectronApp.ts` - existing `setLoginItemSettings` wrapper +- `apps/studio-desktop/src/backend/AxBackendManager.ts` - daemon lifecycle context + +--- + +## 2. Electron API contract (verbatim from source) + +### Internal implementation (platform_util_mac.mm) + +```objc +// agentService → SMAppService.agentServiceWithPlistName: +SMAppService* GetServiceForType(const std::string& type, const std::string& name) { + NSString* service_name = [NSString stringWithUTF8String:name.c_str()]; + if (type == "agentService") + return [SMAppService agentServiceWithPlistName:service_name]; + ... +} + +bool SetLoginItemEnabled(const std::string& type, const std::string& service_name, bool enabled) { + SMAppService* service = GetServiceForType(type, service_name); + NSError* error = nil; + bool result = enabled ? [service registerAndReturnError:&error] + : [service unregisterAndReturnError:&error]; + ... +} +``` + +### JS/TS call shapes (macOS 13+) + +```typescript +// Register (enable) +Electron.app.setLoginItemSettings({ + type: "agentService", + serviceName: "com.necmttn.ax-studio.helper", // plist name, no .plist extension + openAtLogin: true, +}); + +// Unregister (disable) +Electron.app.setLoginItemSettings({ + type: "agentService", + serviceName: "com.necmttn.ax-studio.helper", + openAtLogin: false, +}); + +// Status query +const { status } = Electron.app.getLoginItemSettings({ + type: "agentService", + serviceName: "com.necmttn.ax-studio.helper", +}); +// status: "not-registered" | "enabled" | "requires-approval" | "not-found" +``` + +`serviceName` is the plist filename **without** the `.plist` extension, passed verbatim to +`SMAppService.agentServiceWithPlistName:`. The `type` field defaults to `mainAppService`; +all non-default types require `serviceName`. + +### Matching the existing ElectronApp service style + +The existing wrapper in `apps/studio-desktop/src/electron/ElectronApp.ts` (line 72-77) calls: + +```typescript +Electron.app.setLoginItemSettings({ openAtLogin: enabled, type: "mainAppService" }); +Electron.app.getLoginItemSettings({ type: "mainAppService" }).openAtLogin +``` + +The new `agentService` wrapper must follow the same `Effect.sync(() => ...)` pattern and add +`serviceName` as a parameter. Suggested addition to `ElectronAppShape`: + +```typescript +readonly setAgentServiceEnabled: ( + serviceName: string, + enabled: boolean, +) => Effect.Effect; +readonly getAgentServiceStatus: ( + serviceName: string, +) => Effect.Effect<"not-registered" | "enabled" | "requires-approval" | "not-found">; +``` + +--- + +## 3. Plist placement and structure + +### Where the plist must live + +``` +ax studio.app/ +└── Contents/ + └── Library/ + └── LaunchAgents/ + └── com.necmttn.ax-studio.helper.plist ← MUST be here +``` + +Apple's SMAppService looks for the plist at exactly +`/Contents/Library/LaunchAgents/.plist`. +The `Label` key inside the plist must match the filename (without `.plist`). + +### The `BundleProgram` key + +`BundleProgram` is the bundle-relative path to the helper executable, resolved from the +**app bundle root** (i.e., from `ax studio.app/`, NOT from `Contents/MacOS/`). + +``` +BundleProgram = "Contents/Library/LaunchAgents/ax-serve-helper" + → resolves to: ax studio.app/Contents/Library/LaunchAgents/ax-serve-helper +``` + +**Critical**: `BundleProgram` must be used instead of an absolute `Program` path. +Launchd uses the bundle-relative path to verify the helper's code signature belongs +to the parent app (same Team ID, same Developer ID signing chain). +Absolute `ProgramArguments[0]` paths bypass this check and will be rejected on +hardened/notarized builds. + +### Canonical plist XML shape + +```xml + + + + + + Label + com.necmttn.ax-studio.helper + + + BundleProgram + Contents/Library/LaunchAgents/ax-serve-helper + + + KeepAlive + + + + RunAtLoad + + + + AssociatedBundleIdentifiers + + com.necmttn.ax-studio + + + ProcessType + Background + + + EnvironmentVariables + + AX_DB_URL + ws://127.0.0.1:8521 + AX_DB_NS + ax + AX_DB_DB + main + + + + + +``` + +**Plist filename:** `com.necmttn.ax-studio.helper.plist` (must match `Label` + `serviceName`). + +--- + +## 4. Executable strategy - DECIDED + +### Gate question resolved: can `BundleProgram` point at the main Electron binary? + +**Answer: TECHNICALLY YES, PRACTICALLY NO.** + +- `BundleProgram` can point to any bundle-relative path, including + `Contents/MacOS/ax studio` (the main Electron binary). The `parleq` app + does this for `mainAppService`. +- `ProgramArguments` can coexist with `BundleProgram`: launchd uses `BundleProgram` + as the executable and `ProgramArguments` as argv, so `--background-helper` could + be passed. +- **BUT**: Launching the full Electron binary as a launchd agent is inappropriate: + - Electron expects a display (window server), Cocoa run loop, and renderer processes. + - Launchd background agents have no display environment (`DISPLAY` unset, no + WindowServer session). The Electron binary will crash or hang trying to initialize. + - Even with a `--background-helper` guard, the Electron process model (multi-process, + Chromium IPC) is far heavier than needed for `surreal` + `ax serve` supervision. + +### Decided strategy: **Strategy B - Separate compiled helper binary** + +Place a separate, small compiled binary at: +`ax studio.app/Contents/Library/LaunchAgents/ax-serve-helper` + +This binary's sole job: start `surreal` and `bun axctl serve` in the foreground, +managing their lifetime (similar to what `AxBackendManager` does today). + +**Why this approach:** +1. The established SMAppService agentService pattern universally uses a separate helper + binary (confirmed across 5+ real-world apps). +2. electron-builder automatically signs ALL Mach-O files found anywhere in the app bundle + (including `Contents/Library/LaunchAgents/`) when `hardenedRuntime: true` + a real + Developer ID cert is present - no extra signing step needed. +3. The helper binary is already in scope: the `axctl` compiled bun binary + (`bun build --compile`) is exactly this helper, or a thin wrapper around it. +4. `BundleProgram` code-signature check passes trivially (same Developer ID). +5. No Electron process weight; launchd gets a lightweight background-only binary. + +### Helper binary concrete spec + +**Identity:** A compiled bun binary (output of `bun build --compile`) named `ax-serve-helper` +that runs `ax serve` (starts surreal + serves the ax HTTP API). + +**Source location (to be staged by Task 4):** `apps/studio-desktop/build/LaunchAgents/ax-serve-helper` +(compiled by the `prepackage` script step, placed in the bundle via `extraFiles`). + +**electron-builder extraFiles entry (Task 4 will add):** +```javascript +extraFiles: [ + { + from: "build/LaunchAgents/ax-serve-helper", + to: "Library/LaunchAgents/ax-serve-helper", + // Note: extraFiles `to` is relative to Contents/, so this lands at + // Contents/Library/LaunchAgents/ax-serve-helper + } +] +``` +This places it next to the plist. electron-builder signs it with `hardenedRuntime: true` +automatically. + +### Fallback (if compiled bun binary is infeasible for notarization) + +If `bun build --compile` output fails Apple's notarization scan (e.g., due to unsupported +binary format at notarization time): use a minimal shell launcher at the same path +(`#!/bin/sh` shebang with absolute paths to the bundled `bun` and ax-src entry). Shell +scripts don't go through code-signature verification by `BundleProgram` the same way, but +they ARE executable and `launchd` will run them. This is the accepted fallback documented +in Apple's own developer materials for simple agents. + +--- + +## 5. Plist placement in electron-builder + +The plist file itself (`com.necmttn.ax-studio.helper.plist`) also needs to land at +`Contents/Library/LaunchAgents/`. Add alongside the helper binary: + +```javascript +// in electron-builder.config.cjs, mac section: +extraFiles: [ + { + from: "build/LaunchAgents/com.necmttn.ax-studio.helper.plist", + to: "Library/LaunchAgents/com.necmttn.ax-studio.helper.plist", + }, + { + from: "build/LaunchAgents/ax-serve-helper", + to: "Library/LaunchAgents/ax-serve-helper", + }, +] +``` + +Source files to create (Tasks 2 + 4): +- `apps/studio-desktop/build/LaunchAgents/com.necmttn.ax-studio.helper.plist` - the XML above +- `apps/studio-desktop/build/LaunchAgents/ax-serve-helper` - compiled helper binary (gitignored, built by prepackage) + +--- + +## 6. Registration lifecycle summary + +| Action | Electron call | SMAppService call | Result | +|--------|---------------|-------------------|--------| +| Register (enable at login) | `setLoginItemSettings({ type: 'agentService', serviceName: '…', openAtLogin: true })` | `[service registerAndReturnError:]` | status → `enabled` (or `requires-approval` first time) | +| Check status | `getLoginItemSettings({ type: 'agentService', serviceName: '…' }).status` | `[service status]` | `not-registered` / `enabled` / `requires-approval` / `not-found` | +| Unregister | `setLoginItemSettings({ type: 'agentService', serviceName: '…', openAtLogin: false })` | `[service unregisterAndReturnError:]` | status → `not-registered` | + +`not-found` means the plist file is missing from `Contents/Library/LaunchAgents/` - it +will appear as this until the app is rebuilt with the plist bundled. + +`requires-approval` means the user must approve in System Settings → General → Login Items +(first-time registration on macOS 13+). Electron's `getLoginItemSettings` exposes this +so the app can surface a nudge. + +--- + +## 7. Open concerns for subsequent tasks + +1. **Helper binary build step**: Task 4 must add a `bun build --compile` step in + `prepackage` that produces `build/LaunchAgents/ax-serve-helper`. The helper needs + to know paths to the bundled `surreal` + `bun` binaries relative to itself + (`__dirname`-style resolution from `Contents/Library/LaunchAgents/`). + +2. **Entitlements inheritance**: electron-builder uses `entitlementsInherit` for nested + binaries. The helper binary may need `com.apple.security.cs.allow-jit` removed + (it doesn't run JS in a JIT context). Check `entitlementsInherit` when signing fails. + +3. **`KeepAlive` interaction with app quit**: when the user quits `ax studio`, launchd + will restart the helper immediately (KeepAlive=true). Task 2 must ensure + `AxDaemonArbitration` properly handles the "helper running, app just quitting" + case - the helper is INTENTIONALLY left running. The app must NOT kill it on quit. + +4. **macOS 13 minimum**: `type: 'agentService'` requires macOS 13+. The existing + `ElectronApp.ts` comment already states "macOS 13+" for `mainAppService`; add + the same guard or check `process.platform === 'darwin'` + OS version before calling. + +5. **`StandardOutPath`/`StandardErrorPath` are forbidden** (documented in plist above) for + sandboxed app agents on macOS 14.4+. The helper binary must route logs via `os_log` + or write to `~/.ax/` directly. diff --git a/docs/superpowers/plans/2026-06-24-studio-helper-smappservice.md b/docs/superpowers/plans/2026-06-24-studio-helper-smappservice.md new file mode 100644 index 00000000..41a9f13c --- /dev/null +++ b/docs/superpowers/plans/2026-06-24-studio-helper-smappservice.md @@ -0,0 +1,366 @@ +# Studio Background Helper (SMAppService Option A) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add ONE Developer-ID-signed background helper to `ax studio.app` that owns the surreal + ax serve + ingest backend via launchd (SMAppService `agentService`), so the data plane survives the app being closed or crashing - without re-creating the rejected "unidentified developer bash" Login Items. + +**Architecture:** Invert ownership. Today the UI app supervises surreal/serve as child processes that die with it (`AxBackendManager`). Instead, a launchd-managed **helper** (the same signed app binary launched with `--background-helper`, registered via `app.setLoginItemSettings({ type: 'agentService' })`, `RunAtLoad` + `KeepAlive`) owns the backend. The UI app's existing `AxDaemonArbitration` **attach** path then just connects to the already-running backend. Because the original failure was a *wedge* (alive-but-unresponsive surreal) and `KeepAlive` only restarts on exit, the helper also runs a **real-query watchdog** that force-restarts a hung db. + +**Tech Stack:** Electron 41.5.0, Effect (`effect@beta`), Bun, `@effect/platform-node`, electron-builder, macOS SMAppService / `launchd`, SurrealDB. + +**Resolves:** [#599](https://github.com/Necmttn/ax/issues/599). **Builds on / amends:** `docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md` (this is Option A, the pre-approved escape hatch, now required by the 4-day-stall incident in `memory/ax-data-plane-fragile-ide-model.md`). + +## Global Constraints + +- Electron version: **41.5.0** (no upgrade). `setLoginItemSettings({ type: 'agentService' })` is the API; `mainAppService` is already used for `openAtLogin` (`apps/studio-desktop/src/app/ElectronApp.ts:73`). +- **Helper form - DECIDED by spikes 1 + 1b (Form A):** the agent plist's `BundleProgram` is the **bundled `bun` Mach-O** (`Contents/Resources/bin//bun`), with `ProgramArguments` running an **ax-src helper entry**. There is NO separate compiled helper binary (the main Electron binary can't be a launchd agent - needs WindowServer; and the compiled `axctl` binary can't live-ingest - lmdb won't bundle). The helper MUST run under bundled bun + ax-src. Runtime proven in `.superpowers/sdd/task-1b-microspike-report.md`. +- Helper serviceName == Label == plist filename: **`com.necmttn.ax-studio.helper`**. App id stays `com.necmttn.ax-studio`. Plist at `Contents/Library/LaunchAgents/com.necmttn.ax-studio.helper.plist`. +- Plist MUST NOT set `StandardOutPath`/`StandardErrorPath` (macOS 14.4+ rejects SMAppService jobs that do - `SMAppServiceErrorDomain` 22). The helper opens its own log file under `~/.local/share/ax/logs/`. +- `BundleProgram` is bundle-root-relative (launchd verifies the signature chain to the app's Team ID); never an absolute path. +- Bundle prerequisite: `stage-ax-source.ts` must `bun install` into the staged `ax-src/` (the shipped bundle currently has no `node_modules`, so bundled bun can't run ax-src). Folded into Task 4. +- Data dir is shared and singular: `$AX_DATA_DIR ?? ~/.local/share/ax`. Exactly ONE process owns surreal at a time - enforced by `AxDaemonArbitration` port probes. Never run two surreal on `:8521`. +- Ports: surreal `8521`, ax serve `1738` (`AxDaemonArbitration.ts:21-22`). +- `check:no-node-fs` gate bans `node:fs`/`node:path` in `apps/` - use Effect `FileSystem`/`Path` (see `findDesktopApp` pattern, `install.ts:86`). +- Headless CLI (`ax ingest`, `ax serve`, `ax daemon`) must stay functional and unchanged for non-desktop users. +- macOS-only feature. On non-darwin the helper registration is a no-op (guard on `process.platform === 'darwin'`). + +--- + +### Task 1: Spike - pin the Electron `agentService` contract (de-risk) + +**Why first:** the exact plist placement, the `serviceName`/`name` field, and whether Electron generates the plist or expects it pre-bundled are the only true unknowns. Everything else reuses existing modules. Resolve this before building. + +**Files:** +- Create: `docs/superpowers/notes/2026-06-24-agentservice-contract.md` (findings) +- Reference: `apps/studio-desktop/src/app/ElectronApp.ts:60-80` (existing `setLoginItemSettings`) + +- [ ] **Step 1: Read the Electron 41.5.0 docs for `setLoginItemSettings`** - focus on the macOS `type: 'agentService'` branch. Capture verbatim: required fields (`serviceName`? `name`?), where the agent plist must live in the bundle (`Contents/Library/LaunchAgents/.plist` is the documented SMAppService location), what `BundleProgram` must be relative to, and whether `openAtLogin: true` is what enables it. Source: https://www.electronjs.org/docs/latest/api/app#appsetloginitemsettingssettings-macos-windows + +- [ ] **Step 2: Confirm the BundleProgram-as-main-binary pattern** is valid (launchd runs `…/Contents/MacOS/ax studio --background-helper`). If Electron requires a *separate* helper executable instead, record that as the fallback (a tiny `bun`-shebang launcher staged into the bundle) and which Task 4 variant to use. + +- [ ] **Step 3: Write the contract note** - the exact plist XML shape, the exact `setLoginItemSettings` call args, the registration + `status` + unregister API (`getLoginItemSettings`, `serviceStatus`), and the chosen executable strategy. Every later task references this file. + +- [ ] **Step 4: Commit** + +```bash +git add docs/superpowers/notes/2026-06-24-agentservice-contract.md +git commit -m "docs: pin Electron agentService contract for studio helper (#599)" +``` + +**Gate:** if Step 2 reveals `agentService` cannot point at the main binary AND a bundled bun-launcher is also infeasible on a hardened-runtime/notarized build, STOP and escalate - the approach needs revisiting before more work. + +--- + +### Task 2: `ax serve --managed-db` + `--ingest-every` (the helper runtime) + +**Design (resolved by spikes 1+1b):** the helper plist runs bundled-`bun` against the bundled ax-src `serve` command. So the helper runtime IS `ax serve` with two new flags: `--managed-db` (spawn + supervise bundled surreal as a child, then serve) and `--ingest-every=` (fork an internal ingest loop). Bundle-location independence: `--managed-db` resolves the surreal binary as a **sibling of `process.execPath`** (bundled bun and surreal both live in `Contents/Resources/bin//`), so no absolute paths in the plist. Both flags are also useful for non-desktop users (`ax serve --managed-db` = one-shot self-contained daemon). + +**Files:** +- Create: `apps/axctl/src/dashboard/managed-db.ts` (resolve surreal path + spawn/supervise as a child, readiness-gated) +- Create: `apps/axctl/src/dashboard/serve-ingest-loop.ts` (interval loop calling the in-process ingest entry - the same `runIngest` the `POST /api/ingest` handler forks, NOT an HTTP round-trip) +- Modify: the `serve` command definition (find it: `rg -n "\"serve\"|serveCommand|cmdServe" apps/axctl/src/cli`) to add the two flags + wire them +- Test: `apps/axctl/src/dashboard/managed-db.test.ts`, `apps/axctl/src/dashboard/serve-ingest-loop.test.ts` + +**Interfaces:** +- Consumes: existing serve bootstrap (`apps/axctl/src/dashboard/server.ts`), `@ax/lib/runtime-state` for the db host/port, the existing in-process ingest entry used by `POST /api/ingest` (locate via `rg -n "runIngest|/api/ingest" apps/axctl/src/dashboard`). +- Produces: + - `export const resolveManagedSurrealPath: (execPath: string) => string` - sibling-of-execPath resolution (`/surreal`). + - `export const makeManagedDb: (opts: { surrealPath: string; host: string; port: number; dataDir: string }) => Effect.Effect` - spawns surreal, waits on `/health`, registers a scope finalizer that stops it. + - `export const runIngestLoop: (opts: { every: Duration.Duration; sinceDays: number }) => Effect.Effect` - `repeat(Schedule.spaced(every))`, fail-soft per iteration. + - serve flags: `--managed-db` (boolean), `--ingest-every` (duration string e.g. `2m`, optional). + +- [ ] **Step 1: Write the failing test** - `resolveManagedSurrealPath` returns the sibling path. + +```typescript +import { describe, it, expect } from "bun:test"; +import { resolveManagedSurrealPath } from "./managed-db.ts"; + +describe("resolveManagedSurrealPath", () => { + it("resolves surreal as a sibling of the bun execPath", () => { + expect(resolveManagedSurrealPath("/Applications/ax studio.app/Contents/Resources/bin/arm64/bun")) + .toBe("/Applications/ax studio.app/Contents/Resources/bin/arm64/surreal"); + }); +}); +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `cd apps/axctl && bun test src/dashboard/managed-db.test.ts` +Expected: FAIL - module not found. + +- [ ] **Step 3: Implement `resolveManagedSurrealPath` + `makeManagedDb`** - use Effect `Path` to take `dirname(execPath)` and join `surreal`. `makeManagedDb` spawns via the existing `ChildProcessSpawner` pattern (mirror how the desktop's `SupervisedProcess`/the CLI spawns surreal; reuse the surreal arg shape from `apps/axctl/src/cli/install.ts:108-110` - `start --user root --pass root --bind host:port --log info --allow-experimental=files "rocksdb:///db"`), waits for `GET http://host:port/health`, and adds a `Scope` finalizer that SIGTERMs (then SIGKILLs) the child. + +- [ ] **Step 4: Implement `runIngestLoop`** - fork the in-process ingest entry on `Schedule.spaced(every)`, each iteration `Effect.catchCause`-logged (fail-soft; one bad run never kills the loop). + +- [ ] **Step 5: Wire the flags into `serve`** - when `--managed-db`, run `makeManagedDb` (scoped) BEFORE binding the HTTP server, so surreal is ready first; when `--ingest-every` is set, `Effect.forkScoped(runIngestLoop(...))` after serve is up. Default values: `--ingest-every` unset = no loop (preserve current behavior). Existing `ax serve` with neither flag is byte-for-byte unchanged. + +- [ ] **Step 6: Run tests + a real local smoke on a TEST port** (never 8521 / the live db). + +Run: `cd apps/axctl && bun test src/dashboard/ && AX_DATA_DIR=/tmp/ax-mdb-smoke bun src/cli/index.ts serve --managed-db --port=8531 --ingest-every=2m` then `curl -s 127.0.0.1:8531/api/version`; Ctrl-C and confirm the child surreal is reaped. +Expected: tests pass; `/api/version` answers; no orphan surreal after exit. + +- [ ] **Step 7: Commit** + +```bash +git add apps/axctl/src/dashboard/managed-db.ts apps/axctl/src/dashboard/serve-ingest-loop.ts apps/axctl/src/dashboard/managed-db.test.ts apps/axctl/src/dashboard/serve-ingest-loop.test.ts +git add -p # the serve command file +git commit -m "feat(serve): --managed-db (supervise surreal child) + --ingest-every loop (#599)" +``` + +**Note for Task 3 (watchdog) + Task 6 (UI arbitration):** the wedge watchdog (Task 3) forks inside the `--managed-db` path here (it owns the surreal child it must restart). The UI app does NOT use `--managed-db`; it keeps `AxBackendManager` attach-mode and connects to the helper's serve (Task 6). + +--- + +### Task 3: Surreal wedge watchdog (the incident fix) + +**Why:** the 4-day stall was a *wedge* - surreal held the LISTEN socket but stopped answering. `KeepAlive` never fires (no exit). The helper must detect unresponsiveness via a real query and force-restart. + +**Files:** +- Create: `apps/studio-desktop/src/backend/SurrealWatchdog.ts` +- Modify: `apps/studio-desktop/src/backend/AxBackendManager.ts` (fork the watchdog after surreal-ready in spawn mode; expose a restart hook) +- Test: `apps/studio-desktop/src/backend/SurrealWatchdog.test.ts` + +**Interfaces:** +- Consumes: `HttpClient.HttpClient`; a `restartSurreal: Effect.Effect` callback (the manager already owns the `SupervisedProcess` for surreal - expose its `stop`+`start`, or reuse the supervisor restart). Surreal liveness must be a real query, NOT `/health` (the wedge passed socket checks). Use `POST http://127.0.0.1:8521/sql` with `SELECT 1` (authenticated) and a hard per-probe timeout. +- Produces: `export const makeSurrealWatchdog: (opts: { probe: Effect.Effect; onWedged: Effect.Effect; interval: Duration.Duration; failuresToTrip: number }) => Effect.Effect` (an infinite forked loop). + +- [ ] **Step 1: Write the failing test** - N consecutive failed probes trips exactly one `onWedged`; a success resets the counter. + +```typescript +import { describe, it, expect } from "bun:test"; +import { Effect, Duration, Ref, TestClock, TestContext } from "effect"; +import { makeSurrealWatchdog } from "./SurrealWatchdog.ts"; + +describe("SurrealWatchdog", () => { + it("trips onWedged after failuresToTrip consecutive failures", () => + Effect.gen(function* () { + const trips = yield* Ref.make(0); + yield* Effect.fork(makeSurrealWatchdog({ + probe: Effect.succeed(false), // always wedged + onWedged: Ref.update(trips, (n) => n + 1), + interval: Duration.seconds(5), + failuresToTrip: 3, + })); + yield* TestClock.adjust(Duration.seconds(15)); // 3 ticks → 1 trip + expect(yield* Ref.get(trips)).toBe(1); + }).pipe(Effect.provide(TestContext.TestContext), Effect.runPromise)); +}); +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `cd apps/studio-desktop && bun test src/backend/SurrealWatchdog.test.ts` +Expected: FAIL - module not found. + +- [ ] **Step 3: Implement the watchdog** - a `Ref`-counter loop on `Schedule.spaced(interval)`: probe; on success reset counter to 0; on failure increment; when counter reaches `failuresToTrip`, run `onWedged` and reset counter (so it re-arms after the restart). Pure w.r.t. clock (TestClock-drivable). + +- [ ] **Step 4: Wire into the `--managed-db` path (Task 2)** - inside `makeManagedDb` (`apps/axctl/src/dashboard/managed-db.ts`), after the surreal child is ready, `Effect.forkScoped` a `makeSurrealWatchdog` whose `probe` does a real `SELECT 1` round-trip (1s timeout) and whose `onWedged` force-restarts the managed surreal child (SIGKILL the wedged pid - recall SIGTERM was ignored in the incident - then re-spawn). Add a structured log line on trip. NOTE: this lives only in `--managed-db` (the helper owns surreal); the UI app runs attach-mode and never starts the watchdog. **Files for this task are therefore `apps/axctl/src/dashboard/SurrealWatchdog.ts` (+ `.test.ts`), not studio-desktop** - run tests with `cd apps/axctl && bun test src/dashboard/SurrealWatchdog.test.ts`, and the commit adds `apps/axctl/src/dashboard/SurrealWatchdog*.ts` + `managed-db.ts`. + +- [ ] **Step 5: Run tests, verify pass** + +Run: `cd apps/studio-desktop && bun test src/backend/SurrealWatchdog.test.ts` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add apps/studio-desktop/src/backend/SurrealWatchdog.ts apps/studio-desktop/src/backend/SurrealWatchdog.test.ts apps/studio-desktop/src/backend/AxBackendManager.ts +git commit -m "feat(studio-desktop): real-query surreal wedge watchdog + force-restart (#599)" +``` + +--- + +### Task 4: Bundle the agent plist + place the helper program (electron-builder) + +**Files:** +- Create: `apps/studio-desktop/build/LaunchAgents/com.necmttn.ax-studio.helper.plist` (Form A plist - `BundleProgram` = bundled bun, `ProgramArguments` = ax-src serve entry) +- Modify: `apps/studio-desktop/electron-builder.config.cjs` (`extraFiles` placing the plist at `Contents/Library/LaunchAgents/`) +- Modify: `apps/studio-desktop/scripts/stage-ax-source.ts` (run `bun install --production` in the staged `ax-src/` so bundled bun can resolve ax-src deps - the shipped bundle currently has no `node_modules`; this is the spike-1b blocker) +- Test: `apps/studio-desktop/scripts/verify-helper-bundle.test.ts` (asserts the staged tree has the plist with the right keys) + +**Interfaces (Form A - from `docs/superpowers/notes/2026-06-24-agentservice-contract.md`):** +- `BundleProgram` = `Contents/Resources/bin/${arch}/bun` (the bundled, app-signed bun Mach-O; bundle-root-relative). +- `ProgramArguments` = `[ "ax-src/apps/axctl/src/cli/index.ts", "serve", "--managed-db", "--port=1738", "--ingest-every=2m" ]` - bun runs the bundled ax-src serve entry with the Task-2 flags. (Path is relative to the bundle; bun resolves it from its own `process.execPath` location. Confirm the exact relative form against the staged tree.) +- `Label` == `com.necmttn.ax-studio.helper` (== filename == serviceName). `KeepAlive`=true, `RunAtLoad`=true, `ProcessType`=Background, `AssociatedBundleIdentifiers`=[`com.necmttn.ax-studio`]. +- **NO `StandardOutPath`/`StandardErrorPath`** (macOS 14.4+ rejects them on SMAppService jobs - the helper opens its own log under `~/.local/share/ax/logs/`). + +- [ ] **Step 1: Write the failing test** - given the plist path, assert it parses (`plutil -convert json`) and has: `Label == com.necmttn.ax-studio.helper`, `BundleProgram` ending `/bin//bun`, `ProgramArguments` containing `serve` + `--managed-db`, `RunAtLoad == true`, a truthy `KeepAlive`, and **NO `StandardOutPath`/`StandardErrorPath` keys**. + +```typescript +import { describe, it, expect } from "bun:test"; +import { parseHelperPlist } from "./verify-helper-bundle.ts"; // plutil -convert json wrapper +// asserts the keys above from build/LaunchAgents/com.necmttn.ax-studio.helper.plist +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `cd apps/studio-desktop && bun test scripts/verify-helper-bundle.test.ts` +Expected: FAIL - plist absent. + +- [ ] **Step 3: Author the plist** at `build/LaunchAgents/com.necmttn.ax-studio.helper.plist` using the verbatim Form A shape in the contract note (§3 + §4). Mirror only the `KeepAlive`/`SoftResourceLimits NumberOfFiles 65536`/`ThrottleInterval 5` headroom from the CLI's `dbPlist` (`apps/axctl/src/cli/install.ts:96-157`). Do NOT include `StandardOutPath`/`StandardErrorPath`. + +- [ ] **Step 4: Wire electron-builder + the bundle deps** - (a) add an `extraFiles` entry copying the plist to `Library/LaunchAgents/com.necmttn.ax-studio.helper.plist` (`to` is relative to `Contents/`); (b) update `stage-ax-source.ts` to `bun install` the staged `ax-src/` so bundled bun can run it. electron-builder auto-signs all Mach-Os in the bundle (bun included) under `hardenedRuntime: true` - confirm `entitlementsInherit` covers the bundled bun (it already runs as a child today). + +- [ ] **Step 5: Run the bundle test against a staged build, verify pass** + +Run: `cd apps/studio-desktop && bun run build && bun test scripts/verify-helper-bundle.test.ts` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add apps/studio-desktop/build/com.necmttn.ax-studio-helper.plist apps/studio-desktop/electron-builder.config.cjs apps/studio-desktop/scripts/verify-helper-bundle.* +git commit -m "build(studio-desktop): bundle signed ax-studio-helper LaunchAgent plist (#599)" +``` + +--- + +### Task 5: Register / enable / unregister the helper (SMAppService) + +**Files:** +- Modify: `apps/studio-desktop/src/app/ElectronApp.ts` (add `registerBackgroundHelper` / `unregisterBackgroundHelper` / `helperStatus` to the `ElectronApp` service) +- Modify: `apps/studio-desktop/src/app/DesktopApp.ts:87-96` (call register on prod startup, alongside/instead of the current `setOpenAtLogin`) +- Test: `apps/studio-desktop/src/app/ElectronApp.test.ts` (verify the `setLoginItemSettings` args via a stub Electron) + +**Interfaces:** +- Consumes: Task 1's exact `setLoginItemSettings({ type: 'agentService', serviceName: 'com.necmttn.ax-studio-helper', openAtLogin: true })` arg shape; `app.getLoginItemSettings({ type:'agentService', serviceName })` for status. +- Produces: `ElectronApp.registerBackgroundHelper: Effect.Effect`, `ElectronApp.unregisterBackgroundHelper: Effect.Effect`, `ElectronApp.helperStatus: Effect.Effect<"enabled"|"requiresApproval"|"notRegistered"|"notFound">`. + +- [ ] **Step 1: Write the failing test** - `registerBackgroundHelper` calls the Electron stub's `setLoginItemSettings` exactly once with `type: 'agentService'` and the helper `serviceName`; guarded to no-op off darwin. + +- [ ] **Step 2: Run it, verify it fails** + +Run: `cd apps/studio-desktop && bun test src/app/ElectronApp.test.ts` +Expected: FAIL - method missing. + +- [ ] **Step 3: Implement** the three methods on `ElectronApp` (mirror the existing `setOpenAtLogin` wrapper at line 73). Guard `process.platform === 'darwin'`; fail-soft (log + continue) like the current `setOpenAtLogin` call. + +- [ ] **Step 4: Call on startup** - in `DesktopApp.ts` prod-only startup block, replace the bare `setOpenAtLogin(true)` with `registerBackgroundHelper` (which both registers the agent and keeps the app's own Login Item behavior per Task 1's findings). If `helperStatus` returns `requiresApproval`, surface a one-time tray/notification nudge (System Settings → Login Items). Keep it fail-soft - registration failure must never block the UI. + +- [ ] **Step 5: Run tests, verify pass** + +Run: `cd apps/studio-desktop && bun test src/app/ElectronApp.test.ts` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add apps/studio-desktop/src/app/ElectronApp.ts apps/studio-desktop/src/app/ElectronApp.test.ts apps/studio-desktop/src/app/DesktopApp.ts +git commit -m "feat(studio-desktop): register ax-studio-helper via SMAppService agentService (#599)" +``` + +--- + +### Task 6: UI attaches to the helper-owned backend (no double-spawn) + +**Files:** +- Modify: `apps/studio-desktop/src/backend/AxDaemonArbitration.ts` (ensure a healthy helper backend → `attach`) +- Modify: `apps/studio-desktop/src/backend/AxBackendManager.ts:542-575` (attach path: do not spawn, do not run the watchdog; the helper owns both) +- Test: `apps/studio-desktop/src/backend/AxDaemonArbitration.test.ts` + +**Interfaces:** +- Consumes: existing `decideArbitration` (`AxDaemonArbitration.ts:54-59`) returning `"attach" | "spawn" | "spawn-ax-only" | "conflict"`; probes `probeDaemon` (`/api/version`), `probeSurreal` (`/health`). +- Produces: unchanged decision type. New invariant: when the helper's serve+surreal are healthy, the UI resolves to `attach` and never spawns. + +- [ ] **Step 1: Write the failing/pinning test** - when both `probeDaemon` and `probeSurreal` are healthy, `decideArbitration` returns `"attach"` (covers the helper-already-running case the UI now hits on every launch). + +```typescript +// daemonHealthy=true, surrealHealthy=true, portsFree=false → "attach" +``` + +- [ ] **Step 2: Run it** + +Run: `cd apps/studio-desktop && bun test src/backend/AxDaemonArbitration.test.ts` +Expected: this may already pass (`daemonHealthy → "attach"`). The test pins the invariant regardless; proceed to Step 3. + +- [ ] **Step 3: Verify/adjust the manager attach path** - in `AxBackendManager` attach mode, confirm it opens the window + polls readiness but spawns NO surreal/serve and forks NO `SurrealWatchdog` (Task 3 must be spawn-mode-only). Add a manager-level test if one does not exist. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `cd apps/studio-desktop && bun test src/backend/` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/studio-desktop/src/backend/ +git commit -m "feat(studio-desktop): UI attaches to helper-owned backend, no double-spawn (#599)" +``` + +--- + +### Task 7: Real-query health in `ax daemon status` + doctor (visibility follow-up) + +**Why:** during the incident `ax daemon status`/`doctor` reported `database: listening on :8521` because they check the socket, not a query. A wedge must be visible. + +**Files:** +- Modify: `apps/axctl/src/cli/install.ts:772-843` (`collectDaemonEndpoint` / `collectDaemonStatus` / `formatDaemonStatus`) +- Reference: `@ax/lib/db` for a minimal authenticated `SELECT 1` round-trip +- Test: `apps/axctl/src/cli/install.test.ts` (or the nearest existing daemon-status test) - status maps a socket-up-but-query-timeout db to a distinct `wedged` state. + +**Interfaces:** +- Consumes: existing `probePort` (socket) + a new `probeDbQuery(host, port): Effect.Effect` (real `SELECT 1`, short timeout). +- Produces: status output distinguishes `listening (healthy)` from `listening but NOT answering queries (wedged) - restart with 'ax daemon restart'`. + +- [ ] **Step 1: Write the failing test** - `formatDaemonStatus` with `{ portListening: true, queryOk: false }` includes the word `wedged` and the restart hint. + +- [ ] **Step 2: Run it, verify it fails** + +Run: `cd apps/axctl && bun test src/cli/install.test.ts` +Expected: FAIL. + +- [ ] **Step 3: Implement** `probeDbQuery` (reuse `@ax/lib/db` connect + `SELECT 1`, 1–2s timeout, fail-closed) and fold its result into `collectDaemonStatus`; add the `wedged` line in `formatDaemonStatus`. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `cd apps/axctl && bun test src/cli/install.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/axctl/src/cli/install.ts apps/axctl/src/cli/install.test.ts +git commit -m "feat(daemon): status/doctor flag a wedged db via real SELECT 1 probe (#599)" +``` + +--- + +### Task 8: Build, sign, notarize, and document + +**Files:** +- Modify: `.github/workflows/studio-desktop-release.yml` (verify the bundled plist is signed inside the app) +- Modify: `apps/studio-desktop/README.md` (or the desktop docs) - helper model, how to verify, how to uninstall (SMAppService unregister) +- Modify: `docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md` - flip Option A from "rejected/escape hatch" to "implemented (#599)" with a back-reference to this plan and the incident memory. + +- [ ] **Step 1: Local signed-build smoke** - produce a signed (or ad-hoc) build, install to `/Applications`, launch, quit the app, and confirm: surreal+serve still answer (`curl :1738/api/version`), the agent shows in `launchctl list | rg ax-studio-helper` with exit 0, and `kill -9` of surreal triggers a respawn within `ThrottleInterval`. + +- [ ] **Step 2: Wedge-recovery smoke (incident regression)** - simulate a wedge (`kill -STOP `), confirm the watchdog trips within `failuresToTrip * interval` and force-restarts (`kill -9` + respawn), and queries recover. + +- [ ] **Step 3: Login Item attribution check** - on a signed build, confirm System Settings → General → Login Items shows ONE "ax studio" item attributed to the Developer ID (NOT "bash - unidentified developer"). This is the acceptance criterion distinguishing Option A from the rejected Option B. + +- [ ] **Step 4: Docs + spec flip + commit** + +```bash +git add apps/studio-desktop/README.md docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md +git commit -m "docs: ship Option A studio background helper; flip spec + uninstall notes (#599)" +``` + +- [ ] **Step 5: Tag a release dry-run** (or document the exact secrets/steps if CI secrets are not yet configured - `CSC_LINK`, `APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, `APPLE_TEAM_ID`). + +--- + +## Self-Review + +**Spec coverage (vs #599 acceptance criteria):** +- "Quitting/force-killing the app leaves db running + queryable" → Tasks 2,4,5 (helper owns backend via launchd) + Task 8 Step 1. +- "kill -9 the db triggers respawn (KeepAlive)" → Task 4 (KeepAlive) + Task 8 Step 1. +- "db comes up at login without opening the app (RunAtLoad)" → Task 4 (RunAtLoad) + Task 5 (register). +- "ax daemon status reflects the registered service" → Task 7. +- "clean unregister on uninstall" → Task 5 (`unregisterBackgroundHelper`) + Task 8 docs. +- Beyond #599: wedge detection (Task 3) - the actual incident root cause that KeepAlive alone would miss; and the false-"listening" visibility gap (Task 7). + +**Placeholder scan:** Task 1 is a deliberate spike (the one true unknown - Electron `agentService` exact contract) that gates the executable-strategy choice for Tasks 2/4; its output is a concrete note, not a deferral. No "TODO/handle edge cases" left in implementation steps. + +**Type consistency:** `isHelperInvocation`/`helperProgram` (Task 2), `makeSurrealWatchdog` opts (Task 3), `registerBackgroundHelper`/`unregisterBackgroundHelper`/`helperStatus` (Task 5), `decideArbitration` return union (Task 6), `probeDbQuery` (Task 7) - names are used consistently across the tasks that reference them. + +**Known risks:** (1) Task 1 could invalidate the single-binary `--background-helper` assumption (forcing a separate bundled launcher); the gate in Task 1 catches this before downstream work. (2) `agentService` may require user approval in System Settings on first run (handled as a fail-soft tray nudge in Task 5 Step 4), unlike a loose LaunchAgent - an acceptable trade for Developer-ID attribution. diff --git a/docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md b/docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md index 28e26485..c7c3893c 100644 --- a/docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md +++ b/docs/superpowers/specs/2026-06-16-smappservice-background-helper-design.md @@ -1,8 +1,17 @@ -# Studio desktop - background/daemon model (decided: IDE model) +# Studio desktop - background/daemon model (decided: IDE model → Option A implemented) + +> **UPDATE 2026-06-24 (#599):** Option A is now **IMPLEMENTED**. A 4-day +> data-plane stall (see `memory/ax-data-plane-fragile-ide-model.md`) made +> true app-closed capture a hard requirement; the IDE model's "app must be +> open" accepted tradeoff was no longer acceptable. Option A (SMAppService +> `agentService` helper) ships in #599 as the background daemon. +> Implementation plan: `docs/superpowers/plans/2026-06-24-studio-helper-smappservice.md`. +> Operator guide: see the "Operating the helper" section below. +> Original Option C decision rationale is preserved intact below. **Date:** 2026-06-16 -**Status:** Decided - **Option C (IDE model)**. Background-daemon options (A/B) -rejected. +**Status (original):** Decided - **Option C (IDE model)**. Background-daemon options (A/B) rejected. +**Status (current):** **Option A implemented** via #599 (2026-06-24). Option C IDE-model code remains in place; the helper augments it - the UI app attaches to the already-running helper backend rather than spawning its own. **Decision owner:** Necmttn **Builds on:** `docs/superpowers/specs/2026-06-07-studio-desktop-design.md` @@ -60,9 +69,17 @@ app-closed capture ever becomes a hard requirement, revisit Option A below. `agentService` plist (`BundleProgram`, registered via `setLoginItemSettings({ type: 'agentService', serviceName })`, verified available in Electron 41.5.0) running a headless `AxBackendManager` + polled ingest. - Delivers true app-closed ingest as a single Developer-ID item. **Rejected for - v0:** most work, not the IDE norm, and unnecessary once the app auto-launches + - stays tray-resident. Kept on file as the escape hatch if 24/7 capture is needed. + Delivers true app-closed ingest as a single Developer-ID item. **Originally + rejected for v0** (most work, not the IDE norm, unnecessary once auto-launch + + tray). **NOW IMPLEMENTED via #599 (2026-06-24)** - the 4-day data-plane stall + (IDE model; surreal wedge + app crash; `KeepAlive` alone can't fix a wedge) made + 24/7 app-closed capture a hard requirement. Form A: `BundleProgram` = bundled bun + Mach-O (`Contents/Resources/bin/arm64/bun`); `ProgramArguments` run bundled ax-src + serve entry with `--managed-db --ingest-every=2m`. Helper registered at app boot; + KeepAlive+RunAtLoad in the plist; UI attaches via `AxDaemonArbitration`. A + real-query watchdog SIGKILLs+respawns a hung surreal (the wedge fix KeepAlive + alone can't provide). `ax daemon status` surfaces a wedged db; + `ax daemon restart` triggers recovery. - **B - bundle the existing agents as SMAppService agents.** 2–3 attributed items instead of 5. **Rejected:** still more background items than C, more moving parts. @@ -110,5 +127,75 @@ app-closed capture ever becomes a hard requirement, revisit Option A below. ## Not doing (v0) -Headless background daemon (Option A), Windows/Linux services (separate systemd -track), root `daemonService`. +~~Headless background daemon (Option A)~~ (now implemented via #599), Windows/Linux +services (separate systemd track), root `daemonService`. + +--- + +## Operating the helper (added 2026-06-24, #599) + +The background helper (`com.necmttn.ax-studio.helper`) is an `agentService` launchd +job registered by `ax studio.app` at first launch via `setLoginItemSettings({ type: +'agentService', serviceName: 'com.necmttn.ax-studio.helper', openAtLogin: true })`. +It runs bundled bun → bundled ax-src `ax serve --managed-db --ingest-every=2m`, +keeping surreal + the HTTP API on `:1738` alive even when the UI app is closed or +crashes. A real-query watchdog inside the helper SIGKILLs+respawns a hung surreal +process (the original 4-day-stall scenario that `KeepAlive` alone cannot fix). + +### What the helper does + +- Owns surreal on `127.0.0.1:8521` (spawns it as a child via `--managed-db`). +- Owns `ax serve` on `127.0.0.1:1738` (the same HTTP API `ax studio.app` talks to). +- Runs an ingest loop every 2 minutes (`--ingest-every=2m`) so the graph stays + current even when the app is closed. +- Survives app quit and crashes (launchd `KeepAlive`+`RunAtLoad`). +- Surfaces in System Settings → General → Login Items as **"ax studio"** (one + Developer-ID-attributed item, not anonymous bash). + +### Verifying the helper is running + +```bash +# Check launchd job is registered and running +launchctl list | grep ax-studio-helper + +# Check the ax serve endpoint the helper owns +ax daemon status + +# If the helper is running, /api/version will respond +curl -s http://127.0.0.1:1738/api/version | jq . +``` + +If `ax daemon status` reports a wedged db, run `ax daemon restart` to force a +SIGKILL+respawn of the stuck surreal process. + +### Uninstalling / disabling the helper + +The helper is registered by the app and tied to the app's code signature. To +remove it: + +1. **Via the app:** If an unregister UI is present, use it (calls + `setLoginItemSettings({ type: 'agentService', serviceName: '...', openAtLogin: + false })` internally). +2. **Via System Settings:** General → Login Items → find "ax studio" under "Allow + in the Background" → toggle off or remove. +3. **Via launchctl (manual):** + ```bash + launchctl bootout gui/$(id -u)/com.necmttn.ax-studio.helper + ``` + This stops the job for the current login session. It will re-register on the + next app launch unless the app also calls `SMAppService.unregister`. +4. **Fully remove:** Delete `ax studio.app` from `/Applications`. macOS + automatically unregisters all `agentService` jobs whose parent app bundle is + gone. + +### Open smoke item (deferred to maintainer) + +The plist's `ProgramArguments` use bundle-root-relative paths (e.g. +`Contents/Resources/ax-src/apps/axctl/src/cli/index.ts`). Launchd's working +directory for SMAppService agents is `/` - those paths are NOT auto-resolved by +launchd the way `BundleProgram` is. **The signed-build smoke (Step 1 of the task-8 +brief) must confirm bun actually starts ax serve.** If the agent starts but `ax +serve` fails (check `ax daemon status` + `launchctl list | grep ax-studio-helper` +exit code), switch `BundleProgram` to the shell-wrapper fallback documented in +`docs/superpowers/notes/2026-06-24-agentservice-contract.md §4`, which resolves +paths relative to `$0` before exec.