diff --git a/apps/server/src/db.ts b/apps/server/src/db.ts index 8db193ce05..6542b787f1 100644 --- a/apps/server/src/db.ts +++ b/apps/server/src/db.ts @@ -1,4 +1,10 @@ -import { createConnection, ensurePersonalProject, migrate } from "@bb/db"; +import { + createConnection, + ensurePersonalProject, + getDatabaseAutoVacuumMode, + getDatabaseFreelistStats, + migrate, +} from "@bb/db"; import type { DbConnection, MigrationWarningLogger, @@ -42,5 +48,16 @@ export function initDb( logger: options.logger, }); ensurePersonalProject(db); + // A legacy auto_vacuum=NONE file never shrinks until a full VACUUM + // converts it; surface the resolved mode and the O(1) freelist counters at + // startup so an operator can see which regime this database is in without + // waiting for the hourly maintenance sweep to log. + options.logger?.info( + { + autoVacuumMode: getDatabaseAutoVacuumMode(db), + freelist: getDatabaseFreelistStats(db), + }, + "Database auto-vacuum state", + ); return db; } diff --git a/apps/server/src/internal/events.ts b/apps/server/src/internal/events.ts index 9fde9228b7..394794c83e 100644 --- a/apps/server/src/internal/events.ts +++ b/apps/server/src/internal/events.ts @@ -37,6 +37,7 @@ import { isActivePruneTriggerThreadEventType, maybePruneActiveThreadEventHistory, } from "../services/system/event-pruning.js"; +import { runEventLoopWorkSync } from "../services/system/event-loop-work.js"; import { queueChildThreadTurnNotificationBestEffort } from "../services/threads/child-thread-notifications.js"; import { isParentNotifiableChildThread } from "../services/threads/thread-parent.js"; import { runQueuedMessageAutoSendForThread } from "../services/threads/queued-messages.js"; @@ -194,9 +195,22 @@ interface QueuedMessageAutoSendFollowUp { threadId: string; } +/** + * Opportunistic event-history prune, deferred off the ingest response path. + * Pruning is output-preserving (see the timeline cache notes), so running it + * after the response — instead of inside the hottest handler — only changes + * when the redundant rows disappear, never what any read returns. + */ +interface ActiveThreadEventPruneFollowUp { + kind: "active-thread-event-prune"; + latestPrunableSequence: number; + threadId: string; +} + type EventEffectFollowUp = | ParentTurnNotificationFollowUp - | QueuedMessageAutoSendFollowUp; + | QueuedMessageAutoSendFollowUp + | ActiveThreadEventPruneFollowUp; function isRootTurnStartedEvent( event: Extract, @@ -494,6 +508,16 @@ async function executeEventFollowUpBestEffort( threadId: followUp.threadId, }); return; + case "active-thread-event-prune": + // Synchronous better-sqlite3 work: the wrapper lets the stall + // monitor name the prune instead of blaming whatever ran next. + runEventLoopWorkSync(`event-prune ${followUp.threadId}`, () => + maybePruneActiveThreadEventHistory(deps, { + latestPrunableSequence: followUp.latestPrunableSequence, + threadId: followUp.threadId, + }), + ); + return; } } catch (error) { if (isCommandTimeoutError(error)) { @@ -1023,7 +1047,7 @@ export function registerInternalEventRoutes(app: Hono, deps: AppDeps): void { events: postableEvents, insertedEventIndexes: appendResult.insertedInputIndexes, })) { - maybePruneActiveThreadEventHistory(deps, candidate); + followUps.push({ kind: "active-thread-event-prune", ...candidate }); } deferEventFollowUpBatch(deps, followUps); diff --git a/apps/server/src/routes/threads/base.ts b/apps/server/src/routes/threads/base.ts index d99182e871..4e73cb6127 100644 --- a/apps/server/src/routes/threads/base.ts +++ b/apps/server/src/routes/threads/base.ts @@ -1,4 +1,5 @@ import { + DEFAULT_THREAD_LIST_LIMIT, THREAD_SEARCH_LIMIT_PER_GROUP_DEFAULT, THREAD_SEARCH_LIMIT_PER_GROUP_MAX, countNonDeletedAssignedChildThreads, @@ -233,7 +234,13 @@ export function registerThreadBaseRoutes(app: Hono, deps: AppDeps): void { if (query.sectionId) { requireThreadSection(deps, query.sectionId); } - const threads = listThreadsWithPendingInteractionState(deps.db, { + // The list is capped (default DEFAULT_THREAD_LIST_LIMIT) so one + // long-lived install cannot make every list read scale with its full + // history. The response stays a bare array for compatibility, so the + // truncation signal rides a header: fetch one probe row past the cap and + // report whether it existed. Callers page with limit/offset. + const effectiveLimit = limit ?? DEFAULT_THREAD_LIST_LIMIT; + const threadRows = listThreadsWithPendingInteractionState(deps.db, { ...(query.projectId ? { projectId: query.projectId } : {}), ...(query.parentThreadId ? { parentThreadId: query.parentThreadId } : {}), ...(query.sourceThreadId ? { sourceThreadId: query.sourceThreadId } : {}), @@ -246,9 +253,12 @@ export function registerThreadBaseRoutes(app: Hono, deps: AppDeps): void { query.archived === undefined ? undefined : query.archived === "true", hasParent: query.hasParent === undefined ? undefined : query.hasParent === "true", - ...(limit !== undefined ? { limit } : {}), + limit: effectiveLimit + 1, ...(offset !== undefined ? { offset } : {}), }); + const hasMore = threadRows.length > effectiveLimit; + const threads = hasMore ? threadRows.slice(0, effectiveLimit) : threadRows; + context.header("x-bb-thread-list-has-more", hasMore ? "true" : "false"); return context.json( toThreadListEntryResponses(deps, { threads }) satisfies ThreadListEntry[], ); diff --git a/apps/server/src/routes/threads/data.ts b/apps/server/src/routes/threads/data.ts index 6aa5e42cce..7004a3b577 100644 --- a/apps/server/src/routes/threads/data.ts +++ b/apps/server/src/routes/threads/data.ts @@ -348,8 +348,9 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void { summaryOnly, includeProviderUnhandledOperations, }; + const paramsKey = buildThreadTimelineParamsKey(keyArgs); const full = timelineCache.getOrBuild( - buildThreadTimelineCacheKey({ ...keyArgs, maxSeq }), + { key: buildThreadTimelineCacheKey({ ...keyArgs, maxSeq }), paramsKey }, () => { const { profile, response } = buildThreadTimelineWithProfile( deps.db, @@ -392,7 +393,6 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void { query.afterSequence, "afterSequence", ); - const paramsKey = buildThreadTimelineParamsKey(keyArgs); const previous = afterSequence === undefined ? undefined @@ -401,7 +401,13 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void { previous === undefined ? undefined : computeTimelineRowDelta(previous.rows, full.rows); - timelineLatestRowsCache.set(paramsKey, { maxSeq, rows: full.rows }); + // Keyed by the response's own revision, not the route-computed `maxSeq`: + // the cache's rebuild floor may serve a window built just before the + // newest events, and the snapshot must name the revision the rows reflect. + timelineLatestRowsCache.set(paramsKey, { + maxSeq: full.maxSeq, + rows: full.rows, + }); return context.json( delta === undefined ? full : { ...full, rows: [], delta }, diff --git a/apps/server/src/services/plugins/plugin-api.ts b/apps/server/src/services/plugins/plugin-api.ts index 68d2c57dbc..927006230a 100644 --- a/apps/server/src/services/plugins/plugin-api.ts +++ b/apps/server/src/services/plugins/plugin-api.ts @@ -92,7 +92,7 @@ import type { import type { BbSdk, ThreadForkArgs, ThreadSpawnArgs } from "@bb/sdk"; import type { ServerLogger } from "../../types.js"; import type { PluginInteractionResult } from "../interactions/pending-interactions.js"; -import { appendPluginLogLine } from "./plugin-log.js"; +import { appendPluginLogLine, disposePluginLogWriter } from "./plugin-log.js"; import type { PluginHostArtifactSnapshot } from "./plugin-service-internal.js"; import { readPluginSettingsValues } from "./plugin-settings.js"; @@ -430,6 +430,19 @@ function createStagedRegistrations< }; } +/** + * `bb.realtime.publish` broadcasts to every connected client with no + * per-channel subscription, so a runaway publisher multiplies across the + * whole socket fan-out. Interactive signals are a few per second; anything + * sustained above that is treated as a loop and dropped (signals are + * ephemeral by contract), warning once per window so the log names the + * plugin without flooding. + */ +const REALTIME_PUBLISH_MAX_PAYLOAD_BYTES = 64 * 1024; +const REALTIME_PUBLISH_TOKENS_PER_SECOND = 10; +const REALTIME_PUBLISH_BURST_TOKENS = 20; +const REALTIME_PUBLISH_DROP_WARN_INTERVAL_MS = 10_000; + export function createPluginApi(options: { pluginId: string; logger: ServerLogger; @@ -439,8 +452,10 @@ export function createPluginApi(options: { getSdk: () => BbSdk | undefined; /** Undefined until the server is listening (bb.server is bind-gated too). */ getLoopbackBaseUrl: () => string | undefined; - /** Broadcasts a plugin-signal WS message (hub.notifyPluginSignal). */ - publishSignal: (channel: string, payload: unknown) => void; + /** Broadcasts a plugin-signal WS message (hub.notifyPluginSignal). The + * payload arrives pre-serialized — publish's one JSON.stringify — and is + * embedded verbatim in the wire frame. */ + publishSignal: (channel: string, serializedPayload: string) => void; /** Marks the plugin needs-configuration in the loader's status table. */ reportNeedsConfiguration: (message: string) => void; /** Returns the owning plugin id when another plugin already registered @@ -554,6 +569,10 @@ export function createPluginApi(options: { const pendingAgentToolProblems: string[] = []; const pendingSharedPorts = new Map(); const disposeHooks: Array<() => void | Promise> = []; + // Flush buffered bb.log lines before this generation's resources go away: + // a pending flush timer must not outlive (and race) a plugin data-dir + // removal. + disposeHooks.push(() => disposePluginLogWriter(dataDir, pluginId)); const settingsRecord: PluginApiHandle["settings"] = { descriptors: {}, listeners: [], @@ -866,16 +885,45 @@ export function createPluginApi(options: { }, }; + // Token bucket for realtime publishes; state lives with the handle, so a + // reload starts the plugin with a full burst again. + let realtimePublishTokens = REALTIME_PUBLISH_BURST_TOKENS; + let realtimePublishRefilledAt = Date.now(); + let realtimePublishWarnedAt = 0; + function takeRealtimePublishToken(channel: string): boolean { + const now = Date.now(); + realtimePublishTokens = Math.min( + REALTIME_PUBLISH_BURST_TOKENS, + realtimePublishTokens + + ((now - realtimePublishRefilledAt) / 1000) * + REALTIME_PUBLISH_TOKENS_PER_SECOND, + ); + realtimePublishRefilledAt = now; + if (realtimePublishTokens >= 1) { + realtimePublishTokens -= 1; + return true; + } + if (now - realtimePublishWarnedAt >= REALTIME_PUBLISH_DROP_WARN_INTERVAL_MS) { + realtimePublishWarnedAt = now; + logger.warn( + { pluginId, channel }, + "Dropping bb.realtime.publish signals over the per-plugin rate limit", + ); + } + return false; + } + const realtime: PluginRealtime = { publish(channel, payload) { assertLive(); if (typeof channel !== "string" || channel.length === 0) { throw new Error("realtime channel must be a non-empty string"); } - // JSON round-trip up front: enforces serializability with a clear - // error at the publish site and strips prototypes/getters before the - // payload crosses the WS boundary. - let normalized: unknown = null; + // One serialization up front: it enforces serializability with a clear + // error at the publish site AND produces the exact bytes that cross + // the WS boundary — the hub embeds this string verbatim instead of + // re-parsing and re-stringifying the payload per publish. + let payloadJson = "null"; if (payload !== undefined) { let json: string | undefined; try { @@ -888,9 +936,19 @@ export function createPluginApi(options: { `realtime payload for channel "${channel}" is not JSON-serializable`, ); } - normalized = JSON.parse(json); + payloadJson = json; + } + const payloadBytes = Buffer.byteLength(payloadJson, "utf8"); + if (payloadBytes > REALTIME_PUBLISH_MAX_PAYLOAD_BYTES) { + throw new Error( + `realtime payload for channel "${channel}" is ${payloadBytes} bytes; ` + + `the limit is ${REALTIME_PUBLISH_MAX_PAYLOAD_BYTES} (64KB)`, + ); + } + if (!takeRealtimePublishToken(channel)) { + return; } - publishSignal(channel, normalized); + publishSignal(channel, payloadJson); }, }; diff --git a/apps/server/src/services/plugins/plugin-log.ts b/apps/server/src/services/plugins/plugin-log.ts index 06afbf4a51..4ea6f0b3b0 100644 --- a/apps/server/src/services/plugins/plugin-log.ts +++ b/apps/server/src/services/plugins/plugin-log.ts @@ -1,5 +1,5 @@ -import { appendFileSync, mkdirSync, renameSync, statSync } from "node:fs"; -import { readFile } from "node:fs/promises"; +import { mkdirSync, renameSync, statSync } from "node:fs"; +import { appendFile, readFile } from "node:fs/promises"; import { join } from "node:path"; /** @@ -7,21 +7,134 @@ import { join } from "node:path"; * appended as JSONL to /plugins//logs/plugin.log in addition to * the prefixed server log. Simple size rotation: past 5MB the file is renamed * to plugin.log.1 (one rotated file kept, replacing the previous one). + * + * `bb.log` runs on the server event loop, so lines are buffered and flushed + * as one async append per window (or once the buffer crosses a byte + * threshold) instead of a mkdir + stat + sync append per line. The directory + * is created once per writer lifetime and the rotation size is tracked from + * a cached counter seeded by one stat. Delivery stays best effort — it + * already was, and the prefixed server log still carries every message. */ const PLUGIN_LOG_MAX_BYTES = 5 * 1024 * 1024; const PLUGIN_LOG_FILE = "plugin.log"; const PLUGIN_LOG_ROTATED_FILE = "plugin.log.1"; +const PLUGIN_LOG_FLUSH_INTERVAL_MS = 200; +const PLUGIN_LOG_FLUSH_THRESHOLD_BYTES = 8 * 1024; type PluginLogLevel = "debug" | "info" | "warn" | "error"; +interface PluginLogWriter { + bufferedBytes: number; + bufferedLines: string[]; + dirEnsured: boolean; + /** Serializes flushes so appended lines keep their order. */ + flushChain: Promise; + /** Cached plugin.log size; null until seeded by the writer's one stat. */ + logFileBytes: number | null; + timer: ReturnType | null; +} + +const pluginLogWriters = new Map(); + function pluginLogsDir(dataDir: string, pluginId: string): string { return join(dataDir, "plugins", pluginId, "logs"); } +function pluginLogWriterKey(dataDir: string, pluginId: string): string { + return `${dataDir}\u0000${pluginId}`; +} + +function getPluginLogWriter(dataDir: string, pluginId: string): PluginLogWriter { + const key = pluginLogWriterKey(dataDir, pluginId); + const existing = pluginLogWriters.get(key); + if (existing !== undefined) { + return existing; + } + const writer: PluginLogWriter = { + bufferedBytes: 0, + bufferedLines: [], + dirEnsured: false, + flushChain: Promise.resolve(), + logFileBytes: null, + timer: null, + }; + pluginLogWriters.set(key, writer); + return writer; +} + +function flushPluginLogWriter( + dataDir: string, + pluginId: string, + writer: PluginLogWriter, +): Promise { + if (writer.timer !== null) { + clearTimeout(writer.timer); + writer.timer = null; + } + if (writer.bufferedLines.length === 0) { + return writer.flushChain; + } + const chunk = writer.bufferedLines.join(""); + const chunkBytes = writer.bufferedBytes; + writer.bufferedLines = []; + writer.bufferedBytes = 0; + writer.flushChain = writer.flushChain.then(async () => { + try { + const dir = pluginLogsDir(dataDir, pluginId); + if (!writer.dirEnsured) { + mkdirSync(dir, { recursive: true }); + writer.dirEnsured = true; + } + const file = join(dir, PLUGIN_LOG_FILE); + if (writer.logFileBytes === null) { + try { + writer.logFileBytes = statSync(file).size; + } catch { + // Missing file: nothing logged there yet. + writer.logFileBytes = 0; + } + } + if (writer.logFileBytes > PLUGIN_LOG_MAX_BYTES) { + try { + renameSync(file, join(dir, PLUGIN_LOG_ROTATED_FILE)); + } catch { + // Missing file: nothing to rotate. + } + writer.logFileBytes = 0; + } + await appendFile(file, chunk, "utf8"); + writer.logFileBytes += chunkBytes; + } catch { + // Best effort only — a full disk or permission problem must not break + // the plugin call site; the prefixed server log still carries the + // messages. + } + }); + return writer.flushChain; +} + +/** + * Flush and drop a plugin's buffered log writer. Wired into the plugin API + * handle's dispose hooks so reload/disable/shutdown wait for pending lines + * instead of leaving a flush timer racing the plugin data directory's + * removal. + */ +export async function disposePluginLogWriter( + dataDir: string, + pluginId: string, +): Promise { + const key = pluginLogWriterKey(dataDir, pluginId); + const writer = pluginLogWriters.get(key); + if (writer === undefined) { + return; + } + pluginLogWriters.delete(key); + await flushPluginLogWriter(dataDir, pluginId, writer); +} + /** - * Append one log line synchronously (bb.log is a sync API; lines are tiny). - * Never throws — a full disk or permission problem must not break the plugin - * call site; the prefixed server log still carries the message. + * Buffer one log line (bb.log is a sync API; lines are tiny). Never throws — + * delivery is best effort and happens off the call path. */ export function appendPluginLogLine( dataDir: string, @@ -30,18 +143,21 @@ export function appendPluginLogLine( message: string, ): void { try { - const dir = pluginLogsDir(dataDir, pluginId); - mkdirSync(dir, { recursive: true }); - const file = join(dir, PLUGIN_LOG_FILE); - try { - if (statSync(file).size > PLUGIN_LOG_MAX_BYTES) { - renameSync(file, join(dir, PLUGIN_LOG_ROTATED_FILE)); - } - } catch { - // Missing file: nothing to rotate. + const writer = getPluginLogWriter(dataDir, pluginId); + const line = `${JSON.stringify({ ts: Date.now(), level, message })}\n`; + writer.bufferedLines.push(line); + writer.bufferedBytes += Buffer.byteLength(line, "utf8"); + if (writer.bufferedBytes >= PLUGIN_LOG_FLUSH_THRESHOLD_BYTES) { + void flushPluginLogWriter(dataDir, pluginId, writer); + return; + } + if (writer.timer === null) { + writer.timer = setTimeout(() => { + writer.timer = null; + void flushPluginLogWriter(dataDir, pluginId, writer); + }, PLUGIN_LOG_FLUSH_INTERVAL_MS); + writer.timer.unref?.(); } - const line = JSON.stringify({ ts: Date.now(), level, message }); - appendFileSync(file, `${line}\n`, "utf8"); } catch { // Best effort only. } @@ -53,13 +169,18 @@ function splitLines(content: string): string[] { /** * Last `tail` log lines across the rotated file plus the current one, oldest - * first. Missing files read as empty. + * first. Missing files read as empty. Pending buffered lines are flushed + * first so a tail right after `bb.log` stays read-your-writes. */ export async function readPluginLogTail( dataDir: string, pluginId: string, tail: number, ): Promise { + const writer = pluginLogWriters.get(pluginLogWriterKey(dataDir, pluginId)); + if (writer !== undefined) { + await flushPluginLogWriter(dataDir, pluginId, writer); + } const dir = pluginLogsDir(dataDir, pluginId); const lines: string[] = []; for (const name of [PLUGIN_LOG_ROTATED_FILE, PLUGIN_LOG_FILE]) { diff --git a/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md b/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md index 71f617c36b..2bdac0857d 100644 --- a/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md +++ b/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md @@ -70,7 +70,9 @@ message agents, or inspect projects, providers, and environments. - `bb-app config` and `bb-app env` reload runtime settings in a running server, but the CLI identifies server and launcher settings that are startup-only, including binding/ports, data and the dev-app port, telemetry, inherited skill - roots, and `BB_FF_*` flags. `BB_LOG_LEVEL` is also startup-only. Use + roots, the SQLite memory budgets (`BB_SQLITE_CACHE_SIZE_KIB`, + `BB_SQLITE_MMAP_SIZE_BYTES`; see docs/configuration.md), and `BB_FF_*` + flags. `BB_LOG_LEVEL` is also startup-only. Use `bb-app config`, not `bb-app env`, to change `BB_APP_URL`, `BB_INFERENCE`, `BB_INFERENCE_FALLBACK`, or `BB_TRANSCRIPTION` live. After a startup-only change, run `bb-app stop && bb-app start` or restart the desktop app. Until diff --git a/apps/server/src/services/system/event-pruning.ts b/apps/server/src/services/system/event-pruning.ts index 2f584750e5..c604bdf979 100644 --- a/apps/server/src/services/system/event-pruning.ts +++ b/apps/server/src/services/system/event-pruning.ts @@ -1,4 +1,5 @@ import { performance } from "node:perf_hooks"; +import { and, eq, inArray, lte } from "drizzle-orm"; import { getThread, getLatestThreadSequence, @@ -7,6 +8,7 @@ import { pruneResolvedItemDeltas, pruneTokenUsageEventsBeforeSequence, pruneThreadEventsBeforeSequence, + events as storedEvents, } from "@bb/db"; import type { ThreadEventType } from "@bb/domain"; import { roundDurationMs } from "../lib/duration.js"; @@ -58,7 +60,13 @@ class ThreadEventPruningStepError extends Error { const ACTIVE_THREAD_EVENT_KEEP_RECENT = 1_000; const IDLE_THREAD_EVENT_KEEP_RECENT = 300; -const ARCHIVED_THREAD_EVENT_KEEP_RECENT = 120; +/** + * Keep-recent window for archived threads. Applied by the on-archive prune + * (prunable event classes) and by the periodic archived-thread retention + * sweep (all event classes). Product policy: an archived thread keeps only + * this many recent sequence slots of history. + */ +export const ARCHIVED_THREAD_EVENT_KEEP_RECENT = 120; const ACTIVE_THREAD_EVENT_PRUNE_MIN_SEQUENCE_DELTA = 250; const ACTIVE_THREAD_EVENT_PRUNE_MIN_INTERVAL_MS = 30_000; const SLOW_THREAD_EVENT_PRUNE_LOG_THRESHOLD_MS = 1_000; @@ -85,6 +93,18 @@ const GENERIC_AGE_PRUNABLE_THREAD_EVENT_TYPES: readonly ThreadEventType[] = [ "turn/diff/updated", ] as const; +/** + * Candidate types for `pruneResolvedItemDeltas`. Must stay a superset of the + * delta types that DELETE targets, or the probe below silently disables the + * prune for the missing type. + */ +const RESOLVED_ITEM_DELTA_THREAD_EVENT_TYPES: readonly ThreadEventType[] = [ + "item/agentMessage/delta", + "item/commandExecution/outputDelta", + "item/reasoning/summaryTextDelta", + "item/reasoning/textDelta", +] as const; + const KEEP_RECENT_BY_MODE: Record = { active: ACTIVE_THREAD_EVENT_KEEP_RECENT, idle: IDLE_THREAD_EVENT_KEEP_RECENT, @@ -125,6 +145,49 @@ export function isActivePruneTriggerThreadEventType( return activePruneTriggerThreadEventTypeSet.has(eventType); } +interface HasPrunableCandidateRowsArgs { + threadId: string; + types: readonly ThreadEventType[]; + /** Omit for classes whose DELETE has no sequence bound. */ + sequenceCutoff?: number; +} + +/** + * LIMIT 1 necessary-condition probe for a prune class. The usage-row DELETEs + * walk every row of their type through a correlated-subquery CTE, and the + * resolved-delta/background-progress DELETEs walk their candidates with + * correlated EXISTS checks — even when the thread has no candidate rows at + * all, which is the common case for an active thread pruned every ~30s. This + * probe answers "could that DELETE possibly match?" with one covering seek on + * the (thread_id, type, sequence) index, so the no-op case skips the walk + * outright. A true result only means candidates exist; the DELETE still + * decides what is actually prunable. + */ +function hasPrunableCandidateRows( + db: AppDeps["db"], + args: HasPrunableCandidateRowsArgs, +): boolean { + if (args.sequenceCutoff !== undefined && args.sequenceCutoff <= 0) { + return false; + } + return ( + db + .select({ id: storedEvents.id }) + .from(storedEvents) + .where( + and( + eq(storedEvents.threadId, args.threadId), + inArray(storedEvents.type, [...args.types]), + ...(args.sequenceCutoff === undefined + ? [] + : [lte(storedEvents.sequence, args.sequenceCutoff)]), + ), + ) + .limit(1) + .get() !== undefined + ); +} + export function pruneThreadEventHistory( deps: Pick, args: PruneThreadEventHistoryArgs, @@ -140,17 +203,31 @@ export function pruneThreadEventHistory( const sequenceCutoff = Math.max(0, latestSequence - keepRecent); const removedAgePrunableEvents = runThreadEventPruningStep("prune_context_window_usage", () => - pruneContextWindowUsageEventsBeforeSequence(deps.db, { + hasPrunableCandidateRows(deps.db, { threadId: args.threadId, + types: ["thread/contextWindowUsage/updated"], sequenceCutoff, - }), + }) + ? pruneContextWindowUsageEventsBeforeSequence(deps.db, { + threadId: args.threadId, + sequenceCutoff, + }) + : 0, ) + runThreadEventPruningStep("prune_token_usage", () => - pruneTokenUsageEventsBeforeSequence(deps.db, { + hasPrunableCandidateRows(deps.db, { threadId: args.threadId, + types: ["thread/tokenUsage/updated"], sequenceCutoff, - }), + }) + ? pruneTokenUsageEventsBeforeSequence(deps.db, { + threadId: args.threadId, + sequenceCutoff, + }) + : 0, ) + + // No probe: this DELETE is already a plain (thread_id, type, sequence) + // index-range scan, exactly what the probe would run. runThreadEventPruningStep("prune_generic_age_prunable_events", () => pruneThreadEventsBeforeSequence(deps.db, { threadId: args.threadId, @@ -161,16 +238,26 @@ export function pruneThreadEventHistory( const removedResolvedItemDeltas = runThreadEventPruningStep( "prune_resolved_item_deltas", () => - pruneResolvedItemDeltas(deps.db, { + hasPrunableCandidateRows(deps.db, { threadId: args.threadId, - }), + types: RESOLVED_ITEM_DELTA_THREAD_EVENT_TYPES, + }) + ? pruneResolvedItemDeltas(deps.db, { + threadId: args.threadId, + }) + : 0, ); const removedBackgroundTaskProgressEvents = runThreadEventPruningStep( "prune_background_task_progress", () => - pruneBackgroundTaskProgressEvents(deps.db, { + hasPrunableCandidateRows(deps.db, { threadId: args.threadId, - }), + types: ["item/backgroundTask/progress"], + }) + ? pruneBackgroundTaskProgressEvents(deps.db, { + threadId: args.threadId, + }) + : 0, ); return { diff --git a/apps/server/src/services/system/periodic-sweeps.ts b/apps/server/src/services/system/periodic-sweeps.ts index a6d4dc27af..5157a60e68 100644 --- a/apps/server/src/services/system/periodic-sweeps.ts +++ b/apps/server/src/services/system/periodic-sweeps.ts @@ -1,5 +1,6 @@ import { and, eq, isNull } from "drizzle-orm"; import { + capPromptHistoryEntries, CLOSED_SESSION_ROW_RETENTION_MS, compactDatabase, COMPLETED_EVENT_OUTPUT_RETENTION_MS, @@ -7,21 +8,32 @@ import { DATABASE_COMPACTION_MIN_RECLAIMABLE_RATIO, DATABASE_INCREMENTAL_VACUUM_MAX_PAGES, DATABASE_INCREMENTAL_VACUUM_MIN_FREELIST_PAGES, + DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_ROW_BATCH_SIZE, + DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_THREAD_BATCH_SIZE, DEFAULT_CLOSED_SESSION_PRUNE_BATCH_SIZE, DEFAULT_COMPLETED_EVENT_OUTPUT_TRUNCATION_BATCH_SIZE, DEFAULT_DESTROYED_ENVIRONMENT_PRUNE_BATCH_SIZE, + DEFAULT_PROMPT_HISTORY_CAP_SCOPE_BATCH_SIZE, + DEFAULT_PROVIDER_UNHANDLED_EVENT_PRUNE_BATCH_SIZE, + DEFAULT_SETTLED_PENDING_INTERACTION_PRUNE_BATCH_SIZE, DESTROYED_ENVIRONMENT_TTL_MS, dropDeferredLegacyTables, getDatabaseAutoVacuumMode, - getDatabaseCompactionStats, + getDatabaseCompactionStatsFreelistFirst, getDatabaseFreelistStats, getDatabaseMaintenanceActivity, isDatabaseMaintenanceIdle, listDeferredLegacyTables, environments, + PROMPT_HISTORY_KEEP_PER_SCOPE, + PROVIDER_UNHANDLED_EVENT_RETENTION_MS, + pruneArchivedThreadEvents, pruneClosedSessions, pruneDestroyedEnvironments, + pruneProviderUnhandledEvents, + pruneSettledPendingInteractions, runIncrementalVacuum, + SETTLED_PENDING_INTERACTION_RETENTION_MS, shouldCompactDatabase, shouldRunIncrementalVacuum, sweepManagedEnvironments, @@ -51,6 +63,7 @@ import { advanceThreadProvisioning } from "../threads/thread-provisioning.js"; import { runQueuedMessageAutoSendSweep } from "../threads/queued-messages.js"; import { runDeferredThreadMessageSweep } from "../threads/thread-send-request.js"; import { LIVE_DAEMON_COMMAND_TIMEOUT_MS } from "../hosts/live-command.js"; +import { ARCHIVED_THREAD_EVENT_KEEP_RECENT } from "./event-pruning.js"; import { runEventLoopWork, runEventLoopWorkSync } from "./event-loop-work.js"; type DatabaseMaintenanceSweepDeps = Pick; @@ -349,7 +362,15 @@ export function runDatabaseMaintenanceSweep( return; } - const stats = getDatabaseCompactionStats(deps.db); + // Freelist first: the O(1) counters can justify compaction on their own, + // and only when they cannot is the whole-file dbstat scan worth running. + // The sync wrapper attributes that scan in event-loop stall snapshots. + const stats = runEventLoopWorkSync("db-maintenance:compaction-stats", () => + getDatabaseCompactionStatsFreelistFirst(deps.db, { + minReclaimableBytes: DATABASE_COMPACTION_MIN_RECLAIMABLE_BYTES, + minReclaimableRatio: DATABASE_COMPACTION_MIN_RECLAIMABLE_RATIO, + }), + ); if ( !shouldCompactDatabase({ minReclaimableBytes: DATABASE_COMPACTION_MIN_RECLAIMABLE_BYTES, @@ -549,6 +570,61 @@ async function runDestroyedEnvironmentPruneSweep( } } +function runSettledInteractionPruneSweep( + deps: LoggedPendingInteractionWorkSessionDeps, + now: number, +): void { + pruneSettledPendingInteractions(deps.db, { + createdBefore: now - SETTLED_PENDING_INTERACTION_RETENTION_MS, + limit: DEFAULT_SETTLED_PENDING_INTERACTION_PRUNE_BATCH_SIZE, + }); +} + +// The over-cap probe is a grouped index walk over the whole table, so this +// job runs on a cadence instead of every tick. +const PROMPT_HISTORY_CAP_INTERVAL_MS = 15 * 60_000; + +function runPromptHistoryCapSweep( + deps: LoggedPendingInteractionWorkSessionDeps, +): void { + capPromptHistoryEntries(deps.db, { + keepPerScope: PROMPT_HISTORY_KEEP_PER_SCOPE, + maxScopes: DEFAULT_PROMPT_HISTORY_CAP_SCOPE_BATCH_SIZE, + }); +} + +// Each pass probes up to a thread batch even when nothing is prunable, so +// this job runs on a cadence instead of every tick. A large backlog still +// drains at thousands of rows per minute. +const ARCHIVED_THREAD_EVENT_RETENTION_INTERVAL_MS = 60_000; + +function runArchivedThreadEventRetentionSweep( + deps: LoggedPendingInteractionWorkSessionDeps, + now: number, +): void { + pruneArchivedThreadEvents(deps.db, { + keepRecent: ARCHIVED_THREAD_EVENT_KEEP_RECENT, + maxRows: DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_ROW_BATCH_SIZE, + maxThreads: DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_THREAD_BATCH_SIZE, + now, + }); +} + +// A no-op pass is one covering seek on the partial retention index, but the +// deletes it does perform are write transactions - a cadence keeps them off +// the every-10s tick. +const PROVIDER_UNHANDLED_EVENT_PRUNE_INTERVAL_MS = 60_000; + +function runProviderUnhandledEventPruneSweep( + deps: LoggedPendingInteractionWorkSessionDeps, + now: number, +): void { + pruneProviderUnhandledEvents(deps.db, { + createdBefore: now - PROVIDER_UNHANDLED_EVENT_RETENTION_MS, + limit: DEFAULT_PROVIDER_UNHANDLED_EVENT_PRUNE_BATCH_SIZE, + }); +} + const PERIODIC_SWEEP_JOBS: PeriodicSweepJob[] = [ { cadenceMs: 0, @@ -574,6 +650,30 @@ const PERIODIC_SWEEP_JOBS: PeriodicSweepJob[] = [ name: "destroyed-environment-prune", run: runDestroyedEnvironmentPruneSweep, }, + { + cadenceMs: 0, + category: "retention", + name: "settled-interaction-prune", + run: runSettledInteractionPruneSweep, + }, + { + cadenceMs: PROMPT_HISTORY_CAP_INTERVAL_MS, + category: "retention", + name: "prompt-history-cap", + run: runPromptHistoryCapSweep, + }, + { + cadenceMs: ARCHIVED_THREAD_EVENT_RETENTION_INTERVAL_MS, + category: "retention", + name: "archived-thread-event-retention", + run: runArchivedThreadEventRetentionSweep, + }, + { + cadenceMs: PROVIDER_UNHANDLED_EVENT_PRUNE_INTERVAL_MS, + category: "retention", + name: "provider-unhandled-event-prune", + run: runProviderUnhandledEventPruneSweep, + }, { cadenceMs: 0, category: "orphan-cleanup", diff --git a/apps/server/src/services/threads/timeline-cache.ts b/apps/server/src/services/threads/timeline-cache.ts index f5e1c903ae..e90bb740c7 100644 --- a/apps/server/src/services/threads/timeline-cache.ts +++ b/apps/server/src/services/threads/timeline-cache.ts @@ -22,52 +22,112 @@ import type { ThreadTimelinePageRequest } from "./timeline-pagination.js"; * (`pruneResolvedItemDeltas`, background-task progress) is output-preserving * and never lowers `maxSeq`, so it cannot stale a cached entry. * - * Entries with many rows are not cached: an expanded active turn (the streaming - * case) produces hundreds of rows AND a `maxSeq` that changes on every event, - * so caching it only thrashes the LRU and pins large objects for no reuse. Idle - * windows collapse completed turns to a handful of rows regardless of thread - * size, so the cap excludes exactly the entries that would never be reused. + * Entries with many rows are not stored in the LRU: an expanded active turn + * (the streaming case) produces hundreds of rows AND a `maxSeq` that changes on + * every event, so retaining it only thrashes the LRU and pins large objects for + * no reuse. Idle windows collapse completed turns to a handful of rows + * regardless of thread size, so the cap excludes exactly the entries that would + * never be reused. + * + * Those uncached streaming windows are instead shared through a short-lived + * last-build slot per params key (the key minus `maxSeq`). Several clients + * watch the same streaming thread (desktop, browser, phone, plugin panes), each + * on its own refetch pacing, so one appended event fans out into several + * back-to-back synchronous rebuilds of near-identical windows — they serialize + * on the event loop and stall every other request. The slot serves them all + * from one build: + * + * - Same key inside {@link DEFAULT_SHARE_WINDOW_MS}: the build is shared + * whatever its row count. Identical key means identical output (the build is + * deterministic), so this is invisible except in build count. The build is + * synchronous, so it completes before the next request can start; sharing the + * completed result is what coalesces loop-serialized concurrent requests. + * - New `maxSeq` inside the window: the prior window is returned as-is — a + * rebuild floor. The response carries its own `maxSeq`, so the client simply + * sees the thread as of ≤{@link DEFAULT_SHARE_WINDOW_MS} ago, below client + * refetch pacing. The floor applies ONLY to responses over the LRU row cap: + * LRU-cacheable windows already coalesce same-key storms via the LRU, and + * keeping their rebuilds eager preserves per-request freshness for every + * window the LRU can serve. `thread.status` lives in the params key, so a + * status flip (interrupt, completion) is never floored. */ const DEFAULT_MAX_ENTRIES = 128; const DEFAULT_MAX_CACHEABLE_ROWS = 200; +const DEFAULT_SHARE_WINDOW_MS = 250; interface ThreadTimelineCacheOptions { maxEntries?: number; - /** Responses with more rows than this are returned but not stored. */ + /** Responses with more rows than this are returned but not LRU-stored. */ maxCacheableRows?: number; + /** How long one build is shared across requests of the same params key. */ + shareWindowMs?: number; + /** Clock override for tests; defaults to Date.now. */ + now?: () => number; +} + +interface ThreadTimelineCacheKeys { + /** Full cache key including `maxSeq` ({@link buildThreadTimelineCacheKey}). */ + key: string; + /** The key minus `maxSeq` ({@link buildThreadTimelineParamsKey}). */ + paramsKey: string; } interface ThreadTimelineCache { getOrBuild( - key: string, + keys: ThreadTimelineCacheKeys, build: () => ThreadTimelineResponse, ): ThreadTimelineResponse; - /** Number of currently cached entries (for tests/metrics). */ + /** Number of currently cached LRU entries (for tests/metrics). */ readonly size: number; } +interface RecentBuild { + key: string; + value: ThreadTimelineResponse; + builtAt: number; +} + export function createThreadTimelineCache( options: ThreadTimelineCacheOptions = {}, ): ThreadTimelineCache { const maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES; const maxCacheableRows = options.maxCacheableRows ?? DEFAULT_MAX_CACHEABLE_ROWS; + const shareWindowMs = options.shareWindowMs ?? DEFAULT_SHARE_WINDOW_MS; + const now = options.now ?? Date.now; const entries = new Map(); + // Requests within a params key arrive with monotonic `maxSeq` (the route + // reads the current high-water mark), so one slot per params key holding the + // newest build is enough — no request can want an older revision. Expired + // slots are swept on every store, so the map holds at most the few windows + // built in the last share window. + const recentBuilds = new Map(); return { - getOrBuild(key, build) { - const cached = entries.get(key); + getOrBuild(keys, build) { + const cached = entries.get(keys.key); if (cached !== undefined) { // Re-insert to mark most-recently-used. - entries.delete(key); - entries.set(key, cached); + entries.delete(keys.key); + entries.set(keys.key, cached); return cached; } + const at = now(); + const recent = recentBuilds.get(keys.paramsKey); + if (recent !== undefined && at - recent.builtAt <= shareWindowMs) { + if ( + recent.key === keys.key || + recent.value.rows.length > maxCacheableRows + ) { + return recent.value; + } + } + const value = build(); if (value.rows.length <= maxCacheableRows) { - entries.set(key, value); + entries.set(keys.key, value); while (entries.size > maxEntries) { const oldest = entries.keys().next().value; if (oldest === undefined) { @@ -76,6 +136,13 @@ export function createThreadTimelineCache( entries.delete(oldest); } } + const builtAt = now(); + recentBuilds.set(keys.paramsKey, { key: keys.key, value, builtAt }); + for (const [paramsKey, slot] of recentBuilds) { + if (builtAt - slot.builtAt > shareWindowMs) { + recentBuilds.delete(paramsKey); + } + } return value; }, get size() { diff --git a/apps/server/src/ws/hub.ts b/apps/server/src/ws/hub.ts index 271122f2d5..fb652f8c27 100644 --- a/apps/server/src/ws/hub.ts +++ b/apps/server/src/ws/hub.ts @@ -35,7 +35,14 @@ const TERMINAL_SOCKET_HIGH_WATER_BYTES = 1024 * 1024; // A 16 MiB raw burst expands to about 21.4 MiB as base64 + JSON. Keep // enough bounded headroom for that workload while preventing unbounded growth. const TERMINAL_SOCKET_MAX_QUEUE_BYTES = 32 * 1024 * 1024; -const TERMINAL_SOCKET_DRAIN_POLL_MS = 10; +// App realtime messages are small invalidation hints and ephemeral signals; a +// socket that cannot flush 1 MiB of them is wedged (a suspended phone, a dead +// network), and letting its buffers grow just leaks server memory. Queue a +// short burst, then drop the socket: clients reconnect and catch up via the +// watermark. +const APP_SOCKET_HIGH_WATER_BYTES = 1024 * 1024; +const APP_SOCKET_MAX_QUEUE_BYTES = 4 * 1024 * 1024; +const SOCKET_DRAIN_POLL_MS = 10; /** * A streaming turn appends events ~10 times a second. A client that only * subscribes to the thread list (every open app window, for every thread it @@ -52,18 +59,40 @@ const THREAD_LIST_EVENTS_APPENDED_COALESCE_MS = 1_000; const LIST_RELEVANT_THREAD_EVENT_TYPES: ReadonlySet = new Set(["client/turn/requested", "turn/completed"]); +/** + * The plugin-signal envelope minus the payload: `notifyPluginSignal` embeds + * the publish site's already-serialized payload verbatim, so only the fields + * it splices into the frame go through the outgoing schema. + */ +const pluginSignalEnvelopeSchema = pluginSignalSchema.omit({ payload: true }); + interface HubSocket { close(code?: number, reason?: string): void; raw?: { bufferedAmount: number }; send(data: string): void; } -interface TerminalSocketSendQueue { +interface SocketSendQueue { bytes: number; payloads: string[]; timeout: ReturnType | null; } +/** + * One backpressured send path. The hub runs two: the terminal lane (large + * ordered output bursts, generous queue) and the app lane (small realtime + * notifications, tight queue). Both share the same mechanics — send directly + * below the socket high water, queue and poll-drain above it, and hand the + * socket to `drop` when a send throws or the queue budget overflows. + */ +interface SocketSendLane { + drop: (socket: HubSocket, reason: string) => void; + dropReasons: { queueOverflow: string; sendFailed: string }; + highWaterBytes: number; + maxQueueBytes: number; + queues: Map; +} + type ChangedMessageListener = (message: ChangedMessage) => void; interface PendingThreadListEventsAppended { @@ -225,10 +254,26 @@ export class NotificationHub implements DbNotifier { string, Set >(); - private readonly terminalSocketSendQueues = new Map< - HubSocket, - TerminalSocketSendQueue - >(); + private readonly terminalSendLane: SocketSendLane = { + drop: (socket, reason) => this.dropTerminalSocket(socket, reason), + dropReasons: { + queueOverflow: "terminal-backpressure", + sendFailed: "terminal-send-failed", + }, + highWaterBytes: TERMINAL_SOCKET_HIGH_WATER_BYTES, + maxQueueBytes: TERMINAL_SOCKET_MAX_QUEUE_BYTES, + queues: new Map(), + }; + private readonly appSendLane: SocketSendLane = { + drop: (socket, reason) => this.dropAppSocket(socket, reason), + dropReasons: { + queueOverflow: "app-backpressure", + sendFailed: "app-send-failed", + }, + highWaterBytes: APP_SOCKET_HIGH_WATER_BYTES, + maxQueueBytes: APP_SOCKET_MAX_QUEUE_BYTES, + queues: new Map(), + }; private readonly terminalIdsByClientSocket = new Map< HubSocket, Set @@ -251,6 +296,7 @@ export class NotificationHub implements DbNotifier { unregisterClient(socket: HubSocket): void { this.unregisterTerminalClientSocket(socket); + this.clearSocketSendQueue(this.appSendLane, socket); const keys = this.clientKeysBySocket.get(socket); if (!keys) { return; @@ -314,7 +360,7 @@ export class NotificationHub implements DbNotifier { terminalIds.delete(terminalId); if (terminalIds.size === 0) { this.terminalIdsByClientSocket.delete(socket); - this.clearTerminalSocketSendQueue(socket); + this.clearSocketSendQueue(this.terminalSendLane, socket); } } @@ -337,7 +383,7 @@ export class NotificationHub implements DbNotifier { } this.terminalIdsByClientSocket.delete(socket); - this.clearTerminalSocketSendQueue(socket); + this.clearSocketSendQueue(this.terminalSendLane, socket); } private releaseTerminalResizeOwnership( @@ -363,7 +409,8 @@ export class NotificationHub implements DbNotifier { socket: HubSocket, message: TerminalServerMessage, ): void { - this.sendOrQueueTerminalPayload( + this.sendWithBackpressure( + this.terminalSendLane, socket, JSON.stringify(terminalServerMessageSchema.parse(message)), ); @@ -380,22 +427,32 @@ export class NotificationHub implements DbNotifier { const payload = JSON.stringify(terminalServerMessageSchema.parse(message)); for (const socket of [...sockets]) { - this.sendOrQueueTerminalPayload(socket, payload); + this.sendWithBackpressure(this.terminalSendLane, socket, payload); } } - private sendOrQueueTerminalPayload(socket: HubSocket, payload: string): void { - const existingQueue = this.terminalSocketSendQueues.get(socket); + /** + * Send on a lane, queueing above the socket high water and dropping the + * socket when a send throws or the queue budget overflows — one bad socket + * can neither stall the loop nor grow without bound. Returns false when the + * socket was dropped instead of sent/queued to. + */ + private sendWithBackpressure( + lane: SocketSendLane, + socket: HubSocket, + payload: string, + ): boolean { + const existingQueue = lane.queues.get(socket); if ( !existingQueue && - (socket.raw?.bufferedAmount ?? 0) <= TERMINAL_SOCKET_HIGH_WATER_BYTES + (socket.raw?.bufferedAmount ?? 0) <= lane.highWaterBytes ) { try { socket.send(payload); - return; + return true; } catch { - this.dropTerminalSocket(socket, "terminal-send-failed"); - return; + lane.drop(socket, lane.dropReasons.sendFailed); + return false; } } @@ -405,39 +462,42 @@ export class NotificationHub implements DbNotifier { timeout: null, }; const payloadBytes = Buffer.byteLength(payload, "utf8"); - if (queue.bytes + payloadBytes > TERMINAL_SOCKET_MAX_QUEUE_BYTES) { - this.dropTerminalSocket(socket, "terminal-backpressure"); - return; + if (queue.bytes + payloadBytes > lane.maxQueueBytes) { + lane.drop(socket, lane.dropReasons.queueOverflow); + return false; } queue.payloads.push(payload); queue.bytes += payloadBytes; - this.terminalSocketSendQueues.set(socket, queue); - this.scheduleTerminalSocketDrain(socket, queue); + lane.queues.set(socket, queue); + this.scheduleSocketDrain(lane, socket, queue); + return true; } - private scheduleTerminalSocketDrain( + private scheduleSocketDrain( + lane: SocketSendLane, socket: HubSocket, - queue: TerminalSocketSendQueue, + queue: SocketSendQueue, ): void { if (queue.timeout !== null) { return; } queue.timeout = setTimeout(() => { queue.timeout = null; - this.flushTerminalSocketQueue(socket, queue); - }, TERMINAL_SOCKET_DRAIN_POLL_MS); + this.flushSocketQueue(lane, socket, queue); + }, SOCKET_DRAIN_POLL_MS); } - private flushTerminalSocketQueue( + private flushSocketQueue( + lane: SocketSendLane, socket: HubSocket, - queue: TerminalSocketSendQueue, + queue: SocketSendQueue, ): void { - if (this.terminalSocketSendQueues.get(socket) !== queue) { + if (lane.queues.get(socket) !== queue) { return; } while ( queue.payloads.length > 0 && - (socket.raw?.bufferedAmount ?? 0) <= TERMINAL_SOCKET_HIGH_WATER_BYTES + (socket.raw?.bufferedAmount ?? 0) <= lane.highWaterBytes ) { const payload = queue.payloads[0]; if (payload === undefined) { @@ -446,17 +506,17 @@ export class NotificationHub implements DbNotifier { try { socket.send(payload); } catch { - this.dropTerminalSocket(socket, "terminal-send-failed"); + lane.drop(socket, lane.dropReasons.sendFailed); return; } queue.payloads.shift(); queue.bytes -= Buffer.byteLength(payload, "utf8"); } if (queue.payloads.length === 0) { - this.clearTerminalSocketSendQueue(socket); + this.clearSocketSendQueue(lane, socket); return; } - this.scheduleTerminalSocketDrain(socket, queue); + this.scheduleSocketDrain(lane, socket, queue); } private dropTerminalSocket(socket: HubSocket, reason: string): void { @@ -468,15 +528,29 @@ export class NotificationHub implements DbNotifier { } } - private clearTerminalSocketSendQueue(socket: HubSocket): void { - const queue = this.terminalSocketSendQueues.get(socket); + /** + * Unregister first (which also clears the socket's queues) so the rest of + * the current fan-out and all later ones skip the socket; the client + * reconnects and catches up via the watermark. + */ + private dropAppSocket(socket: HubSocket, reason: string): void { + this.unregisterClient(socket); + try { + socket.close(1013, reason); + } catch { + // The socket is already unusable; registration and queue state are gone. + } + } + + private clearSocketSendQueue(lane: SocketSendLane, socket: HubSocket): void { + const queue = lane.queues.get(socket); if (!queue) { return; } if (queue.timeout !== null) { clearTimeout(queue.timeout); } - this.terminalSocketSendQueues.delete(socket); + lane.queues.delete(socket); } subscribe(socket: HubSocket, target: RealtimeSubscriptionTarget): void { @@ -804,12 +878,7 @@ export class NotificationHub implements DbNotifier { file: request.file, }), ); - let delivered = 0; - for (const socket of this.clientKeysBySocket.keys()) { - socket.send(payload); - delivered += 1; - } - return delivered; + return this.broadcastToAppSockets(payload); } /** Broadcast an ephemeral pane presentation request to every app client. */ @@ -825,12 +894,7 @@ export class NotificationHub implements DbNotifier { action, }), ); - let delivered = 0; - for (const socket of this.clientKeysBySocket.keys()) { - socket.send(payload); - delivered += 1; - } - return delivered; + return this.broadcastToAppSockets(payload); } /** @@ -838,24 +902,36 @@ export class NotificationHub implements DbNotifier { * every connected client. V1 broadcasts to all clients — per-channel * subscriptions arrive with the plugin frontend runtime. Returns how many * clients the signal reached. + * + * `serializedPayload` is the publish site's single JSON serialization of + * the payload, embedded verbatim in the wire frame so the fan-out never + * parses or re-stringifies it. The envelope fields still pass the strict + * outgoing schema. */ notifyPluginSignal( pluginId: string, channel: string, - payload: unknown, + serializedPayload: string, ): number { - const message = JSON.stringify( - pluginSignalSchema.parse({ - type: "plugin-signal", - pluginId, - channel, - payload, - }), - ); + const envelope = pluginSignalEnvelopeSchema.parse({ + type: "plugin-signal", + pluginId, + channel, + }); + const message = `{"type":"plugin-signal","pluginId":${JSON.stringify( + envelope.pluginId, + )},"channel":${JSON.stringify(envelope.channel)},"payload":${serializedPayload}}`; + return this.broadcastToAppSockets(message); + } + + /** Ephemeral broadcast to every app socket; returns how many accepted it. */ + private broadcastToAppSockets(payload: string): number { let delivered = 0; - for (const socket of this.clientKeysBySocket.keys()) { - socket.send(message); - delivered += 1; + // Snapshot: dropping a socket mutates the live registration map. + for (const socket of [...this.clientKeysBySocket.keys()]) { + if (this.sendWithBackpressure(this.appSendLane, socket, payload)) { + delivered += 1; + } } return delivered; } @@ -1055,11 +1131,11 @@ export class NotificationHub implements DbNotifier { return; } const detailKey = subscriptionKey({ kind: "thread-detail", threadId }); - for (const socket of listSockets) { + for (const socket of [...listSockets]) { if (this.clientKeysBySocket.get(socket)?.has(detailKey)) { continue; } - socket.send(payload); + this.sendWithBackpressure(this.appSendLane, socket, payload); } } @@ -1089,8 +1165,9 @@ export class NotificationHub implements DbNotifier { sockets: Iterable, payload: string, ): void { - for (const socket of sockets) { - socket.send(payload); + // Snapshot: dropping a socket mutates the live subscription sets. + for (const socket of [...sockets]) { + this.sendWithBackpressure(this.appSendLane, socket, payload); } } diff --git a/apps/server/test/app/hub-plugin-signal.test.ts b/apps/server/test/app/hub-plugin-signal.test.ts index 3d7e07e881..27a16263ca 100644 --- a/apps/server/test/app/hub-plugin-signal.test.ts +++ b/apps/server/test/app/hub-plugin-signal.test.ts @@ -12,9 +12,13 @@ describe("NotificationHub.notifyPluginSignal", () => { hub.subscribe(first, { kind: "thread-detail", threadId: "thr_1" }); hub.subscribe(second, { kind: "system" }); - const delivered = hub.notifyPluginSignal("linear", "issues-updated", { - count: 42, - }); + // The payload arrives pre-serialized (the publish site's single + // JSON.stringify) and is embedded verbatim in the wire frame. + const delivered = hub.notifyPluginSignal( + "linear", + "issues-updated", + JSON.stringify({ count: 42 }), + ); expect(delivered).toBe(2); for (const socket of [first, second]) { diff --git a/apps/server/test/app/hub.test.ts b/apps/server/test/app/hub.test.ts index 5d8ebcee69..5c5eeab901 100644 --- a/apps/server/test/app/hub.test.ts +++ b/apps/server/test/app/hub.test.ts @@ -250,6 +250,87 @@ describe("NotificationHub", () => { expect(socket.messages).toEqual([]); }); + it("queues app notifications for a socket above high water and flushes on drain", () => { + vi.useFakeTimers(); + const hub = new NotificationHub(); + const healthy = createMockHubSocket(); + const wedged = { + ...createMockHubSocket(), + raw: { bufferedAmount: 2 * 1024 * 1024 }, + }; + hub.subscribe(healthy, { kind: "thread-detail", threadId: "thread-1" }); + hub.subscribe(wedged, { kind: "thread-detail", threadId: "thread-1" }); + + hub.notifyThread("thread-1", ["status-changed"]); + + expect(healthy.messages).toHaveLength(1); + expect(wedged.messages).toHaveLength(0); + + wedged.raw.bufferedAmount = 0; + vi.advanceTimersByTime(10); + expect(wedged.messages).toHaveLength(1); + }); + + it("drops a wedged app socket at the queue cap while siblings still receive", () => { + const hub = new NotificationHub(); + const healthy = createMockHubSocket(); + const wedged = { + ...createMockHubSocket(), + raw: { bufferedAmount: 2 * 1024 * 1024 }, + }; + hub.registerClient(healthy); + hub.registerClient(wedged); + const bigPayload = JSON.stringify("x".repeat(1024 * 1024)); + + for (let index = 0; index < 5; index += 1) { + hub.notifyPluginSignal("plugin-1", "channel-1", bigPayload); + } + + expect(wedged.closed).toContainEqual({ + code: 1013, + reason: "app-backpressure", + }); + expect(wedged.messages).toEqual([]); + expect(healthy.messages).toHaveLength(5); + + // The drop unregistered the socket: later broadcasts skip it entirely. + const delivered = hub.notifyPluginSignal( + "plugin-1", + "channel-1", + JSON.stringify("after"), + ); + expect(delivered).toBe(1); + expect(wedged.closed).toHaveLength(1); + }); + + it("isolates a throwing app socket from healthy subscribers", () => { + const hub = new NotificationHub(); + const healthy = createMockHubSocket(); + const closed: Array<{ code?: number; reason?: string }> = []; + const failing = { + close(code?: number, reason?: string) { + closed.push({ code, reason }); + }, + send() { + throw new Error("socket send failed"); + }, + }; + hub.subscribe(failing, { kind: "thread-detail", threadId: "thread-1" }); + hub.subscribe(healthy, { kind: "thread-detail", threadId: "thread-1" }); + + expect(() => + hub.notifyThread("thread-1", ["status-changed"]), + ).not.toThrow(); + + expect(closed).toEqual([{ code: 1013, reason: "app-send-failed" }]); + expect(healthy.messages).toHaveLength(1); + + // The failing socket was unregistered; the next notify skips it. + hub.notifyThread("thread-1", ["status-changed"]); + expect(closed).toHaveLength(1); + expect(healthy.messages).toHaveLength(2); + }); + it("notifies all clients subscribed to the same thread", () => { const hub = new NotificationHub(); const socket1 = createMockHubSocket(); diff --git a/apps/server/test/public/public-thread-data.test.ts b/apps/server/test/public/public-thread-data.test.ts index 41f204bd43..d59e082f88 100644 --- a/apps/server/test/public/public-thread-data.test.ts +++ b/apps/server/test/public/public-thread-data.test.ts @@ -210,6 +210,33 @@ describe("public thread data routes", () => { }); }); + it("caps thread lists and reports truncation in the has-more header", async () => { + await withTestHarness(async (harness) => { + const { host } = seedHostSession(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + for (let index = 0; index < 3; index += 1) { + seedThread(harness.deps, { + projectId: project.id, + title: `Thread ${index}`, + }); + } + + const capped = await harness.app.request("/api/v1/threads?limit=2"); + expect(capped.status).toBe(200); + expect(capped.headers.get("x-bb-thread-list-has-more")).toBe("true"); + expect(z.array(z.unknown()).parse(await readJson(capped))).toHaveLength( + 2, + ); + + const full = await harness.app.request("/api/v1/threads"); + expect(full.status).toBe(200); + expect(full.headers.get("x-bb-thread-list-has-more")).toBe("false"); + expect(z.array(z.unknown()).parse(await readJson(full))).toHaveLength(3); + }); + }); + it("updates visibility and requires includeHidden to list hidden threads", async () => { await withTestHarness(async (harness) => { const { host } = seedHostSession(harness.deps); diff --git a/apps/server/test/services/plugins/plugin-cli.test.ts b/apps/server/test/services/plugins/plugin-cli.test.ts index 424026a514..5cb9a8c9a8 100644 --- a/apps/server/test/services/plugins/plugin-cli.test.ts +++ b/apps/server/test/services/plugins/plugin-cli.test.ts @@ -365,6 +365,19 @@ describe("plugin CLI commands (bb.cli.register + endpoints + skill + logs)", () }); it("bb.log writes JSONL to the plugin log file and the tail endpoint serves it", async () => { + // The tail endpoint flushes buffered lines (bb.log batches writes off the + // call path), so hit it before reading the raw file. + const response = await harness.app.request( + `${BASE}/api/v1/plugins/acme/logs?tail=1`, + ); + expect(response.status).toBe(200); + const body = (await response.json()) as { ok: boolean; lines: string[] }; + expect(body.ok).toBe(true); + expect(body.lines).toHaveLength(1); + expect(JSON.parse(body.lines[0] ?? "")).toMatchObject({ + message: "cli plugin loaded", + }); + const logFile = join( harness.config.dataDir, "plugins", @@ -383,17 +396,6 @@ describe("plugin CLI commands (bb.cli.register + endpoints + skill + logs)", () expect(parsed.message).toBe("cli plugin loaded"); expect(typeof parsed.ts).toBe("number"); - const response = await harness.app.request( - `${BASE}/api/v1/plugins/acme/logs?tail=1`, - ); - expect(response.status).toBe(200); - const body = (await response.json()) as { ok: boolean; lines: string[] }; - expect(body.ok).toBe(true); - expect(body.lines).toHaveLength(1); - expect(JSON.parse(body.lines[0] ?? "")).toMatchObject({ - message: "cli plugin loaded", - }); - const missing = await harness.app.request( `${BASE}/api/v1/plugins/nope/logs`, ); diff --git a/apps/server/test/services/plugins/plugin-log.test.ts b/apps/server/test/services/plugins/plugin-log.test.ts new file mode 100644 index 0000000000..af76a88d76 --- /dev/null +++ b/apps/server/test/services/plugins/plugin-log.test.ts @@ -0,0 +1,108 @@ +import { existsSync, readFileSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { + appendPluginLogLine, + readPluginLogTail, +} from "../../../src/services/plugins/plugin-log.js"; + +function logFilePath(dataDir: string, pluginId: string): string { + return join(dataDir, "plugins", pluginId, "logs", "plugin.log"); +} + +function fileLines(path: string): string[] { + return readFileSync(path, "utf8") + .split("\n") + .filter((line) => line.length > 0); +} + +async function withDataDir( + run: (dataDir: string) => Promise, +): Promise { + const dataDir = await mkdtemp(join(tmpdir(), "bb-plugin-log-")); + try { + await run(dataDir); + } finally { + await rm(dataDir, { recursive: true, force: true }); + } +} + +describe("plugin log batching", () => { + it("buffers lines off the call path and flushes them as one batch", async () => { + await withDataDir(async (dataDir) => { + for (let index = 0; index < 3; index += 1) { + appendPluginLogLine(dataDir, "batch", "info", `line-${index}`); + } + // Nothing hits the disk synchronously on the call path. + expect(existsSync(logFilePath(dataDir, "batch"))).toBe(false); + + // The tail flushes pending lines first, so reads stay read-your-writes. + const lines = await readPluginLogTail(dataDir, "batch", 10); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[0] ?? "")).toMatchObject({ + level: "info", + message: "line-0", + }); + expect(fileLines(logFilePath(dataDir, "batch"))).toHaveLength(3); + }); + }); + + it("flushes on the timer without a reader", async () => { + await withDataDir(async (dataDir) => { + appendPluginLogLine(dataDir, "timer", "warn", "solo"); + expect(existsSync(logFilePath(dataDir, "timer"))).toBe(false); + + await vi.waitFor( + () => { + expect(existsSync(logFilePath(dataDir, "timer"))).toBe(true); + }, + { timeout: 2_000 }, + ); + expect(fileLines(logFilePath(dataDir, "timer"))).toHaveLength(1); + }); + }); + + it("flushes immediately once the buffer crosses the byte threshold", async () => { + await withDataDir(async (dataDir) => { + const big = "x".repeat(4 * 1024); + appendPluginLogLine(dataDir, "burst", "info", big); + expect(existsSync(logFilePath(dataDir, "burst"))).toBe(false); + + // The second line crosses the 8KB threshold: no timer wait. + appendPluginLogLine(dataDir, "burst", "info", big); + await vi.waitFor( + () => { + expect(existsSync(logFilePath(dataDir, "burst"))).toBe(true); + }, + { timeout: 500 }, + ); + expect(fileLines(logFilePath(dataDir, "burst"))).toHaveLength(2); + }); + }); + + it("rotates past the size cap using the cached size counter", async () => { + await withDataDir(async (dataDir) => { + appendPluginLogLine( + dataDir, + "rotate", + "info", + "x".repeat(6 * 1024 * 1024), + ); + await readPluginLogTail(dataDir, "rotate", 1); + + appendPluginLogLine(dataDir, "rotate", "info", "after-rotation"); + const lines = await readPluginLogTail(dataDir, "rotate", 10); + + const dir = join(dataDir, "plugins", "rotate", "logs"); + expect(existsSync(join(dir, "plugin.log.1"))).toBe(true); + expect(fileLines(logFilePath(dataDir, "rotate"))).toHaveLength(1); + // The tail spans the rotated file plus the current one. + expect(lines).toHaveLength(2); + expect(JSON.parse(lines[1] ?? "")).toMatchObject({ + message: "after-rotation", + }); + }); + }); +}); diff --git a/apps/server/test/services/plugins/plugin-wire.test.ts b/apps/server/test/services/plugins/plugin-wire.test.ts index ff7879dc35..5cbcc29ebd 100644 --- a/apps/server/test/services/plugins/plugin-wire.test.ts +++ b/apps/server/test/services/plugins/plugin-wire.test.ts @@ -25,6 +25,11 @@ const WIRE_SOURCE = ` input: z.object({ channel: z.string(), payload: z.unknown() }), output: z.literal("published"), }, + publishMany: { + input: z.object({ channel: z.string(), count: z.number() }), + output: z.literal("published-many"), + }, + publishHuge: { input: z.record(z.string(), z.unknown()), output: z.null() }, publishBad: { input: z.record(z.string(), z.unknown()), output: z.null() }, invalidOutput: { input: z.null(), output: z.string() }, bigintResult: { input: z.null(), output: z.any() }, @@ -94,6 +99,16 @@ const WIRE_SOURCE = ` bb.realtime.publish(input.channel, input.payload); return "published"; }, + publishMany: async (input: any) => { + for (let index = 0; index < input.count; index += 1) { + bb.realtime.publish(input.channel, { index }); + } + return "published-many"; + }, + publishHuge: async () => { + bb.realtime.publish("huge", "x".repeat(65 * 1024)); + return null; + }, publishBad: async () => { bb.realtime.publish("bad", { n: BigInt(1) }); }, @@ -667,6 +682,52 @@ describe("plugin wire surfaces (http/rpc dispatcher + realtime)", () => { }); }); + it("bb.realtime.publish rejects payloads over the 64KB cap", async () => { + const socket = createMockHubSocket(); + harness.hub.subscribe(socket, { kind: "system" }); + + const response = await rpc(harness, "publishHuge", {}); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ + ok: false, + error: { + code: "handler_error", + message: expect.stringContaining("64KB"), + }, + }); + expect(socket.messages).toHaveLength(0); + }); + + it("bb.realtime.publish drops signals over the per-plugin rate limit with one warning", async () => { + const socket = createMockHubSocket(); + harness.hub.subscribe(socket, { kind: "system" }); + const originalWarn = harness.deps.logger.warn; + const warn = vi.fn(); + harness.deps.logger.warn = warn; + try { + // 40 synchronous publishes land inside one refill tick, so exactly the + // burst allowance goes out and the rest drop with a single warning. + const response = await rpc(harness, "publishMany", { + channel: "flood", + count: 40, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + ok: true, + result: "published-many", + }); + + expect(socket.messages).toHaveLength(20); + const rateLimitWarnings = warn.mock.calls.filter(([, message]) => + String(message).includes("rate limit"), + ); + expect(rateLimitWarnings).toHaveLength(1); + expect(rateLimitWarnings[0]?.[0]).toMatchObject({ pluginId: "wire" }); + } finally { + harness.deps.logger.warn = originalWarn; + } + }); + it("rpc resolves the handler after the body arrives, so a reload during the body read never runs a stale handler", async () => { // The handler closes over its load generation: a binding resolved // before the body read (and invalidated by the mid-read reload) would diff --git a/apps/server/test/services/threads/timeline-cache.test.ts b/apps/server/test/services/threads/timeline-cache.test.ts index 141af64fec..27bec39b8a 100644 --- a/apps/server/test/services/threads/timeline-cache.test.ts +++ b/apps/server/test/services/threads/timeline-cache.test.ts @@ -57,13 +57,18 @@ const baseKeyArgs: ThreadTimelineCacheKeyArgs = { includeProviderUnhandledOperations: false, }; +/** Give each key its own params key to exercise the LRU in isolation. */ +function keys(key: string): { key: string; paramsKey: string } { + return { key, paramsKey: `params:${key}` }; +} + describe("createThreadTimelineCache", () => { it("builds once for the same key and serves cached on repeat", () => { const cache = createThreadTimelineCache(); const build = vi.fn(() => makeResponse(3)); - const first = cache.getOrBuild("k", build); - const second = cache.getOrBuild("k", build); + const first = cache.getOrBuild(keys("k"), build); + const second = cache.getOrBuild(keys("k"), build); expect(build).toHaveBeenCalledTimes(1); expect(second).toBe(first); @@ -74,40 +79,119 @@ describe("createThreadTimelineCache", () => { const cache = createThreadTimelineCache(); const build = vi.fn(() => makeResponse(3)); - cache.getOrBuild("k1", build); - cache.getOrBuild("k2", build); + cache.getOrBuild(keys("k1"), build); + cache.getOrBuild(keys("k2"), build); expect(build).toHaveBeenCalledTimes(2); }); - it("does not cache responses above the row cap (streaming expanded turns)", () => { - const cache = createThreadTimelineCache({ maxCacheableRows: 5 }); + it("does not retain responses above the row cap beyond the share window", () => { + let nowMs = 0; + const cache = createThreadTimelineCache({ + maxCacheableRows: 5, + shareWindowMs: 250, + now: () => nowMs, + }); const build = vi.fn(() => makeResponse(50)); - cache.getOrBuild("k", build); - cache.getOrBuild("k", build); + cache.getOrBuild(keys("k"), build); + nowMs += 251; + cache.getOrBuild(keys("k"), build); expect(build).toHaveBeenCalledTimes(2); expect(cache.size).toBe(0); }); it("evicts least-recently-used entries beyond maxEntries", () => { - const cache = createThreadTimelineCache({ maxEntries: 2 }); + let nowMs = 0; + const cache = createThreadTimelineCache({ maxEntries: 2, now: () => nowMs }); const build = vi.fn(() => makeResponse(1)); - cache.getOrBuild("a", build); // [a] - cache.getOrBuild("b", build); // [a,b] - cache.getOrBuild("a", build); // touch a -> [b,a] - cache.getOrBuild("c", build); // evict b -> [a,c] + cache.getOrBuild(keys("a"), build); // [a] + cache.getOrBuild(keys("b"), build); // [a,b] + cache.getOrBuild(keys("a"), build); // touch a -> [b,a] + cache.getOrBuild(keys("c"), build); // evict b -> [a,c] expect(cache.size).toBe(2); + // Step past the share window so the LRU alone answers the reprobe. + nowMs += 300; const buildAgain = vi.fn(() => makeResponse(1)); - cache.getOrBuild("a", buildAgain); // still cached - cache.getOrBuild("b", buildAgain); // evicted -> rebuild + cache.getOrBuild(keys("a"), buildAgain); // still cached + cache.getOrBuild(keys("b"), buildAgain); // evicted -> rebuild expect(buildAgain).toHaveBeenCalledTimes(1); }); }); +describe("createThreadTimelineCache share window", () => { + function createSharedCache(now: () => number) { + return createThreadTimelineCache({ + maxCacheableRows: 5, + shareWindowMs: 250, + now, + }); + } + + it("shares one build across same-key requests inside the window regardless of row count", () => { + let nowMs = 0; + const cache = createSharedCache(() => nowMs); + const build = vi.fn(() => makeResponse(50)); + + const first = cache.getOrBuild(keys("k"), build); + nowMs += 100; + const second = cache.getOrBuild(keys("k"), build); + + expect(build).toHaveBeenCalledTimes(1); + // Same revision must mean identical rows for every client. + expect(second).toBe(first); + // Shared, not retained: the LRU row cap still applies. + expect(cache.size).toBe(0); + }); + + it("floors rebuilds of over-cap windows: a new maxSeq inside the window gets the prior window", () => { + let nowMs = 0; + const cache = createSharedCache(() => nowMs); + const build = vi.fn(() => makeResponse(50)); + + const first = cache.getOrBuild({ key: "10|p", paramsKey: "p" }, build); + nowMs += 100; + const floored = cache.getOrBuild({ key: "11|p", paramsKey: "p" }, build); + expect(build).toHaveBeenCalledTimes(1); + expect(floored).toBe(first); + + nowMs += 200; // 300ms since the build: past the window. + const rebuilt = cache.getOrBuild({ key: "12|p", paramsKey: "p" }, build); + expect(build).toHaveBeenCalledTimes(2); + expect(rebuilt).not.toBe(first); + }); + + it("keeps rebuilds eager for LRU-cacheable windows (no floor below the row cap)", () => { + let nowMs = 0; + const cache = createSharedCache(() => nowMs); + const build = vi.fn(() => makeResponse(3)); + + cache.getOrBuild({ key: "10|p", paramsKey: "p" }, build); + nowMs += 10; + cache.getOrBuild({ key: "11|p", paramsKey: "p" }, build); + + expect(build).toHaveBeenCalledTimes(2); + }); + + it("never floors across params keys (a status flip builds fresh)", () => { + let nowMs = 0; + const cache = createSharedCache(() => nowMs); + const build = vi.fn(() => makeResponse(50)); + + cache.getOrBuild({ key: "10|p:active", paramsKey: "p:active" }, build); + nowMs += 10; + cache.getOrBuild( + { key: "11|p:interrupted", paramsKey: "p:interrupted" }, + build, + ); + + expect(build).toHaveBeenCalledTimes(2); + }); +}); + describe("buildThreadTimelineCacheKey", () => { it("differs when any projection input differs", () => { const base = buildThreadTimelineCacheKey(baseKeyArgs); diff --git a/apps/server/test/system/event-pruning.test.ts b/apps/server/test/system/event-pruning.test.ts index 5ebe261177..06677b4519 100644 --- a/apps/server/test/system/event-pruning.test.ts +++ b/apps/server/test/system/event-pruning.test.ts @@ -1,5 +1,11 @@ -import { getThread, listEvents } from "@bb/db"; -import { turnScope } from "@bb/domain"; +import { + archiveThread, + getThread, + listEvents, + noopNotifier, + pruneArchivedThreadEvents, +} from "@bb/db"; +import { threadScope, turnScope } from "@bb/domain"; import { groupHostDaemonEvents } from "@bb/host-daemon-contract"; import { describe, expect, it, vi } from "vitest"; import { applyTurnCompletedEvent } from "../../src/internal/turn-completed-events.js"; @@ -130,6 +136,7 @@ function seedResolvedAssistantMessage( for (const sequence of args.deltaSequences) { seedStoredEvent(harness.deps, { threadId: args.threadId, + providerThreadId: "provider-thread-1", sequence, type: "item/agentMessage/delta", scope: turnScope(turnId), @@ -144,6 +151,7 @@ function seedResolvedAssistantMessage( seedStoredEvent(harness.deps, { threadId: args.threadId, + providerThreadId: "provider-thread-1", sequence: args.completedSequence, type: "item/completed", scope: turnScope(turnId), @@ -532,6 +540,7 @@ describe("thread event pruning", () => { for (const sequence of [1_004, 1_005]) { seedStoredEvent(harness.deps, { threadId: thread.id, + providerThreadId: "provider-thread-1", scope: turnScope("turn-active"), sequence, type: "item/agentMessage/delta", @@ -580,6 +589,46 @@ describe("thread event pruning", () => { }); expect(response.status).toBe(200); + + // The prune is deferred off the response path: when the response + // settles, nothing has been removed yet. + expect( + listEventSequencesForType(harness, { + threadId: thread.id, + type: "thread/tokenUsage/updated", + }).at(0), + ).toBe(1); + expect( + listEventSequencesForType(harness, { + threadId: thread.id, + type: "item/agentMessage/delta", + itemId: "msg-completed", + }), + ).toEqual([1_001, 1_002]); + + // Pruning is output-preserving, so a timeline read in the window + // between the append and the deferred prune must project exactly what + // a post-prune read projects. + const timelineOptions = { + eventBudget: 1_000_000, + includeProviderUnhandledOperations: true, + maxInlineOutputChars: null, + maxSeq: 0, + page: { + kind: "latest", + segmentLimit: Number.MAX_SAFE_INTEGER, + }, + } as const; + const beforePrune = buildThreadTimeline( + harness.db, + thread, + timelineOptions, + ); + + // The deferred batch was scheduled with setImmediate during the + // request; one immediate turn later it has run. + await new Promise((resolve) => setImmediate(resolve)); + expect( listEventSequencesForType(harness, { threadId: thread.id, @@ -600,6 +649,152 @@ describe("thread event pruning", () => { itemId: "msg-active", }), ).toEqual([1_004, 1_005]); + + const afterPrune = buildThreadTimeline( + harness.db, + thread, + timelineOptions, + ); + expect(afterPrune.rows).toEqual(beforePrune.rows); + expect(afterPrune.contextWindowUsage).toEqual( + beforePrune.contextWindowUsage, + ); + }); + }); + + it("projects a timeline from the kept window after archived-thread retention", async () => { + await withTestHarness(async (harness) => { + const host = seedHost(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + const environment = seedEnvironment(harness.deps, { + hostId: host.id, + projectId: project.id, + }); + const thread = seedThread(harness.deps, { + projectId: project.id, + environmentId: environment.id, + status: "idle", + }); + + // Ten complete turns; the retention window will cut mid-history, the + // same shape as the timeline event-budget window. + let sequence = 0; + for (let turn = 1; turn <= 10; turn += 1) { + const turnId = `turn-${turn}`; + seedStoredEvent(harness.deps, { + threadId: thread.id, + providerThreadId: "provider-thread-1", + scope: turnScope(turnId), + sequence: (sequence += 1), + type: "turn/started", + itemId: null, + itemKind: null, + data: { providerThreadId: "provider-thread-1" }, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + providerThreadId: "provider-thread-1", + scope: turnScope(turnId), + sequence: (sequence += 1), + type: "item/completed", + itemId: `msg-${turn}`, + itemKind: "agentMessage", + data: { + item: { + id: `msg-${turn}`, + type: "agentMessage", + text: `Answer ${turn}`, + }, + }, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + providerThreadId: "provider-thread-1", + scope: turnScope(turnId), + sequence: (sequence += 1), + type: "turn/completed", + itemId: null, + itemKind: null, + data: { status: "completed" }, + }); + } + + archiveThread(harness.db, noopNotifier, thread.id); + const result = pruneArchivedThreadEvents(harness.db, { + keepRecent: 9, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }); + expect(result.deleted).toBe(21); + + const archivedThread = getThread(harness.db, thread.id); + expect(archivedThread?.archivedAt).toBeTypeOf("number"); + const timeline = buildThreadTimeline(harness.db, archivedThread!, { + eventBudget: 1_000_000, + includeProviderUnhandledOperations: true, + maxInlineOutputChars: null, + maxSeq: 0, + page: { + kind: "latest", + segmentLimit: Number.MAX_SAFE_INTEGER, + }, + }); + + // The kept recent window still projects: the three fully retained + // turns' messages are present, and nothing throws on the cut edge. + const rowText = JSON.stringify(timeline.rows); + expect(timeline.rows.length).toBeGreaterThan(0); + expect(rowText).toContain("Answer 10"); + expect(rowText).toContain("Answer 8"); + expect(rowText).not.toContain("Answer 7"); + }); + }); + + it("prunes superseded background-task progress snapshots", async () => { + await withTestHarness(async (harness) => { + const host = seedHost(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + const environment = seedEnvironment(harness.deps, { + hostId: host.id, + projectId: project.id, + }); + const thread = seedThread(harness.deps, { + projectId: project.id, + environmentId: environment.id, + }); + + // Two snapshots for the same item: only the latest is load-bearing. + for (const sequence of [1, 2]) { + seedStoredEvent(harness.deps, { + threadId: thread.id, + sequence, + type: "item/backgroundTask/progress", + scope: threadScope(), + itemId: "task-1", + itemKind: "backgroundTask", + data: { + item: { id: "task-1", type: "backgroundTask", status: "running" }, + }, + }); + } + + const result = pruneThreadEventHistory(harness.deps, { + mode: "idle", + threadId: thread.id, + }); + + expect(result.removedBackgroundTaskProgressEvents).toBe(1); + expect( + listEventSequencesForType(harness, { + threadId: thread.id, + type: "item/backgroundTask/progress", + }), + ).toEqual([2]); }); }); }); diff --git a/docs/configuration.md b/docs/configuration.md index cd5f4e96fb..1f40559764 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,8 @@ signal it, so a stale file left by a crash cannot stop an unrelated process. | `BB_SERVER_PORT` | `bb-app env`, environment, or `--server-port` | Startup-only | HTTP listener port. Defaults to `38886`. A full launcher or desktop app restart is required after a persistent set or unset. | | `BB_HOST_DAEMON_PORT` | `bb-app env`, environment, or `--host-daemon-port` | Startup-only | Local host-daemon API port. Defaults to `38887`. A full launcher or desktop app restart is required after a persistent set or unset. | | `BB_LOG_LEVEL` | `bb-app config` | Startup-only debugging | Log level: `trace`, `debug`, `info`, `warn`, `error`, or `fatal`. A full launcher or desktop app restart is required. | +| `BB_SQLITE_CACHE_SIZE_KIB` | `bb-app env`, or environment | Startup-only tuning | SQLite page-cache budget for the server database, in KiB. Defaults to `262144` (256 MiB). `0` disables the extra cache; malformed values fall back to the default. A full launcher or desktop app restart is required. | +| `BB_SQLITE_MMAP_SIZE_BYTES` | `bb-app env`, or environment | Startup-only tuning | SQLite memory-map window for the server database, in bytes. Defaults to `1073741824` (1 GiB). `0` disables memory mapping; malformed values fall back to the default. Lowering it reduces server RSS at the cost of more file reads on a large database. A full launcher or desktop app restart is required. | | `OPENAI_API_KEY` | `bb-app env` | OpenAI opt-in routes | Required only when selecting explicit OpenAI provider routes such as `openai/gpt-4o-mini` or `openai/gpt-transcribe`. | By default, helper inference and voice transcription use Codex credentials from diff --git a/packages/bb-app/src/launcher.ts b/packages/bb-app/src/launcher.ts index 500feb1a3a..525c49f850 100644 --- a/packages/bb-app/src/launcher.ts +++ b/packages/bb-app/src/launcher.ts @@ -121,6 +121,8 @@ const STARTUP_ONLY_MANAGED_ENV_KEYS = new Set([ "BB_POSTHOG_API_KEY", "BB_SERVER_BIND_HOST", "BB_SERVER_PORT", + "BB_SQLITE_CACHE_SIZE_KIB", + "BB_SQLITE_MMAP_SIZE_BYTES", "BB_TELEMETRY", "BB_TRANSCRIPTION", ]); @@ -1560,7 +1562,8 @@ Startup-only server and launcher keys: BB_EXTERNAL_URL, BB_HOST_DAEMON_PORT, BB_INFERENCE, BB_INFERENCE_FALLBACK, BB_INHERITED_SKILLS_ROOTS, BB_LOG_LEVEL, BB_MANAGED_DEV_BUILTIN_PLUGIN_HOT_RELOAD, BB_POSTHOG_API_KEY, - BB_SERVER_BIND_HOST, BB_SERVER_PORT, BB_TELEMETRY, BB_TRANSCRIPTION, + BB_SERVER_BIND_HOST, BB_SERVER_PORT, BB_SQLITE_CACHE_SIZE_KIB, + BB_SQLITE_MMAP_SIZE_BYTES, BB_TELEMETRY, BB_TRANSCRIPTION, and BB_FF_* feature flags. Changes require a full bb-app restart with bb-app stop && bb-app start, or a desktop app restart. BB_APP_URL, BB_INFERENCE, diff --git a/packages/db/drizzle/0109_provider_unhandled_retention_index.sql b/packages/db/drizzle/0109_provider_unhandled_retention_index.sql new file mode 100644 index 0000000000..9a36b99e2c --- /dev/null +++ b/packages/db/drizzle/0109_provider_unhandled_retention_index.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS `events_provider_unhandled_created_idx` ON `events` (`created_at`,`id`) WHERE "events"."type" = 'provider/unhandled'; diff --git a/packages/db/drizzle/meta/0109_snapshot.json b/packages/db/drizzle/meta/0109_snapshot.json new file mode 100644 index 0000000000..e0a32ceb53 --- /dev/null +++ b/packages/db/drizzle/meta/0109_snapshot.json @@ -0,0 +1,3800 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4206cb53-833c-4479-aa10-3497e87da5fd", + "prevId": "6e7bcf0b-8ad1-473a-b856-292027ad2441", + "tables": { + "app_settings": { + "name": "app_settings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caffeinate": { + "name": "caffeinate", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_keyboard_hints": { + "name": "show_keyboard_hints", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "steer_active_thread_on_enter": { + "name": "steer_active_thread_on_enter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_unhandled_provider_events": { + "name": "show_unhandled_provider_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "codex_memory_enabled": { + "name": "codex_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "claude_code_memory_enabled": { + "name": "claude_code_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "codex_subagents_disabled": { + "name": "codex_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_subagents_disabled": { + "name": "claude_code_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_workflows_disabled": { + "name": "claude_code_workflows_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "keybinding_overrides": { + "name": "keybinding_overrides", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_settings_values": { + "name": "app_settings_values", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_theme": { + "name": "app_theme", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "theme_id": { + "name": "theme_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "favicon_color": { + "name": "favicon_color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refillInterval": { + "name": "refillInterval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refillAmount": { + "name": "refillAmount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRefillAt": { + "name": "lastRefillAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitEnabled": { + "name": "rateLimitEnabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitTimeWindow": { + "name": "rateLimitTimeWindow", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitMax": { + "name": "rateLimitMax", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestCount": { + "name": "requestCount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRequest": { + "name": "lastRequest", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "configId": { + "name": "configId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_unique": { + "name": "apikey_key_unique", + "columns": [ + "key" + ], + "isUnique": true + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "referenceId" + ], + "isUnique": false + }, + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "configId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "apikey_referenceId_user_id_fk": { + "name": "apikey_referenceId_user_id_fk", + "tableFrom": "apikey", + "tableTo": "user", + "columnsFrom": [ + "referenceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deferred_thread_messages": { + "name": "deferred_thread_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "deferred_thread_messages_thread_created_idx": { + "name": "deferred_thread_messages_thread_created_idx", + "columns": [ + "thread_id", + "created_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "deferred_thread_messages_thread_id_threads_id_fk": { + "name": "deferred_thread_messages_thread_id_threads_id_fk", + "tableFrom": "deferred_thread_messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environments": { + "name": "environments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "managed": { + "name": "managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_worktree": { + "name": "is_worktree", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "merge_base_branch": { + "name": "merge_base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destroy_attempt_id": { + "name": "destroy_attempt_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retire_requested_at": { + "name": "retire_requested_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace_provision_type": { + "name": "workspace_provision_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provisioning'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "environments_project_host_path_idx": { + "name": "environments_project_host_path_idx", + "columns": [ + "project_id", + "host_id", + "path" + ], + "isUnique": true + }, + "environments_host_path_lookup_idx": { + "name": "environments_host_path_lookup_idx", + "columns": [ + "host_id", + "path" + ], + "isUnique": false + }, + "environments_project_idx": { + "name": "environments_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "environments_status_idx": { + "name": "environments_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "environments_project_id_projects_id_fk": { + "name": "environments_project_id_projects_id_fk", + "tableFrom": "environments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environments_host_id_hosts_id_fk": { + "name": "environments_host_id_hosts_id_fk", + "tableFrom": "environments", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope_kind": { + "name": "scope_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_id": { + "name": "item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "events_thread_sequence_idx": { + "name": "events_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": true + }, + "events_delegating_item_lookup_idx": { + "name": "events_delegating_item_lookup_idx", + "columns": [ + "thread_id", + "item_id", + "sequence", + "item_kind" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" IN ('toolCall', 'delegation')" + }, + "events_plan_steps_thread_sequence_idx": { + "name": "events_plan_steps_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "(\"events\".\"item_kind\" = 'planSteps' AND \"events\".\"type\" = 'item/completed') OR \"events\".\"type\" = 'turn/plan/updated'" + }, + "events_parent_tool_call_thread_parent_sequence_idx": { + "name": "events_parent_tool_call_thread_parent_sequence_idx", + "columns": [ + "thread_id", + "parent_tool_call_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"parent_tool_call_id\" IS NOT NULL" + }, + "events_thread_type_item_kind_sequence_idx": { + "name": "events_thread_type_item_kind_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_kind", + "sequence" + ], + "isUnique": false + }, + "events_background_task_thread_type_item_sequence_idx": { + "name": "events_background_task_thread_type_item_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" = 'backgroundTask'" + }, + "events_thread_type_sequence_idx": { + "name": "events_thread_type_sequence_idx", + "columns": [ + "thread_id", + "type", + "sequence" + ], + "isUnique": false + }, + "events_thread_turn_type_item_sequence_idx": { + "name": "events_thread_turn_type_item_sequence_idx", + "columns": [ + "thread_id", + "turn_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false + }, + "events_item_lifecycle_thread_item_sequence_idx": { + "name": "events_item_lifecycle_thread_item_sequence_idx", + "columns": [ + "thread_id", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('item/started', 'item/completed', 'item/backgroundTask/completed')" + }, + "events_environment_idx": { + "name": "events_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "events_completed_item_truncation_idx": { + "name": "events_completed_item_truncation_idx", + "columns": [ + "item_kind", + "created_at", + "id" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'item/completed'" + }, + "events_provider_unhandled_created_idx": { + "name": "events_provider_unhandled_created_idx", + "columns": [ + "created_at", + "id" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'provider/unhandled'" + }, + "events_thread_state_thread_sequence_idx": { + "name": "events_thread_state_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('thread/goal/updated', 'thread/goal/cleared', 'thread/extensionState/updated')" + } + }, + "foreignKeys": { + "events_thread_id_threads_id_fk": { + "name": "events_thread_id_threads_id_fk", + "tableFrom": "events", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "events_environment_id_environments_id_fk": { + "name": "events_environment_id_environments_id_fk", + "tableFrom": "events", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "events_scope_shape_check": { + "name": "events_scope_shape_check", + "value": "(\n (\"events\".\"scope_kind\" = 'turn' AND \"events\".\"turn_id\" IS NOT NULL)\n OR\n (\"events\".\"scope_kind\" = 'thread' AND \"events\".\"turn_id\" IS NULL)\n )" + } + } + }, + "host_daemon_sessions": { + "name": "host_daemon_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_name": { + "name": "host_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_type": { + "name": "host_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_dir": { + "name": "data_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heartbeat_interval_ms": { + "name": "heartbeat_interval_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_timeout_ms": { + "name": "lease_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "closed_at": { + "name": "closed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "host_daemon_sessions_host_status_idx": { + "name": "host_daemon_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "host_daemon_sessions_host_latest_idx": { + "name": "host_daemon_sessions_host_latest_idx", + "columns": [ + "host_id", + "updated_at", + "created_at", + "id" + ], + "isUnique": false + }, + "host_daemon_sessions_closed_prune_idx": { + "name": "host_daemon_sessions_closed_prune_idx", + "columns": [ + "status", + "closed_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "host_daemon_sessions_host_id_hosts_id_fk": { + "name": "host_daemon_sessions_host_id_hosts_id_fk", + "tableFrom": "host_daemon_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hosts": { + "name": "hosts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connect_machine_id": { + "name": "connect_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_permission_mode": { + "name": "max_permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'full'" + }, + "destroyed_at": { + "name": "destroyed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_rejected_protocol_version": { + "name": "last_rejected_protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "hosts_last_seen_idx": { + "name": "hosts_last_seen_idx", + "columns": [ + "last_seen_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugins": { + "name": "plugins", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "catalog_entry_id": { + "name": "catalog_entry_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "catalog_marketplace_name": { + "name": "catalog_marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'path'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_builtin_name": { + "name": "source_builtin_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_package": { + "name": "source_npm_package", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_registry": { + "name": "source_npm_registry", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_requested_spec": { + "name": "source_npm_requested_spec", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_spec_kind": { + "name": "source_npm_spec_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_url": { + "name": "source_git_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_subdirectory": { + "name": "source_git_subdirectory", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_requested_ref": { + "name": "source_git_requested_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_ref_kind": { + "name": "source_git_ref_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_range": { + "name": "source_git_range", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_tag_prefix": { + "name": "source_git_tag_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_resolved_tag": { + "name": "source_git_resolved_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_integrity": { + "name": "npm_integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_update_check_at": { + "name": "last_update_check_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "available_compatible_version": { + "name": "available_compatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "newest_incompatible_version": { + "name": "newest_incompatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "update_status_detail": { + "name": "update_status_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_version": { + "name": "last_failure_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_detail": { + "name": "last_failure_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_artifact_id": { + "name": "active_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "normalization_version": { + "name": "normalization_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "root_dir": { + "name": "root_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "removed_at": { + "name": "removed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "installed_at": { + "name": "installed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "plugins_active_artifact_id_plugin_artifacts_id_fk": { + "name": "plugins_active_artifact_id_plugin_artifacts_id_fk", + "tableFrom": "plugins", + "tableTo": "plugin_artifacts", + "columnsFrom": [ + "active_artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_scan_cursors": { + "name": "maintenance_scan_cursors", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_path": { + "name": "output_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_created_at": { + "name": "last_created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_event_id": { + "name": "last_event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "maintenance_scan_cursors_path_idx": { + "name": "maintenance_scan_cursors_path_idx", + "columns": [ + "policy", + "version", + "item_kind", + "output_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_interactions": { + "name": "pending_interactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provider'" + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_request_id": { + "name": "provider_request_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "renderer_id": { + "name": "renderer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolution": { + "name": "resolution", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_reason": { + "name": "status_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pending_interactions_provider_request_idx": { + "name": "pending_interactions_provider_request_idx", + "columns": [ + "provider_id", + "provider_thread_id", + "provider_request_id" + ], + "isUnique": true + }, + "pending_interactions_thread_created_idx": { + "name": "pending_interactions_thread_created_idx", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_thread_status_created_idx": { + "name": "pending_interactions_thread_status_created_idx", + "columns": [ + "thread_id", + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_status_created_idx": { + "name": "pending_interactions_status_created_idx", + "columns": [ + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_plugin_status_created_idx": { + "name": "pending_interactions_plugin_status_created_idx", + "columns": [ + "plugin_id", + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pending_interactions_thread_id_threads_id_fk": { + "name": "pending_interactions_thread_id_threads_id_fk", + "tableFrom": "pending_interactions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_artifacts": { + "name": "plugin_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_checkout_root": { + "name": "git_checkout_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrity": { + "name": "integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "validation_result": { + "name": "validation_result", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "validated_at": { + "name": "validated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "plugin_artifacts_plugin_idx": { + "name": "plugin_artifacts_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_kv": { + "name": "plugin_kv", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_kv_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_kv_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplace_icons": { + "name": "plugin_marketplace_icons", + "columns": { + "marketplace_name": { + "name": "marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_marketplace_icons_marketplace_name_entry_id_pk": { + "columns": [ + "marketplace_name", + "entry_id" + ], + "name": "plugin_marketplace_icons_marketplace_name_entry_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplaces": { + "name": "plugin_marketplaces", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'https'" + }, + "manifest_url": { + "name": "manifest_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_git_ref": { + "name": "source_git_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_commit": { + "name": "source_git_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest_json": { + "name": "manifest_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_successful_refresh_at": { + "name": "last_successful_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_attempted_refresh_at": { + "name": "last_attempted_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_schedules": { + "name": "plugin_schedules", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_schedules_plugin_id_name_pk": { + "columns": [ + "plugin_id", + "name" + ], + "name": "plugin_schedules_plugin_id_name_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_settings": { + "name": "plugin_settings", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_settings_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_settings_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_state_snapshots": { + "name": "plugin_state_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_artifact_id": { + "name": "from_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "to_artifact_id": { + "name": "to_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_path": { + "name": "snapshot_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "database_path": { + "name": "database_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state_path": { + "name": "state_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secrets_path": { + "name": "secrets_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registration_path": { + "name": "registration_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rollback_candidate_version": { + "name": "rollback_candidate_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_source_fingerprint": { + "name": "rollback_source_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_bb_version": { + "name": "rollback_bb_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_sdk_version": { + "name": "rollback_sdk_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_detail": { + "name": "rollback_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_until": { + "name": "retained_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "plugin_state_snapshots_plugin_idx": { + "name": "plugin_state_snapshots_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + }, + "plugin_state_snapshots_retention_idx": { + "name": "plugin_state_snapshots_retention_idx", + "columns": [ + "retained_until" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_execution_defaults": { + "name": "project_execution_defaults", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_execution_defaults_project_idx": { + "name": "project_execution_defaults_project_idx", + "columns": [ + "project_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_execution_defaults_project_id_projects_id_fk": { + "name": "project_execution_defaults_project_id_projects_id_fk", + "tableFrom": "project_execution_defaults", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_sources": { + "name": "project_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_sources_project_idx": { + "name": "project_sources_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "project_sources_host_idx": { + "name": "project_sources_host_idx", + "columns": [ + "host_id" + ], + "isUnique": false + }, + "project_sources_project_host_idx": { + "name": "project_sources_project_host_idx", + "columns": [ + "project_id", + "host_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_sources_project_id_projects_id_fk": { + "name": "project_sources_project_id_projects_id_fk", + "tableFrom": "project_sources", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_sources_host_id_hosts_id_fk": { + "name": "project_sources_host_id_hosts_id_fk", + "tableFrom": "project_sources", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "project_sources_shape_check": { + "name": "project_sources_shape_check", + "value": "(\n \"project_sources\".\"type\" = 'local_path' AND \"project_sources\".\"host_id\" IS NOT NULL AND \"project_sources\".\"path\" IS NOT NULL\n )" + } + } + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'standard'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "git_remote_url": { + "name": "git_remote_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'V'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "projects_updated_idx": { + "name": "projects_updated_idx", + "columns": [ + "updated_at" + ], + "isUnique": false + }, + "projects_deleted_idx": { + "name": "projects_deleted_idx", + "columns": [ + "deleted_at" + ], + "isUnique": false + }, + "projects_sort_idx": { + "name": "projects_sort_idx", + "columns": [ + "sort_key", + "id" + ], + "isUnique": false + }, + "projects_personal_singleton_idx": { + "name": "projects_personal_singleton_idx", + "columns": [ + "kind" + ], + "isUnique": true, + "where": "\"projects\".\"kind\" = 'personal'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "prompt_history_entries": { + "name": "prompt_history_entries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input": { + "name": "input", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "prompt_history_entries_thread_request_idx": { + "name": "prompt_history_entries_thread_request_idx", + "columns": [ + "thread_id", + "request_sequence" + ], + "isUnique": true + }, + "prompt_history_entries_project_scope_created_idx": { + "name": "prompt_history_entries_project_scope_created_idx", + "columns": [ + "project_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + }, + "prompt_history_entries_thread_scope_created_idx": { + "name": "prompt_history_entries_thread_scope_created_idx", + "columns": [ + "thread_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "prompt_history_entries_project_id_projects_id_fk": { + "name": "prompt_history_entries_project_id_projects_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "prompt_history_entries_thread_id_threads_id_fk": { + "name": "prompt_history_entries_thread_id_threads_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "queued_thread_messages": { + "name": "queued_thread_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sender_thread_id": { + "name": "sender_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_with_next": { + "name": "group_with_next", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "queued_thread_messages_thread_created_idx": { + "name": "queued_thread_messages_thread_created_idx", + "columns": [ + "thread_id", + "created_at", + "id" + ], + "isUnique": false + }, + "queued_thread_messages_thread_sort_idx": { + "name": "queued_thread_messages_thread_sort_idx", + "columns": [ + "thread_id", + "sort_key", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "queued_thread_messages_thread_id_threads_id_fk": { + "name": "queued_thread_messages_thread_id_threads_id_fk", + "tableFrom": "queued_thread_messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_experiments": { + "name": "system_experiments", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_sessions": { + "name": "terminal_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "daemon_session_id": { + "name": "daemon_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "initial_cwd": { + "name": "initial_cwd", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cols": { + "name": "cols", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rows": { + "name": "rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_user_input_at": { + "name": "last_user_input_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "terminal_sessions_thread_status_updated_idx": { + "name": "terminal_sessions_thread_status_updated_idx", + "columns": [ + "thread_id", + "status", + "updated_at" + ], + "isUnique": false + }, + "terminal_sessions_environment_status_idx": { + "name": "terminal_sessions_environment_status_idx", + "columns": [ + "environment_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_host_status_idx": { + "name": "terminal_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_daemon_session_idx": { + "name": "terminal_sessions_daemon_session_idx", + "columns": [ + "daemon_session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "terminal_sessions_thread_id_threads_id_fk": { + "name": "terminal_sessions_thread_id_threads_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_environment_id_environments_id_fk": { + "name": "terminal_sessions_environment_id_environments_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_host_id_hosts_id_fk": { + "name": "terminal_sessions_host_id_hosts_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk": { + "name": "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "host_daemon_sessions", + "columnsFrom": [ + "daemon_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_dynamic_context_file_states": { + "name": "thread_dynamic_context_file_states", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_key": { + "name": "file_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_status": { + "name": "content_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shown_at": { + "name": "shown_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_dynamic_context_file_states_thread_file_idx": { + "name": "thread_dynamic_context_file_states_thread_file_idx", + "columns": [ + "thread_id", + "file_key" + ], + "isUnique": true + } + }, + "foreignKeys": { + "thread_dynamic_context_file_states_thread_id_threads_id_fk": { + "name": "thread_dynamic_context_file_states_thread_id_threads_id_fk", + "tableFrom": "thread_dynamic_context_file_states", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_search_segments": { + "name": "thread_search_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_seq": { + "name": "source_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_search_segments_source_idx": { + "name": "thread_search_segments_source_idx", + "columns": [ + "thread_id", + "source_kind", + "source_key" + ], + "isUnique": true + }, + "thread_search_segments_thread_source_seq_idx": { + "name": "thread_search_segments_thread_source_seq_idx", + "columns": [ + "thread_id", + "source_seq" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_search_segments_thread_id_threads_id_fk": { + "name": "thread_search_segments_thread_id_threads_id_fk", + "tableFrom": "thread_search_segments", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_sections": { + "name": "thread_sections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_sections_name_idx": { + "name": "thread_sections_name_idx", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tabs": { + "name": "thread_tabs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tabs_json": { + "name": "tabs_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tabs_thread_id_threads_id_fk": { + "name": "thread_tabs_thread_id_threads_id_fk", + "tableFrom": "thread_tabs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_override": { + "name": "model_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_level_override": { + "name": "reasoning_level_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title_fallback": { + "name": "title_fallback", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'starting'" + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_plugin_id": { + "name": "origin_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'visible'" + }, + "archived_at": { + "name": "archived_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pinned_at": { + "name": "pinned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pin_sort_key": { + "name": "pin_sort_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latest_attention_at": { + "name": "latest_attention_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "threads_project_updated_idx": { + "name": "threads_project_updated_idx", + "columns": [ + "project_id", + "updated_at" + ], + "isUnique": false + }, + "threads_project_archived_deleted_idx": { + "name": "threads_project_archived_deleted_idx", + "columns": [ + "project_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_pin_sort_idx": { + "name": "threads_pin_sort_idx", + "columns": [ + "archived_at", + "deleted_at", + "pin_sort_key", + "id" + ], + "isUnique": false, + "where": "\"threads\".\"pinned_at\" IS NOT NULL" + }, + "threads_environment_idx": { + "name": "threads_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "threads_parent_idx": { + "name": "threads_parent_idx", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "threads_source_origin_idx": { + "name": "threads_source_origin_idx", + "columns": [ + "source_thread_id", + "origin_kind" + ], + "isUnique": false + }, + "threads_origin_plugin_archived_idx": { + "name": "threads_origin_plugin_archived_idx", + "columns": [ + "origin_plugin_id", + "archived_at" + ], + "isUnique": false + }, + "threads_section_archived_deleted_idx": { + "name": "threads_section_archived_deleted_idx", + "columns": [ + "section_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_archived_status_idx": { + "name": "threads_archived_status_idx", + "columns": [ + "archived_at", + "status" + ], + "isUnique": false + }, + "threads_environment_archived_deleted_idx": { + "name": "threads_environment_archived_deleted_idx", + "columns": [ + "environment_id", + "archived_at", + "deleted_at" + ], + "isUnique": false + }, + "threads_active_maintenance_idx": { + "name": "threads_active_maintenance_idx", + "columns": [ + "status" + ], + "isUnique": false, + "where": "\"threads\".\"deleted_at\" IS NULL" + } + }, + "foreignKeys": { + "threads_project_id_projects_id_fk": { + "name": "threads_project_id_projects_id_fk", + "tableFrom": "threads", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_environment_id_environments_id_fk": { + "name": "threads_environment_id_environments_id_fk", + "tableFrom": "threads", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_section_id_thread_sections_id_fk": { + "name": "threads_section_id_thread_sections_id_fk", + "tableFrom": "threads", + "tableTo": "thread_sections", + "columnsFrom": [ + "section_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_parent_thread_id_threads_id_fk": { + "name": "threads_parent_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "parent_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_source_thread_id_threads_id_fk": { + "name": "threads_source_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "source_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 693796a569..0f7db600df 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -764,6 +764,13 @@ "when": 1787613751578, "tag": "0108_deferred_thread_messages", "breakpoints": true + }, + { + "idx": 109, + "version": "6", + "when": 1787642080729, + "tag": "0109_provider_unhandled_retention_index", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/connection.ts b/packages/db/src/connection.ts index 4c3a6b0c32..32dcb5c623 100644 --- a/packages/db/src/connection.ts +++ b/packages/db/src/connection.ts @@ -42,10 +42,47 @@ interface TimedStatementOperationArgs { const DEFAULT_SLOW_DB_QUERY_LOG_THRESHOLD_MS = 100; /** 256 MiB page cache. Negative cache_size is kibibytes. */ -export const SQLITE_CACHE_SIZE_KIB = 262_144; +export const DEFAULT_SQLITE_CACHE_SIZE_KIB = 262_144; /** Memory-map the first 1 GiB of the database file. */ -export const SQLITE_MMAP_SIZE_BYTES = 1_073_741_824; +export const DEFAULT_SQLITE_MMAP_SIZE_BYTES = 1_073_741_824; +/** Overrides the page-cache budget in KiB (`0` disables the extra cache). */ +export const SQLITE_CACHE_SIZE_KIB_ENV_VAR = "BB_SQLITE_CACHE_SIZE_KIB"; +/** Overrides the mmap window in bytes (`0` disables memory mapping). */ +export const SQLITE_MMAP_SIZE_BYTES_ENV_VAR = "BB_SQLITE_MMAP_SIZE_BYTES"; export const SQLITE_BUSY_TIMEOUT_MS = 5_000; + +/** + * Reads a non-negative integer override from the environment. An unset, + * empty, or malformed value falls back to the default: these are operator + * tuning knobs read before any logger exists, so a bad value must never + * prevent the database from opening. + */ +function resolveNonNegativeIntegerEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (raw === undefined || raw === "") { + return fallback; + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 0) { + return fallback; + } + return value; +} + +/** Resolved per connection so an env override applies without a rebuild. */ +export function resolveSqliteCacheSizeKib(): number { + return resolveNonNegativeIntegerEnv( + SQLITE_CACHE_SIZE_KIB_ENV_VAR, + DEFAULT_SQLITE_CACHE_SIZE_KIB, + ); +} + +export function resolveSqliteMmapSizeBytes(): number { + return resolveNonNegativeIntegerEnv( + SQLITE_MMAP_SIZE_BYTES_ENV_VAR, + DEFAULT_SQLITE_MMAP_SIZE_BYTES, + ); +} const MAX_LOGGED_SQL_LENGTH = 1_000; const SQL_TRUNCATION_SUFFIX = "..."; // Keep ORM-generated quoted identifiers intact. SQLite accepts double-quoted @@ -186,8 +223,8 @@ export function createConnection( // transactions; it cannot corrupt the file. cache_size/mmap_size replace // the 2 MiB default cache so a multi-GB database is not re-read per query. sqlite.pragma("synchronous = NORMAL"); - sqlite.pragma(`cache_size = -${SQLITE_CACHE_SIZE_KIB}`); - sqlite.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`); + sqlite.pragma(`cache_size = -${resolveSqliteCacheSizeKib()}`); + sqlite.pragma(`mmap_size = ${resolveSqliteMmapSizeBytes()}`); sqlite.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`); instrumentSqliteClient(sqlite, options); diff --git a/packages/db/src/data/index.ts b/packages/db/src/data/index.ts index 4cadc15b1a..70c90851f3 100644 --- a/packages/db/src/data/index.ts +++ b/packages/db/src/data/index.ts @@ -29,9 +29,12 @@ export { renameThreadSection, } from "./thread-sections.js"; export { + capPromptHistoryEntries, createPromptHistoryEntry, + DEFAULT_PROMPT_HISTORY_CAP_SCOPE_BATCH_SIZE, listStoredProjectPromptHistoryRows, listStoredThreadPromptHistoryRows, + PROMPT_HISTORY_KEEP_PER_SCOPE, } from "./prompt-history.js"; export type { StoredPromptHistoryEntryRow, @@ -55,6 +58,7 @@ export { createThread, countLiveThreadsInEnvironment, countNonDeletedAssignedChildThreads, + DEFAULT_THREAD_LIST_LIMIT, getThread, getThreadExecutionOverride, hasActiveThreadAttention, @@ -322,6 +326,7 @@ export type { export { createPendingInteraction, + DEFAULT_SETTLED_PENDING_INTERACTION_PRUNE_BATCH_SIZE, getActivePendingInteractionForThread, getPendingInteraction, getPendingInteractionByProviderRequest, @@ -330,6 +335,8 @@ export { interruptPendingInteractionsForPlugin, listActivePluginPendingInteractions, listPendingInteractionsByThread, + pruneSettledPendingInteractions, + SETTLED_PENDING_INTERACTION_RETENTION_MS, setPendingInteractionInterrupted, setPendingInteractionResolving, setPendingInteractionResolved, @@ -389,12 +396,18 @@ export type { export { CLOSED_SESSION_ROW_RETENTION_MS, COMPLETED_EVENT_OUTPUT_RETENTION_MS, + DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_ROW_BATCH_SIZE, + DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_THREAD_BATCH_SIZE, DEFAULT_CLOSED_SESSION_PRUNE_BATCH_SIZE, DEFAULT_COMPLETED_EVENT_OUTPUT_TRUNCATION_BATCH_SIZE, DEFAULT_DESTROYED_ENVIRONMENT_PRUNE_BATCH_SIZE, + DEFAULT_PROVIDER_UNHANDLED_EVENT_PRUNE_BATCH_SIZE, DESTROYED_ENVIRONMENT_TTL_MS, + PROVIDER_UNHANDLED_EVENT_RETENTION_MS, + pruneArchivedThreadEvents, pruneClosedSessions, pruneDestroyedEnvironments, + pruneProviderUnhandledEvents, truncateCompletedEventItemOutputs, sweepManagedEnvironments, } from "./sweeps.js"; @@ -409,6 +422,7 @@ export { DATABASE_INCREMENTAL_VACUUM_MAX_PAGES, DATABASE_MAINTENANCE_BUSY_TIMEOUT_MS, getDatabaseCompactionStats, + getDatabaseCompactionStatsFreelistFirst, getDatabaseFreelistStats, getDatabaseMaintenanceActivity, isDatabaseMaintenanceIdle, diff --git a/packages/db/src/data/maintenance.ts b/packages/db/src/data/maintenance.ts index 666c861136..69eb655a4b 100644 --- a/packages/db/src/data/maintenance.ts +++ b/packages/db/src/data/maintenance.ts @@ -360,6 +360,38 @@ export function getDatabaseCompactionStats( }; } +/** + * Compaction stats with the freelist checked first. The freelist/page + * counters are O(1) PRAGMAs, while the dbstat unused-bytes scan walks every + * page of the file — seconds of synchronous work on a multi-hundred-MB + * database. When the freelist alone already crosses the caller's compaction + * thresholds the decision cannot change (dbstat's unused bytes only add + * reclaimable space), so the scan is skipped and the returned stats carry + * `unusedBytes: 0`. Only when the freelist alone is below the thresholds is + * the full dbstat scan needed to decide. + */ +export function getDatabaseCompactionStatsFreelistFirst( + db: DbConnection, + args: Omit, +): DatabaseCompactionStats { + const freelistStats = getDatabaseFreelistStats(db); + const freelistOnlyStats: DatabaseCompactionStats = { + ...freelistStats, + reclaimableBytes: freelistStats.freelistBytes, + unusedBytes: 0, + }; + if ( + shouldCompactDatabase({ + minReclaimableBytes: args.minReclaimableBytes, + minReclaimableRatio: args.minReclaimableRatio, + stats: freelistOnlyStats, + }) + ) { + return freelistOnlyStats; + } + return getDatabaseCompactionStats(db); +} + export function shouldCompactDatabase( args: DatabaseCompactionDecisionArgs, ): boolean { diff --git a/packages/db/src/data/pending-interactions.ts b/packages/db/src/data/pending-interactions.ts index b55509197f..57d8983a39 100644 --- a/packages/db/src/data/pending-interactions.ts +++ b/packages/db/src/data/pending-interactions.ts @@ -76,8 +76,80 @@ export interface InterruptPendingInteractionsForPluginArgs { statusReason: string; } +export interface PruneSettledPendingInteractionsArgs { + createdBefore: number; + limit: number; +} + +export interface PruneSettledPendingInteractionsResult { + deleted: number; +} + const SQLITE_IN_CLAUSE_BATCH_SIZE = 900; +/** + * Settled interaction rows stay inspectable for a month, then their payloads + * (full approval diffs and command text) are reclaimed. Retention policy — + * revisit deliberately, not incidentally. + */ +export const SETTLED_PENDING_INTERACTION_RETENTION_MS = 30 * 24 * 60 * 60_000; +export const DEFAULT_SETTLED_PENDING_INTERACTION_PRUNE_BATCH_SIZE = 1_000; + +/** Terminal statuses; rows in these statuses are never read for live work. */ +const SETTLED_PENDING_INTERACTION_STATUSES = [ + "resolved", + "interrupted", +] as const satisfies readonly PendingInteractionStatus[]; + +type SettledPendingInteractionDeleteParameters = [ + PendingInteractionStatus, + number, + number, +]; + +/** + * Deletes settled (resolved or interrupted) interaction rows created before + * the cutoff. Their payloads hold full approval prompts (diffs, command + * text), so without a prune they accumulate forever. Retention keys on + * `created_at` rather than `resolved_at` because the covering + * `pending_interactions_status_created_idx` exists for `(status, created_at)` + * and a settled row's resolution follows its creation within the same + * approval flow, so a creation-time cutoff is equivalent at retention + * horizons. Pending/resolving rows are never touched. + */ +export function pruneSettledPendingInteractions( + db: DbConnection, + args: PruneSettledPendingInteractionsArgs, +): PruneSettledPendingInteractionsResult { + let deleted = 0; + for (const status of SETTLED_PENDING_INTERACTION_STATUSES) { + const remainingLimit = args.limit - deleted; + if (remainingLimit <= 0) { + break; + } + // Keep the prune plan pinned to the retention index; this path runs + // periodically and can otherwise regress into a scan plus temp sort. + const result = db.$client + .prepare( + ` + DELETE FROM pending_interactions + WHERE id IN ( + SELECT id + FROM pending_interactions INDEXED BY pending_interactions_status_created_idx + WHERE status = ? + AND created_at < ? + ORDER BY created_at + LIMIT ? + ) + `, + ) + .run(status, args.createdBefore, remainingLimit); + deleted += result.changes; + } + + return { deleted }; +} + function sliceInClauseBatches(values: readonly T[]): T[][] { const batches: T[][] = []; diff --git a/packages/db/src/data/prompt-history.ts b/packages/db/src/data/prompt-history.ts index 7f495c91d6..16e5298060 100644 --- a/packages/db/src/data/prompt-history.ts +++ b/packages/db/src/data/prompt-history.ts @@ -4,7 +4,7 @@ import { type PromptHistoryScope, type PromptInput, } from "@bb/domain"; -import type { DbQueryConnection } from "../connection.js"; +import type { DbConnection, DbQueryConnection } from "../connection.js"; import { promptHistoryEntries, threads } from "../schema.js"; import { createPromptHistoryEntryId } from "../ids.js"; @@ -104,6 +104,81 @@ export function listStoredProjectPromptHistoryRows( .all(); } +/** + * Newest prompt-history entries kept per `(thread, scope)`. Four times the + * largest read window (`PROMPT_HISTORY_ENTRY_LIMIT * 2`), so capping never + * changes what the history pickers can page to. Retention policy — revisit + * deliberately, not incidentally. + */ +export const PROMPT_HISTORY_KEEP_PER_SCOPE = 200; +export const DEFAULT_PROMPT_HISTORY_CAP_SCOPE_BATCH_SIZE = 100; + +export interface CapPromptHistoryEntriesArgs { + keepPerScope: number; + maxScopes: number; +} + +export interface CapPromptHistoryEntriesResult { + deleted: number; + scopesCapped: number; +} + +interface OverCapPromptHistoryScopeRow { + scope: PromptHistoryScope; + threadId: string; +} + +type OverCapScopeParameters = [number, number]; +type CapDeleteParameters = [string, PromptHistoryScope, number]; + +/** + * Caps stored prompt history at the newest `keepPerScope` entries per + * `(thread, scope)` pair — the exact granularity the history reads use — so a + * long-lived thread stops accumulating a second full copy of every prompt. + * Deletion order matches the read order (newest by `created_at`, + * `request_sequence`, `id`), so the kept window is exactly what the reads + * page over. Bounded per pass by `maxScopes`; the sweep converges across + * passes. + */ +export function capPromptHistoryEntries( + db: DbConnection, + args: CapPromptHistoryEntriesArgs, +): CapPromptHistoryEntriesResult { + const overCapScopes = db.$client + .prepare( + ` + SELECT thread_id AS threadId, scope + FROM prompt_history_entries + GROUP BY thread_id, scope + HAVING COUNT(*) > ? + LIMIT ? + `, + ) + .all(args.keepPerScope, args.maxScopes); + + let deleted = 0; + for (const overCapScope of overCapScopes) { + const result = db.$client + .prepare( + ` + DELETE FROM prompt_history_entries + WHERE id IN ( + SELECT id + FROM prompt_history_entries + WHERE thread_id = ? + AND scope = ? + ORDER BY created_at DESC, request_sequence DESC, id DESC + LIMIT -1 OFFSET ? + ) + `, + ) + .run(overCapScope.threadId, overCapScope.scope, args.keepPerScope); + deleted += result.changes; + } + + return { deleted, scopesCapped: overCapScopes.length }; +} + export function listStoredThreadPromptHistoryRows( db: DbQueryConnection, args: ListStoredThreadPromptHistoryArgs, diff --git a/packages/db/src/data/sweeps.ts b/packages/db/src/data/sweeps.ts index 1ddbda31d4..ff07c5eae4 100644 --- a/packages/db/src/data/sweeps.ts +++ b/packages/db/src/data/sweeps.ts @@ -1,6 +1,9 @@ import { eq, and, + gt, + isNotNull, + isNull, sql, lt, asc, @@ -8,7 +11,8 @@ import { import { type ThreadEventItemType } from "@bb/domain"; import type { DbConnection } from "../connection.js"; import type { DbNotifier } from "../notifier.js"; -import { environments, maintenanceScanCursors } from "../schema.js"; +import { environments, maintenanceScanCursors, threads } from "../schema.js"; +import { getLatestThreadSequence } from "./events.js"; /** Destroyed environments are hard-deleted after 7 days. */ export const DESTROYED_ENVIRONMENT_TTL_MS = 7 * 24 * 60 * 60_000; @@ -344,6 +348,222 @@ export function truncateCompletedEventItemOutputs( }; } +/** + * `provider/unhandled` rows are raw provider events bb could not translate, + * persisted for diagnostics. Stored rows are read back only by the + * diagnostics timeline path (development builds or the + * `showUnhandledProviderEvents` setting) and by the legacy claude-code + * model-fallback extraction for rows persisted before `provider/modelFallback` + * existed; nothing replays them to a provider. They are capped by age + * regardless of thread archival. Retention policy — revisit deliberately, + * not incidentally. + */ +export const PROVIDER_UNHANDLED_EVENT_RETENTION_MS = 30 * 24 * 60 * 60_000; +export const DEFAULT_PROVIDER_UNHANDLED_EVENT_PRUNE_BATCH_SIZE = 1_000; + +export interface PruneProviderUnhandledEventsArgs { + createdBefore: number; + limit: number; +} + +export interface PruneProviderUnhandledEventsResult { + deleted: number; +} + +type ProviderUnhandledDeleteParameters = [number, number]; + +/** + * Deletes `provider/unhandled` event rows created before the cutoff, + * oldest-first, bounded per pass. Deleting from the head of the partial + * retention index means each pass resumes where the last one stopped + * without a cursor. + */ +export function pruneProviderUnhandledEvents( + db: DbConnection, + args: PruneProviderUnhandledEventsArgs, +): PruneProviderUnhandledEventsResult { + // Keep the prune plan pinned to the partial retention index; the literal + // type predicate is what makes the partial index usable. + const result = db.$client + .prepare( + ` + DELETE FROM events + WHERE id IN ( + SELECT id + FROM events INDEXED BY events_provider_unhandled_created_idx + WHERE type = 'provider/unhandled' + AND created_at < ? + ORDER BY created_at + LIMIT ? + ) + `, + ) + .run(args.createdBefore, args.limit); + + return { deleted: result.changes }; +} + +const ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_POLICY = + "archived_thread_event_retention"; +const ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_VERSION = 1; +const ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_ID = [ + ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_POLICY, + `v${ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_VERSION}`, +].join(":"); + +/** Archived threads examined per retention pass. */ +export const DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_THREAD_BATCH_SIZE = 50; +/** + * Total event rows deleted per retention pass. Bounds the write transaction + * so a backlog of hundreds of thousands of rows drains across passes without + * stalling foreground work on the synchronous SQLite writer. + */ +export const DEFAULT_ARCHIVED_THREAD_EVENT_PRUNE_ROW_BATCH_SIZE = 2_000; + +export interface PruneArchivedThreadEventsArgs { + /** + * Number of most recent sequence slots kept per archived thread. The + * caller (the server) owns this product policy. + */ + keepRecent: number; + maxRows: number; + maxThreads: number; + now: number; +} + +export interface PruneArchivedThreadEventsResult { + /** + * True when this pass reached the end of the archived-thread walk; the + * cursor has wrapped and the next pass starts over. + */ + completedCycle: boolean; + deleted: number; + scannedThreads: number; +} + +type ArchivedThreadEventDeleteParameters = [string, number, number]; + +function getArchivedThreadEventRetentionCursorThreadId( + db: DbConnection, +): string { + const row = db + .select({ lastEventId: maintenanceScanCursors.lastEventId }) + .from(maintenanceScanCursors) + .where( + eq(maintenanceScanCursors.id, ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_ID), + ) + .get(); + + return row?.lastEventId ?? ""; +} + +function setArchivedThreadEventRetentionCursorThreadId( + db: DbConnection, + args: { threadId: string; updatedAt: number }, +): void { + db.insert(maintenanceScanCursors) + .values({ + id: ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_ID, + policy: ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_POLICY, + version: ARCHIVED_THREAD_EVENT_RETENTION_CURSOR_VERSION, + itemKind: "", + outputPath: "", + lastCreatedAt: 0, + lastEventId: args.threadId, + updatedAt: args.updatedAt, + }) + .onConflictDoUpdate({ + target: maintenanceScanCursors.id, + set: { + lastEventId: args.threadId, + updatedAt: args.updatedAt, + }, + }) + .run(); +} + +/** + * Hard-deletes archived threads' events beyond the caller's keep-recent + * window (the same window the on-archive prune already applies to its + * prunable event classes — the product accepts that an archived thread keeps + * only its recent history). Walks archived, non-deleted threads in id order + * behind a durable keyset cursor, so a large backlog drains across passes + * and newly archived threads are picked up on the next cycle. Unarchiving a + * thread removes it from the walk — rows already pruned stay gone, which is + * the same contract the on-archive prune implies. + */ +export function pruneArchivedThreadEvents( + db: DbConnection, + args: PruneArchivedThreadEventsArgs, +): PruneArchivedThreadEventsResult { + if (args.keepRecent < 0 || args.maxThreads <= 0 || args.maxRows <= 0) { + return { completedCycle: false, deleted: 0, scannedThreads: 0 }; + } + + const cursorThreadId = getArchivedThreadEventRetentionCursorThreadId(db); + const threadRows = db + .select({ id: threads.id }) + .from(threads) + .where( + and( + isNotNull(threads.archivedAt), + isNull(threads.deletedAt), + gt(threads.id, cursorThreadId), + ), + ) + .orderBy(threads.id) + .limit(args.maxThreads) + .all(); + + let deleted = 0; + let scannedThreads = 0; + let lastFinishedThreadId = cursorThreadId; + let exhaustedRowBudget = false; + for (const threadRow of threadRows) { + scannedThreads += 1; + const latestSequence = getLatestThreadSequence(db, { + threadId: threadRow.id, + }); + const sequenceCutoff = latestSequence - args.keepRecent; + if (sequenceCutoff > 0) { + // Keep the delete plan pinned to the (thread_id, sequence) index; the + // subquery is a bounded oldest-first range scan on it. + const result = db.$client + .prepare( + ` + DELETE FROM events + WHERE id IN ( + SELECT id + FROM events INDEXED BY events_thread_sequence_idx + WHERE thread_id = ? + AND sequence <= ? + ORDER BY sequence + LIMIT ? + ) + `, + ) + .run(threadRow.id, sequenceCutoff, args.maxRows - deleted); + deleted += result.changes; + if (deleted >= args.maxRows) { + // The thread may still hold prunable rows; leave the cursor before + // it so the next pass resumes here. + exhaustedRowBudget = true; + break; + } + } + lastFinishedThreadId = threadRow.id; + } + + const completedCycle = + !exhaustedRowBudget && threadRows.length < args.maxThreads; + setArchivedThreadEventRetentionCursorThreadId(db, { + threadId: completedCycle ? "" : lastFinishedThreadId, + updatedAt: args.now, + }); + + return { completedCycle, deleted, scannedThreads }; +} + /** * Sweep retiring managed environments with zero non-archived threads. * Returns the list of environment records that are candidates for cleanup. diff --git a/packages/db/src/data/threads.ts b/packages/db/src/data/threads.ts index b09c32fc75..3df5356b35 100644 --- a/packages/db/src/data/threads.ts +++ b/packages/db/src/data/threads.ts @@ -373,6 +373,15 @@ export function listThreadMentionRowsByIds( .all(); } +/** + * Default row cap for thread listings. The sidebar and every SDK/CLI list + * ride these queries; without a cap one long-lived install (thousands of + * archived-then-restored threads) makes every list read scale with history. + * Generous enough that hitting it means paging, not filtering — callers pass + * an explicit `limit` to page or to read more. + */ +export const DEFAULT_THREAD_LIST_LIMIT = 200; + export interface ListThreadsOptions { projectId?: string; archived?: boolean; @@ -400,6 +409,12 @@ type ThreadRow = typeof threads.$inferSelect; export interface ListThreadsForProjectsOptions { projectIds: readonly string[]; archived?: boolean; + /** + * Row cap across all listed projects combined (the query orders by the + * active-thread sidebar order, so the cap keeps the most relevant rows). + * Defaults to {@link DEFAULT_THREAD_LIST_LIMIT}. + */ + limit?: number; } export interface PinThreadArgs { @@ -1204,10 +1219,8 @@ export function listThreadsWithPendingInteractionState( let query = threadWithPendingInteractionBaseQuery(db) .where(and(...buildListThreadsFilters(options))) .orderBy(...buildListThreadsOrderBy(options)) - .$dynamic(); - if (options.limit !== undefined) { - query = query.limit(options.limit); - } + .$dynamic() + .limit(options.limit ?? DEFAULT_THREAD_LIST_LIMIT); if (options.offset !== undefined) { query = query.offset(options.offset); } @@ -1261,6 +1274,7 @@ export function listThreadsWithPendingInteractionStateForProjects( const rows = threadWithPendingInteractionBaseQuery(db) .where(and(...buildListThreadsForProjectsFilters(options))) .orderBy(...buildListThreadsForProjectsOrderBy(options)) + .limit(options.limit ?? DEFAULT_THREAD_LIST_LIMIT) .all(); return rows.map(toThreadWithPendingInteractionState); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 7d7ecfa802..b3a591d8f5 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -801,6 +801,13 @@ export const events = sqliteTable( index("events_completed_item_truncation_idx") .on(table.itemKind, table.createdAt, table.id) .where(sql`${table.type} = 'item/completed'`), + // The provider/unhandled retention sweep deletes these diagnostic rows + // oldest-first by age. The partial index keeps that delete an index-only + // range scan and stays tiny: it holds only the retained rows of this one + // type. + index("events_provider_unhandled_created_idx") + .on(table.createdAt, table.id) + .where(sql`${table.type} = 'provider/unhandled'`), // Latest-thread-state lookup (listLatestThreadStateEventRowsByThreadIds) // runs over every listed thread on each sidebar bootstrap: the newest // plugin thread-state snapshot of one kind (codex goals today), plus the @@ -823,13 +830,20 @@ export const events = sqliteTable( ], ); +// Generic keyset cursors for maintenance policies. `item_kind` and +// `output_path` are per-policy scan dimensions (the completed-output +// truncation scan uses an event item kind and a JSON output path); a policy +// without those dimensions stores empty strings. `last_created_at` and +// `last_event_id` hold the policy's keyset position (the archived-thread +// event retention policy stores its last finished thread id in +// `last_event_id` and leaves `last_created_at` at 0). export const maintenanceScanCursors = sqliteTable( "maintenance_scan_cursors", { id: text("id").primaryKey(), policy: text("policy").notNull(), version: integer("version").notNull(), - itemKind: text("item_kind").$type().notNull(), + itemKind: text("item_kind").notNull(), outputPath: text("output_path").notNull(), lastCreatedAt: integer("last_created_at").notNull().default(0), lastEventId: text("last_event_id").notNull().default(""), diff --git a/packages/db/test/connection.test.ts b/packages/db/test/connection.test.ts index 898ab2d250..66b61d90ab 100644 --- a/packages/db/test/connection.test.ts +++ b/packages/db/test/connection.test.ts @@ -5,9 +5,11 @@ import { describe, expect, it } from "vitest"; import { eq } from "drizzle-orm"; import { createConnection, + DEFAULT_SQLITE_CACHE_SIZE_KIB, + DEFAULT_SQLITE_MMAP_SIZE_BYTES, SQLITE_BUSY_TIMEOUT_MS, - SQLITE_CACHE_SIZE_KIB, - SQLITE_MMAP_SIZE_BYTES, + SQLITE_CACHE_SIZE_KIB_ENV_VAR, + SQLITE_MMAP_SIZE_BYTES_ENV_VAR, type SlowDbQueryLogger, type SlowDbQueryLogFields, } from "../src/connection.js"; @@ -140,7 +142,7 @@ describe("createConnection", () => { db.$client.prepare("PRAGMA cache_size").get() as { cache_size: number; }, - ).toEqual({ cache_size: -SQLITE_CACHE_SIZE_KIB }); + ).toEqual({ cache_size: -DEFAULT_SQLITE_CACHE_SIZE_KIB }); expect( db.$client.prepare("PRAGMA synchronous").get() as { synchronous: number; @@ -148,7 +150,7 @@ describe("createConnection", () => { ).toEqual({ synchronous: 1 }); expect( db.$client.prepare("PRAGMA mmap_size").get() as { mmap_size: number }, - ).toEqual({ mmap_size: SQLITE_MMAP_SIZE_BYTES }); + ).toEqual({ mmap_size: DEFAULT_SQLITE_MMAP_SIZE_BYTES }); expect( db.$client.prepare("PRAGMA busy_timeout").get() as { timeout: number }, ).toEqual({ timeout: SQLITE_BUSY_TIMEOUT_MS }); @@ -162,4 +164,61 @@ describe("createConnection", () => { rmSync(directory, { force: true, recursive: true }); } }); + + it("honors env overrides for the cache and mmap budgets and ignores malformed values", () => { + const originalCache = process.env[SQLITE_CACHE_SIZE_KIB_ENV_VAR]; + const originalMmap = process.env[SQLITE_MMAP_SIZE_BYTES_ENV_VAR]; + const directory = mkdtempSync(join(tmpdir(), "bb-db-pragma-env-")); + + function readMemoryPragmas(db: ReturnType): { + cacheSize: number; + mmapSize: number; + } { + const cacheRow = db.$client.prepare("PRAGMA cache_size").get() as { + cache_size: number; + }; + const mmapRow = db.$client.prepare("PRAGMA mmap_size").get() as { + mmap_size: number; + }; + return { cacheSize: cacheRow.cache_size, mmapSize: mmapRow.mmap_size }; + } + + try { + process.env[SQLITE_CACHE_SIZE_KIB_ENV_VAR] = "65536"; + process.env[SQLITE_MMAP_SIZE_BYTES_ENV_VAR] = "0"; + const overridden = createConnection(join(directory, "override.db")); + try { + expect(readMemoryPragmas(overridden)).toEqual({ + cacheSize: -65_536, + mmapSize: 0, + }); + } finally { + overridden.$client.close(); + } + + process.env[SQLITE_CACHE_SIZE_KIB_ENV_VAR] = "not-a-number"; + process.env[SQLITE_MMAP_SIZE_BYTES_ENV_VAR] = "-5"; + const malformed = createConnection(join(directory, "malformed.db")); + try { + expect(readMemoryPragmas(malformed)).toEqual({ + cacheSize: -DEFAULT_SQLITE_CACHE_SIZE_KIB, + mmapSize: DEFAULT_SQLITE_MMAP_SIZE_BYTES, + }); + } finally { + malformed.$client.close(); + } + } finally { + if (originalCache === undefined) { + delete process.env[SQLITE_CACHE_SIZE_KIB_ENV_VAR]; + } else { + process.env[SQLITE_CACHE_SIZE_KIB_ENV_VAR] = originalCache; + } + if (originalMmap === undefined) { + delete process.env[SQLITE_MMAP_SIZE_BYTES_ENV_VAR]; + } else { + process.env[SQLITE_MMAP_SIZE_BYTES_ENV_VAR] = originalMmap; + } + rmSync(directory, { force: true, recursive: true }); + } + }); }); diff --git a/packages/db/test/data/maintenance.test.ts b/packages/db/test/data/maintenance.test.ts index 3d7fc5b878..708789e6d4 100644 --- a/packages/db/test/data/maintenance.test.ts +++ b/packages/db/test/data/maintenance.test.ts @@ -12,6 +12,7 @@ import { DATABASE_INCREMENTAL_VACUUM_MIN_FREELIST_PAGES, dropDeferredLegacyTables, getDatabaseAutoVacuumMode, + getDatabaseCompactionStatsFreelistFirst, getDatabaseFreelistStats, getDatabaseMaintenanceActivity, isDatabaseMaintenanceIdle, @@ -271,6 +272,76 @@ describe("database maintenance", () => { expect(getDatabaseAutoVacuumMode(db)).toBe("incremental"); }); + it("skips the dbstat scan when the freelist alone justifies compaction", () => { + const { db } = setup(); + + // Build a freelist the same way the incremental-vacuum test does. + db.$client.exec( + "CREATE TABLE scratch_blobs (id INTEGER PRIMARY KEY, blob TEXT)", + ); + const insert = db.$client.prepare( + "INSERT INTO scratch_blobs (blob) VALUES (?)", + ); + const blob = "x".repeat(8 * 1024); + const insertMany = db.$client.transaction((count: number) => { + for (let index = 0; index < count; index += 1) { + insert.run(blob); + } + }); + insertMany(3_000); + db.$client.exec("DELETE FROM scratch_blobs"); + + const freelistStats = getDatabaseFreelistStats(db); + expect(freelistStats.freelistBytes).toBeGreaterThan(0); + + const preparedSql: string[] = []; + const raw = db.$client; + const originalPrepare = raw.prepare.bind(raw); + Object.defineProperty(raw, "prepare", { + configurable: true, + value: (source: string) => { + preparedSql.push(source); + return originalPrepare(source); + }, + writable: true, + }); + let freelistOnly: ReturnType; + let fullScan: ReturnType; + let dbstatPreparedAfterFreelistShortCircuit: boolean; + try { + // Thresholds beneath the freelist: the decision is already made, so + // no dbstat statement may be prepared. + freelistOnly = getDatabaseCompactionStatsFreelistFirst(db, { + minReclaimableBytes: 1, + minReclaimableRatio: 0.000_001, + }); + dbstatPreparedAfterFreelistShortCircuit = preparedSql.some((source) => + source.includes("dbstat"), + ); + // Thresholds above anything reclaimable: the freelist alone cannot + // decide, so the full dbstat-backed stats are computed. + fullScan = getDatabaseCompactionStatsFreelistFirst(db, { + minReclaimableBytes: Number.MAX_SAFE_INTEGER, + minReclaimableRatio: 1, + }); + } finally { + Object.defineProperty(raw, "prepare", { + configurable: true, + value: originalPrepare, + writable: true, + }); + } + + expect(dbstatPreparedAfterFreelistShortCircuit).toBe(false); + expect(freelistOnly.unusedBytes).toBe(0); + expect(freelistOnly.reclaimableBytes).toBe(freelistOnly.freelistBytes); + + expect(preparedSql.some((source) => source.includes("dbstat"))).toBe(true); + expect(fullScan.reclaimableBytes).toBe( + fullScan.freelistBytes + fullScan.unusedBytes, + ); + }); + it("does not schedule incremental vacuum for internal fragmentation without enough freelist pages", () => { const freelistStats = { databaseBytes: 1_000, diff --git a/packages/db/test/data/pending-interactions.test.ts b/packages/db/test/data/pending-interactions.test.ts index 2577050750..295e2478e2 100644 --- a/packages/db/test/data/pending-interactions.test.ts +++ b/packages/db/test/data/pending-interactions.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; import { noopNotifier } from "../../src/notifier.js"; import { createEnvironment } from "../../src/data/environments.js"; import { upsertHost } from "../../src/data/hosts.js"; @@ -10,9 +11,12 @@ import { interruptPendingInteractionsForThreadIds, interruptPendingInteractionsForThreads, listPendingInteractionsByThread, + pruneSettledPendingInteractions, + setPendingInteractionInterrupted, setPendingInteractionResolved, } from "../../src/data/pending-interactions.js"; import { createThread } from "../../src/data/threads.js"; +import { pendingInteractions } from "../../src/schema.js"; import { createMigratedConnection } from "../helpers/migrated-connection.js"; function setup() { @@ -282,3 +286,157 @@ describe("pending interactions", () => { ); }); + +describe("pruneSettledPendingInteractions", () => { + interface SeedSettledInteractionArgs { + createdAt: number; + requestId: string; + settle: "resolved" | "interrupted" | null; + threadId: string; + } + + function seedInteractionAt( + db: ReturnType["db"], + args: SeedSettledInteractionArgs, + ): string { + const created = createPendingInteraction(db, { + threadId: args.threadId, + providerId: "codex", + providerThreadId: "provider-thread-1", + providerRequestId: args.requestId, + turnId: "turn-1", + payload: commandApprovalPayload("rm -rf build", args.requestId), + }); + if (args.settle === "resolved") { + setPendingInteractionResolved(db, { + id: created.id, + resolution: JSON.stringify({ decision: "allow_once" }), + }); + } else if (args.settle === "interrupted") { + setPendingInteractionInterrupted(db, { + id: created.id, + statusReason: "Thread stopped", + }); + } + db.update(pendingInteractions) + .set({ createdAt: args.createdAt }) + .where(eq(pendingInteractions.id, created.id)) + .run(); + return created.id; + } + + function listRemainingIds(db: ReturnType["db"]): string[] { + return db + .select({ id: pendingInteractions.id }) + .from(pendingInteractions) + .all() + .map((row) => row.id) + .sort(); + } + + it("deletes settled rows older than the cutoff and never touches live rows", () => { + const { db, thread } = setup(); + const now = Date.now(); + const staleCreatedAt = now - 10_000; + + const staleResolved = seedInteractionAt(db, { + createdAt: staleCreatedAt, + requestId: "req-stale-resolved", + settle: "resolved", + threadId: thread.id, + }); + const staleInterrupted = seedInteractionAt(db, { + createdAt: staleCreatedAt, + requestId: "req-stale-interrupted", + settle: "interrupted", + threadId: thread.id, + }); + const freshResolved = seedInteractionAt(db, { + createdAt: now - 1_000, + requestId: "req-fresh-resolved", + settle: "resolved", + threadId: thread.id, + }); + const stalePending = seedInteractionAt(db, { + createdAt: staleCreatedAt, + requestId: "req-stale-pending", + settle: null, + threadId: thread.id, + }); + + expect( + pruneSettledPendingInteractions(db, { + createdBefore: now - 5_000, + limit: 100, + }), + ).toEqual({ deleted: 2 }); + const remaining = listRemainingIds(db); + expect(remaining).toEqual([freshResolved, stalePending].sort()); + expect(remaining).not.toContain(staleResolved); + expect(remaining).not.toContain(staleInterrupted); + }); + + it("honors the delete batch limit across both settled statuses", () => { + const { db, thread } = setup(); + const now = Date.now(); + const staleCreatedAt = now - 10_000; + + for (let index = 0; index < 3; index += 1) { + seedInteractionAt(db, { + createdAt: staleCreatedAt, + requestId: `req-batch-resolved-${index}`, + settle: "resolved", + threadId: thread.id, + }); + seedInteractionAt(db, { + createdAt: staleCreatedAt, + requestId: `req-batch-interrupted-${index}`, + settle: "interrupted", + threadId: thread.id, + }); + } + + // A limit of 4 spans the resolved batch (3) and part of the interrupted + // batch (1), proving the limit is shared across statuses. + expect( + pruneSettledPendingInteractions(db, { + createdBefore: now - 5_000, + limit: 4, + }), + ).toEqual({ deleted: 4 }); + expect(listRemainingIds(db)).toHaveLength(2); + expect( + pruneSettledPendingInteractions(db, { + createdBefore: now - 5_000, + limit: 100, + }), + ).toEqual({ deleted: 2 }); + expect(listRemainingIds(db)).toHaveLength(0); + }); + + it("keeps a resolving row even when it is old", () => { + const { db, thread } = setup(); + const now = Date.now(); + + const created = createPendingInteraction(db, { + threadId: thread.id, + providerId: "codex", + providerThreadId: "provider-thread-1", + providerRequestId: "req-old-resolving", + turnId: "turn-1", + payload: commandApprovalPayload("git push", "req-old-resolving"), + }); + db.update(pendingInteractions) + .set({ createdAt: now - 10_000, status: "resolving" }) + .where(eq(pendingInteractions.id, created.id)) + .run(); + + expect( + pruneSettledPendingInteractions(db, { + createdBefore: now - 5_000, + limit: 100, + }), + ).toEqual({ deleted: 0 }); + expect(listRemainingIds(db)).toEqual([created.id]); + }); +}); diff --git a/packages/db/test/data/prompt-history.test.ts b/packages/db/test/data/prompt-history.test.ts new file mode 100644 index 0000000000..1452ecb520 --- /dev/null +++ b/packages/db/test/data/prompt-history.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "vitest"; +import type { PromptHistoryScope } from "@bb/domain"; +import { noopNotifier } from "../../src/notifier.js"; +import { upsertHost } from "../../src/data/hosts.js"; +import { createProject } from "../../src/data/projects.js"; +import { createThread } from "../../src/data/threads.js"; +import { + capPromptHistoryEntries, + createPromptHistoryEntry, + listStoredThreadPromptHistoryRows, +} from "../../src/data/prompt-history.js"; +import { promptHistoryEntries } from "../../src/schema.js"; +import { createMigratedConnection } from "../helpers/migrated-connection.js"; + +function setup() { + const db = createMigratedConnection(); + const host = upsertHost(db, noopNotifier, { + name: "test-host", + type: "persistent", + }); + const { project } = createProject(db, noopNotifier, { + name: "test-project", + source: { type: "local_path", hostId: host.id, path: "/tmp/test" }, + }); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + status: "idle", + }); + const siblingThread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + status: "idle", + }); + return { db, project, thread, siblingThread }; +} + +interface SeedEntriesArgs { + count: number; + db: ReturnType["db"]; + projectId: string; + scope: PromptHistoryScope; + startRequestSequence?: number; + threadId: string; +} + +function seedEntries(args: SeedEntriesArgs): void { + const startRequestSequence = args.startRequestSequence ?? 1; + const baseCreatedAt = Date.now() - args.count * 1_000; + for (let index = 0; index < args.count; index += 1) { + createPromptHistoryEntry(args.db, { + createdAt: baseCreatedAt + index * 1_000, + input: [{ type: "text", text: `prompt ${index}`, mentions: [] }], + projectId: args.projectId, + requestSequence: startRequestSequence + index, + scope: args.scope, + threadId: args.threadId, + }); + } +} + +function countEntries( + db: ReturnType["db"], +): number { + return db.select().from(promptHistoryEntries).all().length; +} + +describe("capPromptHistoryEntries", () => { + it("keeps only the newest entries per (thread, scope) and leaves under-cap scopes alone", () => { + const { db, project, thread, siblingThread } = setup(); + + seedEntries({ + count: 12, + db, + projectId: project.id, + scope: "thread", + threadId: thread.id, + }); + seedEntries({ + count: 3, + db, + projectId: project.id, + scope: "project", + startRequestSequence: 100, + threadId: thread.id, + }); + seedEntries({ + count: 5, + db, + projectId: project.id, + scope: "thread", + threadId: siblingThread.id, + }); + + const result = capPromptHistoryEntries(db, { + keepPerScope: 10, + maxScopes: 100, + }); + + expect(result).toEqual({ deleted: 2, scopesCapped: 1 }); + expect(countEntries(db)).toBe(18); + + // The kept window is exactly the newest rows in read order. + const keptRows = listStoredThreadPromptHistoryRows(db, { + limit: 50, + threadId: thread.id, + }); + expect(keptRows).toHaveLength(10); + expect(keptRows[0]?.requestSequence).toBe(12); + expect(keptRows.at(-1)?.requestSequence).toBe(3); + }); + + it("bounds each pass by maxScopes and converges across passes", () => { + const { db, project, thread, siblingThread } = setup(); + + seedEntries({ + count: 4, + db, + projectId: project.id, + scope: "thread", + threadId: thread.id, + }); + seedEntries({ + count: 4, + db, + projectId: project.id, + scope: "thread", + threadId: siblingThread.id, + }); + + const firstPass = capPromptHistoryEntries(db, { + keepPerScope: 2, + maxScopes: 1, + }); + expect(firstPass).toEqual({ deleted: 2, scopesCapped: 1 }); + + const secondPass = capPromptHistoryEntries(db, { + keepPerScope: 2, + maxScopes: 1, + }); + expect(secondPass).toEqual({ deleted: 2, scopesCapped: 1 }); + + expect( + capPromptHistoryEntries(db, { keepPerScope: 2, maxScopes: 1 }), + ).toEqual({ deleted: 0, scopesCapped: 0 }); + expect(countEntries(db)).toBe(4); + }); +}); diff --git a/packages/db/test/data/sweeps.test.ts b/packages/db/test/data/sweeps.test.ts index 98dcdb411f..6d0f26326d 100644 --- a/packages/db/test/data/sweeps.test.ts +++ b/packages/db/test/data/sweeps.test.ts @@ -9,8 +9,10 @@ import { COMPLETED_EVENT_OUTPUT_RETAINED_TAIL_CHARS, COMPLETED_EVENT_OUTPUT_TRUNCATION_THRESHOLD_CHARS, DESTROYED_ENVIRONMENT_TTL_MS, + pruneArchivedThreadEvents, pruneClosedSessions, pruneDestroyedEnvironments, + pruneProviderUnhandledEvents, sweepManagedEnvironments, truncateCompletedEventItemOutputs, } from "../../src/data/sweeps.js"; @@ -20,6 +22,7 @@ import { createThread, archiveThread, markThreadDeleted, + unarchiveThread, } from "../../src/data/threads.js"; import { createEnvironment, @@ -549,6 +552,326 @@ describe("pruneClosedSessions", () => { }); }); +describe("pruneArchivedThreadEvents", () => { + interface SeedTurnEventsArgs { + db: DbConnection; + endingSequence: number; + startingSequence?: number; + threadId: string; + } + + function seedTurnEvents(args: SeedTurnEventsArgs): void { + const startingSequence = args.startingSequence ?? 1; + const createdAt = Date.now(); + for ( + let sequence = startingSequence; + sequence <= args.endingSequence; + sequence += 1 + ) { + args.db + .insert(events) + .values({ + id: createEventId(), + threadId: args.threadId, + scopeKind: "turn", + turnId: `turn-${sequence}`, + providerThreadId: "provider-thread-1", + sequence, + type: "thread/tokenUsage/updated", + itemId: null, + itemKind: null, + data: JSON.stringify({ tokenUsage: { totalTokens: sequence } }), + createdAt, + }) + .run(); + } + } + + function listSequences(db: DbConnection, threadId: string): number[] { + return db + .select({ sequence: events.sequence }) + .from(events) + .where(eq(events.threadId, threadId)) + .all() + .map((row) => row.sequence) + .sort((left, right) => left - right); + } + + function createIdleThread(context: ReturnType) { + return createThread(context.db, noopNotifier, { + projectId: context.project.id, + providerId: "codex", + status: "idle", + }); + } + + it("prunes archived threads to the keep-recent window and leaves live threads alone", () => { + const context = setup(); + const { db } = context; + const archived = createIdleThread(context); + const live = createIdleThread(context); + + seedTurnEvents({ db, endingSequence: 30, threadId: archived.id }); + seedTurnEvents({ db, endingSequence: 30, threadId: live.id }); + archiveThread(db, noopNotifier, archived.id); + + const result = pruneArchivedThreadEvents(db, { + keepRecent: 10, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }); + + expect(result).toEqual({ + completedCycle: true, + deleted: 20, + scannedThreads: 1, + }); + // keepRecent is a sequence window: rows above latest - keepRecent stay. + expect(listSequences(db, archived.id)).toEqual([ + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + ]); + expect(listSequences(db, live.id)).toHaveLength(30); + }); + + it("stops pruning a thread once it is unarchived", () => { + const context = setup(); + const { db } = context; + const thread = createIdleThread(context); + + seedTurnEvents({ db, endingSequence: 30, threadId: thread.id }); + archiveThread(db, noopNotifier, thread.id); + unarchiveThread(db, noopNotifier, thread.id); + + const result = pruneArchivedThreadEvents(db, { + keepRecent: 10, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }); + + expect(result).toEqual({ + completedCycle: true, + deleted: 0, + scannedThreads: 0, + }); + expect(listSequences(db, thread.id)).toHaveLength(30); + }); + + it("bounds each pass by the row budget and resumes the same thread next pass", () => { + const context = setup(); + const { db } = context; + const first = createIdleThread(context); + const second = createIdleThread(context); + + seedTurnEvents({ db, endingSequence: 30, threadId: first.id }); + seedTurnEvents({ db, endingSequence: 30, threadId: second.id }); + archiveThread(db, noopNotifier, first.id); + archiveThread(db, noopNotifier, second.id); + const [earlierThreadId, laterThreadId] = [first.id, second.id].sort(); + + const firstPass = pruneArchivedThreadEvents(db, { + keepRecent: 10, + maxRows: 15, + maxThreads: 10, + now: Date.now(), + }); + expect(firstPass.deleted).toBe(15); + expect(firstPass.completedCycle).toBe(false); + // The row budget cut the earlier thread mid-delete: it still holds + // prunable rows, and the later thread is untouched. + expect(listSequences(db, earlierThreadId!)).toHaveLength(15); + expect(listSequences(db, laterThreadId!)).toHaveLength(30); + + const secondPass = pruneArchivedThreadEvents(db, { + keepRecent: 10, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }); + expect(secondPass.deleted).toBe(25); + expect(listSequences(db, earlierThreadId!)).toHaveLength(10); + expect(listSequences(db, laterThreadId!)).toHaveLength(10); + }); + + it("wraps the cursor after a completed cycle so newly archived threads are picked up", () => { + const context = setup(); + const { db } = context; + const first = createIdleThread(context); + + seedTurnEvents({ db, endingSequence: 20, threadId: first.id }); + archiveThread(db, noopNotifier, first.id); + + expect( + pruneArchivedThreadEvents(db, { + keepRecent: 5, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }), + ).toMatchObject({ completedCycle: true, deleted: 15 }); + + const second = createIdleThread(context); + seedTurnEvents({ db, endingSequence: 20, threadId: second.id }); + archiveThread(db, noopNotifier, second.id); + + expect( + pruneArchivedThreadEvents(db, { + keepRecent: 5, + maxRows: 1_000, + maxThreads: 10, + now: Date.now(), + }), + ).toMatchObject({ deleted: 15 }); + expect(listSequences(db, second.id)).toEqual([16, 17, 18, 19, 20]); + }); +}); + +describe("pruneProviderUnhandledEvents", () => { + interface SeedEventAtArgs { + createdAt: number; + db: DbConnection; + sequence: number; + threadId: string; + type: "provider/unhandled" | "item/completed"; + } + + function seedEventAt(args: SeedEventAtArgs): string { + const id = createEventId(); + args.db + .insert(events) + .values({ + id, + threadId: args.threadId, + scopeKind: "turn", + turnId: "turn-1", + providerThreadId: "provider-thread-1", + sequence: args.sequence, + type: args.type, + itemId: args.type === "item/completed" ? "item-1" : null, + itemKind: args.type === "item/completed" ? "agentMessage" : null, + data: + args.type === "provider/unhandled" + ? JSON.stringify({ + providerId: "claude-code", + providerThreadId: "provider-thread-1", + rawType: "sdk/unknown", + rawEvent: { jsonrpc: "2.0", method: "sdk/unknown" }, + }) + : JSON.stringify({ + item: { id: "item-1", type: "agentMessage", text: "hi" }, + }), + createdAt: args.createdAt, + }) + .run(); + return id; + } + + function listRemainingEventIds(db: DbConnection, threadId: string): string[] { + return db + .select({ id: events.id }) + .from(events) + .where(eq(events.threadId, threadId)) + .all() + .map((row) => row.id) + .sort(); + } + + it("deletes old provider/unhandled rows on an active thread and keeps everything else", () => { + const { db, project } = setup(); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + status: "active", + }); + const now = Date.now(); + const staleCreatedAt = now - 10_000; + + const staleUnhandled = seedEventAt({ + createdAt: staleCreatedAt, + db, + sequence: 1, + threadId: thread.id, + type: "provider/unhandled", + }); + const staleCompleted = seedEventAt({ + createdAt: staleCreatedAt, + db, + sequence: 2, + threadId: thread.id, + type: "item/completed", + }); + const freshUnhandled = seedEventAt({ + createdAt: now - 1_000, + db, + sequence: 3, + threadId: thread.id, + type: "provider/unhandled", + }); + + expect( + pruneProviderUnhandledEvents(db, { + createdBefore: now - 5_000, + limit: 100, + }), + ).toEqual({ deleted: 1 }); + + const remaining = listRemainingEventIds(db, thread.id); + expect(remaining).toEqual([staleCompleted, freshUnhandled].sort()); + expect(remaining).not.toContain(staleUnhandled); + }); + + it("honors the delete batch limit oldest-first and converges across passes", () => { + const { db, project } = setup(); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + status: "idle", + }); + const now = Date.now(); + + const oldest = seedEventAt({ + createdAt: now - 30_000, + db, + sequence: 1, + threadId: thread.id, + type: "provider/unhandled", + }); + const middle = seedEventAt({ + createdAt: now - 20_000, + db, + sequence: 2, + threadId: thread.id, + type: "provider/unhandled", + }); + const newest = seedEventAt({ + createdAt: now - 10_000, + db, + sequence: 3, + threadId: thread.id, + type: "provider/unhandled", + }); + + expect( + pruneProviderUnhandledEvents(db, { + createdBefore: now - 5_000, + limit: 2, + }), + ).toEqual({ deleted: 2 }); + expect(listRemainingEventIds(db, thread.id)).toEqual([newest]); + expect(listRemainingEventIds(db, thread.id)).not.toContain(oldest); + expect(listRemainingEventIds(db, thread.id)).not.toContain(middle); + + expect( + pruneProviderUnhandledEvents(db, { + createdBefore: now - 5_000, + limit: 2, + }), + ).toEqual({ deleted: 1 }); + expect(listRemainingEventIds(db, thread.id)).toEqual([]); + }); +}); + describe("sweepManagedEnvironments", () => { it("returns retiring managed environments with zero non-archived threads", () => { const { db, host, project } = setup(); diff --git a/packages/db/test/data/threads.test.ts b/packages/db/test/data/threads.test.ts index 69349b53fd..5256021f2a 100644 --- a/packages/db/test/data/threads.test.ts +++ b/packages/db/test/data/threads.test.ts @@ -7,6 +7,7 @@ import { createThread, countLiveThreadsInEnvironment, countNonDeletedAssignedChildThreads, + DEFAULT_THREAD_LIST_LIMIT, getThread, getThreadExecutionOverride, hasActiveThreadAttention, @@ -235,6 +236,38 @@ describe("threads", () => { ).toBe(2); }); + it("caps thread listings at the default limit and honors explicit limits", () => { + const { db, project } = setup(); + for (let index = 0; index < DEFAULT_THREAD_LIST_LIMIT + 1; index += 1) { + createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + }); + } + + expect( + listThreadsWithPendingInteractionState(db, { projectId: project.id }), + ).toHaveLength(DEFAULT_THREAD_LIST_LIMIT); + expect( + listThreadsWithPendingInteractionState(db, { + projectId: project.id, + limit: DEFAULT_THREAD_LIST_LIMIT + 1, + }), + ).toHaveLength(DEFAULT_THREAD_LIST_LIMIT + 1); + + expect( + listThreadsWithPendingInteractionStateForProjects(db, { + projectIds: [project.id], + }), + ).toHaveLength(DEFAULT_THREAD_LIST_LIMIT); + expect( + listThreadsWithPendingInteractionStateForProjects(db, { + projectIds: [project.id], + limit: 5, + }), + ).toHaveLength(5); + }); + it("allows hidden threads to belong to sections", () => { const { db, project } = setup(); const section = mustCreateThreadSection(db, "Work"); diff --git a/packages/db/test/migrate.test.ts b/packages/db/test/migrate.test.ts index 3886ca009b..3077fe49ec 100644 --- a/packages/db/test/migrate.test.ts +++ b/packages/db/test/migrate.test.ts @@ -4072,6 +4072,7 @@ describe("migrate", () => { "events_item_lifecycle_thread_item_sequence_idx", "events_parent_tool_call_thread_parent_sequence_idx", "events_plan_steps_thread_sequence_idx", + "events_provider_unhandled_created_idx", "events_thread_sequence_idx", "events_thread_state_thread_sequence_idx", "events_thread_turn_type_item_sequence_idx", diff --git a/packages/templates/src/templates/bb-guide-customization.md b/packages/templates/src/templates/bb-guide-customization.md index 86e46c810e..29630a63a2 100644 --- a/packages/templates/src/templates/bb-guide-customization.md +++ b/packages/templates/src/templates/bb-guide-customization.md @@ -42,7 +42,9 @@ Packaged launcher settings `bb-app config` and `bb-app env` reload runtime settings in a running server, but the CLI identifies server and launcher settings that are startup-only, including binding/ports, data and the dev-app port, telemetry, inherited skill -roots, and `BB_FF_*` flags. `BB_LOG_LEVEL` is also startup-only. Use +roots, the SQLite memory budgets (`BB_SQLITE_CACHE_SIZE_KIB`, +`BB_SQLITE_MMAP_SIZE_BYTES`), and `BB_FF_*` flags. `BB_LOG_LEVEL` is also +startup-only. Use `bb-app config`, not `bb-app env`, to change `BB_APP_URL`, `BB_INFERENCE`, `BB_INFERENCE_FALLBACK`, or `BB_TRANSCRIPTION` live. After a startup-only change, run `bb-app stop && bb-app start` or restart the desktop app. Until