diff --git a/AgentSourceCore/package.json b/AgentSourceCore/package.json new file mode 100644 index 000000000..cdd2583db --- /dev/null +++ b/AgentSourceCore/package.json @@ -0,0 +1,13 @@ +{ + "name": "@memmy/agent-source-core", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + } +} diff --git a/AgentSourceCore/src/index.test.ts b/AgentSourceCore/src/index.test.ts new file mode 100644 index 000000000..3b85da8c7 --- /dev/null +++ b/AgentSourceCore/src/index.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { conversationContentHash, orderedTurns, splitTurn, type ConversationMessage } from "./index.js"; + +const message = (id: string, role: ConversationMessage["role"], content: string, createdAt: string): ConversationMessage => ({ + messageId: id, sourceId: "fixture", conversationId: "conversation", role, content, createdAt, + workspacePath: null, gitRoot: null, rawMeta: {} +}); + +describe("agent source core", () => { + it("emits stable turns across page boundaries", async () => { + const pages = (async function*() { + yield message("u1", "user", "hello", "2026-01-01T00:00:00Z"); + yield message("t1", "tool", "tool", "2026-01-01T00:00:01Z"); + yield message("a1", "assistant", "world", "2026-01-01T00:00:02Z"); + yield message("u2", "user", "next", "2026-01-01T00:00:03Z"); + yield message("a2", "assistant", "done", "2026-01-01T00:00:04Z"); + })(); + const turns = []; + for await (const turn of orderedTurns(pages)) turns.push(turn); + expect(turns).toHaveLength(2); + expect(turns.map((turn) => turn.messages[0]?.messageId)).toEqual(["u1", "u2"]); + }); + + it("splits oversized content with unique part hashes", () => { + const turn = { sourceId: "fixture", conversationId: "conversation", turnIndex: 0, messages: [message("u", "user", "x".repeat(30_000), "2026-01-01T00:00:00Z"), message("a", "assistant", "ok", "2026-01-01T00:00:01Z")] }; + const parts = splitTurn(turn, 4000, 1_000_000); + expect(parts.length).toBeGreaterThan(1); + expect(new Set(parts.map((part) => part.contentHash)).size).toBe(parts.length); + expect(parts.every((part) => Buffer.byteLength(part.content) <= 1_000_000)).toBe(true); + expect(conversationContentHash(turn.messages)).toHaveLength(64); + }); +}); diff --git a/AgentSourceCore/src/index.ts b/AgentSourceCore/src/index.ts new file mode 100644 index 000000000..e6f510fdb --- /dev/null +++ b/AgentSourceCore/src/index.ts @@ -0,0 +1,397 @@ +import { createHash } from "node:crypto"; + +export interface ConversationMessage { + messageId: string; + sourceId: string; + conversationId: string; + role: "user" | "assistant" | "tool" | "system"; + content: string; + createdAt: string; + workspacePath: string | null; + gitRoot: string | null; + rawMeta: Readonly>; + ordinal?: number; +} + +export interface SourceDescriptor { + sourceId: string; + displayName: string; + builtin: boolean; + dataPath: string; +} + +export interface ScanProgress { + sourceId: string; + phase: "discover" | "read" | "redact" | "emit" | "scan" | "prepare" | "add" | "summarize" | "done" | "stopped"; + current: number; + total: number; + message?: string; +} + +export interface ScanOptions { + since?: string; + maxMessages?: number; + maxScanTargets?: number; + order?: "source_default" | "recent_first"; + signal?: AbortSignal; + /** Production scanners set this to bypass legacy whole-window buffering. */ + fullHistory?: boolean; + onProgress?: (progress: ScanProgress) => void; +} + +export interface SourceAdapter { + readonly descriptor: SourceDescriptor; + detect(): Promise; + scan(options: ScanOptions): AsyncIterable; +} + +export interface ScanStore { + stage(message: ConversationMessage): boolean; + stageBatch(messages: readonly ConversationMessage[]): number; + messages(sourceId: string, cursor?: MessageCursor, limit?: number): Iterable; + saveScanCursor(sourceId: string, cursor: MessageCursor): void; + getScanCursor(sourceId: string): MessageCursor | null; + saveSourceState(state: ScanSourceState): void; + getSourceState(sourceId: string): ScanSourceState | null; + sourceCount(): number; + count(sourceId?: string): number; + saveCheckpoint(checkpoint: ConversationCheckpoint): void; + getCheckpoint(sourceId: string, conversationId: string): ConversationCheckpoint | null; + saveConversationMeta(meta: PreparedConversation): void; + getConversationMeta(sourceId: string, conversationId: string): PreparedConversation | null; + selectAllConversations(sourceId: string): void; + saveTurnMeta(meta: PreparedTurn): void; + getTurnMeta(sourceId: string, conversationId: string, turnId: string): PreparedTurn | null; + selectInitialTurns(sourceIds: readonly string[], globalLimit: number, absentSourceLimit: number): void; + saveResult(result: ScanStoredResult): void; + resultCount(sourceId?: string): number; + results(sourceId?: string, cursor?: string, limit?: number): Iterable; + close(): void; + remove(): void; +} + +export interface PreparedTurn { + sourceId: string; + conversationId: string; + turnId: string; + firstMessageId: string; + firstCreatedAt: string; + lastMessageId: string; + lastCreatedAt: string; + selected: boolean; +} + +export interface PreparedConversation { + sourceId: string; + conversationId: string; + lastMessageId: string; + lastCreatedAt: string; + contentHash: string; + selected: boolean; +} + +export interface MessageCursor { + conversationId: string; + createdAt: string; + messageId: string; + ordinal: number; +} + +export interface ConversationCheckpoint { + sourceId: string; + conversationId: string; + lastMessageId: string; + lastCreatedAt: string; + contentHash: string; + updatedAt: string; +} + +export interface ScanStoredResult { + sourceId: string; + conversationId: string; + memoryId?: string; + error?: string; + /** Opaque keyset cursor populated when a result is read from a store. */ + cursor?: string; +} + +export type ScanStage = "stage" | "prepare" | "ingest" | "summarize" | "done" | "failed" | "paused" | "canceled"; + +export interface ScanSourceState { + sourceId: string; + mode: string; + phase: ScanStage; + messageCount: number; + resultCount: number; + errorCount: number; + scanStartedAt?: string; + watermarkedSince?: string; + updatedAt: string; + error?: string; +} + +export interface ImportedTurn { + sourceId: string; + conversationId: string; + turnIndex: number; + messages: ConversationMessage[]; +} + +export interface TurnPart extends ImportedTurn { + parentTurnId: string; + partIndex: number; + partCount: number; + content: string; + contentHash: string; +} + +export function compareMessageOrder(left: ConversationMessage, right: ConversationMessage): number { + return left.conversationId.localeCompare(right.conversationId) + || Date.parse(left.createdAt) - Date.parse(right.createdAt) + || left.messageId.localeCompare(right.messageId) + || (left.ordinal ?? 0) - (right.ordinal ?? 0); +} + +export function compareCursor(left: ConversationMessage, right: MessageCursor): number { + return left.conversationId.localeCompare(right.conversationId) + || Date.parse(left.createdAt) - Date.parse(right.createdAt) + || left.messageId.localeCompare(right.messageId) + || (left.ordinal ?? 0) - right.ordinal; +} + +export async function* orderedTurns(messages: AsyncIterable): AsyncIterable { + let current: ConversationMessage[] = []; + let conversationId = ""; + let turnIndex = 0; + for await (const message of messages) { + if (message.conversationId !== conversationId) { + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + current = []; + conversationId = message.conversationId; + turnIndex = 0; + } + if (message.role === "user" && current.length > 0) { + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + turnIndex += 1; + current = []; + } + current.push(message); + } + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; +} + +export function isCompleteTurn(messages: readonly ConversationMessage[]): boolean { + const first = messages[0]; + const last = messages[messages.length - 1]; + return first?.role === "user" && Boolean(first.content.trim()) + && last?.role === "assistant" && Boolean(last.content.trim()); +} + +export function renderMessageContent(message: ConversationMessage): string { + if (message.role !== "tool" || /^Tool:\s*/im.test(message.content)) return message.content; + const toolName = stringMeta(message.rawMeta, "toolName") ?? stringMeta(message.rawMeta, "hermesToolName"); + const callId = stringMeta(message.rawMeta, "toolCallId") ?? stringMeta(message.rawMeta, "hermesToolCallId"); + return [toolName ? `Tool: ${toolName}` : undefined, callId ? `Call ID: ${callId}` : undefined, message.content] + .filter(Boolean).join("\n\n"); +} + +export function renderTurn(messages: readonly ConversationMessage[]): string { + return messages.map((message) => `## ${message.role}\n\n${renderMessageContent(message)}`).join("\n\n"); +} + +export function conversationContentHash(messages: Iterable): string { + const hash = createHash("sha256"); + hash.update("["); + let first = true; + for (const message of messages) { + if (!first) hash.update(","); + first = false; + hash.update(JSON.stringify({ + messageId: message.messageId, + role: message.role, + content: message.content, + createdAt: message.createdAt, + toolName: hashMetaString(message.rawMeta, "toolName") ?? hashMetaString(message.rawMeta, "hermesToolName"), + toolCallId: hashMetaString(message.rawMeta, "toolCallId") ?? hashMetaString(message.rawMeta, "hermesToolCallId") + })); + } + hash.update("]"); + return hash.digest("hex"); +} + +export function stableTurnIdentity(turn: ImportedTurn): string { + const firstUser = turn.messages.find((message) => message.role === "user"); + if (!firstUser) throw new Error("turn is missing user message"); + return `${turn.sourceId}::${turn.conversationId}::${firstUser.messageId}`; +} + +/** Preserves the pre-staging idempotency key for an unsplit turn. */ +export function legacyTurnRequestId(turn: ImportedTurn): string { + const first = turn.messages[0]; + if (!first) throw new Error("turn is empty"); + return createHash("sha256").update([stableTurnIdentity(turn), first.createdAt, renderTurn(turn.messages)].join("\u0000")).digest("hex"); +} + +/** Preserves the pre-staging stable turn id for an unsplit turn. */ +export function legacyTurnId(turn: ImportedTurn): string { + return `${turn.sourceId}:${createHash("sha256").update(stableTurnIdentity(turn)).digest("hex").slice(0, 24)}`; +} + +export function splitTurn(turn: ImportedTurn, maxTokens = 4000, maxBytes = 1024 * 1024): TurnPart[] { + const chunks: ConversationMessage[][] = []; + let current: ConversationMessage[] = []; + const fits = (candidate: readonly ConversationMessage[]) => { + const content = renderTurn(candidate); + return estimateTokens(content) <= maxTokens && Buffer.byteLength(content) <= maxBytes; + }; + for (const message of turn.messages) { + if (fits([...current, message])) { + current.push(message); + continue; + } + if (current.length > 0) { chunks.push(current); current = []; } + const pieces = splitMessage(message, maxTokens, maxBytes); + if (current.length === 0 && chunks.length > 0) { + // A user prefix followed by an oversized assistant/tool message should + // remain one logical part whenever the prefix can share any content. + const previous = chunks[chunks.length - 1]; + if (previous && !isCompleteTurn(previous)) { + const combined = combinePrefix(previous, pieces[0]!, maxTokens, maxBytes); + if (combined) { + chunks[chunks.length - 1] = combined.messages; + if (combined.remainder) chunks.push(...splitMessage(combined.remainder, maxTokens, maxBytes).map((piece) => [piece])); + for (const piece of pieces.slice(1)) chunks.push([piece]); + continue; + } + } + } + for (const piece of pieces) chunks.push([piece]); + } + if (current.length > 0) chunks.push(current); + if (chunks.length === 0) chunks.push([...turn.messages]); + const parentTurnId = createHash("sha256").update(stableTurnIdentity(turn)).digest("hex").slice(0, 24); + return chunks.map((messages, partIndex) => { + const content = renderTurn(messages); + return { + ...turn, + messages, + parentTurnId, + partIndex, + partCount: chunks.length, + content, + contentHash: createHash("sha256").update(content).digest("hex") + }; + }); +} + +function combinePrefix( + prefix: readonly ConversationMessage[], + piece: ConversationMessage, + maxTokens: number, + maxBytes: number +): { messages: ConversationMessage[]; remainder?: ConversationMessage } | null { + const fits = (content: string) => { + const candidate = [...prefix, { ...piece, content }]; + const rendered = renderTurn(candidate); + return estimateTokens(rendered) <= maxTokens && Buffer.byteLength(rendered) <= maxBytes; + }; + if (fits(piece.content)) return { messages: [...prefix, piece] }; + const characters = Array.from(piece.content); + let low = 0; + let high = characters.length; + let best = 0; + while (low <= high) { + const middle = Math.floor((low + high) / 2); + if (fits(characters.slice(0, middle).join(""))) { + best = middle; + low = middle + 1; + } else { + high = middle - 1; + } + } + if (best === 0) return null; + const content = characters.slice(0, best).join(""); + const remainder = characters.slice(best).join(""); + return { + messages: [...prefix, { ...piece, content }], + ...(remainder ? { remainder: { ...piece, content: remainder } } : {}) + }; +} + +function splitMessage(message: ConversationMessage, maxTokens: number, maxBytes: number): ConversationMessage[] { + const emptyRendered = renderTurn([{ ...message, content: "" }]); + const bodyTokenLimit = Math.max(1, maxTokens - estimateTokens(emptyRendered)); + const bodyByteLimit = Math.max(1, maxBytes - Buffer.byteLength(emptyRendered)); + const fitsContent = (content: string) => { + const rendered = renderTurn([{ ...message, content }]); + return estimateTokens(rendered) <= maxTokens && Buffer.byteLength(rendered) <= maxBytes; + }; + if (fitsContent(message.content)) return [message]; + const pieces: string[] = []; + for (const paragraph of message.content.split(/\n\s*\n/)) { + if (!paragraph) continue; + if (paragraph && estimateTokens(paragraph) <= bodyTokenLimit && Buffer.byteLength(paragraph) <= bodyByteLimit && fitsContent(paragraph)) { + pieces.push(paragraph); + } else { + pieces.push(...splitText(paragraph, bodyTokenLimit, bodyByteLimit)); + } + } + return (pieces.length > 0 ? pieces : [""]).map((content) => ({ ...message, content })); +} + +function splitText(value: string, maxTokens: number, maxBytes: number): string[] { + const maxChars = Math.max(1, maxTokens * 4); + const chunks: string[] = []; + let current = ""; + for (const line of value.split(/\r?\n/u)) { + const candidate = current ? `${current}\n${line}` : line; + if (current && (estimateTokens(candidate) > maxTokens || Buffer.byteLength(candidate) > maxBytes)) { + chunks.push(...splitUtf8(current, maxBytes)); + current = ""; + } + if (line.length > maxChars || Buffer.byteLength(line) > maxBytes) { + if (current) { chunks.push(...splitUtf8(current, maxBytes)); current = ""; } + let part = ""; + for (const character of line) { + if (part && (part.length >= maxChars || Buffer.byteLength(part + character) > maxBytes)) { + chunks.push(part); + part = ""; + } + part += character; + } + if (part) chunks.push(part); + } else { + current = candidate; + } + } + if (current) chunks.push(...splitUtf8(current, maxBytes)); + return chunks.length > 0 ? chunks : [""]; +} + +function splitUtf8(value: string, maxBytes: number): string[] { + const parts: string[] = []; + let current = ""; + for (const character of value) { + const candidate = current + character; + if (current && Buffer.byteLength(candidate) > maxBytes) { + parts.push(current); + current = character; + } else { + current = candidate; + } + } + if (current) parts.push(current); + return parts.length > 0 ? parts : [""]; +} + +export function estimateTokens(value: string): number { return Math.ceil(value.length / 4); } + +function stringMeta(meta: Readonly>, key: string): string | undefined { + const value = meta[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function hashMetaString(meta: Readonly>, key: string): string | undefined { + const value = meta[key]; + return typeof value === "string" ? value : undefined; +} diff --git a/AgentSourceCore/tsconfig.json b/AgentSourceCore/tsconfig.json new file mode 100644 index 000000000..6c185e9e9 --- /dev/null +++ b/AgentSourceCore/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/App/backend/local-api-contracts/src/index.ts b/App/backend/local-api-contracts/src/index.ts index d43532c07..23ff36e2c 100644 --- a/App/backend/local-api-contracts/src/index.ts +++ b/App/backend/local-api-contracts/src/index.ts @@ -535,6 +535,9 @@ export const ScanResultSchema = z.object({ emittedMessages: z.number().int().nonnegative(), skipped: z.number().int().nonnegative(), memoryIds: z.array(z.string().min(1)).optional(), + memoryIdCount: z.number().int().nonnegative().optional(), + errorCount: z.number().int().nonnegative().optional(), + detailsTruncated: z.boolean().optional(), errors: z.array( z.object({ conversationId: z.string().min(1), @@ -544,6 +547,18 @@ export const ScanResultSchema = z.object({ }); export type ScanResult = z.infer; +/** Schema for paged persisted scan details. */ +export const ScanResultPageSchema = z.object({ + items: z.array(z.object({ + sourceId: z.string().min(1), + conversationId: z.string().min(1), + memoryId: z.string().min(1).optional(), + error: z.string().min(1).optional() + })), + nextCursor: z.string().nullable() +}); +export type ScanResultPage = z.infer; + /** Schema for legal agreement locale urls. */ export const LegalAgreementLocaleUrlsSchema = z.object({ "zh-CN": z.string().url(), diff --git a/App/backend/package.json b/App/backend/package.json index bbac18037..235b64f1f 100644 --- a/App/backend/package.json +++ b/App/backend/package.json @@ -14,14 +14,15 @@ "scripts": { "workspace-bridge:build": "node src/adapters/outbound/skill-writer/workspace-bridge/build-runtime.mjs", "workspace-bridge:build:dist": "node src/adapters/outbound/skill-writer/workspace-bridge/build-runtime.mjs --dist", - "build": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node -e \"const fs=require('node:fs'); fs.cpSync('src/infrastructure/app-state-store/migrations', 'dist/src/infrastructure/app-state-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/agent-source-store/migrations', 'dist/src/infrastructure/agent-source-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/idempotency-store/migrations', 'dist/src/infrastructure/idempotency-store/migrations', { recursive: true }); if (fs.existsSync('src/adapters/outbound/agent-adapter/plugins')) fs.cpSync('src/adapters/outbound/agent-adapter/plugins', 'dist/src/adapters/outbound/agent-adapter/plugins', { recursive: true });\" && npm run workspace-bridge:build:dist", + "build": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node -e \"const fs=require('node:fs'); fs.cpSync('src/infrastructure/app-state-store/migrations', 'dist/src/infrastructure/app-state-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/agent-source-store/migrations', 'dist/src/infrastructure/agent-source-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/idempotency-store/migrations', 'dist/src/infrastructure/idempotency-store/migrations', { recursive: true }); if (fs.existsSync('src/adapters/outbound/agent-adapter/plugins')) fs.cpSync('src/adapters/outbound/agent-adapter/plugins', 'dist/src/adapters/outbound/agent-adapter/plugins', { recursive: true });\" && npm run workspace-bridge:build:dist", "lint": "eslint \"src/**/*.ts\" \"vitest.config.ts\"", - "typecheck": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && tsc -p tsconfig.json --noEmit", - "test": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && npm run workspace-bridge:build && vitest run", + "typecheck": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && tsc -p tsconfig.json --noEmit", + "test": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && npm run workspace-bridge:build && vitest run", "test:agent-adapter:coverage": "npm run build -w @memmy/local-api-contracts && vitest run src/adapters/outbound/agent-adapter/tests --coverage", "db:migrate": "tsx src/infrastructure/app-state-store/cli/migrate.ts" }, "dependencies": { + "@memmy/agent-source-core": "0.0.0", "@memmy/local-api-contracts": "0.0.0", "@memmy/migrations": "0.0.0", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts b/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts index 781e2342c..0a20fd3f1 100644 --- a/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts +++ b/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts @@ -9,6 +9,7 @@ import { AgentSourceScanJobResponseSchema, AgentSourceScanStatusResponseSchema, AgentSourceViewSchema, + ScanResultPageSchema, ManagedAgentSourceImportInputSchema, ManagedAgentSourceImportResultSchema, ManagedAgentSourceUpdateInputSchema, @@ -23,9 +24,12 @@ import { withErrorEnvelope } from "../../../../services/error-envelope.js"; import type { AgentSourceAutoInjectService } from "../../../../services/agent-source-auto-inject-service.js"; import type { AgentSourceService } from "../../../../services/agent-source-service.js"; import { + deleteDurableScanStore, deletePersistedScanResume, + readDurableScanResults, readLatestPersistedScanResume } from "../../../../services/agent-source-scan-journal.js"; +import { migrateLegacyScanJournals } from "../../../../services/agent-source-scan-migration.js"; import type { ProgressBus } from "../../../../services/progress-bus.js"; import { type AgentSourceScanJobState, @@ -67,6 +71,7 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe lastProgress: PipelineProgress & { jobId: string }; resume: RouteScanResumeState | null; }; + if (options.scanProcess?.databasePath) migrateLegacyScanJournals(options.scanProcess.databasePath); const restoredScanJob = toPausedScanJob(readLatestPersistedScanResume(options.scanProcess?.databasePath)); let pausedScanJob: PausedScanJob | null = restoredScanJob; let lastScanProgress: (PipelineProgress & { jobId: string }) | null = restoredScanJob?.lastProgress ?? null; @@ -135,6 +140,15 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe })); }); + app.get("/api/agent-sources/scan/jobs/:jobId/results", { preHandler: options.authenticateRuntimeToken }, async (request, reply) => { + if (!options.scanProcess) return reply.send({ items: [], nextCursor: null }); + const params = request.params as { jobId: string }; + const query = request.query as { cursor?: string; limit?: string }; + const limit = Math.min(500, Math.max(1, Number.parseInt(query.limit ?? "100", 10) || 100)); + const cursor = query.cursor ?? "0"; + return reply.send(ScanResultPageSchema.parse(readDurableScanResults(options.scanProcess.databasePath, params.jobId, cursor, limit))); + }); + app.post("/api/agent-sources/scan", { preHandler: options.authenticateRuntimeToken }, async (request, reply) => { const input = AgentSourceScanInputSchema.parse(request.body); const sourceId = input.sourceId; @@ -156,7 +170,10 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe return reply.send(AgentSourceScanJobResponseSchema.parse({ jobId: activeScanJob.jobId })); } - const pausedJob = pausedScanJob?.resume && pausedScanJob.sourceId === sourceId && pausedScanJob.mode === mode ? pausedScanJob : null; + const pausedJob = pausedScanJob && pausedScanJob.sourceId === sourceId && pausedScanJob.mode === mode + && (pausedScanJob.resume || options.scanProcess) + ? pausedScanJob + : null; if (!pausedJob) { cleanupResumeState(pausedScanJob?.resume ?? null); pausedScanJob = null; @@ -218,8 +235,10 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe activeScanJob = null; abortScanJob(canceledJob); cleanupResumeState(canceledJob.resume); + deleteDurableScanStore(options.scanProcess?.databasePath, canceledJob.jobId); } cleanupResumeState(pausedScanJob?.resume ?? null); + if (pausedScanJob) deleteDurableScanStore(options.scanProcess?.databasePath, pausedScanJob.jobId); pausedScanJob = null; lastScanProgress = null; return reply.send(OkResponseSchema.parse({ ok: true })); @@ -547,6 +566,7 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe deletePersistedScanResume(options.scanProcess?.databasePath, resume.jobId); } + } function toPausedScanJob( diff --git a/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts b/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts index b358a4d15..d18b5fc42 100644 --- a/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveClaudeCodeProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverClaudeCodeSessions } from "./project-discovery.js"; @@ -68,13 +68,13 @@ export function createClaudeCodeSourceAdapter(deps: CreateClaudeCodeSourceAdapte message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readClaudeCodeTranscript(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts b/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts index 0bff8b96e..29d88e7e1 100644 --- a/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts @@ -34,7 +34,7 @@ export async function discoverClaudeCodeSessions( const projectPath = join(options.root, projectEntry.name); const files = await readDirectoryIfExists(projectPath); for (const file of files) { - if (!file.isFile() || !file.name.endsWith(".jsonl")) { + if (!file.isFile() || !file.name.endsWith(".jsonl") || /\.jsonl\.bak-/u.test(file.name)) { continue; } diff --git a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts index 3b9680ae4..62b5dcab9 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCodexSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readCodexRollout, type RawCodexMessage } from "./rollout-reader.js"; @@ -67,13 +67,13 @@ export function createCodexSourceAdapter(deps: CreateCodexSourceAdapterDeps = {} message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readCodexRollout(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts index 956d10c4f..6dd193392 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts @@ -2,6 +2,8 @@ import { basename } from "node:path"; import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; +const MAX_TOOL_NAME_ENTRIES = 4096; + export interface RawCodexMessage { /** Message id. */ messageId: string; @@ -73,6 +75,10 @@ function toToolMessage( const name = getString(payload.name) ?? "tool"; if (callId) { toolNamesByCallId.set(callId, name); + if (toolNamesByCallId.size > MAX_TOOL_NAME_ENTRIES) { + const oldest = toolNamesByCallId.keys().next().value; + if (typeof oldest === "string") toolNamesByCallId.delete(oldest); + } } return { messageId: `${rolloutId}:${lineNumber}`, diff --git a/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts index 57d2b41df..b6d0bbfbd 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts @@ -56,7 +56,7 @@ async function listRolloutFiles( continue; } - if (entry.isFile() && entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } diff --git a/App/backend/src/adapters/outbound/agent-source/conversation-window.ts b/App/backend/src/adapters/outbound/agent-source/conversation-window.ts index 78b331deb..14cdd8c05 100644 --- a/App/backend/src/adapters/outbound/agent-source/conversation-window.ts +++ b/App/backend/src/adapters/outbound/agent-source/conversation-window.ts @@ -45,6 +45,29 @@ export async function collectConversationWindow included.has(message.conversationId)); } +/** Streams a source target without materializing its complete history. */ +export async function* streamConversationWindow( + input: AsyncIterable, + since?: string, + signal?: AbortSignal, + maxMessages?: number, + fullHistory = false +): AsyncIterable { + if (maxMessages !== undefined && maxMessages <= 0) return; + if (fullHistory) { + let emitted = 0; + for await (const message of input) { + signal?.throwIfAborted(); + if (maxMessages !== undefined && emitted >= maxMessages) break; + emitted += 1; + yield message; + } + return; + } + // Keep the legacy complete-conversation semantics for non-production callers. + for (const message of await collectConversationWindow(input, since, signal, maxMessages)) yield message; +} + export function remainingMessageCapacity(limit: number | undefined, emitted: number): number | undefined { return limit === undefined ? undefined : Math.max(0, limit - emitted); } diff --git a/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts b/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts index 3db120766..f46b4255b 100644 --- a/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCursorDataPaths } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; +import { readCursorVscdb, streamCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; import { discoverCursorWorkspaces, type CursorWorkspace } from "./workspace-discovery.js"; const CURSOR_SOURCE_ID = "cursor"; @@ -84,13 +84,13 @@ export function createCursorSourceAdapter(deps: CreateCursorSourceAdapterDeps = message: target.storageHash }); - const messages = await collectConversationWindow( - readCursorVscdb(target.stateDbPath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamCursorVscdb(target.stateDbPath) : readCursorVscdb(target.stateDbPath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts b/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts index 1f257f5f0..af24e7ebf 100644 --- a/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts @@ -3,6 +3,7 @@ import { DatabaseSync } from "node:sqlite"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw cursor message. */ export interface RawCursorMessage { @@ -61,6 +62,36 @@ export async function* readCursorVscdb(path: string): AsyncIterable { + const db = new DatabaseSync(path, { readOnly: true }); + try { + if (hasTable(db, "ItemTable")) { + const statement = db.prepare("SELECT key, value FROM ItemTable WHERE value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + for (const message of extractMessagesFromItemRow(row)) yield message; + } + } + if (hasTable(db, "cursorDiskKV")) { + const statement = db.prepare("SELECT key, value FROM cursorDiskKV WHERE key LIKE 'bubbleId:%' AND value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + const message = extractMessageFromBubbleRow(row); + if (message) yield message; + } + } + } finally { + db.close(); + } +} + /** Reads read item table messages. */ async function readItemTableMessages(db: DatabaseSync): Promise { if (!hasTable(db, "ItemTable")) { @@ -76,7 +107,7 @@ async function readItemTableMessages(db: DatabaseSync): Promise MAX_RECORD_BYTES) continue; const message = extractMessageFromBubbleRow(row); if (message) { messages.push(message); diff --git a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts index 278fdc8f2..62fcee55b 100644 --- a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts @@ -1,11 +1,11 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveDeepseekHarnessHomeDirectory, resolveDeepseekHarnessSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverDeepseekHarnessSessions } from "./session-discovery.js"; -import { readDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; +import { streamDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; const SOURCE_ID = "deepseek_harness"; @@ -61,13 +61,13 @@ export function createDeepseekHarnessSourceAdapter( total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( - toAsyncIterable(await readDeepseekHarnessSession(session.sessionFilePath, options.signal)), + for await (const rawMessage of streamConversationWindow( + streamDeepseekHarnessSession(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { options.signal?.throwIfAborted(); emittedMessages += 1; options.onProgress?.({ sourceId: SOURCE_ID, phase: "emit", current: emittedMessages, total: emittedMessages }); @@ -91,10 +91,6 @@ function toConversationMessage( }; } -async function* toAsyncIterable(values: readonly T[]): AsyncIterable { - for (const value of values) yield value; -} - function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error; } diff --git a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts index 63efdcba2..00b5dae73 100644 --- a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts @@ -1,6 +1,8 @@ +import { createReadStream } from "node:fs"; import { readFile } from "node:fs/promises"; import { basename } from "node:path"; -import { decompress, ZstdErrorCode } from "fzstd"; +import { decompress, Decompress, ZstdErrorCode } from "fzstd"; +import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; const ZSTD_FRAME_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); @@ -25,6 +27,108 @@ export async function readDeepseekHarnessSession( return parseSessionRows(text, filePath, signal); } +/** Streams uncompressed sessions; compressed legacy files use the existing decoder. */ +export async function* streamDeepseekHarnessSession( + filePath: string, + signal?: AbortSignal +): AsyncIterable { + if (filePath.endsWith(".zstd")) { + let conversationId = basename(filePath).replace(/\.jsonl\.zstd$/u, ""); + let workspacePath: string | null = null; + for await (const record of streamZstdJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } + return; + } + let conversationId = basename(filePath).replace(/\.jsonl$/u, ""); + let workspacePath: string | null = null; + for await (const record of readJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } +} + +/** Streams zstd frames and parses bounded JSONL records without materializing the file. */ +async function* streamZstdJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { + const input = createReadStream(filePath); + const output: Buffer[] = []; + let outputBytes = 0; + let carry: Buffer = Buffer.alloc(0); + let overLimit = false; + const decoder = new Decompress((chunk) => { + outputBytes += chunk.byteLength; + if (outputBytes > 8 * 1024 * 1024) throw new Error("DeepSeek Harness decompressed chunk exceeds 8 MiB staging limit"); + output.push(Buffer.from(chunk)); + }); + try { + for await (const chunk of input) { + signal?.throwIfAborted(); + decoder.push(chunk as Buffer); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length > 64 * 1024 * 1024) continue; + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + if (carry.length > 64 * 1024 * 1024) { carry = Buffer.alloc(0); overLimit = true; } + } + } + decoder.push(new Uint8Array(), true); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + } + } + if (!overLimit && carry.length > 0 && carry.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(carry); + if (parsed) yield parsed; + } + } finally { + input.destroy(); + } +} + +function parseJsonObject(line: Buffer): JsonObject | null { + try { + const parsed = JSON.parse(line.toString("utf8").trim()) as unknown; + return isRecord(parsed) ? parsed as JsonObject : null; + } catch { + return null; + } +} + function decompressFrames(bytes: Buffer): string { if (!bytes.subarray(0, ZSTD_FRAME_MAGIC.length).equals(ZSTD_FRAME_MAGIC)) { throw new Error("DeepSeek Harness session has no Zstandard frame header"); diff --git a/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts b/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts index cdbc95155..77e3d1ce9 100644 --- a/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts @@ -2,7 +2,7 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveHermesHomeDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readHermesRollout, type RawHermesRolloutMessage } from "./rollout-reader.js"; @@ -79,13 +79,13 @@ export function createHermesSourceAdapter(deps: CreateHermesSourceAdapterDeps = ? streamJsonlMessages(target.session, options.signal) : streamStateDbMessages(target.stateDbPath); - const messages = await collectConversationWindow( + for await (const message of streamConversationWindow( iterable, options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const message of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts index 36e5b2f57..5153c022e 100644 --- a/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts @@ -58,7 +58,7 @@ async function listJsonlFiles(root: string, order: "path_asc" | "recent_first", continue; } - if (entry.isFile() && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } diff --git a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts index 7d3ef80a6..c0fd7cbd6 100644 --- a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts +++ b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts @@ -1,6 +1,5 @@ /** Jsonl lines module. */ import { createReadStream } from "node:fs"; -import { createInterface } from "node:readline"; export type JsonPrimitive = string | number | boolean | null; export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; @@ -15,34 +14,63 @@ export type JsonObject = { readonly [key: string]: JsonValue }; * @returns The JSON objects parsed line by line. */ export async function* readJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ - input: stream, - crlfDelay: Number.POSITIVE_INFINITY - }); + const stream = createReadStream(filePath); + const maxRecordBytes = 64 * 1024 * 1024; + let segments: Buffer[] = []; + let recordBytes = 0; + let overLimit = false; - try { - for await (const line of lines) { - throwIfAborted(signal, filePath); - if (line.trim().length === 0) { - continue; - } + const append = (segment: Buffer): void => { + if (overLimit || segment.length === 0) return; + recordBytes += segment.length; + if (recordBytes > maxRecordBytes) { + segments = []; + overLimit = true; + return; + } + segments.push(segment); + }; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - continue; - } + const reset = (): void => { + segments = []; + recordBytes = 0; + overLimit = false; + }; - if (!isJsonObject(parsed)) { - continue; - } + const parseSegments = (): JsonObject | null => { + if (overLimit) return null; + const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); + const text = line.toString("utf8").trim(); + if (!text) return null; + try { + const parsed = JSON.parse(text) as unknown; + return isJsonObject(parsed) ? parsed : null; + } catch { + return null; + } + }; - yield parsed; + try { + for await (const chunk of stream) { + throwIfAborted(signal, filePath); + const buffer = chunk as Buffer; + let start = 0; + while (start <= buffer.length) { + const newline = buffer.indexOf(0x0a, start); + if (newline < 0) { + append(buffer.subarray(start)); + break; + } + append(buffer.subarray(start, newline)); + const parsed = parseSegments(); + reset(); + if (parsed) yield parsed; + start = newline + 1; + } } + const parsed = parseSegments(); + if (parsed) yield parsed; } finally { - lines.close(); stream.destroy(); } } diff --git a/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts b/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts index 2a95d9527..6123da83a 100644 --- a/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts +++ b/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts @@ -33,7 +33,7 @@ export async function discoverJsonlSessionFiles( const path = join(directory, entry.name); if (entry.isDirectory()) { directories.push(path); - } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + } else if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { files.push({ path, mtimeMs: (await stat(path)).mtimeMs }); } } diff --git a/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts b/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts index cf6259b48..488b3c1c2 100644 --- a/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts +++ b/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts @@ -133,7 +133,7 @@ export function createWorkbuddyInsightSampler(input: { root: string }): Onboardi sourceId: "workbuddy", displayName: "WorkBuddy", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: isPotentialWorkbuddyMessageLine, extractMessage: extractWorkbuddySampledMessage }); @@ -144,7 +144,7 @@ export function createPiInsightSampler(input: { root: string }): OnboardingInsig sourceId: "pi", displayName: "Pi", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: (line) => /"type"\s*:\s*"message"/u.test(line), extractMessage: extractPiSampledMessage }); @@ -155,7 +155,7 @@ export function createQwenworkInsightSampler(input: { root: string }): Onboardin sourceId: "qwenwork", displayName: "QwenWork", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: (line) => /"type"\s*:\s*"(?:user|assistant|system)"/u.test(line), extractMessage: extractQwenworkSampledMessage }); @@ -166,7 +166,7 @@ export function createCodexInsightSampler(input: { root: string }): OnboardingIn sourceId: "codex", displayName: "Codex", root: input.root, - matchesFile: (name) => name.startsWith("rollout-") && name.endsWith(".jsonl"), + matchesFile: (name) => name.startsWith("rollout-") && name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: isPotentialCodexMessageLine, extractMessage: extractCodexMessage }); @@ -177,7 +177,7 @@ export function createClaudeCodeInsightSampler(input: { root: string }): Onboard sourceId: "claude_code", displayName: "Claude Code", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), extractMessage: extractClaudeCodeMessage }); } @@ -188,7 +188,7 @@ export function createHermesInsightSampler(input: { root: string }): OnboardingI sourceId: "hermes", displayName: "Hermes", root: join(input.root, "sessions"), - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), extractMessage: extractGenericJsonlMessage }); diff --git a/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts b/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts index 1e96b4439..23c1c5a1e 100644 --- a/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpenclawStateDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverOpenclawDatabases } from "./db-discovery.js"; @@ -65,13 +65,13 @@ export function createOpenclawSourceAdapter(deps: CreateOpenclawSourceAdapterDep message: database.databasePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readOpenclawDatabase(database.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts b/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts index e71486ba2..293fa1e94 100644 --- a/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpencodeDatabasePath } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; +import { readOpencodeDatabase, streamOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; const OPENCODE_SOURCE_ID = "opencode"; @@ -58,13 +58,13 @@ export function createOpencodeSourceAdapter(deps: CreateOpencodeSourceAdapterDep message: target.databasePath }); - const messages = await collectConversationWindow( - readOpencodeDatabase(target.databasePath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamOpencodeDatabase(target.databasePath) : readOpencodeDatabase(target.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts b/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts index c651dce53..388e68892 100644 --- a/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw opencode database message. */ export interface RawOpencodeDatabaseMessage { @@ -58,6 +59,39 @@ export async function* readOpencodeDatabase(path: string): AsyncIterable { + const db = new DatabaseSync(path, { readOnly: true }); + try { + if (!hasTable(db, "message") || !hasTable(db, "part") || !hasTable(db, "session")) return; + const statement = db.prepare(` + SELECT m.id AS message_id, m.session_id AS session_id, m.time_created AS message_time_created, + m.data AS message_data, s.directory AS session_directory, p.id AS part_id, + p.time_created AS part_time_created, p.data AS part_data + FROM message m LEFT JOIN session s ON s.id = m.session_id LEFT JOIN part p ON p.message_id = m.id + ORDER BY m.session_id ASC, m.time_created ASC, m.id ASC, p.time_created ASC, p.id ASC + `); + let current: MessageAccumulator | null = null; + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (!current || current.messageId !== row.message_id) { + if (current?.contentParts.length) yield toDatabaseMessage(current); + current = Buffer.byteLength(row.message_data) <= MAX_RECORD_BYTES ? createAccumulator(row) : null; + } + if (!current || !row.part_data || !row.part_id || Buffer.byteLength(row.part_data) > MAX_RECORD_BYTES) continue; + const text = getPartText(parseJson(row.part_data)); + if (!text) continue; + current.partIds.push(row.part_id); + current.contentParts.push(text); + } + if (current?.contentParts.length) yield toDatabaseMessage(current); + } finally { + db.close(); + } +} + async function readMessages(db: DatabaseSync): Promise { const statement = db.prepare(` SELECT @@ -84,7 +118,7 @@ async function readMessages(db: DatabaseSync): Promise MAX_RECORD_BYTES) { continue; } @@ -117,6 +151,7 @@ function getOrCreateAccumulator( accumulators: Map, row: OpencodePartRow ): MessageAccumulator | null { + if (Buffer.byteLength(row.message_data) > MAX_RECORD_BYTES) return null; const existing = accumulators.get(row.message_id); if (existing) { return existing; @@ -135,7 +170,21 @@ function getOrCreateAccumulator( const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; const explicitRoot = getNestedString(messageData, "path", "root"); const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; - const accumulator: MessageAccumulator = { + const accumulator = createAccumulator(row); + if (!accumulator) return null; + accumulators.set(row.message_id, accumulator); + return accumulator; +} + +function createAccumulator(row: OpencodePartRow): MessageAccumulator | null { + const messageData = parseJson(row.message_data); + if (!isRecord(messageData)) return null; + const role = normalizeRole(messageData.role); + if (!role) return null; + const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; + const explicitRoot = getNestedString(messageData, "path", "root"); + const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; + return { messageId: row.message_id, conversationId: row.session_id, role, @@ -145,8 +194,19 @@ function getOrCreateAccumulator( partIds: [], contentParts: [] }; - accumulators.set(row.message_id, accumulator); - return accumulator; +} + +function toDatabaseMessage(message: MessageAccumulator): RawOpencodeDatabaseMessage { + return { + messageId: message.messageId, + conversationId: message.conversationId, + role: message.role, + content: message.contentParts.join("\n"), + createdAt: message.createdAt, + workspacePath: message.workspacePath, + gitRoot: message.gitRoot, + rawMeta: Object.freeze({ opencodePartIds: message.partIds }) + }; } function getPartText(partData: unknown): string | null { diff --git a/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts b/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts index b3ba25852..301455660 100644 --- a/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolvePiAgentDirectory, resolvePiSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createPiSourceAdapter(deps: CreatePiSourceAdapterDeps = {}): Sou total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readPiHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts b/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts index 19e12d2d0..2d794c53e 100644 --- a/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolveQwenworkHomeDirectory, resolveQwenworkProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createQwenworkSourceAdapter(deps: CreateQwenworkSourceAdapterDep total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readQwenworkHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/App/backend/src/adapters/outbound/agent-source/types.ts b/App/backend/src/adapters/outbound/agent-source/types.ts index ce4bce8bb..bc7f26ee5 100644 --- a/App/backend/src/adapters/outbound/agent-source/types.ts +++ b/App/backend/src/adapters/outbound/agent-source/types.ts @@ -1,34 +1,20 @@ -/** Types module. */ +/** Types module. Shared with the standalone runtime. */ +import type { + ConversationMessage as CoreConversationMessage, + SourceDescriptor as CoreSourceDescriptor, + ScanOptions as CoreScanOptions, + ScanProgress as CoreScanProgress, + SourceAdapter as CoreSourceAdapter +} from "@memmy/agent-source-core"; /** Contract for source descriptor. */ -export interface SourceDescriptor { - sourceId: string; - displayName: string; - builtin: boolean; - dataPath: string; -} +export type SourceDescriptor = CoreSourceDescriptor; /** Contract for conversation message. */ -export interface ConversationMessage { - messageId: string; - sourceId: string; - conversationId: string; - role: "user" | "assistant" | "tool" | "system"; - content: string; - createdAt: string; - workspacePath: string | null; - gitRoot: string | null; - rawMeta: Readonly>; -} +export type ConversationMessage = CoreConversationMessage; /** Contract for scan progress. */ -export interface ScanProgress { - sourceId: string; - phase: "discover" | "read" | "redact" | "emit" | "scan" | "add" | "summarize" | "done" | "stopped"; - current: number; - total: number; - message?: string; -} +export type ScanProgress = CoreScanProgress; /** Contract for scan result. */ export interface ScanResult { @@ -40,18 +26,7 @@ export interface ScanResult { } /** Contract for source adapter. */ -export interface SourceAdapter { - readonly descriptor: SourceDescriptor; - detect(): Promise; - scan(options: ScanOptions): AsyncIterable; -} +export type SourceAdapter = CoreSourceAdapter; /** Contract for scan options. */ -export interface ScanOptions { - since?: string; - maxMessages?: number; - maxScanTargets?: number; - order?: "source_default" | "recent_first"; - signal?: AbortSignal; - onProgress?: (progress: ScanProgress) => void; -} +export type ScanOptions = CoreScanOptions; diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts index ae6d07c00..cf6e7bd97 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts @@ -4,7 +4,7 @@ import { resolveWorkbuddyHomeDirectory, resolveWorkbuddyProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readWorkbuddyHistory, type RawWorkbuddyMessage } from "./history-reader.js"; @@ -73,13 +73,13 @@ export function createWorkbuddySourceAdapter(deps: CreateWorkbuddySourceAdapterD message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readWorkbuddyHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts index 1b69de428..bb26c279b 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts @@ -1,6 +1,5 @@ -import { createReadStream } from "node:fs"; import { basename } from "node:path"; -import { createInterface } from "node:readline"; +import { readJsonlObjects } from "../jsonl-lines.js"; export interface RawWorkbuddyMessage { messageId: string; @@ -18,27 +17,13 @@ export async function* readWorkbuddyHistory( signal?: AbortSignal ): AsyncIterable { const fallbackConversationId = basename(filePath, ".jsonl"); - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Number.POSITIVE_INFINITY }); let lineNumber = 0; - - try { - for await (const line of lines) { - lineNumber += 1; - throwIfAborted(signal, filePath); - const record = parseRecord(line); - if (!record) { - continue; - } - - const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); - if (message) { - yield message; - } + for await (const record of readJsonlObjects(filePath, signal)) { + lineNumber += 1; + const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); + if (message) { + yield message; } - } finally { - lines.close(); - stream.destroy(); } } @@ -278,18 +263,6 @@ function formatStructuredValue(value: unknown): string { } } -function parseRecord(line: string): Record | null { - if (!line.trim()) { - return null; - } - try { - const parsed: unknown = JSON.parse(line); - return isRecord(parsed) ? parsed : null; - } catch { - return null; - } -} - function isRoleEvent(value: string): boolean { return ["user", "human", "assistant", "ai", "tool", "function", "system", "developer"].includes(value); } @@ -325,9 +298,3 @@ function recordValue(value: unknown): Record | null { function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } - -function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { - if (signal?.aborted) { - throw new DOMException(`WorkBuddy history read aborted: ${filePath}`, "AbortError"); - } -} diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts index 03c78af87..f79e6ae2d 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts @@ -55,7 +55,7 @@ async function listJsonlFiles(root: string): Promise MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); + const result = insert.run(job.jobId, message.sourceId, message.conversationId, message.messageId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, JSON.stringify(message.rawMeta), ordinal++); + return Number(result.changes) > 0; + }, + stageBatch(messages) { + let inserted = 0; + db.exec("BEGIN IMMEDIATE"); + try { for (const message of messages) if (store.stage(message)) inserted += 1; db.exec("COMMIT"); } + catch (error) { db.exec("ROLLBACK"); throw error; } + return inserted; + }, + messages(sourceId, cursor, limit = 500) { + limit = Number.isFinite(limit) ? Math.min(500, Math.max(1, Math.floor(limit))) : 500; + const parameters: SQLInputValue[] = [job.jobId, sourceId]; + let where = "job_id=? AND source_id=?"; + if (cursor) { + where += " AND ((conversation_id > ?) OR (conversation_id = ? AND (created_at > ? OR (created_at = ? AND (message_id > ? OR (message_id = ? AND ordinal > ?))))))"; + parameters.push(cursor.conversationId, cursor.conversationId, cursor.createdAt, cursor.createdAt, cursor.messageId, cursor.messageId, cursor.ordinal); + } + const iterator = db.prepare(`SELECT source_id AS sourceId, conversation_id AS conversationId, message_id AS messageId, role, content, created_at AS createdAt, workspace_path AS workspacePath, git_root AS gitRoot, raw_meta_json AS rawMetaJson, ordinal FROM staged_messages WHERE ${where} ORDER BY conversation_id, created_at, message_id, ordinal LIMIT ?`).iterate(...parameters, limit) as Iterable>; + return (function*() { + let bytes = 0; + let count = 0; + for (const row of iterator) { + const message = rowToMessage(row); + yield message; + count += 1; + bytes += Buffer.byteLength(JSON.stringify(message)); + if (count >= 500 || bytes >= MAX_PAGE_BYTES) break; + } + })(); + }, + saveScanCursor(sourceId, cursor) { db.prepare("INSERT INTO scan_cursors(source_id,conversation_id,created_at,message_id,ordinal) VALUES(?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET conversation_id=excluded.conversation_id,created_at=excluded.created_at,message_id=excluded.message_id,ordinal=excluded.ordinal").run(sourceId, cursor.conversationId, cursor.createdAt, cursor.messageId, cursor.ordinal); }, + getScanCursor(sourceId) { const row = db.prepare("SELECT conversation_id AS conversationId,created_at AS createdAt,message_id AS messageId,ordinal FROM scan_cursors WHERE source_id=?").get(sourceId) as MessageCursor|undefined; return row ?? null; }, + saveSourceState(state) { + db.prepare(`INSERT INTO scan_source_state(source_id,mode,phase,message_count,result_count,error_count,scan_started_at,watermarked_since,updated_at,error) + VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET mode=excluded.mode,phase=excluded.phase,message_count=excluded.message_count,result_count=excluded.result_count,error_count=excluded.error_count,scan_started_at=excluded.scan_started_at,watermarked_since=excluded.watermarked_since,updated_at=excluded.updated_at,error=excluded.error`) + .run(state.sourceId, state.mode, state.phase, state.messageCount, state.resultCount, state.errorCount, state.scanStartedAt ?? null, state.watermarkedSince ?? null, state.updatedAt, state.error ?? null); + }, + getSourceState(sourceId) { + const row = db.prepare("SELECT source_id AS sourceId,mode,phase,message_count AS messageCount,result_count AS resultCount,error_count AS errorCount,scan_started_at AS scanStartedAt,watermarked_since AS watermarkedSince,updated_at AS updatedAt,error FROM scan_source_state WHERE source_id=?").get(sourceId) as ScanSourceState | undefined; + return row ?? null; + }, + sourceCount() { return Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count); }, + count(sourceId) { + const row = db.prepare(`SELECT COUNT(*) AS count FROM staged_messages WHERE job_id=?${sourceId ? " AND source_id=?" : ""}`).get(job.jobId, ...(sourceId ? [sourceId] : [])) as { count: number }; + return Number(row.count); + }, + conversationCount(sourceId) { + const row = db.prepare("SELECT COUNT(DISTINCT conversation_id) AS count FROM staged_messages WHERE job_id=? AND source_id=?").get(job.jobId, sourceId) as { count: number }; + return Number(row.count); + }, + saveCheckpoint(checkpoint) { db.prepare("INSERT INTO checkpoints(source_id,conversation_id,last_message_id,last_created_at,content_hash,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,updated_at=excluded.updated_at").run(checkpoint.sourceId, checkpoint.conversationId, checkpoint.lastMessageId, checkpoint.lastCreatedAt, checkpoint.contentHash, checkpoint.updatedAt); }, + getCheckpoint(sourceId, conversationId) { return (db.prepare("SELECT source_id AS sourceId, conversation_id AS conversationId, last_message_id AS lastMessageId, last_created_at AS lastCreatedAt, content_hash AS contentHash, updated_at AS updatedAt FROM checkpoints WHERE source_id=? AND conversation_id=?").get(sourceId, conversationId) as ConversationCheckpoint | undefined) ?? null; }, + saveConversationMeta(meta) { db.prepare("INSERT INTO conversation_meta(source_id,conversation_id,last_message_id,last_created_at,content_hash,selected) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,selected=excluded.selected").run(meta.sourceId,meta.conversationId,meta.lastMessageId,meta.lastCreatedAt,meta.contentHash,meta.selected?1:0); }, + getConversationMeta(sourceId, conversationId) { const row = db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,content_hash AS contentHash,selected FROM conversation_meta WHERE source_id=? AND conversation_id=?").get(sourceId,conversationId) as (Omit & {selected:number})|undefined; return row ? {...row, selected: row.selected === 1} : null; }, + selectAllConversations(sourceId) { db.prepare("UPDATE conversation_meta SET selected=1 WHERE source_id=?").run(sourceId); }, + saveTurnMeta(meta) { db.prepare("INSERT INTO turn_meta(source_id,conversation_id,turn_id,first_message_id,first_created_at,last_message_id,last_created_at,selected) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id,turn_id) DO UPDATE SET first_message_id=excluded.first_message_id,first_created_at=excluded.first_created_at,last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,selected=excluded.selected").run(meta.sourceId,meta.conversationId,meta.turnId,meta.firstMessageId,meta.firstCreatedAt,meta.lastMessageId,meta.lastCreatedAt,meta.selected?1:0); }, + getTurnMeta(sourceId, conversationId, turnId) { const row = db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,turn_id AS turnId,first_message_id AS firstMessageId,first_created_at AS firstCreatedAt,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,selected FROM turn_meta WHERE source_id=? AND conversation_id=? AND turn_id=?").get(sourceId,conversationId,turnId) as (Omit & {selected:number})|undefined; return row ? {...row,selected:row.selected===1} : null; }, + selectInitialTurns(sourceIds, globalLimit, absentSourceLimit) { + if (sourceIds.length === 0) return; + const placeholders = sourceIds.map(() => "?").join(","); + db.exec("BEGIN IMMEDIATE"); + try { + db.prepare(`UPDATE turn_meta SET selected=0 WHERE source_id IN (${placeholders})`).run(...sourceIds); + db.prepare(`UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT rowid FROM turn_meta WHERE source_id IN (${placeholders}) ORDER BY first_created_at DESC,source_id,conversation_id,first_message_id,turn_id LIMIT ?)`).run(...sourceIds, globalLimit); + db.prepare(`WITH ranked AS (SELECT source_id,turn_id,ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY first_created_at DESC,conversation_id,first_message_id,turn_id) AS rank FROM turn_meta WHERE source_id IN (${placeholders})), absent AS (SELECT source_id FROM turn_meta WHERE source_id IN (${placeholders}) GROUP BY source_id HAVING MAX(selected)=0) UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT t.rowid FROM turn_meta t JOIN ranked r ON r.source_id=t.source_id AND r.turn_id=t.turn_id JOIN absent a ON a.source_id=t.source_id WHERE r.rank <= ?)`).run(...sourceIds, ...sourceIds, absentSourceLimit); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + }, + saveResult(result) { + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT ?,?,?,? WHERE NOT EXISTS ( + SELECT 1 FROM scan_results WHERE source_id=? AND conversation_id=? AND memory_id IS ? AND error IS ? + )`).run(result.sourceId, result.conversationId, result.memoryId ?? null, result.error ?? null, result.sourceId, result.conversationId, result.memoryId ?? null, result.error ?? null); + }, + resultCount(sourceId) { + const row = db.prepare(`SELECT COUNT(*) AS count FROM scan_results${sourceId ? " WHERE source_id=?" : ""}`).get(...(sourceId ? [sourceId] : [])) as { count: number }; + return Number(row.count); + }, + results(sourceId, cursor, limit = 100) { + const safeLimit = Math.min(500, Math.max(1, Math.floor(limit))); + const numericCursor = Number(cursor); + const safeCursor = Number.isFinite(numericCursor) && numericCursor >= 0 ? Math.floor(numericCursor) : 0; + const parameters: SQLInputValue[] = [sourceId ?? "%", safeCursor]; + const iterator = db.prepare("SELECT id, source_id AS sourceId, conversation_id AS conversationId, memory_id AS memoryId, error FROM scan_results WHERE source_id LIKE ? AND id > ? ORDER BY id LIMIT ?").iterate(...parameters, safeLimit) as Iterable>; + return (function*() { + for (const row of iterator) { + const result = { sourceId: String(row.sourceId), conversationId: String(row.conversationId), ...(row.memoryId ? { memoryId: String(row.memoryId) } : {}), ...(row.error ? { error: String(row.error) } : {}) } as ScanStoredResult; + Object.defineProperty(result, "cursor", { value: String(row.id), enumerable: false }); + yield result; + } + })(); + }, + saveMeta(meta) { db.prepare("UPDATE scan_meta SET job_id=?,source_id=?,mode=?,phase=?,created_at=?,updated_at=?,error=? WHERE id=1").run(meta.jobId, meta.sourceId, meta.mode, meta.phase, meta.createdAt, meta.updatedAt, meta.error ?? null); }, + getMeta() { return (db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, created_at AS createdAt, updated_at AS updatedAt, error FROM scan_meta WHERE id=1").get() as AppScanJobMeta | undefined) ?? null; }, + clearMeta() { db.prepare("DELETE FROM scan_meta WHERE id=1").run(); }, + close() { db.close(); }, + remove() { db.close(); rmSync(path, { force: true }); rmSync(`${path}-wal`, { force: true }); rmSync(`${path}-shm`, { force: true }); } + }; + return store; +} + +export function removeAppAgentSourceScanStore(path: string): void { + rmSync(path, { force: true }); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); +} + +function rowToMessage(row: Record): ConversationMessage { + return { + sourceId: String(row.sourceId), conversationId: String(row.conversationId), messageId: String(row.messageId), + role: row.role as ConversationMessage["role"], content: String(row.content), createdAt: String(row.createdAt), + workspacePath: row.workspacePath == null ? null : String(row.workspacePath), gitRoot: row.gitRoot == null ? null : String(row.gitRoot), + rawMeta: JSON.parse(String(row.rawMetaJson)) as Record, ordinal: Number(row.ordinal) + }; +} diff --git a/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts new file mode 100644 index 000000000..1c3076700 --- /dev/null +++ b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts @@ -0,0 +1,45 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openAppAgentSourceScanStore } from "../index.js"; + +let directory: string | undefined; +afterEach(() => { if (directory) rmSync(directory, { recursive: true, force: true }); directory = undefined; }); + +describe("durable scan store", () => { + it("deduplicates staged rows and reads keyset pages", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-store-")); + const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "fixture", mode: "full", phase: "stage", createdAt: "2026-01-01", updatedAt: "2026-01-01" }); + const message = { messageId: "m1", sourceId: "fixture", conversationId: "c1", role: "user" as const, content: "hello", createdAt: "2026-01-01T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: {} }; + expect(store.stageBatch([message, message])).toBe(1); + store.saveSourceState({ sourceId: "fixture", mode: "full", phase: "stage", messageCount: 1, resultCount: 0, errorCount: 0, updatedAt: "2026-01-01" }); + expect(store.sourceCount()).toBe(1); + expect([...store.messages("fixture", undefined, 1)]).toHaveLength(1); + store.saveResult({ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }); + store.saveResult({ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }); + expect([...store.results("fixture", "0", 1)]).toEqual([{ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }]); + store.remove(); + }); + + it("selects global recent turns and keeps an absent source fallback", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-store-")); + const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "all", mode: "initial_subset", phase: "prepare", createdAt: "2026-01-01", updatedAt: "2026-01-01" }); + const addTurn = (sourceId: string, index: number, day: string) => store.saveTurnMeta({ + sourceId, + conversationId: `conversation-${sourceId}-${index}`, + turnId: `${sourceId}::conversation-${sourceId}-${index}::user-${index}`, + firstMessageId: `user-${index}`, + firstCreatedAt: `2026-01-${day}T00:00:00Z`, + lastMessageId: `assistant-${index}`, + lastCreatedAt: `2026-01-${day}T00:01:00Z`, + selected: true + }); + addTurn("source-a", 1, "01"); + addTurn("source-b", 1, "02"); + store.selectInitialTurns(["source-a", "source-b"], 1, 1); + expect(store.getTurnMeta("source-b", "conversation-source-b-1", "source-b::conversation-source-b-1::user-1")?.selected).toBe(true); + expect(store.getTurnMeta("source-a", "conversation-source-a-1", "source-a::conversation-source-a-1::user-1")?.selected).toBe(true); + store.remove(); + }); +}); diff --git a/App/backend/src/services/agent-source-scan-journal.ts b/App/backend/src/services/agent-source-scan-journal.ts index e441db02d..4dc15c2f0 100644 --- a/App/backend/src/services/agent-source-scan-journal.ts +++ b/App/backend/src/services/agent-source-scan-journal.ts @@ -1,8 +1,13 @@ /** Agent source scan journal service helpers. */ import { DatabaseSync } from "node:sqlite"; +import { readdirSync, existsSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; import { createAgentSourceScanJournal } from "../infrastructure/agent-source-scan-journal/index.js"; +import { openAppAgentSourceScanStore } from "../infrastructure/agent-source-scan-store/index.js"; import type { ScanResumeStateReference } from "./agent-source-scan-runner.js"; +const COMPLETED_DETAILS_RETENTION_MS = 60 * 60 * 1000; + export interface PersistedScanResume { jobId: string; sourceId: string; @@ -20,7 +25,9 @@ export function readLatestPersistedScanResume(databasePath: string | undefined): PRAGMA busy_timeout = 5000; `); const job = createAgentSourceScanJournal(db).findLatestJob(); - if (!job) return null; + if (!job) { + return readLatestDurableScanResume(databasePath); + } return { jobId: job.jobId, sourceId: job.sourceId, @@ -47,6 +54,43 @@ export function readLatestPersistedScanResume(databasePath: string | undefined): } } +function readLatestDurableScanResume(databasePath: string): PersistedScanResume | null { + const directory = join(dirname(databasePath), "agent-source-scans"); + if (!existsSync(directory)) return null; + const files = readdirSync(directory).filter((file) => file.endsWith(".sqlite")); + let latest: PersistedScanResume | null = null; + let latestUpdatedAt = Number.NEGATIVE_INFINITY; + for (const file of files) { + const path = join(directory, file); + try { + const db = new DatabaseSync(path); + const row = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, updated_at AS updatedAt FROM scan_meta WHERE id=1").get() as { jobId: string; sourceId: string; mode?: "initial_subset" | "incremental" | "full"; phase: string; updatedAt: string } | undefined; + const count = Number((db.prepare("SELECT COUNT(*) AS count FROM staged_messages").get() as { count: number }).count); + let sourceCount = 1; + try { + sourceCount = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count) || 1; + } catch { + // Stores created by the first staging build did not have source state. + } + const resultCount = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count); + db.close(); + if (!row) continue; + const updatedAt = Date.parse(row.updatedAt); + if (row.phase === "done") { + if (Number.isFinite(updatedAt) && Date.now() - updatedAt > COMPLETED_DETAILS_RETENTION_MS) deleteDurableScanStore(databasePath, row.jobId); + continue; + } + if (updatedAt >= latestUpdatedAt) { + latestUpdatedAt = updatedAt; + latest = row.phase === "summarize" + ? { jobId: row.jobId, sourceId: row.sourceId, mode: row.mode, resume: { storage: "sqlite", phase: "summarize", jobId: row.jobId, sourceId: row.sourceId, resultCount } } + : { jobId: row.jobId, sourceId: row.sourceId, mode: row.mode, resume: { storage: "sqlite", phase: "add", jobId: row.jobId, sourceId: row.sourceId, messageCount: count, sourceCount: sourceCount || 1 } }; + } + } catch { /* corrupt stores are retained for diagnostics */ } + } + return latest; +} + /** Deletes persisted scan resume state for one job. */ export function deletePersistedScanResume(databasePath: string | undefined, jobId: string): void { if (!databasePath) { @@ -64,3 +108,31 @@ export function deletePersistedScanResume(databasePath: string | undefined, jobI db.close(); } } + +export function deleteDurableScanStore(databasePath: string | undefined, jobId: string): void { + if (!databasePath) return; + const path = join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`); + rmSync(path, { force: true }); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); +} + +export function readDurableScanResults(databasePath: string | undefined, jobId: string, cursor = "0", limit = 100): { items: unknown[]; nextCursor: string | null } { + if (!databasePath) return { items: [], nextCursor: null }; + const path = join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`); + if (!existsSync(path)) return { items: [], nextCursor: null }; + const store = openAppAgentSourceScanStore(path, { jobId, sourceId: "all", mode: "incremental", phase: "ingest", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + let removeAfterRead = false; + try { + const safeLimit = Math.min(500, Math.max(1, limit)); + const safeCursor = Number.isFinite(Number(cursor)) && Number(cursor) >= 0 ? String(Math.floor(Number(cursor))) : "0"; + const rows = [...store.results(undefined, safeCursor, safeLimit)]; + const items = rows.map(({ cursor: _cursor, ...item }) => item); + const nextCursor = rows.length < safeLimit ? null : rows.at(-1)?.cursor ?? null; + removeAfterRead = nextCursor === null && store.getMeta()?.phase === "done"; + return { items, nextCursor }; + } finally { + store.close(); + if (removeAfterRead) deleteDurableScanStore(databasePath, jobId); + } +} diff --git a/App/backend/src/services/agent-source-scan-migration.ts b/App/backend/src/services/agent-source-scan-migration.ts new file mode 100644 index 000000000..5c7e4c08d --- /dev/null +++ b/App/backend/src/services/agent-source-scan-migration.ts @@ -0,0 +1,91 @@ +import { existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +/** Migrates one legacy app-state journal into the durable per-job store. */ +export function migrateLegacyScanJournal(databasePath: string, storePath: string, jobId: string): boolean { + if (!existsSync(databasePath)) return false; + mkdirSync(dirname(storePath), { recursive: true }); + const db = new DatabaseSync(storePath); + try { + db.exec(` + PRAGMA busy_timeout = 5000; + CREATE TABLE IF NOT EXISTS schema_meta (version INTEGER NOT NULL); + INSERT INTO schema_meta(version) SELECT 2 WHERE NOT EXISTS (SELECT 1 FROM schema_meta); + UPDATE schema_meta SET version = 2 WHERE version < 2; + CREATE TABLE IF NOT EXISTS scan_meta (id INTEGER PRIMARY KEY CHECK (id = 1), job_id TEXT NOT NULL, source_id TEXT NOT NULL, mode TEXT NOT NULL, phase TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS staged_messages (job_id TEXT NOT NULL, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, message_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL, created_at TEXT NOT NULL, workspace_path TEXT, git_root TEXT, raw_meta_json TEXT NOT NULL, ordinal INTEGER NOT NULL, PRIMARY KEY(job_id,source_id,message_id)); + CREATE TABLE IF NOT EXISTS scan_source_state (source_id TEXT PRIMARY KEY, mode TEXT NOT NULL, phase TEXT NOT NULL, message_count INTEGER NOT NULL DEFAULT 0, result_count INTEGER NOT NULL DEFAULT 0, error_count INTEGER NOT NULL DEFAULT 0, scan_started_at TEXT, watermarked_since TEXT, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS scan_results (id INTEGER PRIMARY KEY AUTOINCREMENT, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, memory_id TEXT, error TEXT); + CREATE INDEX IF NOT EXISTS scan_result_identity ON scan_results(source_id,conversation_id,memory_id,error); + `); + db.exec(`ATTACH DATABASE '${databasePath.replaceAll("'", "''")}' AS legacy`); + const oldMessages = Number((db.prepare("SELECT COUNT(*) AS count FROM legacy.account_agent_source_scan_messages WHERE job_id=?").get(jobId) as { count: number }).count); + const oldSources = Number((db.prepare("SELECT COUNT(*) AS count FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").get(jobId) as { count: number }).count); + const oldResults = Number((db.prepare("SELECT COALESCE(SUM(COALESCE(json_array_length(memory_ids_json),0) + COALESCE(json_array_length(errors_json),0)),0) AS count FROM legacy.account_agent_source_scan_results WHERE job_id=?").get(jobId) as { count: number }).count); + const oldSourceErrors = Number((db.prepare("SELECT COALESCE(SUM(COALESCE(json_array_length(errors_json),0)),0) AS count FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").get(jobId) as { count: number }).count); + const oldJob = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, COALESCE(mode,'incremental') AS mode, phase, created_at AS createdAt, updated_at AS updatedAt FROM legacy.account_agent_source_scan_jobs WHERE job_id=?").get(jobId) as { jobId: string; sourceId: string; mode: string; phase: string; createdAt: string; updatedAt: string } | undefined; + if (!oldJob) return false; + db.prepare("INSERT OR IGNORE INTO scan_meta(id,job_id,source_id,mode,phase,created_at,updated_at,error) VALUES(1,?,?,?,?,?,?,NULL)").run(oldJob.jobId, oldJob.sourceId, oldJob.mode, oldJob.phase, oldJob.createdAt, oldJob.updatedAt); + const existingResults = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count); + db.exec("BEGIN"); + db.prepare(`INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) + SELECT job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,message_order + FROM legacy.account_agent_source_scan_messages WHERE job_id=?`).run(jobId); + db.prepare(`INSERT OR IGNORE INTO scan_source_state(source_id,mode,phase,message_count,result_count,error_count,scan_started_at,watermarked_since,updated_at,error) + SELECT s.source_id, COALESCE(s.scan_mode, ?), ?, + (SELECT COUNT(*) FROM legacy.account_agent_source_scan_messages m WHERE m.job_id=s.job_id AND m.source_id=s.source_id), + (SELECT COUNT(*) FROM legacy.account_agent_source_scan_results r WHERE r.job_id=s.job_id AND r.source_id=s.source_id), + COALESCE(json_array_length(s.errors_json),0), s.scan_started_at, s.watermarked_since, s.updated_at, NULL + FROM legacy.account_agent_source_scan_source_state s WHERE s.job_id=?`).run(oldJob.mode, oldJob.phase, jobId); + // json_each keeps legacy result arrays out of JavaScript memory. + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT r.source_id, 'scan', json_each.value, NULL + FROM legacy.account_agent_source_scan_results r, json_each(r.memory_ids_json) WHERE r.job_id=?`).run(jobId); + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT r.source_id, json_extract(json_each.value,'$.conversationId'), NULL, json_extract(json_each.value,'$.reason') + FROM legacy.account_agent_source_scan_results r, json_each(r.errors_json) WHERE r.job_id=?`).run(jobId); + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT s.source_id, json_extract(json_each.value,'$.conversationId'), NULL, json_extract(json_each.value,'$.reason') + FROM legacy.account_agent_source_scan_source_state s, json_each(s.errors_json) WHERE s.job_id=?`).run(jobId); + const newMessages = Number((db.prepare("SELECT COUNT(*) AS count FROM staged_messages WHERE job_id=?").get(jobId) as { count: number }).count); + const newSources = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count); + const newResults = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count) - existingResults; + if (newMessages < oldMessages || newSources < oldSources || newResults < oldResults + oldSourceErrors) { db.exec("ROLLBACK"); return false; } + db.prepare("DELETE FROM legacy.account_agent_source_scan_messages WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_results WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_jobs WHERE job_id=?").run(jobId); + db.exec("COMMIT"); + return true; + } catch { + try { db.exec("ROLLBACK"); } catch { /* preserve legacy journal */ } + return false; + } finally { + try { db.exec("DETACH DATABASE legacy"); } catch { /* no attachment */ } + db.close(); + } +} + +/** Migrates every legacy scan job during App startup without loading journal arrays. */ +export function migrateLegacyScanJournals(databasePath: string): number { + if (!existsSync(databasePath)) return 0; + let jobIds: string[] = []; + try { + const db = new DatabaseSync(databasePath); + try { + jobIds = (db.prepare("SELECT job_id AS jobId FROM account_agent_source_scan_jobs ORDER BY job_id").all() as Array<{ jobId: string }>) + .map((row) => row.jobId) + .filter((jobId) => typeof jobId === "string" && jobId.length > 0); + } finally { + db.close(); + } + } catch { + return 0; + } + let migrated = 0; + for (const jobId of jobIds) { + if (migrateLegacyScanJournal(databasePath, join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`), jobId)) migrated += 1; + } + return migrated; +} diff --git a/App/backend/src/services/agent-source-scan-process.ts b/App/backend/src/services/agent-source-scan-process.ts index 0ee8207b3..7dd0898f0 100644 --- a/App/backend/src/services/agent-source-scan-process.ts +++ b/App/backend/src/services/agent-source-scan-process.ts @@ -9,21 +9,14 @@ import { } from "../analytics/agent-source-analytics.js"; import { createMemoryDesktopAddAnalytics } from "../analytics/memory-add-analytics.js"; import { createAppStateStore, type AppStateStore } from "../infrastructure/app-state-store/index.js"; -import { createAgentSourceScanJournal, type AgentSourceScanJournal } from "../infrastructure/agent-source-scan-journal/index.js"; import { createAgentSourceService } from "./agent-source-service.js"; import { createBuiltinAgentSourceRegistry } from "./builtin-agent-source-registry.js"; import { createBuiltinSkillTargetRegistry } from "./builtin-skill-target-registry.js"; import { createIngestionService } from "./ingestion-service.js"; import { createSkillDistributionService } from "./skill-distribution-service.js"; -import { - type AgentSourceScanProcessCommand, - type AgentSourceScanProcessData, - type AgentSourceScanProcessMessage, - isScanResumeStateReference, - type ScanResumeState, - type ScanResumeStateReference, - runAgentSourceScanJob -} from "./agent-source-scan-runner.js"; +import { isScanResumeStateReference, type AgentSourceScanProcessCommand, type AgentSourceScanProcessData, type AgentSourceScanProcessMessage, runAgentSourceScanJob } from "./agent-source-scan-runner.js"; +import { dirname, join } from "node:path"; +import { migrateLegacyScanJournal } from "./agent-source-scan-migration.js"; const DEFAULT_MEMORY_LAYER_TIMEOUT_MS = 20_000; @@ -62,13 +55,21 @@ async function runProcess(data: AgentSourceScanProcessData): Promise { let appStateStore: AppStateStore | null = null; try { appStateStore = createAppStateStore({ databasePath: data.databasePath }); - const scanJournal = createAgentSourceScanJournal(appStateStore.db); + const legacyMigrationSucceeded = migrateLegacyScanJournal( + data.databasePath, + join(dirname(data.databasePath), "agent-source-scans", `${data.job.jobId}.sqlite`), + data.job.jobId + ); + const preservedResume = data.job.resume && isScanResumeStateReference(data.job.resume) ? data.job.resume : null; + const preserveLegacyResume = Boolean(preservedResume && !legacyMigrationSucceeded); const memoryClient = createDefaultMemoryClient(process.env); const agentSources = createAgentSources(appStateStore, memoryClient); await runAgentSourceScanJob( { ...data.job, - resume: readResumeState(scanJournal, data.job.resume), + // The durable per-job store is the source of truth. Do not hydrate the + // legacy journal arrays into JavaScript while resuming an old job. + resume: null, controller }, agentSources, @@ -77,7 +78,7 @@ async function runProcess(data: AgentSourceScanProcessData): Promise { postProcessMessage({ type: "progress", progress }); }, onResumeChanged(resume) { - postProcessMessage({ type: "resume", resume: resume ? writeResumeState(scanJournal, data, resume) : null }); + postProcessMessage({ type: "resume", resume: preserveLegacyResume ? preservedResume : null }); }, onCompleted(results) { postProcessMessage({ type: "completed", results }); @@ -121,6 +122,7 @@ function createAgentSources(appStateStore: AppStateStore, memoryClient: MemoryCl skillDistributionService: createSkillDistributionService({ targetRegistry: createBuiltinSkillTargetRegistry() }), + scanStoreDirectory: `${dataDirectoryForScanStore(appStateStore)}`, agentSourceAnalytics: createAgentSourceLifecycleAnalytics({ getUserId: resolveAnalyticsUserId, getUserMode: resolveAnalyticsUserMode, @@ -128,6 +130,10 @@ function createAgentSources(appStateStore: AppStateStore, memoryClient: MemoryCl }); } +function dataDirectoryForScanStore(appStateStore: AppStateStore): string { + return join(dirname(appStateStore.databasePath), "agent-source-scans"); +} + function createDefaultMemoryClient(env: NodeJS.ProcessEnv): MemoryClient { const memoryLayerConfig = readMemoryLayerConfig(env); if (memoryLayerConfig) { @@ -151,50 +157,6 @@ function readMemoryLayerConfig(env: NodeJS.ProcessEnv): MemoryLayerConfig | null }; } -function readResumeState(scanJournal: AgentSourceScanJournal, resume: AgentSourceScanProcessData["job"]["resume"]): ScanResumeState | null { - if (!resume) { - return null; - } - - if (!isScanResumeStateReference(resume)) { - return resume; - } - - return scanJournal.readResume(resume.jobId); -} - -function writeResumeState( - scanJournal: AgentSourceScanJournal, - data: AgentSourceScanProcessData, - resume: ScanResumeState -): ScanResumeStateReference { - scanJournal.writeResume({ - jobId: data.job.jobId, - sourceId: data.job.sourceId, - mode: data.job.mode, - resume - }); - - if (resume.phase === "add") { - return { - storage: "sqlite", - phase: "add", - jobId: data.job.jobId, - sourceId: resume.collected[0]?.sourceId ?? data.job.sourceId, - messageCount: resume.collected.reduce((sum, source) => sum + source.messages.length, 0), - sourceCount: resume.collected.length - }; - } - - return { - storage: "sqlite", - phase: "summarize", - jobId: data.job.jobId, - sourceId: data.job.sourceId, - resultCount: resume.results.length - }; -} - function postProcessMessage(message: AgentSourceScanProcessMessage): void { if (process.connected) process.send?.(message); } diff --git a/App/backend/src/services/agent-source-scan-runner.ts b/App/backend/src/services/agent-source-scan-runner.ts index e5fc95c7b..1f3ce842d 100644 --- a/App/backend/src/services/agent-source-scan-runner.ts +++ b/App/backend/src/services/agent-source-scan-runner.ts @@ -120,39 +120,26 @@ export async function runAgentSourceScanJob( }; let results: ScanResult[]; - if (job.resume?.phase === "summarize") { - results = job.resume.results; + let legacyPipeline = false; + if (agentSources.supportsPersistentScan) { + results = job.sourceId === "all" + ? await agentSources.scanAll({ ...scanOptions, scanJobId: job.jobId }) + : [await agentSources.scanOne(job.sourceId, { ...scanOptions, scanJobId: job.jobId })]; + callbacks.onResumeChanged(null); } else { + legacyPipeline = true; const collected = job.resume?.phase === "add" ? job.resume.collected - : job.sourceId === "all" - ? await agentSources.collectAll(scanOptions) - : [await agentSources.collectOne(job.sourceId, scanOptions)]; - if (job.controller.signal.aborted) { - return; - } + : job.sourceId === "all" ? await agentSources.collectAll(scanOptions) : [await agentSources.collectOne(job.sourceId, scanOptions)]; callbacks.onResumeChanged({ phase: "add", collected }); results = await agentSources.ingestCollected(collected, scanOptions); - if (job.controller.signal.aborted) { - return; - } - } - - callbacks.onResumeChanged({ phase: "summarize", results }); - const failures = await agentSources.processImportSummaries( - results.flatMap((result) => result.memoryIds ?? []), - { ...scanOptions, progressSourceId: job.sourceId } - ); - const resultByMemoryId = new Map(); - for (const result of results) { - for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result); } - for (const failure of failures) { - const result = resultByMemoryId.get(failure.memoryId); - result?.errors.push({ - conversationId: failure.memoryId, - reason: failure.reason - }); + if (legacyPipeline) { + callbacks.onResumeChanged({ phase: "summarize", results }); + const failures = await agentSources.processImportSummaries(results.flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: job.sourceId }); + const resultByMemoryId = new Map(); + for (const result of results) for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result); + for (const failure of failures) resultByMemoryId.get(failure.memoryId)?.errors.push({ conversationId: failure.memoryId, reason: failure.reason }); } if (job.controller.signal.aborted) { return; diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index 0ebb48411..478afb1b6 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -21,7 +21,8 @@ import type { import type { ConversationMessage, ScanOptions, - ScanProgress + ScanProgress, + SourceAdapter } from "../adapters/outbound/agent-source/types.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; import type { SourceRegistry } from "../adapters/outbound/agent-source/source-registry.js"; @@ -43,6 +44,15 @@ import { extractManagedAgentHistory, selectIncrementalManagedMessages } from "./managed-agent-history.js"; +import { + orderedTurns, + splitTurn, + stableTurnIdentity, + isCompleteTurn, + legacyTurnId, + legacyTurnRequestId +} from "@memmy/agent-source-core"; +import { openAppAgentSourceScanStore, type AppAgentSourceScanStore } from "../infrastructure/agent-source-scan-store/index.js"; export type { ScanProgress } from "../adapters/outbound/agent-source/types.js"; @@ -60,6 +70,7 @@ const INITIAL_SOURCE_MEMORY_LIMIT = 1_000; /** Contract for agent source service. */ export interface AgentSourceService { + readonly supportsPersistentScan?: true; list(): Promise; scanAll(options?: AgentSourceScanOptions): Promise; scanOne(sourceId: string, options?: AgentSourceScanOptions): Promise; @@ -95,6 +106,7 @@ export interface AgentSourceScanOptions { signal?: AbortSignal; onProgress?: (progress: ScanProgress) => void; progressSourceId?: string; + scanJobId?: string; } /** Contract for create agent source service options. */ @@ -108,6 +120,7 @@ export interface CreateAgentSourceServiceOptions { getScanPermission?: () => Promise; now?: () => string; createId?: () => string; + scanStoreDirectory?: string; } /** Creates create agent source service. */ @@ -117,19 +130,20 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption const agentSourceAnalytics = options.agentSourceAnalytics ?? createAgentSourceLifecycleAnalytics(); return { + supportsPersistentScan: true, async list() { return await listSources(options); }, async scanAll(scanOptions = {}) { - const collected = await this.collectAll(scanOptions); - const results = await this.ingestCollected(collected, scanOptions); - const failures = await this.processImportSummaries( - results.flatMap((result) => result.memoryIds ?? []), - { ...scanOptions, progressSourceId: "all" } - ); - appendProcessingFailuresToResults(results, failures); - return results; + if (!scanOptions.scanJobId && !options.scanStoreDirectory) { + const collected = await this.collectAll(scanOptions); + const results = await this.ingestCollected(collected, scanOptions); + const failures = await this.processImportSummaries(results.flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: "all" }); + appendProcessingFailuresToResults(results, failures); + return results; + } + return scanPersistent(options, "all", scanOptions, now); }, async collectAll(scanOptions = {}) { @@ -160,15 +174,21 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption }, async scanOne(sourceId, scanOptions = {}) { - const collected = await this.collectOne(sourceId, scanOptions); - const result = await ingestCollectedSource(options, collected, scanOptions, now); - const failures = await processPendingImportSummaries( - options, - result.memoryIds ?? [], - { ...scanOptions, progressSourceId: sourceId } - ); - appendProcessingFailures(result, failures); - return result; + if (!scanOptions.scanJobId && !options.scanStoreDirectory) { + const collected = await this.collectOne(sourceId, scanOptions); + const result = await ingestCollectedSource(options, collected, scanOptions, now); + const failures = await processPendingImportSummaries(options, result.memoryIds ?? [], { ...scanOptions, progressSourceId: sourceId }); + appendProcessingFailures(result, failures); + return result; + } + const results = await scanPersistent(options, sourceId, scanOptions, now); + return results[0] ?? { + sourceId, + discoveredConversations: 0, + emittedMessages: 0, + skipped: 0, + errors: [{ conversationId: "scan", reason: "No result" }] + }; }, async addManual(input) { @@ -513,6 +533,406 @@ async function detectAvailableSourceAdapters(options: CreateAgentSourceServiceOp return detected.filter((entry) => entry.available).map((entry) => entry.adapter); } +interface PersistentSourceStage { + adapter: SourceAdapter; + sourceId: string; + mode: AgentSourceScanMode; + since?: string; + errors: Array<{ conversationId: string; reason: string }>; + scanErrorCount: number; +} + +/** Runs the production scan through a durable, bounded staging store. */ +async function scanPersistent( + options: CreateAgentSourceServiceOptions, + requestedSourceId: string, + scanOptions: AgentSourceScanOptions, + now: () => string +): Promise { + const adapters = requestedSourceId === "all" + ? options.sourceRegistry.list() + : [options.sourceRegistry.require(requestedSourceId)]; + const jobId = scanOptions.scanJobId ?? randomUUID(); + const directory = options.scanStoreDirectory ?? `${process.cwd()}/agent-source-scans`; + const store = openAppAgentSourceScanStore(`${directory}/${jobId}.sqlite`, { + jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: "stage", createdAt: now(), updatedAt: now() + }); + const results: ScanResult[] = []; + let completed = false; + try { + const available: SourceAdapter[] = []; + for (const adapter of adapters) { + scanOptions.signal?.throwIfAborted(); + if (await adapter.detect()) available.push(adapter); + else if (requestedSourceId !== "all") throw new AgentSourceUnavailableError(adapter.descriptor.displayName); + } + const globalInitial = requestedSourceId === "all" && available.length > 0 && + (scanOptions.mode === "initial_subset" || (scanOptions.mode === undefined && available.every((adapter) => !options.agentSourceRepository.getScanWatermark(adapter.descriptor.sourceId)))); + const stages: PersistentSourceStage[] = []; + for (const adapter of available) { + scanOptions.signal?.throwIfAborted(); + const sourceId = adapter.descriptor.sourceId; + const watermark = options.agentSourceRepository.getScanWatermark(sourceId); + const mode = scanOptions.mode ?? (watermark ? "incremental" : "initial_subset"); + const since = scanOptions.since ?? (mode === "incremental" ? watermarkCursor(watermark) : undefined); + stages.push(await stagePersistentSource(options, store, adapter, mode, since, scanOptions, now)); + } + if (globalInitial) { + for (const stage of stages) { + scanOptions.signal?.throwIfAborted(); + store.saveMeta({ jobId: store.getMeta()?.jobId ?? jobId, sourceId: store.getMeta()?.sourceId ?? requestedSourceId, mode: stage.mode, phase: "prepare", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const sourceState = store.getSourceState(stage.sourceId); + store.saveSourceState({ ...(sourceState ?? { sourceId: stage.sourceId, mode: stage.mode, messageCount: store.count(stage.sourceId), resultCount: store.resultCount(stage.sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "prepare", updatedAt: now() }); + await preparePersistentSource(options, store, stage.sourceId, stage.mode); + } + store.selectInitialTurns(stages.map((stage) => stage.sourceId), INITIAL_GLOBAL_MEMORY_LIMIT, INITIAL_ABSENT_SOURCE_MEMORY_LIMIT); + } + for (const stage of stages) { + const result = await ingestPersistentStagedSource(options, store, stage, scanOptions, now, globalInitial); + results.push(result); + store.saveSourceState({ + sourceId: stage.sourceId, + mode: stage.mode, + phase: result.errorCount && result.errorCount > 0 ? "failed" : "done", + messageCount: result.emittedMessages, + resultCount: store.resultCount(stage.sourceId), + errorCount: result.errorCount ?? result.errors.length, + updatedAt: now(), + ...(stage.since ? { watermarkedSince: stage.since } : {}) + }); + } + const hasErrors = results.some((result) => (result.errorCount ?? result.errors.length) > 0); + store.saveMeta({ jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: hasErrors ? "failed" : "done", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now(), ...(hasErrors ? { error: "Agent source scan completed with errors" } : {}) }); + // Keep large result details available for the paged results endpoint until + // the client consumes the final page. Small jobs can release their store + // immediately as before. + completed = !hasErrors && !results.some((result) => result.detailsTruncated); + return results; + } catch (error) { + store.saveMeta({ ...(store.getMeta() ?? { jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: "stage", createdAt: now(), updatedAt: now() }), phase: "failed", updatedAt: now(), error: error instanceof Error ? error.message : "Agent source scan failed" }); + throw error; + } finally { + if (completed) store.remove(); + else store.close(); + } +} + +async function stagePersistentSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + adapter: SourceAdapter, + mode: AgentSourceScanMode, + since: string | undefined, + scanOptions: AgentSourceScanOptions, + now: () => string +): Promise { + const sourceId = adapter.descriptor.sourceId; + options.agentSourceRepository.upsertSource({ sourceId, displayName: adapter.descriptor.displayName, dataPath: adapter.descriptor.dataPath, builtin: adapter.descriptor.builtin }); + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "stage", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + store.saveSourceState({ sourceId, mode, phase: "stage", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: 0, updatedAt: now(), ...(since ? { watermarkedSince: since } : {}) }); + const errors: Array<{ conversationId: string; reason: string }> = []; + let scanErrorCount = 0; + let batch: ConversationMessage[] = []; + let bytes = 0; + let emittedOrdinal = 0; + try { + for await (const message of adapter.scan({ + since, + order: scanOptions.order ?? (mode === "initial_subset" ? "recent_first" : "source_default"), + fullHistory: true, + signal: scanOptions.signal, + onProgress: (progress) => emitProgress(scanOptions, { ...progress, phase: "scan" }) + })) { + scanOptions.signal?.throwIfAborted(); + const messageBytes = Buffer.byteLength(JSON.stringify(message)); + if (messageBytes > 64 * 1024 * 1024) { + scanErrorCount += 1; + if (errors.length < 1000) errors.push({ conversationId: message.conversationId, reason: "scan record exceeds 64 MiB limit" }); + store.saveResult({ sourceId, conversationId: message.conversationId, error: "scan record exceeds 64 MiB limit" }); + continue; + } + if (batch.length > 0 && (batch.length >= 500 || bytes + messageBytes > 8 * 1024 * 1024)) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch = []; + bytes = 0; + } + batch.push({ ...message, ordinal: emittedOrdinal++ }); + bytes += messageBytes; + if (batch.length >= 500 || bytes >= 8 * 1024 * 1024) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch = []; + bytes = 0; + } + } + if (batch.length > 0) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + } + } catch (error) { + if (scanOptions.signal?.aborted) throw error; + scanErrorCount += 1; + const reason = error instanceof Error ? error.message : "Agent source scan failed"; + if (errors.length < 1000) errors.push({ conversationId: "scan", reason }); + store.saveResult({ sourceId, conversationId: "scan", error: reason }); + } + store.saveSourceState({ sourceId, mode, phase: scanErrorCount > 0 ? "failed" : "stage", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: scanErrorCount, updatedAt: now(), ...(since ? { watermarkedSince: since } : {}) }); + return { adapter, sourceId, mode, since, errors, scanErrorCount }; +} + +async function ingestPersistentStagedSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + stage: PersistentSourceStage, + scanOptions: AgentSourceScanOptions, + now: () => string, + globalInitial: boolean +): Promise { + const { sourceId, mode, since } = stage; + if (!globalInitial) { + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "prepare", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const sourceState = store.getSourceState(sourceId); + store.saveSourceState({ ...(sourceState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "prepare", updatedAt: now() }); + await preparePersistentSource(options, store, sourceId, mode); + if (mode === "initial_subset") store.selectInitialTurns([sourceId], INITIAL_SOURCE_MEMORY_LIMIT, 0); + } + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "ingest", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const preparedState = store.getSourceState(sourceId); + store.saveSourceState({ ...(preparedState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "ingest", updatedAt: now() }); + const ingestion = await ingestPersistentSource(options, store, sourceId, scanOptions, []); + const scannedAt = now(); + options.agentSourceRepository.setLastScannedAt(sourceId, scannedAt); + const skillResult = await ingestSourceSkills(options, sourceId, scanOptions, store); + if (stage.errors.length === 0 && ingestion.errors.length === 0 && skillResult.errorCount === 0) updatePersistentWatermark(options, sourceId, mode, ingestion.latestSeenAt, scannedAt, since); + const allErrors = [...stage.errors, ...ingestion.errors, ...skillResult.errors]; + const errorCount = stage.scanErrorCount + ingestion.errorCount + skillResult.errorCount; + const memoryIdCount = ingestion.memoryIdCount + skillResult.memoryIdCount; + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "summarize", createdAt: store.getMeta()?.createdAt ?? scannedAt, updatedAt: scannedAt }); + store.saveSourceState({ sourceId, mode, phase: "summarize", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount, updatedAt: scannedAt, ...(since ? { watermarkedSince: since } : {}) }); + return { + sourceId, + discoveredConversations: store.conversationCount(sourceId), + emittedMessages: store.count(sourceId), + skipped: ingestion.deduped, + memoryIds: ingestion.memoryIds, + memoryIdCount, + errorCount, + detailsTruncated: errorCount > 1000 || memoryIdCount > 1000, + errors: allErrors.slice(0, 1000) + }; +} + +async function preparePersistentSource(options: CreateAgentSourceServiceOptions, store: AppAgentSourceScanStore, sourceId: string, mode: AgentSourceScanMode): Promise { + let cursor: { conversationId: string; createdAt: string; messageId: string; ordinal: number } | undefined; + let currentId: string | null = null; + let currentTurn: ConversationMessage[] = []; + let turnIndex = 0; + let hash = createHash("sha256"); + let first = true; + let latest: ConversationMessage | null = null; + const flushTurn = () => { + if (!currentTurn.length || !isCompleteTurn(currentTurn)) return; + const firstMessage = currentTurn[0]!; + const lastMessage = currentTurn[currentTurn.length - 1]!; + const turn = { sourceId, conversationId: firstMessage.conversationId, turnIndex, messages: currentTurn }; + store.saveTurnMeta({ + sourceId, + conversationId: firstMessage.conversationId, + turnId: stableTurnIdentity(turn), + firstMessageId: firstMessage.messageId, + firstCreatedAt: firstMessage.createdAt, + lastMessageId: lastMessage.messageId, + lastCreatedAt: lastMessage.createdAt, + selected: true + }); + turnIndex += 1; + }; + const flush = () => { + if (!currentId || !latest) return; + hash.update("]"); + const contentHash = hash.digest("hex"); + const checkpoint = options.agentSourceRepository.getConversationCheckpoint(sourceId, currentId); + const selected = mode === "full" || mode === "initial_subset" || !checkpoint + || Date.parse(latest.createdAt) > Date.parse(checkpoint.lastCreatedAt) + || (Date.parse(latest.createdAt) === Date.parse(checkpoint.lastCreatedAt) && latest.messageId.localeCompare(checkpoint.lastMessageId) > 0) + || checkpoint.contentHash !== contentHash; + store.saveConversationMeta({ sourceId, conversationId: currentId, lastMessageId: latest.messageId, lastCreatedAt: latest.createdAt, contentHash, selected }); + }; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + if (message.conversationId !== currentId) { + flushTurn(); + flush(); + currentId = message.conversationId; + currentTurn = []; + turnIndex = 0; + hash = createHash("sha256"); + hash.update("["); + first = true; + } + if (message.role === "user" && currentTurn.length > 0) { + flushTurn(); + currentTurn = []; + } + currentTurn.push(message); + if (!first) hash.update(","); + first = false; + hash.update(JSON.stringify({ messageId: message.messageId, role: message.role, content: message.content, createdAt: message.createdAt, toolName: hashMetaString(message, "toolName") ?? hashMetaString(message, "hermesToolName"), toolCallId: hashMetaString(message, "toolCallId") ?? hashMetaString(message, "hermesToolCallId") })); + latest = message; + } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; + } + flushTurn(); + if (currentId && latest) flush(); +} + +function hashMetaString(message: ConversationMessage, key: string): string | undefined { + const value = message.rawMeta[key]; + return typeof value === "string" ? value : undefined; +} + +async function ingestPersistentSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + sourceId: string, + scanOptions: AgentSourceScanOptions, + initialErrors: readonly { conversationId: string; reason: string }[] +): Promise<{ memoryIds: string[]; memoryIdCount: number; deduped: number; errorCount: number; errors: Array<{ conversationId: string; reason: string }>; latestSeenAt: string | null }> { + const memoryIds: string[] = []; + let memoryIdCount = 0; + const pendingIds: string[] = []; + const errors = initialErrors.slice(0, 1000); + let errorCount = initialErrors.length; + let deduped = 0; + let latestSeenAt: string | null = null; + let activeConversationId: string | null = null; + let activeConversationFailed = false; + const commitConversation = () => { + if (!activeConversationId || activeConversationFailed) return; + const meta = store.getConversationMeta(sourceId, activeConversationId); + if (!meta) return; + const updatedAt = new Date().toISOString(); + const checkpoint = { sourceId, conversationId: activeConversationId, lastMessageId: meta.lastMessageId, lastCreatedAt: meta.lastCreatedAt, contentHash: meta.contentHash, updatedAt }; + store.saveCheckpoint(checkpoint); + options.agentSourceRepository.upsertConversationCheckpoint(checkpoint); + }; + const pages = (async function*() { + let cursor: { conversationId: string; createdAt: string; messageId: string; ordinal: number } | undefined; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + yield message; + } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; + } + })(); + for await (const turn of orderedTurns(pages)) { + scanOptions.signal?.throwIfAborted(); + const turnLatest = turn.messages[turn.messages.length - 1]?.createdAt ?? null; + if (turnLatest && (!latestSeenAt || Date.parse(turnLatest) > Date.parse(latestSeenAt))) latestSeenAt = turnLatest; + if (turn.conversationId !== activeConversationId) { + commitConversation(); + activeConversationId = turn.conversationId; + activeConversationFailed = false; + } + const conversationMeta = store.getConversationMeta(sourceId, turn.conversationId); + if (conversationMeta?.selected === false) continue; + const selectedTurn = store.getTurnMeta(sourceId, turn.conversationId, stableTurnIdentity(turn)); + if (selectedTurn && !selectedTurn.selected) continue; + let turnSucceeded = true; + // Leave ample room for JSON escaping and the add-memory envelope while + // keeping every request below the 1 MiB wire limit. + const parts = splitTurn(turn, 4000, 512 * 1024); + for (const part of parts) { + const contentHash = createHash("sha256").update(part.content).digest("hex"); + const requestId = parts.length === 1 + ? legacyTurnRequestId(turn) + : createHash("sha256").update([stableTurnIdentity(turn), String(part.partIndex), contentHash].join("\u0000")).digest("hex"); + const turnId = parts.length === 1 ? legacyTurnId(turn) : `${sourceId}:${part.parentTurnId}:${part.partIndex}`; + try { + const added = await options.memoryClient.addMemory({ + requestId, + adapterId: `agent-source:${sourceId}`, + content: part.content, + layer: "L1", + title: firstTurnLine(part.messages) ?? `${sourceId} conversation`, + tags: ["agent-source", sourceId], + source: sourceId, + turnId, + createdAt: part.messages[0]!.createdAt, + deferProcessing: true + }); + if (added.duplicate) deduped += part.messages.length; + else { + memoryIdCount += 1; + if (memoryIds.length < 1000) memoryIds.push(added.id); + pendingIds.push(added.id); + if (pendingIds.length >= IMPORT_PROCESSING_COHORT_SIZE) { + const cohort = pendingIds.splice(0, pendingIds.length); + const failures = await processPendingImportSummaries(options, cohort, { ...scanOptions, progressSourceId: sourceId }); + if (failures.length > 0) activeConversationFailed = true; + const mapped = failures.map((failure) => ({ conversationId: failure.memoryId, reason: failure.reason })); + errorCount += mapped.length; + errors.push(...mapped.slice(0, Math.max(0, 1000 - errors.length))); + for (const failure of failures) store.saveResult({ sourceId, conversationId: failure.memoryId, error: failure.reason }); + } + } + store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId: added.id }); + } catch (error) { + turnSucceeded = false; + activeConversationFailed = true; + const reason = error instanceof Error ? error.message : "Agent source ingestion failed"; + errorCount += 1; + if (errors.length < 1000) errors.push({ conversationId: turn.conversationId, reason }); + store.saveResult({ sourceId, conversationId: turn.conversationId, error: reason }); + } + } + if (!turnSucceeded) activeConversationFailed = true; + emitProgress(scanOptions, { sourceId, phase: "add", current: memoryIds.length + deduped, total: store.count(sourceId), message: "Adding raw memories" }); + } + if (pendingIds.length > 0) { + const failures = await processPendingImportSummaries(options, pendingIds, { ...scanOptions, progressSourceId: sourceId }); + if (failures.length > 0) activeConversationFailed = true; + const mapped = failures.map((failure) => ({ conversationId: failure.memoryId, reason: failure.reason })); + errorCount += mapped.length; + errors.push(...mapped.slice(0, Math.max(0, 1000 - errors.length))); + for (const failure of failures) store.saveResult({ sourceId, conversationId: failure.memoryId, error: failure.reason }); + } + commitConversation(); + return { memoryIds, memoryIdCount, deduped, errorCount, errors, latestSeenAt }; +} + +function readScanPage(store: AppAgentSourceScanStore, sourceId: string, cursor?: { conversationId: string; createdAt: string; messageId: string; ordinal: number }): ConversationMessage[] { + const page: ConversationMessage[] = []; + let bytes = 0; + for (const message of store.messages(sourceId, cursor, 500)) { + page.push(message); + bytes += Buffer.byteLength(JSON.stringify(message)); + if (page.length >= 500 || bytes >= 8 * 1024 * 1024) break; + } + return page; +} + +function firstTurnLine(messages: readonly ConversationMessage[]): string | undefined { + const value = messages.find((message) => message.role === "user")?.content; + const line = value?.split(/\r?\n/).map((part) => part.trim()).find(Boolean); + return line ? (line.length <= 120 ? line : `${line.slice(0, 117)}...`) : undefined; +} + +function updatePersistentWatermark(options: CreateAgentSourceServiceOptions, sourceId: string, mode: AgentSourceScanMode, latestSeenAt: string | null, scannedAt: string, since?: string): void { + const existing = options.agentSourceRepository.getScanWatermark(sourceId); + options.agentSourceRepository.upsertScanWatermark({ sourceId, mode, baselineAt: existing?.baselineAt ?? since ?? scannedAt, latestSeenCreatedAt: maxIso(existing?.latestSeenCreatedAt ?? null, latestSeenAt), updatedAt: scannedAt }); +} + export interface CollectedSourceScan { sourceId: string; scanMode?: AgentSourceScanMode; @@ -692,7 +1112,7 @@ async function ingestCollectedSource( ) { updateScanWatermark(options, collected, scanOptions, scannedAt); } - errors.push(...await ingestSourceSkills(options, collected.sourceId, scanOptions)); + errors.push(...(await ingestSourceSkills(options, collected.sourceId, scanOptions)).errors); return { sourceId: collected.sourceId, discoveredConversations: collected.conversationIds.length, @@ -703,28 +1123,39 @@ async function ingestCollectedSource( }; } +interface SkillScanOutcome { + errors: Array<{ conversationId: string; reason: string }>; + errorCount: number; + memoryIdCount: number; +} + async function ingestSourceSkills( options: CreateAgentSourceServiceOptions, sourceId: string, - scanOptions: AgentSourceScanOptions -): Promise> { - if (!options.skillDistributionService.listSkills) return []; + scanOptions: AgentSourceScanOptions, + store?: AppAgentSourceScanStore +): Promise { + if (!options.skillDistributionService.listSkills) return { errors: [], errorCount: 0, memoryIdCount: 0 }; const errors: Array<{ conversationId: string; reason: string }> = []; + let errorCount = 0; + let memoryIdCount = 0; let skills; try { skills = await options.skillDistributionService.listSkills(sourceId); } catch (error) { - return [{ + const detail = { conversationId: "skills", reason: error instanceof Error ? error.message : "Agent Skill scan failed" - }]; + }; + store?.saveResult({ sourceId, conversationId: detail.conversationId, error: detail.reason }); + return { errors: [detail], errorCount: 1, memoryIdCount: 0 }; } for (const skill of skills) { scanOptions.signal?.throwIfAborted(); try { - await options.memoryClient.addMemory({ + const added = await options.memoryClient.addMemory({ requestId: `agent-source-skill:${sourceId}:${skill.sourceSkillId}:${skill.sourceContentHash}`, adapterId: `agent-source:${sourceId}`, content: skill.content, @@ -740,14 +1171,19 @@ async function ingestSourceSkills( sourceSkillVersion: skill.sourceSkillVersion, sourceContentHash: skill.sourceContentHash }); + memoryIdCount += 1; + store?.saveResult({ sourceId, conversationId: `skill:${skill.sourceSkillId}`, memoryId: added.id }); } catch (error) { - errors.push({ + errorCount += 1; + const detail = { conversationId: `skill:${skill.sourceSkillId}`, reason: error instanceof Error ? error.message : "Agent Skill import failed" - }); + }; + store?.saveResult({ sourceId, conversationId: detail.conversationId, error: detail.reason }); + if (errors.length < 1000) errors.push(detail); } } - return errors; + return { errors, errorCount, memoryIdCount }; } function filterCheckpointedConversations( diff --git a/App/backend/src/services/index.ts b/App/backend/src/services/index.ts index f697d0e8c..bdd32dabe 100644 --- a/App/backend/src/services/index.ts +++ b/App/backend/src/services/index.ts @@ -1,4 +1,5 @@ import type { AccountChannel } from "@memmy/local-api-contracts"; +import { dirname, join } from "node:path"; import type { AppStateStore } from "../infrastructure/app-state-store/index.js"; import { type MemmyConfigWriter } from "../infrastructure/memmy-config/index.js"; import type { ScanPreferencesStore } from "../infrastructure/memmy-config/agent-access.js"; @@ -165,6 +166,7 @@ export function createBackendServices(options: CreateBackendServicesOptions): Ba getUserId: resolveAnalyticsUserId, getUserMode: resolveAnalyticsUserMode, }), + scanStoreDirectory: join(dirname(options.appStateStore.databasePath), "agent-source-scans"), }); const toolConnectionAnalytics = createToolConnectionAnalytics({ getUserId: resolveAnalyticsUserId, diff --git a/App/frontend/desktop/src/api/agent-source-client.ts b/App/frontend/desktop/src/api/agent-source-client.ts index d3a747388..5bae7edbf 100644 --- a/App/frontend/desktop/src/api/agent-source-client.ts +++ b/App/frontend/desktop/src/api/agent-source-client.ts @@ -4,6 +4,7 @@ import { AgentSourceScanJobResponseSchema, AgentSourceScanInputSchema, AgentSourceScanStatusResponseSchema, + ScanResultPageSchema, AgentSourceViewSchema, ManagedAgentSourceImportResultSchema, OkResponseSchema, @@ -15,6 +16,7 @@ import { type AgentSourceScanInput, type AgentSourceScanStatusResponse, type AgentSourceView, + type ScanResultPage, type ManagedAgentSourceImportResult, type RuntimeConfig } from "@memmy/local-api-contracts"; @@ -24,6 +26,7 @@ export interface AgentSourceClient { listSources(): Promise; startScan(input?: AgentSourceScanInput): Promise; getScanStatus(): Promise; + getScanResults(jobId: string, cursor?: string, limit?: number): Promise; stopScan(): Promise; cancelScan(): Promise; addManualSource(input: AddManualInput): Promise; @@ -63,6 +66,14 @@ export function createHttpAgentSourceClient(config: RuntimeConfig): AgentSourceC }); }, + async getScanResults(jobId, cursor = "0", limit = 100) { + return requestJson({ + config, + path: `/api/agent-sources/scan/jobs/${encodeURIComponent(jobId)}/results?cursor=${encodeURIComponent(cursor)}&limit=${encodeURIComponent(String(limit))}`, + schema: ScanResultPageSchema + }); + }, + async getMemoryPluginConflicts() { const response = await requestJson({ config, diff --git a/Memory/package.json b/Memory/package.json index ede0f6d54..97359eea7 100644 --- a/Memory/package.json +++ b/Memory/package.json @@ -8,7 +8,7 @@ "memmy-memory": "./dist/src/cli/index.js" }, "scripts": { - "build": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && npm run viewer:build && tsc -p tsconfig.json && npm run integration:build:dist && npm run copy-cli-assets", + "build": "npm run build -w @memmy/agent-source-core && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && npm run viewer:build && tsc -p tsconfig.json && npm run integration:build:dist && npm run copy-cli-assets", "postbuild": "node src/cli/scripts/set-executable.mjs dist/src/cli/index.js", "copy-cli-assets": "node src/cli/scripts/copy-assets.mjs", "dev": "tsx src/server/index.ts", @@ -19,10 +19,10 @@ "preserve:dev": "npm run integration:build", "worker:run": "node dist/src/cli/index.js raw POST /worker/run", "test": "vitest run --dir tests", - "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p viewer/tsconfig.json --noEmit", + "typecheck": "npm run build -w @memmy/agent-source-core && tsc -p tsconfig.json --noEmit && tsc -p viewer/tsconfig.json --noEmit", "viewer:build": "vite build --config viewer/vite.config.ts", "viewer:dev": "vite --config viewer/vite.config.ts --host 127.0.0.1", - "pretest": "npm run viewer:build && npm run integration:build", + "pretest": "npm run build -w @memmy/agent-source-core && npm run viewer:build && npm run integration:build", "integration:build": "node src/agent-source/integration/workspace-bridge/build-runtime.mjs", "integration:build:dist": "node src/agent-source/integration/workspace-bridge/build-runtime.mjs --dist", "package:npm": "node src/cli/npm/build-package.mjs", @@ -35,6 +35,7 @@ "node": ">=20" }, "dependencies": { + "@memmy/agent-source-core": "0.0.0", "@huggingface/transformers": "^3.8.0", "better-sqlite3": "^12.6.3", "dotenv": "^16.6.1", diff --git a/Memory/src/agent-source/adapters/claude-code/adapter.ts b/Memory/src/agent-source/adapters/claude-code/adapter.ts index b358a4d15..d18b5fc42 100644 --- a/Memory/src/agent-source/adapters/claude-code/adapter.ts +++ b/Memory/src/agent-source/adapters/claude-code/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveClaudeCodeProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverClaudeCodeSessions } from "./project-discovery.js"; @@ -68,13 +68,13 @@ export function createClaudeCodeSourceAdapter(deps: CreateClaudeCodeSourceAdapte message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readClaudeCodeTranscript(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/Memory/src/agent-source/adapters/claude-code/project-discovery.ts b/Memory/src/agent-source/adapters/claude-code/project-discovery.ts index 0bff8b96e..29d88e7e1 100644 --- a/Memory/src/agent-source/adapters/claude-code/project-discovery.ts +++ b/Memory/src/agent-source/adapters/claude-code/project-discovery.ts @@ -34,7 +34,7 @@ export async function discoverClaudeCodeSessions( const projectPath = join(options.root, projectEntry.name); const files = await readDirectoryIfExists(projectPath); for (const file of files) { - if (!file.isFile() || !file.name.endsWith(".jsonl")) { + if (!file.isFile() || !file.name.endsWith(".jsonl") || /\.jsonl\.bak-/u.test(file.name)) { continue; } diff --git a/Memory/src/agent-source/adapters/codex/adapter.ts b/Memory/src/agent-source/adapters/codex/adapter.ts index 3b9680ae4..62b5dcab9 100644 --- a/Memory/src/agent-source/adapters/codex/adapter.ts +++ b/Memory/src/agent-source/adapters/codex/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCodexSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readCodexRollout, type RawCodexMessage } from "./rollout-reader.js"; @@ -67,13 +67,13 @@ export function createCodexSourceAdapter(deps: CreateCodexSourceAdapterDeps = {} message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readCodexRollout(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/Memory/src/agent-source/adapters/codex/rollout-reader.ts b/Memory/src/agent-source/adapters/codex/rollout-reader.ts index 956d10c4f..6dd193392 100644 --- a/Memory/src/agent-source/adapters/codex/rollout-reader.ts +++ b/Memory/src/agent-source/adapters/codex/rollout-reader.ts @@ -2,6 +2,8 @@ import { basename } from "node:path"; import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; +const MAX_TOOL_NAME_ENTRIES = 4096; + export interface RawCodexMessage { /** Message id. */ messageId: string; @@ -73,6 +75,10 @@ function toToolMessage( const name = getString(payload.name) ?? "tool"; if (callId) { toolNamesByCallId.set(callId, name); + if (toolNamesByCallId.size > MAX_TOOL_NAME_ENTRIES) { + const oldest = toolNamesByCallId.keys().next().value; + if (typeof oldest === "string") toolNamesByCallId.delete(oldest); + } } return { messageId: `${rolloutId}:${lineNumber}`, diff --git a/Memory/src/agent-source/adapters/codex/session-discovery.ts b/Memory/src/agent-source/adapters/codex/session-discovery.ts index 57d2b41df..b6d0bbfbd 100644 --- a/Memory/src/agent-source/adapters/codex/session-discovery.ts +++ b/Memory/src/agent-source/adapters/codex/session-discovery.ts @@ -56,7 +56,7 @@ async function listRolloutFiles( continue; } - if (entry.isFile() && entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } diff --git a/Memory/src/agent-source/adapters/conversation-window.ts b/Memory/src/agent-source/adapters/conversation-window.ts index 78b331deb..038c2965e 100644 --- a/Memory/src/agent-source/adapters/conversation-window.ts +++ b/Memory/src/agent-source/adapters/conversation-window.ts @@ -45,6 +45,28 @@ export async function collectConversationWindow included.has(message.conversationId)); } +/** Streams a source target without materializing its complete history. */ +export async function* streamConversationWindow( + input: AsyncIterable, + since?: string, + signal?: AbortSignal, + maxMessages?: number, + fullHistory = false +): AsyncIterable { + if (maxMessages !== undefined && maxMessages <= 0) return; + if (fullHistory) { + let emitted = 0; + for await (const message of input) { + signal?.throwIfAborted(); + if (maxMessages !== undefined && emitted >= maxMessages) break; + emitted += 1; + yield message; + } + return; + } + for (const message of await collectConversationWindow(input, since, signal, maxMessages)) yield message; +} + export function remainingMessageCapacity(limit: number | undefined, emitted: number): number | undefined { return limit === undefined ? undefined : Math.max(0, limit - emitted); } diff --git a/Memory/src/agent-source/adapters/cursor/adapter.ts b/Memory/src/agent-source/adapters/cursor/adapter.ts index 3db120766..f46b4255b 100644 --- a/Memory/src/agent-source/adapters/cursor/adapter.ts +++ b/Memory/src/agent-source/adapters/cursor/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCursorDataPaths } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; +import { readCursorVscdb, streamCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; import { discoverCursorWorkspaces, type CursorWorkspace } from "./workspace-discovery.js"; const CURSOR_SOURCE_ID = "cursor"; @@ -84,13 +84,13 @@ export function createCursorSourceAdapter(deps: CreateCursorSourceAdapterDeps = message: target.storageHash }); - const messages = await collectConversationWindow( - readCursorVscdb(target.stateDbPath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamCursorVscdb(target.stateDbPath) : readCursorVscdb(target.stateDbPath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/Memory/src/agent-source/adapters/cursor/vscdb-reader.ts b/Memory/src/agent-source/adapters/cursor/vscdb-reader.ts index b2ea463d5..1806e6e01 100644 --- a/Memory/src/agent-source/adapters/cursor/vscdb-reader.ts +++ b/Memory/src/agent-source/adapters/cursor/vscdb-reader.ts @@ -3,6 +3,7 @@ import Database from "better-sqlite3"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw cursor message. */ export interface RawCursorMessage { @@ -61,6 +62,36 @@ export async function* readCursorVscdb(path: string): AsyncIterable { + const db = new Database(path, { readonly: true }); + try { + if (hasTable(db, "ItemTable")) { + const statement = db.prepare("SELECT key, value FROM ItemTable WHERE value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + for (const message of extractMessagesFromItemRow(row)) yield message; + } + } + if (hasTable(db, "cursorDiskKV")) { + const statement = db.prepare("SELECT key, value FROM cursorDiskKV WHERE key LIKE 'bubbleId:%' AND value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + const message = extractMessageFromBubbleRow(row); + if (message) yield message; + } + } + } finally { + db.close(); + } +} + /** Reads read item table messages. */ async function readItemTableMessages(db: Database.Database): Promise { if (!hasTable(db, "ItemTable")) { @@ -76,7 +107,7 @@ async function readItemTableMessages(db: Database.Database): Promise MAX_RECORD_BYTES) continue; const message = extractMessageFromBubbleRow(row); if (message) { messages.push(message); diff --git a/Memory/src/agent-source/adapters/deepseek-harness/adapter.ts b/Memory/src/agent-source/adapters/deepseek-harness/adapter.ts index 278fdc8f2..62fcee55b 100644 --- a/Memory/src/agent-source/adapters/deepseek-harness/adapter.ts +++ b/Memory/src/agent-source/adapters/deepseek-harness/adapter.ts @@ -1,11 +1,11 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveDeepseekHarnessHomeDirectory, resolveDeepseekHarnessSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverDeepseekHarnessSessions } from "./session-discovery.js"; -import { readDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; +import { streamDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; const SOURCE_ID = "deepseek_harness"; @@ -61,13 +61,13 @@ export function createDeepseekHarnessSourceAdapter( total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( - toAsyncIterable(await readDeepseekHarnessSession(session.sessionFilePath, options.signal)), + for await (const rawMessage of streamConversationWindow( + streamDeepseekHarnessSession(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { options.signal?.throwIfAborted(); emittedMessages += 1; options.onProgress?.({ sourceId: SOURCE_ID, phase: "emit", current: emittedMessages, total: emittedMessages }); @@ -91,10 +91,6 @@ function toConversationMessage( }; } -async function* toAsyncIterable(values: readonly T[]): AsyncIterable { - for (const value of values) yield value; -} - function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error; } diff --git a/Memory/src/agent-source/adapters/deepseek-harness/session-reader.ts b/Memory/src/agent-source/adapters/deepseek-harness/session-reader.ts index 63efdcba2..00b5dae73 100644 --- a/Memory/src/agent-source/adapters/deepseek-harness/session-reader.ts +++ b/Memory/src/agent-source/adapters/deepseek-harness/session-reader.ts @@ -1,6 +1,8 @@ +import { createReadStream } from "node:fs"; import { readFile } from "node:fs/promises"; import { basename } from "node:path"; -import { decompress, ZstdErrorCode } from "fzstd"; +import { decompress, Decompress, ZstdErrorCode } from "fzstd"; +import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; const ZSTD_FRAME_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); @@ -25,6 +27,108 @@ export async function readDeepseekHarnessSession( return parseSessionRows(text, filePath, signal); } +/** Streams uncompressed sessions; compressed legacy files use the existing decoder. */ +export async function* streamDeepseekHarnessSession( + filePath: string, + signal?: AbortSignal +): AsyncIterable { + if (filePath.endsWith(".zstd")) { + let conversationId = basename(filePath).replace(/\.jsonl\.zstd$/u, ""); + let workspacePath: string | null = null; + for await (const record of streamZstdJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } + return; + } + let conversationId = basename(filePath).replace(/\.jsonl$/u, ""); + let workspacePath: string | null = null; + for await (const record of readJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } +} + +/** Streams zstd frames and parses bounded JSONL records without materializing the file. */ +async function* streamZstdJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { + const input = createReadStream(filePath); + const output: Buffer[] = []; + let outputBytes = 0; + let carry: Buffer = Buffer.alloc(0); + let overLimit = false; + const decoder = new Decompress((chunk) => { + outputBytes += chunk.byteLength; + if (outputBytes > 8 * 1024 * 1024) throw new Error("DeepSeek Harness decompressed chunk exceeds 8 MiB staging limit"); + output.push(Buffer.from(chunk)); + }); + try { + for await (const chunk of input) { + signal?.throwIfAborted(); + decoder.push(chunk as Buffer); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length > 64 * 1024 * 1024) continue; + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + if (carry.length > 64 * 1024 * 1024) { carry = Buffer.alloc(0); overLimit = true; } + } + } + decoder.push(new Uint8Array(), true); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + } + } + if (!overLimit && carry.length > 0 && carry.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(carry); + if (parsed) yield parsed; + } + } finally { + input.destroy(); + } +} + +function parseJsonObject(line: Buffer): JsonObject | null { + try { + const parsed = JSON.parse(line.toString("utf8").trim()) as unknown; + return isRecord(parsed) ? parsed as JsonObject : null; + } catch { + return null; + } +} + function decompressFrames(bytes: Buffer): string { if (!bytes.subarray(0, ZSTD_FRAME_MAGIC.length).equals(ZSTD_FRAME_MAGIC)) { throw new Error("DeepSeek Harness session has no Zstandard frame header"); diff --git a/Memory/src/agent-source/adapters/hermes/adapter.ts b/Memory/src/agent-source/adapters/hermes/adapter.ts index cdbc95155..77e3d1ce9 100644 --- a/Memory/src/agent-source/adapters/hermes/adapter.ts +++ b/Memory/src/agent-source/adapters/hermes/adapter.ts @@ -2,7 +2,7 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveHermesHomeDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readHermesRollout, type RawHermesRolloutMessage } from "./rollout-reader.js"; @@ -79,13 +79,13 @@ export function createHermesSourceAdapter(deps: CreateHermesSourceAdapterDeps = ? streamJsonlMessages(target.session, options.signal) : streamStateDbMessages(target.stateDbPath); - const messages = await collectConversationWindow( + for await (const message of streamConversationWindow( iterable, options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const message of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/Memory/src/agent-source/adapters/hermes/session-discovery.ts b/Memory/src/agent-source/adapters/hermes/session-discovery.ts index 36e5b2f57..5153c022e 100644 --- a/Memory/src/agent-source/adapters/hermes/session-discovery.ts +++ b/Memory/src/agent-source/adapters/hermes/session-discovery.ts @@ -58,7 +58,7 @@ async function listJsonlFiles(root: string, order: "path_asc" | "recent_first", continue; } - if (entry.isFile() && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } diff --git a/Memory/src/agent-source/adapters/jsonl-lines.ts b/Memory/src/agent-source/adapters/jsonl-lines.ts index 7d3ef80a6..f6a9cea41 100644 --- a/Memory/src/agent-source/adapters/jsonl-lines.ts +++ b/Memory/src/agent-source/adapters/jsonl-lines.ts @@ -1,6 +1,5 @@ /** Jsonl lines module. */ import { createReadStream } from "node:fs"; -import { createInterface } from "node:readline"; export type JsonPrimitive = string | number | boolean | null; export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; @@ -15,34 +14,60 @@ export type JsonObject = { readonly [key: string]: JsonValue }; * @returns The JSON objects parsed line by line. */ export async function* readJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ - input: stream, - crlfDelay: Number.POSITIVE_INFINITY - }); + const stream = createReadStream(filePath); + const maxRecordBytes = 64 * 1024 * 1024; + let segments: Buffer[] = []; + let recordBytes = 0; + let overLimit = false; + const append = (segment: Buffer): void => { + if (overLimit || segment.length === 0) return; + recordBytes += segment.length; + if (recordBytes > maxRecordBytes) { + segments = []; + overLimit = true; + return; + } + segments.push(segment); + }; + const reset = (): void => { + segments = []; + recordBytes = 0; + overLimit = false; + }; + const parseSegments = (): JsonObject | null => { + if (overLimit) return null; + const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); + const text = line.toString("utf8").trim(); + if (!text) return null; + try { + const parsed = JSON.parse(text) as unknown; + return isJsonObject(parsed) ? parsed : null; + } catch { + return null; + } + }; try { - for await (const line of lines) { + for await (const chunk of stream) { throwIfAborted(signal, filePath); - if (line.trim().length === 0) { - continue; - } - - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - continue; + const buffer = chunk as Buffer; + let start = 0; + while (start <= buffer.length) { + const newline = buffer.indexOf(0x0a, start); + if (newline < 0) { + append(buffer.subarray(start)); + break; + } + append(buffer.subarray(start, newline)); + const parsed = parseSegments(); + reset(); + if (parsed) yield parsed; + start = newline + 1; } - - if (!isJsonObject(parsed)) { - continue; - } - - yield parsed; } + const parsed = parseSegments(); + if (parsed) yield parsed; } finally { - lines.close(); stream.destroy(); } } diff --git a/Memory/src/agent-source/adapters/jsonl-session-files.ts b/Memory/src/agent-source/adapters/jsonl-session-files.ts index 2a95d9527..6123da83a 100644 --- a/Memory/src/agent-source/adapters/jsonl-session-files.ts +++ b/Memory/src/agent-source/adapters/jsonl-session-files.ts @@ -33,7 +33,7 @@ export async function discoverJsonlSessionFiles( const path = join(directory, entry.name); if (entry.isDirectory()) { directories.push(path); - } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + } else if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { files.push({ path, mtimeMs: (await stat(path)).mtimeMs }); } } diff --git a/Memory/src/agent-source/adapters/openclaw/adapter.ts b/Memory/src/agent-source/adapters/openclaw/adapter.ts index 1e96b4439..23c1c5a1e 100644 --- a/Memory/src/agent-source/adapters/openclaw/adapter.ts +++ b/Memory/src/agent-source/adapters/openclaw/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpenclawStateDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverOpenclawDatabases } from "./db-discovery.js"; @@ -65,13 +65,13 @@ export function createOpenclawSourceAdapter(deps: CreateOpenclawSourceAdapterDep message: database.databasePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readOpenclawDatabase(database.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/Memory/src/agent-source/adapters/opencode/adapter.ts b/Memory/src/agent-source/adapters/opencode/adapter.ts index e71486ba2..293fa1e94 100644 --- a/Memory/src/agent-source/adapters/opencode/adapter.ts +++ b/Memory/src/agent-source/adapters/opencode/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpencodeDatabasePath } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; +import { readOpencodeDatabase, streamOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; const OPENCODE_SOURCE_ID = "opencode"; @@ -58,13 +58,13 @@ export function createOpencodeSourceAdapter(deps: CreateOpencodeSourceAdapterDep message: target.databasePath }); - const messages = await collectConversationWindow( - readOpencodeDatabase(target.databasePath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamOpencodeDatabase(target.databasePath) : readOpencodeDatabase(target.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/Memory/src/agent-source/adapters/opencode/db-reader.ts b/Memory/src/agent-source/adapters/opencode/db-reader.ts index 48d8729fd..84172d833 100644 --- a/Memory/src/agent-source/adapters/opencode/db-reader.ts +++ b/Memory/src/agent-source/adapters/opencode/db-reader.ts @@ -5,6 +5,7 @@ import Database from "better-sqlite3"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw opencode database message. */ export interface RawOpencodeDatabaseMessage { @@ -58,6 +59,39 @@ export async function* readOpencodeDatabase(path: string): AsyncIterable { + const db = new Database(path, { readonly: true }); + try { + if (!hasTable(db, "message") || !hasTable(db, "part") || !hasTable(db, "session")) return; + const statement = db.prepare(` + SELECT m.id AS message_id, m.session_id AS session_id, m.time_created AS message_time_created, + m.data AS message_data, s.directory AS session_directory, p.id AS part_id, + p.time_created AS part_time_created, p.data AS part_data + FROM message m LEFT JOIN session s ON s.id = m.session_id LEFT JOIN part p ON p.message_id = m.id + ORDER BY m.session_id ASC, m.time_created ASC, m.id ASC, p.time_created ASC, p.id ASC + `); + let current: MessageAccumulator | null = null; + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (!current || current.messageId !== row.message_id) { + if (current?.contentParts.length) yield toDatabaseMessage(current); + current = Buffer.byteLength(row.message_data) <= MAX_RECORD_BYTES ? createAccumulator(row) : null; + } + if (!current || !row.part_data || !row.part_id || Buffer.byteLength(row.part_data) > MAX_RECORD_BYTES) continue; + const text = getPartText(parseJson(row.part_data)); + if (!text) continue; + current.partIds.push(row.part_id); + current.contentParts.push(text); + } + if (current?.contentParts.length) yield toDatabaseMessage(current); + } finally { + db.close(); + } +} + async function readMessages(db: Database.Database): Promise { const statement = db.prepare(` SELECT @@ -84,7 +118,7 @@ async function readMessages(db: Database.Database): Promise MAX_RECORD_BYTES) { continue; } @@ -117,6 +151,7 @@ function getOrCreateAccumulator( accumulators: Map, row: OpencodePartRow ): MessageAccumulator | null { + if (Buffer.byteLength(row.message_data) > MAX_RECORD_BYTES) return null; const existing = accumulators.get(row.message_id); if (existing) { return existing; @@ -135,7 +170,21 @@ function getOrCreateAccumulator( const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; const explicitRoot = getNestedString(messageData, "path", "root"); const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; - const accumulator: MessageAccumulator = { + const accumulator = createAccumulator(row); + if (!accumulator) return null; + accumulators.set(row.message_id, accumulator); + return accumulator; +} + +function createAccumulator(row: OpencodePartRow): MessageAccumulator | null { + const messageData = parseJson(row.message_data); + if (!isRecord(messageData)) return null; + const role = normalizeRole(messageData.role); + if (!role) return null; + const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; + const explicitRoot = getNestedString(messageData, "path", "root"); + const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; + return { messageId: row.message_id, conversationId: row.session_id, role, @@ -145,8 +194,19 @@ function getOrCreateAccumulator( partIds: [], contentParts: [] }; - accumulators.set(row.message_id, accumulator); - return accumulator; +} + +function toDatabaseMessage(message: MessageAccumulator): RawOpencodeDatabaseMessage { + return { + messageId: message.messageId, + conversationId: message.conversationId, + role: message.role, + content: message.contentParts.join("\n"), + createdAt: message.createdAt, + workspacePath: message.workspacePath, + gitRoot: message.gitRoot, + rawMeta: Object.freeze({ opencodePartIds: message.partIds }) + }; } function getPartText(partData: unknown): string | null { diff --git a/Memory/src/agent-source/adapters/pi/adapter.ts b/Memory/src/agent-source/adapters/pi/adapter.ts index b3ba25852..301455660 100644 --- a/Memory/src/agent-source/adapters/pi/adapter.ts +++ b/Memory/src/agent-source/adapters/pi/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolvePiAgentDirectory, resolvePiSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createPiSourceAdapter(deps: CreatePiSourceAdapterDeps = {}): Sou total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readPiHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/Memory/src/agent-source/adapters/qwenwork/adapter.ts b/Memory/src/agent-source/adapters/qwenwork/adapter.ts index 19e12d2d0..2d794c53e 100644 --- a/Memory/src/agent-source/adapters/qwenwork/adapter.ts +++ b/Memory/src/agent-source/adapters/qwenwork/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolveQwenworkHomeDirectory, resolveQwenworkProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createQwenworkSourceAdapter(deps: CreateQwenworkSourceAdapterDep total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readQwenworkHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/Memory/src/agent-source/adapters/types.ts b/Memory/src/agent-source/adapters/types.ts index ce4bce8bb..6259826b9 100644 --- a/Memory/src/agent-source/adapters/types.ts +++ b/Memory/src/agent-source/adapters/types.ts @@ -1,34 +1,20 @@ -/** Types module. */ +/** Types module. Shared with the backend scanner. */ +import type { + ConversationMessage as CoreConversationMessage, + SourceDescriptor as CoreSourceDescriptor, + ScanOptions as CoreScanOptions, + ScanProgress as CoreScanProgress, + SourceAdapter as CoreSourceAdapter +} from "@memmy/agent-source-core"; /** Contract for source descriptor. */ -export interface SourceDescriptor { - sourceId: string; - displayName: string; - builtin: boolean; - dataPath: string; -} +export type SourceDescriptor = CoreSourceDescriptor; /** Contract for conversation message. */ -export interface ConversationMessage { - messageId: string; - sourceId: string; - conversationId: string; - role: "user" | "assistant" | "tool" | "system"; - content: string; - createdAt: string; - workspacePath: string | null; - gitRoot: string | null; - rawMeta: Readonly>; -} +export type ConversationMessage = CoreConversationMessage; /** Contract for scan progress. */ -export interface ScanProgress { - sourceId: string; - phase: "discover" | "read" | "redact" | "emit" | "scan" | "add" | "summarize" | "done" | "stopped"; - current: number; - total: number; - message?: string; -} +export type ScanProgress = CoreScanProgress; /** Contract for scan result. */ export interface ScanResult { @@ -40,18 +26,7 @@ export interface ScanResult { } /** Contract for source adapter. */ -export interface SourceAdapter { - readonly descriptor: SourceDescriptor; - detect(): Promise; - scan(options: ScanOptions): AsyncIterable; -} +export type SourceAdapter = CoreSourceAdapter; /** Contract for scan options. */ -export interface ScanOptions { - since?: string; - maxMessages?: number; - maxScanTargets?: number; - order?: "source_default" | "recent_first"; - signal?: AbortSignal; - onProgress?: (progress: ScanProgress) => void; -} +export type ScanOptions = CoreScanOptions; diff --git a/Memory/src/agent-source/adapters/workbuddy/adapter.ts b/Memory/src/agent-source/adapters/workbuddy/adapter.ts index ae6d07c00..cf6e7bd97 100644 --- a/Memory/src/agent-source/adapters/workbuddy/adapter.ts +++ b/Memory/src/agent-source/adapters/workbuddy/adapter.ts @@ -4,7 +4,7 @@ import { resolveWorkbuddyHomeDirectory, resolveWorkbuddyProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readWorkbuddyHistory, type RawWorkbuddyMessage } from "./history-reader.js"; @@ -73,13 +73,13 @@ export function createWorkbuddySourceAdapter(deps: CreateWorkbuddySourceAdapterD message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readWorkbuddyHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/Memory/src/agent-source/adapters/workbuddy/history-reader.ts b/Memory/src/agent-source/adapters/workbuddy/history-reader.ts index 1b69de428..bb26c279b 100644 --- a/Memory/src/agent-source/adapters/workbuddy/history-reader.ts +++ b/Memory/src/agent-source/adapters/workbuddy/history-reader.ts @@ -1,6 +1,5 @@ -import { createReadStream } from "node:fs"; import { basename } from "node:path"; -import { createInterface } from "node:readline"; +import { readJsonlObjects } from "../jsonl-lines.js"; export interface RawWorkbuddyMessage { messageId: string; @@ -18,27 +17,13 @@ export async function* readWorkbuddyHistory( signal?: AbortSignal ): AsyncIterable { const fallbackConversationId = basename(filePath, ".jsonl"); - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Number.POSITIVE_INFINITY }); let lineNumber = 0; - - try { - for await (const line of lines) { - lineNumber += 1; - throwIfAborted(signal, filePath); - const record = parseRecord(line); - if (!record) { - continue; - } - - const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); - if (message) { - yield message; - } + for await (const record of readJsonlObjects(filePath, signal)) { + lineNumber += 1; + const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); + if (message) { + yield message; } - } finally { - lines.close(); - stream.destroy(); } } @@ -278,18 +263,6 @@ function formatStructuredValue(value: unknown): string { } } -function parseRecord(line: string): Record | null { - if (!line.trim()) { - return null; - } - try { - const parsed: unknown = JSON.parse(line); - return isRecord(parsed) ? parsed : null; - } catch { - return null; - } -} - function isRoleEvent(value: string): boolean { return ["user", "human", "assistant", "ai", "tool", "function", "system", "developer"].includes(value); } @@ -325,9 +298,3 @@ function recordValue(value: unknown): Record | null { function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } - -function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { - if (signal?.aborted) { - throw new DOMException(`WorkBuddy history read aborted: ${filePath}`, "AbortError"); - } -} diff --git a/Memory/src/agent-source/adapters/workbuddy/session-discovery.ts b/Memory/src/agent-source/adapters/workbuddy/session-discovery.ts index 03c78af87..f79e6ae2d 100644 --- a/Memory/src/agent-source/adapters/workbuddy/session-discovery.ts +++ b/Memory/src/agent-source/adapters/workbuddy/session-discovery.ts @@ -55,7 +55,7 @@ async function listJsonlFiles(root: string): Promise; cancelScan(): Promise<{ ok: true }>; + scanResults?(jobId: string, cursor?: string, limit?: number): Promise<{ items: Array<{ sourceId: string; conversationId: string; memoryId?: string; error?: string }>; nextCursor: string | null }>; mutateConnection(sourceId: string, kind: "plugin" | "skill", method: "POST" | "DELETE"): Promise; startAutomation(): void; dispose(): void; @@ -90,11 +105,11 @@ interface PersistedSourceState { messageCount: number; lastScannedAt: string | null; latestSeenAt: string | null; - importedRequestIds?: string[]; + contentHash?: string; } interface PersistedState { - version: 1; + version: 2; sources: Record; } @@ -107,6 +122,7 @@ export interface CreateAgentSourceExecutorOptions { scheduledScanIntervalMs?: number; scheduleWorker?: () => void; integrationRegistry?: SkillTargetRegistry; + scanStoreDirectory?: string; } export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOptions): AgentSourceExecutor { @@ -117,6 +133,7 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti ); const integrationRegistry = options.integrationRegistry ?? createBuiltinIntegrationRegistry(configPath); const statePath = options.statePath ?? join(dirname(configPath), "memory-service", "agent-sources.json"); + const scanStoreDirectory = options.scanStoreDirectory ?? join(dirname(statePath), "agent-source-scans"); let statePromise: Promise | undefined; let scan: AgentSourceScanState = emptyScanState(); let scanTimer: ReturnType | undefined; @@ -185,7 +202,8 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti if (scanPaused) { throw new MemoryServiceError("conflict", "Stop the paused Agent source scan before starting another scan"); } - const jobId = `agent-scan-${Date.now().toString(36)}`; + const jobId = findReusableScanJob(scanStoreDirectory, request.sourceId, request.mode) + ?? `agent-scan-${Date.now().toString(36)}`; scan = { running: true, jobId, @@ -200,7 +218,7 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti progressBeforePause = null; const controller = new AbortController(); scanAbortController = controller; - void runScan(request, controller.signal).then(() => { + void runScan(request, controller.signal, jobId).then(() => { if (scan.jobId !== jobId) return; scan = { ...scan, running: false, completedAt: new Date().toISOString() }; logger.info("scan.completed", { jobId, sourceId: request.sourceId }); @@ -228,83 +246,114 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti async function runScan( request: ReturnType, - signal: AbortSignal + signal: AbortSignal, + jobId: string ): Promise { const failures: string[] = []; + let failureCount = 0; const adapters = request.sourceId === "all" ? registry.list() : [registry.require(request.sourceId)]; const state = await readState(); - for (const adapter of adapters) { - await waitWhilePaused(signal); - signal.throwIfAborted(); - if (!(await adapter.detect())) { - if (request.sourceId !== "all") { - throw new MemoryServiceError("not_found", `${adapter.descriptor.displayName} is not installed`); - } - continue; - } - const stored = state.sources[adapter.descriptor.sourceId] ?? emptySourceState(); - const mode = request.mode ?? (stored.lastScannedAt ? "incremental" : "initial_subset"); - const messages: ConversationMessage[] = []; - for await (const message of adapter.scan({ - ...(mode === "incremental" && stored.latestSeenAt ? { since: stored.latestSeenAt } : {}), - ...(mode === "initial_subset" ? { maxMessages: INITIAL_SCAN_MESSAGE_LIMIT, maxScanTargets: INITIAL_SCAN_MESSAGE_LIMIT } : {}), - order: mode === "initial_subset" ? "recent_first" : "source_default", - signal, - onProgress(progress) { - if (!scanPaused) { - progressBeforePause = progress; - scan = { ...scan, progress }; - } - } - })) { + let store: MemoryAgentSourceScanStore | undefined; + let completed = false; + const preparedContentHashes = new Map(); + try { + store = await openMemoryAgentSourceScanStore(join(scanStoreDirectory, `${jobId}.sqlite`), { + jobId, sourceId: request.sourceId, mode: request.mode ?? "incremental", phase: "stage", + createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() + }); + const available: SourceAdapter[] = []; + for (const adapter of adapters) { await waitWhilePaused(signal); signal.throwIfAborted(); - messages.push(message); + if (await adapter.detect()) available.push(adapter); + else if (request.sourceId !== "all") throw new MemoryServiceError("not_found", `${adapter.descriptor.displayName} is not installed`); } - await waitWhilePaused(signal); - signal.throwIfAborted(); - const importedRequestIds = new Set(stored.importedRequestIds ?? []); - const result = ingestMessages( - options.service, - adapter.descriptor.sourceId, - messages, - importedRequestIds - ); - const skillResult = await ingestAgentSkills( - options.service, - adapter.descriptor.sourceId, - importedRequestIds - ); - failures.push(...result.errors, ...skillResult.errors); - const now = new Date().toISOString(); - state.sources[adapter.descriptor.sourceId] = { - ...stored, - messageCount: stored.messageCount + result.messageCount, - lastScannedAt: now, - latestSeenAt: maxCreatedAt(messages) ?? stored.latestSeenAt, - importedRequestIds: [...importedRequestIds] - }; - await persist(state); - const memoryIds = [...result.memoryIds, ...skillResult.memoryIds]; - if (memoryIds.length > 0) { - options.service.enqueuePendingImportSummaries(INITIAL_SCAN_MESSAGE_LIMIT, memoryIds); - options.scheduleWorker?.(); + const globalInitial = request.sourceId === "all" && available.length > 0 && + (request.mode === "initial_subset" || (request.mode === undefined && available.every((adapter) => !state.sources[adapter.descriptor.sourceId]?.lastScannedAt))); + const stages: StandaloneSourceStage[] = []; + for (const adapter of available) { + const stored = state.sources[adapter.descriptor.sourceId] ?? emptySourceState(); + const mode = request.mode ?? (stored.lastScannedAt ? "incremental" : "initial_subset"); + store.saveMeta({ jobId, sourceId: store.getMeta()?.sourceId ?? request.sourceId, mode, phase: "stage", createdAt: store.getMeta()?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }); + stages.push(await stageStandaloneSource(adapter, stored, mode, store, signal, () => waitWhilePaused(signal), (progress) => { + if (!scanPaused) { progressBeforePause = progress; scan = { ...scan, progress }; } + })); } - scan = { - ...scan, - progress: { - sourceId: adapter.descriptor.sourceId, - phase: "done", - current: messages.length, - total: messages.length, - message: `Imported ${result.written} memories and ${skillResult.written} skills` + for (const stage of stages) { + await waitWhilePaused(signal); + signal.throwIfAborted(); + const { sourceId, mode } = stage; + store.saveMeta({ jobId, sourceId: store.getMeta()?.sourceId ?? request.sourceId, mode, phase: "prepare", createdAt: store.getMeta()?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }); + const sourceState = store.getSourceState(sourceId); + store.saveSourceState({ ...(sourceState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: new Date().toISOString() }), phase: "prepare", updatedAt: new Date().toISOString() }); + preparedContentHashes.set(sourceId, await prepareStandaloneSource(store, sourceId, mode, stage.stored.latestSeenAt, stage.stored.contentHash)); + } + if (globalInitial) store.selectInitialTurns(stages.map((stage) => stage.sourceId), INITIAL_SCAN_MESSAGE_LIMIT, 200); + for (const stage of stages) { + await waitWhilePaused(signal); + signal.throwIfAborted(); + const { adapter, stored, mode, sourceId, staged } = stage; + if (mode === "initial_subset" && !globalInitial) store.selectInitialTurns([sourceId], INITIAL_SCAN_MESSAGE_LIMIT, 0); + store.saveMeta({ jobId, sourceId: store.getMeta()?.sourceId ?? request.sourceId, mode, phase: "ingest", createdAt: store.getMeta()?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }); + const preparedState = store.getSourceState(sourceId); + store.saveSourceState({ ...(preparedState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: new Date().toISOString() }), phase: "ingest", updatedAt: new Date().toISOString() }); + const result = await ingestStagedMessages(options.service, store, sourceId, signal, (progress) => { + if (!scanPaused) { progressBeforePause = progress; scan = { ...scan, progress }; } + }, options.scheduleWorker); + const skillResult = await ingestAgentSkills(options.service, sourceId, store, options.scheduleWorker); + const sourceErrorCount = stage.scanErrorCount + result.errorCount + skillResult.errorCount; + failureCount += sourceErrorCount; + for (const detail of [...stage.errors, ...result.errors, ...skillResult.errors]) { + if (failures.length >= 1000) break; + failures.push(detail); } - }; + const now = new Date().toISOString(); + state.sources[sourceId] = { + ...stored, + messageCount: stored.messageCount + result.messageCount, + lastScannedAt: now, + ...(stage.scanErrorCount === 0 && result.errorCount === 0 && skillResult.errorCount === 0 && preparedContentHashes.has(sourceId) + ? { contentHash: preparedContentHashes.get(sourceId) } + : stored.contentHash ? { contentHash: stored.contentHash } : {}), + latestSeenAt: stage.scanErrorCount === 0 && result.errorCount === 0 && skillResult.errorCount === 0 + ? (result.latestSeenAt ?? stored.latestSeenAt) + : stored.latestSeenAt + }; + await persist(state); + store.saveSourceState({ + sourceId, + mode, + phase: sourceErrorCount > 0 ? "failed" : "done", + messageCount: result.messageCount, + resultCount: store.resultCount(sourceId), + errorCount: sourceErrorCount, + updatedAt: now + }); + store.saveMeta({ jobId, sourceId: store.getMeta()?.sourceId ?? request.sourceId, mode, phase: "summarize", createdAt: store.getMeta()?.createdAt ?? now, updatedAt: now }); + scan = { ...scan, progress: { sourceId, phase: "done", current: staged, total: staged, message: `Imported ${result.written} memories and ${skillResult.written} skills` } }; + } + store.saveMeta({ jobId, sourceId: request.sourceId, mode: request.mode ?? "incremental", phase: "done", createdAt: store.getMeta()?.createdAt ?? new Date().toISOString(), updatedAt: new Date().toISOString() }); + if (failureCount > 0) { + const meta = store.getMeta(); + if (meta) store.saveMeta({ ...meta, phase: "failed", updatedAt: new Date().toISOString(), error: failures.slice(0, 3).join("; ") }); + } + completed = failureCount === 0 && store.resultCount() <= INITIAL_SCAN_MESSAGE_LIMIT; + } catch (error) { + if (store) { + const meta = store.getMeta(); + if (meta) store.saveMeta({ ...meta, phase: "failed", updatedAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) }); + } + throw error; + } finally { + if (store) { + if (completed) store.remove(); + else store.close(); + } } - if (failures.length > 0) { - throw new Error(`Agent source scan completed with ${failures.length} import failure${failures.length === 1 ? "" : "s"}: ${failures.slice(0, 3).join("; ")}`); + if (failureCount > 0) { + throw new Error(`Agent source scan completed with ${failureCount} import failure${failureCount === 1 ? "" : "s"}: ${failures.slice(0, 3).join("; ")}`); } } @@ -337,6 +386,14 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti const sourceId = activeScanRequest?.sourceId; scanPaused = false; controller?.abort(); + if (jobId) { + const path = join(scanStoreDirectory, `${jobId}.sqlite`); + await Promise.all([ + rm(path, { force: true }), + rm(`${path}-wal`, { force: true }), + rm(`${path}-shm`, { force: true }) + ]); + } resumePausedScan?.(); resumePausedScan = undefined; scan = emptyScanState(); @@ -346,6 +403,27 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti return { ok: true }; } + async function scanResults(jobId: string, cursor = "0", limit = 100): Promise<{ items: Array<{ sourceId: string; conversationId: string; memoryId?: string; error?: string }>; nextCursor: string | null }> { + const path = join(scanStoreDirectory, `${jobId}.sqlite`); + if (!existsSync(path)) return { items: [], nextCursor: null }; + const store = await openMemoryAgentSourceScanStore(path, { jobId, sourceId: "all", mode: "incremental", phase: "ingest", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + let removeAfterRead = false; + try { + const safeLimit = Math.min(500, Math.max(1, limit)); + const safeCursor = Number.isFinite(Number(cursor)) && Number(cursor) >= 0 ? String(Math.floor(Number(cursor))) : "0"; + const rows = [...store.results(undefined, safeCursor, safeLimit)]; + const items = rows.map(({ cursor: _cursor, ...item }) => item); + const nextCursor = rows.length < safeLimit ? null : rows.at(-1)?.cursor ?? null; + removeAfterRead = nextCursor === null && store.getMeta()?.phase === "done"; + return { items, nextCursor }; + } finally { + store.close(); + if (removeAfterRead) { + await Promise.all([rm(path, { force: true }), rm(`${path}-wal`, { force: true }), rm(`${path}-shm`, { force: true })]); + } + } + } + async function waitWhilePaused(signal: AbortSignal): Promise { while (scanPaused) { await new Promise((resolve, reject) => { @@ -439,6 +517,7 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti scanStatus: () => scan, pauseScan, cancelScan, + scanResults, mutateConnection, startAutomation() { if (scanTimer || disposed) return; @@ -463,6 +542,33 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti }; } +function findReusableScanJob(directory: string, sourceId: string, mode?: string): string | null { + if (!existsSync(directory)) return null; + let selected: { jobId: string; updatedAt: number } | null = null; + for (const name of readdirSync(directory).filter((value) => value.endsWith(".sqlite"))) { + const path = join(directory, name); + try { + const db = new Database(path, { readonly: true }); + const row = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, updated_at AS updatedAt FROM scan_meta WHERE id=1").get() as { jobId: string; sourceId: string; mode?: string; phase: string; updatedAt: string } | undefined; + db.close(); + if (row?.phase === "done") { + const updatedAt = Date.parse(row.updatedAt); + if (Number.isFinite(updatedAt) && Date.now() - updatedAt > COMPLETED_DETAILS_RETENTION_MS) { + rmSync(path, { force: true }); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); + } + continue; + } + if (row && row.sourceId === sourceId && (!mode || !row.mode || row.mode === mode)) { + const updatedAt = Date.parse(row.updatedAt); + if (!selected || updatedAt > selected.updatedAt) selected = { jobId: row.jobId, updatedAt }; + } + } catch { /* leave corrupt stores for explicit diagnostics */ } + } + return selected?.jobId ?? null; +} + function sameScanRequest( left: ReturnType, right: ReturnType @@ -500,82 +606,332 @@ export function createBuiltinIntegrationRegistry(configPath: string): SkillTarge ]); } -function ingestMessages( +interface StandaloneSourceStage { + adapter: SourceAdapter; + stored: PersistedSourceState; + sourceId: string; + mode: "initial_subset" | "incremental" | "full"; + staged: number; + scanErrorCount: number; + errors: string[]; +} + +async function stageStandaloneSource( + adapter: SourceAdapter, + stored: PersistedSourceState, + mode: "initial_subset" | "incremental" | "full", + store: MemoryAgentSourceScanStore, + signal: AbortSignal, + waitIfPaused: () => Promise, + onProgress: (progress: ScanProgress) => void +): Promise { + const sourceId = adapter.descriptor.sourceId; + const errors: string[] = []; + let scanErrorCount = 0; + const batch: ConversationMessage[] = []; + let batchBytes = 0; + let staged = 0; + let emittedOrdinal = 0; + store.saveSourceState({ + sourceId, + mode, + phase: "stage", + messageCount: store.count(sourceId), + resultCount: store.resultCount(sourceId), + errorCount: 0, + updatedAt: new Date().toISOString(), + ...(stored.latestSeenAt ? { watermarkedSince: stored.latestSeenAt } : {}) + }); + try { + for await (const message of adapter.scan({ + ...(mode === "incremental" && stored.latestSeenAt ? { since: stored.latestSeenAt } : {}), + order: mode === "initial_subset" ? "recent_first" : "source_default", + fullHistory: true, + signal, + onProgress + })) { + await waitIfPaused(); + signal.throwIfAborted(); + const bytes = Buffer.byteLength(JSON.stringify(message)); + if (bytes > 64 * 1024 * 1024) { + const reason = "record exceeds 64 MiB"; + scanErrorCount += 1; + if (errors.length < 1000) errors.push(`${sourceId}:${message.conversationId}: ${reason}`); + store.saveResult({ sourceId, conversationId: message.conversationId, error: reason }); + continue; + } + if (batch.length > 0 && (batch.length >= 500 || batchBytes + bytes > 8 * 1024 * 1024)) { + staged += store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch.length = 0; + batchBytes = 0; + } + batch.push({ ...message, ordinal: emittedOrdinal++ }); + batchBytes += bytes; + if (batch.length >= 500 || batchBytes >= 8 * 1024 * 1024) { + staged += store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch.length = 0; + batchBytes = 0; + } + } + if (batch.length > 0) { + staged += store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + } + } catch (error) { + if (signal.aborted) throw error; + scanErrorCount += 1; + const reason = error instanceof Error ? error.message : "Agent source scan failed"; + if (errors.length < 1000) errors.push(`${sourceId}: ${reason}`); + store.saveResult({ sourceId, conversationId: "scan", error: reason }); + } + store.saveSourceState({ + sourceId, + mode, + phase: scanErrorCount > 0 ? "failed" : "stage", + messageCount: store.count(sourceId), + resultCount: store.resultCount(sourceId), + errorCount: scanErrorCount, + updatedAt: new Date().toISOString(), + ...(stored.latestSeenAt ? { watermarkedSince: stored.latestSeenAt } : {}) + }); + return { adapter, stored, sourceId, mode, staged, scanErrorCount, errors }; +} + +async function ingestStagedMessages( service: MemoryService, + store: MemoryAgentSourceScanStore, sourceId: string, - messages: readonly ConversationMessage[], - importedRequestIds: Set -): { written: number; messageCount: number; memoryIds: string[]; errors: string[] } { - const memoryIds: string[] = []; - const errors: string[] = []; + signal: AbortSignal, + onProgress: (progress: ScanProgress) => void, + scheduleWorker?: () => void +): Promise<{ written: number; messageCount: number; errors: string[]; errorCount: number; latestSeenAt: string | null }> { + let written = 0; let messageCount = 0; - for (const turn of completeTurns(messages)) { - const content = turn - .map((message) => `## ${message.role}\n\n${renderMessageContent(message)}`) - .join("\n\n"); - const identity = `${sourceId}::${turn[0]!.conversationId}::${turn[0]!.messageId}`; - const turnHash = createHash("sha256").update(identity).digest("hex"); - const requestId = createHash("sha256") - .update([identity, turn[0]!.createdAt, content].join("\u0000")) - .digest("hex"); - if (importedRequestIds.has(requestId)) continue; - try { - const added = service.addMemory({ - requestId, - adapterId: `agent-source:${sourceId}`, - content, - layer: "L1", - title: titleForTurn(sourceId, turn), - tags: ["agent-source", sourceId], - source: sourceId, - turnId: `${sourceId}:${turnHash.slice(0, 24)}`, - createdAt: turn[0]!.createdAt, - deferProcessing: true - }); - importedRequestIds.add(requestId); - memoryIds.push(added.id); - messageCount += turn.length; - } catch (error) { - errors.push(`${turn[0]!.conversationId}: ${error instanceof Error ? error.message : String(error)}`); + let processed = 0; + let latestSeenAt: string | null = null; + const errors: string[] = []; + let errorCount = 0; + let activeConversationId: string | null = null; + let activeConversationFailed = false; + const commitConversation = () => { + if (!activeConversationId || activeConversationFailed) return; + const meta = store.getConversationMeta(sourceId, activeConversationId); + if (!meta) return; + const checkpoint = { + sourceId, + conversationId: activeConversationId, + lastMessageId: meta.lastMessageId, + lastCreatedAt: meta.lastCreatedAt, + contentHash: meta.contentHash, + updatedAt: new Date().toISOString() + }; + store.saveCheckpoint(checkpoint); + }; + const memoryIds: string[] = []; + const flush = (force = false) => { + if (memoryIds.length === 0 || (!force && memoryIds.length < 100)) return; + service.enqueuePendingImportSummaries(INITIAL_SCAN_MESSAGE_LIMIT, memoryIds.splice(0)); + scheduleWorker?.(); + }; + const pages = (async function*() { + let cursor: Parameters[1]; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + yield message; + } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; + } + })(); + for await (const turn of orderedTurns(pages)) { + signal.throwIfAborted(); + const turnLatest = turn.messages[turn.messages.length - 1]?.createdAt ?? null; + if (turnLatest && (latestSeenAt === null || Date.parse(turnLatest) > Date.parse(latestSeenAt))) latestSeenAt = turnLatest; + if (turn.conversationId !== activeConversationId) { + commitConversation(); + activeConversationId = turn.conversationId; + activeConversationFailed = false; + } + const conversationMeta = store.getConversationMeta(sourceId, turn.conversationId); + if (conversationMeta?.selected === false) continue; + const selectedTurn = store.getTurnMeta(sourceId, turn.conversationId, stableTurnIdentity(turn)); + if (selectedTurn && !selectedTurn.selected) continue; + let succeeded = true; + // Leave ample room for JSON escaping and the add-memory envelope while + // keeping every request below the 1 MiB wire limit. + const parts = splitTurn(turn, 4000, 512 * 1024); + for (const part of parts) { + const requestId = parts.length === 1 + ? legacyTurnRequestId(turn) + : createHash("sha256").update([stableTurnIdentity(turn), String(part.partIndex), part.contentHash].join("\u0000")).digest("hex"); + const turnId = parts.length === 1 ? legacyTurnId(turn) : `${sourceId}:${part.parentTurnId}:${part.partIndex}`; + try { + const added = service.addMemory({ + requestId, adapterId: `agent-source:${sourceId}`, content: part.content, layer: "L1", + title: titleForTurn(sourceId, part.messages), tags: ["agent-source", sourceId], source: sourceId, + turnId, createdAt: part.messages[0]!.createdAt, deferProcessing: true + }); + if (added.duplicate) store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId: added.id }); + else { store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId: added.id }); memoryIds.push(added.id); written += 1; } + } catch (error) { + succeeded = false; + activeConversationFailed = true; + const reason = error instanceof Error ? error.message : String(error); + errorCount += 1; + if (errors.length < 1000) errors.push(`${turn.conversationId}: ${reason}`); + store.saveResult({ sourceId, conversationId: turn.conversationId, error: reason }); + } + } + if (succeeded) { + messageCount += turn.messages.length; + flush(); + } + processed += turn.messages.length; + onProgress({ sourceId, phase: "add", current: processed, total: store.count(sourceId), message: "Adding raw memories" }); + } + flush(true); + commitConversation(); + return { written, messageCount, errors, errorCount, latestSeenAt }; +} + +async function prepareStandaloneSource( + store: MemoryAgentSourceScanStore, + sourceId: string, + mode: "initial_subset" | "incremental" | "full", + latestSeenAt: string | null, + previousContentHash?: string +): Promise { + let cursor: Parameters[1]; + let currentConversation: string | null = null; + let currentTurn: ConversationMessage[] = []; + let hash = createHash("sha256"); + let first = true; + let latest: ConversationMessage | null = null; + const sourceHash = createHash("sha256"); + sourceHash.update("["); + let firstSourceMessage = true; + const flushTurn = () => { + if (!currentTurn.length || !isCompleteTurn(currentTurn)) return; + const firstMessage = currentTurn[0]!; + const lastMessage = currentTurn[currentTurn.length - 1]!; + const turn = { sourceId, conversationId: firstMessage.conversationId, turnIndex: 0, messages: currentTurn }; + store.saveTurnMeta({ + sourceId, + conversationId: firstMessage.conversationId, + turnId: stableTurnIdentity(turn), + firstMessageId: firstMessage.messageId, + firstCreatedAt: firstMessage.createdAt, + lastMessageId: lastMessage.messageId, + lastCreatedAt: lastMessage.createdAt, + selected: true + }); + }; + const flushConversation = () => { + if (!currentConversation || !latest) return; + hash.update("]"); + const selected = mode !== "incremental" || !latestSeenAt || Date.parse(latest.createdAt) > Date.parse(latestSeenAt); + store.saveConversationMeta({ + sourceId, + conversationId: currentConversation, + lastMessageId: latest.messageId, + lastCreatedAt: latest.createdAt, + contentHash: hash.digest("hex"), + selected + }); + }; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + if (message.conversationId !== currentConversation) { + flushTurn(); + flushConversation(); + currentConversation = message.conversationId; + currentTurn = []; + hash = createHash("sha256"); + hash.update("["); + first = true; + } + if (message.role === "user" && currentTurn.length > 0) { + flushTurn(); + currentTurn = []; + } + currentTurn.push(message); + if (!first) hash.update(","); + first = false; + const hashable = { + messageId: message.messageId, + role: message.role, + content: message.content, + createdAt: message.createdAt, + toolName: hashMeta(message, "toolName") ?? hashMeta(message, "hermesToolName"), + toolCallId: hashMeta(message, "toolCallId") ?? hashMeta(message, "hermesToolCallId") + }; + const serialized = JSON.stringify(hashable); + if (!firstSourceMessage) sourceHash.update(","); + firstSourceMessage = false; + sourceHash.update(serialized); + hash.update(serialized); + latest = message; } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; } - return { written: memoryIds.length, messageCount, memoryIds, errors }; + flushTurn(); + flushConversation(); + sourceHash.update("]"); + const contentHash = sourceHash.digest("hex"); + if (mode === "incremental" && previousContentHash !== contentHash) store.selectAllConversations(sourceId); + return contentHash; } -function renderMessageContent(message: ConversationMessage): string { - if (message.role !== "tool" || /^Tool:\s*/im.test(message.content)) return message.content; - const toolName = stringMeta(message.rawMeta, "toolName") ?? stringMeta(message.rawMeta, "hermesToolName"); - const callId = stringMeta(message.rawMeta, "toolCallId") ?? stringMeta(message.rawMeta, "hermesToolCallId"); - if (!toolName && !callId) return message.content; - return [ - toolName ? `Tool: ${toolName}` : undefined, - callId ? `Call ID: ${callId}` : undefined, - message.content - ].filter(Boolean).join("\n\n"); +function hashMeta(message: ConversationMessage, key: string): string | undefined { + const value = message.rawMeta[key]; + return typeof value === "string" ? value : undefined; } -function stringMeta(meta: Readonly>, key: string): string | undefined { - const value = meta[key]; - return typeof value === "string" && value.length > 0 ? value : undefined; +function readScanPage(store: MemoryAgentSourceScanStore, sourceId: string, cursor?: { conversationId: string; createdAt: string; messageId: string; ordinal: number }): ConversationMessage[] { + const page: ConversationMessage[] = []; + let bytes = 0; + for (const message of store.messages(sourceId, cursor, 500)) { + page.push(message); + bytes += Buffer.byteLength(JSON.stringify(message)); + if (page.length >= 500 || bytes >= 8 * 1024 * 1024) break; + } + return page; } async function ingestAgentSkills( service: MemoryService, sourceId: string, - importedRequestIds: Set -): Promise<{ written: number; memoryIds: string[]; errors: string[] }> { + store: MemoryAgentSourceScanStore, + scheduleWorker?: () => void +): Promise<{ written: number; memoryIdCount: number; errorCount: number; errors: string[] }> { const root = agentRootDirectory(sourceId); - if (!root) return { written: 0, memoryIds: [], errors: [] }; + if (!root) return { written: 0, memoryIdCount: 0, errorCount: 0, errors: [] }; const skillsRoot = join(root, "skills"); - const files = await findSkillFiles(skillsRoot); - const memoryIds: string[] = []; const errors: string[] = []; - for (const filePath of files) { + let written = 0; + let memoryIdCount = 0; + let errorCount = 0; + const pendingIds: string[] = []; + const flush = (force = false) => { + if (pendingIds.length === 0 || (!force && pendingIds.length < 100)) return; + service.enqueuePendingImportSummaries(INITIAL_SCAN_MESSAGE_LIMIT, pendingIds.splice(0)); + scheduleWorker?.(); + }; + for await (const filePath of findSkillFiles(skillsRoot)) { const content = await readFile(filePath, "utf8"); const contentHash = createHash("sha256").update(content).digest("hex"); const sourceSkillId = relative(skillsRoot, dirname(filePath)).replaceAll("\\", "/"); const requestId = `agent-source-skill:${sourceId}:${sourceSkillId}:${contentHash}`; - if (importedRequestIds.has(requestId)) continue; const fileStat = await stat(filePath); try { const added = service.addMemory({ @@ -595,21 +951,28 @@ async function ingestAgentSkills( sourceContentHash: contentHash, deferProcessing: true }); - importedRequestIds.add(requestId); - memoryIds.push(added.id); + written += 1; + memoryIdCount += 1; + store.saveResult({ sourceId, conversationId: `skill:${sourceSkillId}`, memoryId: added.id }); + if (!added.duplicate) { + pendingIds.push(added.id); + flush(); + } } catch (error) { - errors.push(`skill ${sourceSkillId}: ${error instanceof Error ? error.message : String(error)}`); + const reason = `skill ${sourceSkillId}: ${error instanceof Error ? error.message : String(error)}`; + errorCount += 1; + if (errors.length < 1000) errors.push(reason); + store.saveResult({ sourceId, conversationId: `skill:${sourceSkillId}`, error: reason }); } } - return { written: memoryIds.length, memoryIds, errors }; + flush(true); + return { written, memoryIdCount, errorCount, errors }; } -async function findSkillFiles(root: string): Promise { - const files: string[] = []; - await visit(root, 0); - return files.sort(); +async function* findSkillFiles(root: string): AsyncGenerator { + yield* visit(root, 0); - async function visit(directory: string, depth: number): Promise { + async function* visit(directory: string, depth: number): AsyncGenerator { let entries; try { entries = await readdir(directory, { withFileTypes: true }); @@ -617,11 +980,11 @@ async function findSkillFiles(root: string): Promise { if (isNodeError(error) && error.code === "ENOENT") return; throw error; } - for (const entry of entries) { + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { if (entry.name === "memmy-memory" || entry.name === "node_modules" || entry.name === ".git") continue; const path = join(directory, entry.name); - if (entry.isFile() && entry.name.toLowerCase() === "skill.md") files.push(path); - else if (depth < 2 && entry.isDirectory()) await visit(path, depth + 1); + if (entry.isFile() && entry.name.toLowerCase() === "skill.md") yield path; + else if (depth < 2 && entry.isDirectory()) yield* visit(path, depth + 1); } } } @@ -651,45 +1014,12 @@ function frontmatterValue(content: string, key: string): string | undefined { ?.trim(); } -function completeTurns(messages: readonly ConversationMessage[]): ConversationMessage[][] { - const sorted = [...messages].sort((left, right) => - left.conversationId.localeCompare(right.conversationId) - || Date.parse(left.createdAt) - Date.parse(right.createdAt) - || left.messageId.localeCompare(right.messageId) - ); - const turns: ConversationMessage[][] = []; - let current: ConversationMessage[] = []; - let conversationId = ""; - for (const message of sorted) { - if (message.conversationId !== conversationId || (message.role === "user" && current.length > 0)) { - if (isCompleteTurn(current)) turns.push(current); - current = []; - conversationId = message.conversationId; - } - current.push(message); - } - if (isCompleteTurn(current)) turns.push(current); - return turns; -} - -function isCompleteTurn(messages: readonly ConversationMessage[]): boolean { - return messages[0]?.role === "user" - && Boolean(messages[0].content.trim()) - && messages[messages.length - 1]?.role === "assistant" - && Boolean(messages[messages.length - 1]?.content.trim()); -} - function titleForTurn(sourceId: string, messages: readonly ConversationMessage[]): string { const firstLine = messages[0]?.content.split(/\r?\n/).map((line) => line.trim()).find(Boolean); const title = firstLine || `${sourceId} conversation`; return title.length <= 120 ? title : `${title.slice(0, 117)}...`; } -function maxCreatedAt(messages: readonly ConversationMessage[]): string | null { - return messages.reduce((latest, message) => - !latest || message.createdAt > latest ? message.createdAt : latest, null); -} - function normalizeScanInput(value: unknown): { sourceId: string; mode?: "initial_subset" | "incremental" | "full"; @@ -720,25 +1050,113 @@ function emptySourceState(): PersistedSourceState { status: "not_connected", messageCount: 0, lastScannedAt: null, - latestSeenAt: null, - importedRequestIds: [] + latestSeenAt: null }; } async function loadState(path: string): Promise { try { - const parsed = JSON.parse(await readFile(path, "utf8")) as unknown; + const parsed = JSON.parse(await readStateWithoutLegacyIds(path)) as unknown; const value = record(parsed); - return { - version: 1, - sources: record(value.sources) as Record + const sourceValues = record(value.sources); + const hasLegacyIds = Object.values(sourceValues).some((raw) => Object.hasOwn(record(raw), "importedRequestIds")); + const sources = Object.fromEntries(Object.entries(sourceValues).map(([sourceId, raw]) => { + const source = record(raw); + const status = source.status === "skill_installed" || source.status === "plugin_installed" ? source.status : "not_connected"; + return [sourceId, { + status, + messageCount: typeof source.messageCount === "number" && Number.isFinite(source.messageCount) ? Math.max(0, Math.floor(source.messageCount)) : 0, + lastScannedAt: typeof source.lastScannedAt === "string" ? source.lastScannedAt : null, + latestSeenAt: typeof source.latestSeenAt === "string" ? source.latestSeenAt : null, + ...(typeof source.contentHash === "string" ? { contentHash: source.contentHash } : {}) + } satisfies PersistedSourceState]; + })); + const state: PersistedState = { + version: 2, + sources }; + if (value.version !== 2 || hasLegacyIds) await writeState(path, state); + return state; } catch (error) { - if (isNodeError(error) && error.code === "ENOENT") return { version: 1, sources: {} }; + if (isNodeError(error) && error.code === "ENOENT") return { version: 2, sources: {} }; throw error; } } +/** Streams legacy state while replacing the unbounded ID arrays with empty arrays. */ +async function readStateWithoutLegacyIds(path: string): Promise { + const temporaryPath = `${path}.v2-migration-${process.pid}-${Date.now()}-${randomUUID()}`; + const input = createReadStream(path, { encoding: "utf8" }); + await new Promise((resolve, reject) => { + input.once("open", () => resolve()); + input.once("error", reject); + }); + const output = createWriteStream(temporaryPath, { encoding: "utf8" }); + let inString = false; + let escaped = false; + let pendingLegacyArray = false; + let skipDepth = 0; + let skipString = false; + let skipEscaped = false; + let keyBuffer = ""; + try { + for await (const chunk of input) { + const text = String(chunk); + let emitted = ""; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]!; + if (skipDepth > 0) { + if (skipString) { + if (skipEscaped) skipEscaped = false; + else if (char === "\\") skipEscaped = true; + else if (char === '"') skipString = false; + } else if (char === '"') skipString = true; + else if (char === "[") skipDepth += 1; + else if (char === "]") skipDepth -= 1; + continue; + } + if (inString) { + emitted += char; + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') { inString = false; keyBuffer += char; } + else if (keyBuffer.length < 32) keyBuffer += char; + continue; + } + if (char === '"') { + inString = true; + keyBuffer = '"'; + emitted += char; + continue; + } + if (pendingLegacyArray) { + emitted += char; + if (/\s/u.test(char)) continue; + if (char === "[") { + emitted = emitted.slice(0, -1) + "[]"; + pendingLegacyArray = false; + skipDepth = 1; + skipString = false; + skipEscaped = false; + } else { + pendingLegacyArray = false; + } + continue; + } + emitted += char; + if (char === ":" && keyBuffer === '"importedRequestIds"') pendingLegacyArray = true; + if (!/\s/u.test(char)) keyBuffer = ""; + } + if (emitted && !output.write(emitted)) await once(output, "drain"); + } + output.end(); + await once(output, "close"); + return await readFile(temporaryPath, "utf8"); + } finally { + await rm(temporaryPath, { force: true }); + } +} + async function writeState(path: string, state: PersistedState): Promise { await mkdir(dirname(path), { recursive: true }); const temporary = `${path}.${process.pid}.${Date.now()}.tmp`; diff --git a/Memory/src/agent-source/scan-store.ts b/Memory/src/agent-source/scan-store.ts new file mode 100644 index 000000000..dc3f06255 --- /dev/null +++ b/Memory/src/agent-source/scan-store.ts @@ -0,0 +1,73 @@ +import { mkdir } from "node:fs/promises"; +import { rmSync } from "node:fs"; +import { dirname } from "node:path"; +import Database from "better-sqlite3"; +import type { ConversationCheckpoint, ConversationMessage, MessageCursor, PreparedConversation, PreparedTurn, ScanSourceState, ScanStore, ScanStoredResult } from "@memmy/agent-source-core"; + +const MAX_RECORD_BYTES = 64 * 1024 * 1024; +const MAX_PAGE_BYTES = 8 * 1024 * 1024; + +export interface MemoryScanJobMeta { jobId: string; sourceId: string; mode: string; phase: string; createdAt: string; updatedAt: string; error?: string; } +export interface MemoryAgentSourceScanStore extends ScanStore { readonly path: string; saveMeta(meta: MemoryScanJobMeta): void; getMeta(): MemoryScanJobMeta | null; remove(): void; } + +export async function openMemoryAgentSourceScanStore(path: string, job: MemoryScanJobMeta): Promise { + await mkdir(dirname(path), { recursive: true }); + const db = new Database(path); + db.pragma("journal_mode = WAL"); + db.pragma("synchronous = NORMAL"); + db.pragma("busy_timeout = 5000"); + db.exec(`CREATE TABLE IF NOT EXISTS scan_meta (id INTEGER PRIMARY KEY CHECK(id=1), job_id TEXT NOT NULL, source_id TEXT NOT NULL, mode TEXT NOT NULL, phase TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS schema_meta (version INTEGER NOT NULL); + INSERT INTO schema_meta(version) SELECT 2 WHERE NOT EXISTS (SELECT 1 FROM schema_meta); + UPDATE schema_meta SET version=2 WHERE version<2; + CREATE TABLE IF NOT EXISTS staged_messages (job_id TEXT NOT NULL, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, message_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL, created_at TEXT NOT NULL, workspace_path TEXT, git_root TEXT, raw_meta_json TEXT NOT NULL, ordinal INTEGER NOT NULL, PRIMARY KEY(job_id,source_id,message_id)); + CREATE INDEX IF NOT EXISTS staged_order ON staged_messages(job_id,source_id,conversation_id,created_at,message_id,ordinal); + CREATE TABLE IF NOT EXISTS scan_source_state (source_id TEXT PRIMARY KEY, mode TEXT NOT NULL, phase TEXT NOT NULL, message_count INTEGER NOT NULL DEFAULT 0, result_count INTEGER NOT NULL DEFAULT 0, error_count INTEGER NOT NULL DEFAULT 0, scan_started_at TEXT, watermarked_since TEXT, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS scan_cursors (source_id TEXT PRIMARY KEY, conversation_id TEXT NOT NULL, created_at TEXT NOT NULL, message_id TEXT NOT NULL, ordinal INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS checkpoints (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, content_hash TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(source_id,conversation_id)); + CREATE TABLE IF NOT EXISTS conversation_meta (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, content_hash TEXT NOT NULL, selected INTEGER NOT NULL, PRIMARY KEY(source_id,conversation_id)); + CREATE TABLE IF NOT EXISTS turn_meta (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, turn_id TEXT NOT NULL, first_message_id TEXT NOT NULL, first_created_at TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, selected INTEGER NOT NULL, PRIMARY KEY(source_id,conversation_id,turn_id)); + CREATE INDEX IF NOT EXISTS turn_selection_order ON turn_meta(first_created_at DESC,source_id,conversation_id,first_message_id,turn_id); + CREATE TABLE IF NOT EXISTS scan_results (id INTEGER PRIMARY KEY AUTOINCREMENT, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, memory_id TEXT, error TEXT); + CREATE INDEX IF NOT EXISTS scan_result_identity ON scan_results(source_id,conversation_id,memory_id,error);`); + if (!db.prepare("SELECT 1 FROM scan_meta WHERE id=1").get()) db.prepare("INSERT INTO scan_meta(id,job_id,source_id,mode,phase,created_at,updated_at,error) VALUES(1,@jobId,@sourceId,@mode,@phase,@createdAt,@updatedAt,@error)").run({ ...job, error: job.error ?? null }); + let ordinal = Number((db.prepare("SELECT COALESCE(MAX(ordinal),-1) AS value FROM staged_messages WHERE job_id=?").get(job.jobId) as { value: number }).value) + 1; + const insert = db.prepare("INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) VALUES(?,?,?,?,?,?,?,?,?,?,?)"); + const store: MemoryAgentSourceScanStore = { + path, + stage(message) { const bytes = Buffer.byteLength(JSON.stringify(message)); if (bytes > MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); return Number(insert.run(job.jobId,message.sourceId,message.conversationId,message.messageId,message.role,message.content,message.createdAt,message.workspacePath,message.gitRoot,JSON.stringify(message.rawMeta),ordinal++).changes)>0; }, + stageBatch(messages) { const tx = db.transaction(() => { let count = 0; for (const message of messages) if (store.stage(message)) count += 1; return count; }); return tx(); }, + messages(sourceId, cursor, limit=500) { limit=Math.min(500,Math.max(1,limit)); const params: unknown[]=[job.jobId,sourceId]; let where="job_id=? AND source_id=?"; if(cursor){where += " AND ((conversation_id > ?) OR (conversation_id = ? AND (created_at > ? OR (created_at = ? AND (message_id > ? OR (message_id = ? AND ordinal > ?))))))"; params.push(cursor.conversationId,cursor.conversationId,cursor.createdAt,cursor.createdAt,cursor.messageId,cursor.messageId,cursor.ordinal);} const rows = db.prepare(`SELECT source_id AS sourceId,conversation_id AS conversationId,message_id AS messageId,role,content,created_at AS createdAt,workspace_path AS workspacePath,git_root AS gitRoot,raw_meta_json AS rawMetaJson,ordinal FROM staged_messages WHERE ${where} ORDER BY conversation_id,created_at,message_id,ordinal LIMIT ?`).iterate(...params,limit) as Iterable>; return (function*(){let bytes=0; let count=0; for(const row of rows){const message=rowToMessage(row); yield message; count+=1; bytes+=Buffer.byteLength(JSON.stringify(message)); if(count>=500||bytes>=MAX_PAGE_BYTES) break;}})(); }, + saveScanCursor(s,c){db.prepare("INSERT INTO scan_cursors(source_id,conversation_id,created_at,message_id,ordinal) VALUES(?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET conversation_id=excluded.conversation_id,created_at=excluded.created_at,message_id=excluded.message_id,ordinal=excluded.ordinal").run(s,c.conversationId,c.createdAt,c.messageId,c.ordinal);}, + getScanCursor(s){return (db.prepare("SELECT conversation_id AS conversationId,created_at AS createdAt,message_id AS messageId,ordinal FROM scan_cursors WHERE source_id=?").get(s) as MessageCursor|undefined)??null;}, + saveSourceState(s){db.prepare("INSERT INTO scan_source_state(source_id,mode,phase,message_count,result_count,error_count,scan_started_at,watermarked_since,updated_at,error) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET mode=excluded.mode,phase=excluded.phase,message_count=excluded.message_count,result_count=excluded.result_count,error_count=excluded.error_count,scan_started_at=excluded.scan_started_at,watermarked_since=excluded.watermarked_since,updated_at=excluded.updated_at,error=excluded.error").run(s.sourceId,s.mode,s.phase,s.messageCount,s.resultCount,s.errorCount,s.scanStartedAt??null,s.watermarkedSince??null,s.updatedAt,s.error??null);}, + getSourceState(s){return(db.prepare("SELECT source_id AS sourceId,mode,phase,message_count AS messageCount,result_count AS resultCount,error_count AS errorCount,scan_started_at AS scanStartedAt,watermarked_since AS watermarkedSince,updated_at AS updatedAt,error FROM scan_source_state WHERE source_id=?").get(s) as ScanSourceState|undefined)??null;}, + sourceCount(){return Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as {count:number}).count);}, + count(sourceId){const row=db.prepare(`SELECT COUNT(*) AS count FROM staged_messages WHERE job_id=?${sourceId?" AND source_id=?":""}`).get(job.jobId,...(sourceId?[sourceId]:[])) as {count:number}; return Number(row.count);}, + saveCheckpoint(c){db.prepare("INSERT INTO checkpoints(source_id,conversation_id,last_message_id,last_created_at,content_hash,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,updated_at=excluded.updated_at").run(c.sourceId,c.conversationId,c.lastMessageId,c.lastCreatedAt,c.contentHash,c.updatedAt);}, + getCheckpoint(s,c){return (db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,content_hash AS contentHash,updated_at AS updatedAt FROM checkpoints WHERE source_id=? AND conversation_id=?").get(s,c) as ConversationCheckpoint|undefined)??null;}, + saveConversationMeta(m){db.prepare("INSERT INTO conversation_meta(source_id,conversation_id,last_message_id,last_created_at,content_hash,selected) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,selected=excluded.selected").run(m.sourceId,m.conversationId,m.lastMessageId,m.lastCreatedAt,m.contentHash,m.selected?1:0);}, + getConversationMeta(s,c){const row=db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,content_hash AS contentHash,selected FROM conversation_meta WHERE source_id=? AND conversation_id=?").get(s,c) as (Omit&{selected:number})|undefined; return row?{...row,selected:row.selected===1}:null;}, selectAllConversations(s){db.prepare("UPDATE conversation_meta SET selected=1 WHERE source_id=?").run(s);}, + saveTurnMeta(m){db.prepare("INSERT INTO turn_meta(source_id,conversation_id,turn_id,first_message_id,first_created_at,last_message_id,last_created_at,selected) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id,turn_id) DO UPDATE SET first_message_id=excluded.first_message_id,first_created_at=excluded.first_created_at,last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,selected=excluded.selected").run(m.sourceId,m.conversationId,m.turnId,m.firstMessageId,m.firstCreatedAt,m.lastMessageId,m.lastCreatedAt,m.selected?1:0);}, + getTurnMeta(s,c,t){const row=db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,turn_id AS turnId,first_message_id AS firstMessageId,first_created_at AS firstCreatedAt,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,selected FROM turn_meta WHERE source_id=? AND conversation_id=? AND turn_id=?").get(s,c,t) as (Omit&{selected:number})|undefined; return row?{...row,selected:row.selected===1}:null;}, + selectInitialTurns(sourceIds,globalLimit,absentSourceLimit){ + if(sourceIds.length===0)return; + const placeholders=sourceIds.map(()=>"?").join(","); + const transaction=db.transaction(()=>{ + db.prepare(`UPDATE turn_meta SET selected=0 WHERE source_id IN (${placeholders})`).run(...sourceIds); + db.prepare(`UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT rowid FROM turn_meta WHERE source_id IN (${placeholders}) ORDER BY first_created_at DESC,source_id,conversation_id,first_message_id,turn_id LIMIT ?)`).run(...sourceIds,globalLimit); + db.prepare(`WITH ranked AS (SELECT source_id,turn_id,ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY first_created_at DESC,conversation_id,first_message_id,turn_id) AS rank FROM turn_meta WHERE source_id IN (${placeholders})), absent AS (SELECT source_id FROM turn_meta WHERE source_id IN (${placeholders}) GROUP BY source_id HAVING MAX(selected)=0) UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT t.rowid FROM turn_meta t JOIN ranked r ON r.source_id=t.source_id AND r.turn_id=t.turn_id JOIN absent a ON a.source_id=t.source_id WHERE r.rank <= ?)`).run(...sourceIds,...sourceIds,absentSourceLimit); + }); + transaction(); + }, + saveResult(r){db.prepare("INSERT INTO scan_results(source_id,conversation_id,memory_id,error) SELECT ?,?,?,? WHERE NOT EXISTS (SELECT 1 FROM scan_results WHERE source_id=? AND conversation_id=? AND memory_id IS ? AND error IS ?)").run(r.sourceId,r.conversationId,r.memoryId??null,r.error??null,r.sourceId,r.conversationId,r.memoryId??null,r.error??null);}, resultCount(s){const row=db.prepare(`SELECT COUNT(*) AS count FROM scan_results${s?" WHERE source_id=?":""}`).get(...(s?[s]:[])) as {count:number}; return Number(row.count);}, + results(sourceId,cursor,limit=100){const safeLimit=Math.min(500,Math.max(1,Math.floor(limit))); const numericCursor=Number(cursor); const safeCursor=Number.isFinite(numericCursor)&&numericCursor>=0?Math.floor(numericCursor):0; const rows=db.prepare("SELECT id,source_id AS sourceId,conversation_id AS conversationId,memory_id AS memoryId,error FROM scan_results WHERE source_id LIKE ? AND id>? ORDER BY id LIMIT ?").iterate(sourceId??"%",safeCursor,safeLimit) as Iterable>; return (function*(){for(const row of rows){const result={sourceId:String(row.sourceId),conversationId:String(row.conversationId),...(row.memoryId?{memoryId:String(row.memoryId)}:{}),...(row.error?{error:String(row.error)}:{})} as ScanStoredResult; Object.defineProperty(result,"cursor",{value:String(row.id),enumerable:false}); yield result;}})();}, + saveMeta(meta){db.prepare("UPDATE scan_meta SET job_id=@jobId,source_id=@sourceId,mode=@mode,phase=@phase,created_at=@createdAt,updated_at=@updatedAt,error=@error WHERE id=1").run({...meta,error:meta.error??null});}, + getMeta(){return (db.prepare("SELECT job_id AS jobId,source_id AS sourceId,mode,phase,created_at AS createdAt,updated_at AS updatedAt,error FROM scan_meta WHERE id=1").get() as MemoryScanJobMeta|undefined)??null;}, + close(){db.close();}, + remove(){db.close(); rmSync(path,{force:true}); rmSync(`${path}-wal`,{force:true}); rmSync(`${path}-shm`,{force:true});} + }; + return store; +} + +function rowToMessage(row: Record): ConversationMessage { return { sourceId:String(row.sourceId), conversationId:String(row.conversationId), messageId:String(row.messageId), role:row.role as ConversationMessage["role"], content:String(row.content), createdAt:String(row.createdAt), workspacePath:row.workspacePath==null?null:String(row.workspacePath), gitRoot:row.gitRoot==null?null:String(row.gitRoot), rawMeta:JSON.parse(String(row.rawMetaJson)) as Record, ordinal:Number(row.ordinal) }; } diff --git a/Memory/src/server/viewer-api.ts b/Memory/src/server/viewer-api.ts index b47a3cda5..692ed4cd5 100644 --- a/Memory/src/server/viewer-api.ts +++ b/Memory/src/server/viewer-api.ts @@ -37,6 +37,7 @@ export const VIEWER_API_ROUTES = [ "GET /api/v1/agent-sources", "POST /api/v1/agent-sources/scan", "GET /api/v1/agent-sources/scan/status", + "GET /api/v1/agent-sources/scan/jobs/:jobId/results", "POST /api/v1/agent-sources/scan/stop", "POST /api/v1/agent-sources/scan/cancel", "POST /api/v1/agent-sources/:id/plugin", @@ -223,6 +224,12 @@ export async function routeViewerRequest( if (method === "GET" && path === "/api/v1/agent-sources/scan/status") { return { body: context.agentSources.scanStatus() }; } + const scanResults = path.match(/^\/api\/v1\/agent-sources\/scan\/jobs\/([^/]+)\/results$/); + if (method === "GET" && scanResults?.[1]) { + return { body: context.agentSources.scanResults + ? await context.agentSources.scanResults(decodeURIComponent(scanResults[1]), query(url, "cursor") ?? "0", numberQuery(url, "limit") ?? 100) + : { items: [], nextCursor: null } }; + } if (method === "POST" && path === "/api/v1/agent-sources/scan/stop") { return { body: await context.agentSources.pauseScan() }; } diff --git a/Memory/tests/agent-source-runtime.test.ts b/Memory/tests/agent-source-runtime.test.ts index ad0ce2cc5..36448d4d4 100644 --- a/Memory/tests/agent-source-runtime.test.ts +++ b/Memory/tests/agent-source-runtime.test.ts @@ -105,12 +105,11 @@ describe("standalone Agent source executor", () => { await waitForScan(executor); expect(addMemory).toHaveBeenCalledTimes(1); expect(JSON.parse(readFileSync(statePath, "utf8"))).toMatchObject({ - version: 1, + version: 2, sources: { "fixture-agent": { messageCount: 2, latestSeenAt: "2026-08-28T01:01:00.000Z", - importedRequestIds: [expect.any(String)] } } }); diff --git a/package-lock.json b/package-lock.json index 4597451a3..3ad0d765f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.1.1", "workspaces": [ "Migrations", + "AgentSourceCore", "Memory", "App/backend/local-api-contracts", "App/backend", @@ -36,10 +37,15 @@ "node": ">=20" } }, + "AgentSourceCore": { + "name": "@memmy/agent-source-core", + "version": "0.0.0" + }, "App/backend": { "name": "@memmy/backend", "version": "0.0.0", "dependencies": { + "@memmy/agent-source-core": "0.0.0", "@memmy/local-api-contracts": "0.0.0", "@memmy/migrations": "0.0.0", "@modelcontextprotocol/sdk": "^1.29.0", @@ -716,6 +722,7 @@ "version": "2.1.0", "dependencies": { "@huggingface/transformers": "^3.8.0", + "@memmy/agent-source-core": "0.0.0", "better-sqlite3": "^12.6.3", "dotenv": "^16.6.1", "fast-xml-parser": "^5.8.0", @@ -3369,6 +3376,10 @@ "node": ">=10" } }, + "node_modules/@memmy/agent-source-core": { + "resolved": "AgentSourceCore", + "link": true + }, "node_modules/@memmy/backend": { "resolved": "App/backend", "link": true diff --git a/package.json b/package.json index 8f98c32d0..f88ed479b 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "description": "Local-first agent memory substrate with desktop and CLI surfaces.", "workspaces": [ "Migrations", + "AgentSourceCore", "Memory", "App/backend/local-api-contracts", "App/backend", diff --git a/scripts/internal/linux/build-cli-archive.sh b/scripts/internal/linux/build-cli-archive.sh index b03d88014..4170456ec 100755 --- a/scripts/internal/linux/build-cli-archive.sh +++ b/scripts/internal/linux/build-cli-archive.sh @@ -47,18 +47,21 @@ if [ "${MEMMY_LINUX_CLI_SKIP_BUILD:-0}" != "1" ]; then rm -rf \ "$REPO_ROOT/App/memmy-agent/dist" \ "$REPO_ROOT/App/backend/dist" \ + "$REPO_ROOT/AgentSourceCore/dist" \ "$REPO_ROOT/Memory/dist" \ "$REPO_ROOT/Migrations/dist" \ "$REPO_ROOT/App/backend/local-api-contracts/dist" npm --prefix "$REPO_ROOT/Migrations" run build npm --prefix "$REPO_ROOT/App/backend/local-api-contracts" run build npm --prefix "$REPO_ROOT/App/backend" run build + npm --prefix "$REPO_ROOT/AgentSourceCore" run build npm --prefix "$REPO_ROOT/Memory" run build npm --prefix "$REPO_ROOT/App/memmy-agent" run build fi for required in \ "$REPO_ROOT/App/memmy-agent/dist/main.js" \ + "$REPO_ROOT/AgentSourceCore/dist/src/index.js" \ "$REPO_ROOT/Memory/dist/src/server/index.js" \ "$REPO_ROOT/Memory/dist/src/cli/index.js" \ "$REPO_ROOT/App/backend/dist/src/analytics/analytics-transport.js" \ @@ -80,6 +83,7 @@ trap cleanup EXIT PAYLOAD_DIR="$BUILD_DIR/payload" mkdir -p \ "$PAYLOAD_DIR/App/memmy-agent" \ + "$PAYLOAD_DIR/AgentSourceCore" \ "$PAYLOAD_DIR/App/backend/dist/src/analytics" \ "$PAYLOAD_DIR/App/backend/dist/src/adapters/outbound" \ "$PAYLOAD_DIR/App/backend/dist/src/services" \ @@ -93,6 +97,8 @@ cp "$REPO_ROOT/package-lock.json" "$PAYLOAD_DIR/package-lock.json" cp "$REPO_ROOT/App/memmy-agent/package.json" "$PAYLOAD_DIR/App/memmy-agent/package.json" cp "$REPO_ROOT/App/memmy-agent/package-lock.json" "$PAYLOAD_DIR/App/memmy-agent/package-lock.json" cp -R "$REPO_ROOT/App/memmy-agent/dist" "$PAYLOAD_DIR/App/memmy-agent/dist" +cp "$REPO_ROOT/AgentSourceCore/package.json" "$PAYLOAD_DIR/AgentSourceCore/package.json" +cp -R "$REPO_ROOT/AgentSourceCore/dist" "$PAYLOAD_DIR/AgentSourceCore/dist" cp "$REPO_ROOT/App/backend/package.json" "$PAYLOAD_DIR/App/backend/package.json" cp -R "$REPO_ROOT/App/backend/dist/src/adapters/outbound/skill-writer" \ "$PAYLOAD_DIR/App/backend/dist/src/adapters/outbound/skill-writer" @@ -119,6 +125,7 @@ import { readFileSync, writeFileSync } from "node:fs"; const manifestPath = process.argv[2]; const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); manifest.workspaces = [ + "AgentSourceCore", "Memory", "Migrations", "App/backend/local-api-contracts" diff --git a/tests/linux-cli-packaging.test.mjs b/tests/linux-cli-packaging.test.mjs index 35b4d2e07..4eaa3850f 100644 --- a/tests/linux-cli-packaging.test.mjs +++ b/tests/linux-cli-packaging.test.mjs @@ -231,6 +231,7 @@ describe("Linux CLI package boundary", () => { const installer = readFileSync(installerPath, "utf8"); expect(builder).toContain("App/memmy-agent/dist/main.js"); + expect(builder).toContain("AgentSourceCore/dist/src/index.js"); expect(builder).toContain("Memory/dist/src/server/index.js"); expect(builder).toContain("Memory/dist/src/cli/index.js"); expect(builder).toContain("builtin-skill-target-registry.js"); @@ -290,6 +291,7 @@ describe("Linux CLI package boundary", () => { const listing = spawnSync("tar", ["-tzf", archive], { encoding: "utf8" }); expect(listing.status, listing.stderr).toBe(0); expect(listing.stdout).toContain("App/memmy-agent/dist/main.js"); + expect(listing.stdout).toContain("AgentSourceCore/dist/src/index.js"); expect(listing.stdout).toContain("Memory/dist/src/server/index.js"); expect(listing.stdout).toContain("Memory/dist/src/cli/index.js"); expect(listing.stdout).toContain("App/backend/dist/src/services/builtin-skill-target-registry.js"); @@ -335,6 +337,8 @@ describe("Linux CLI package boundary", () => { env: cleanNpmLifecycleEnv(), }); expect(runtimeInstall.status, runtimeInstall.stderr).toBe(0); + expect(existsSync(path.join(extracted, "AgentSourceCore", "dist", "src", "index.js"))).toBe(true); + expect(existsSync(path.join(extracted, "node_modules", "@memmy", "agent-source-core"))).toBe(true); const migrationModuleUrl = pathToFileURL(path.join( extracted,