diff --git a/.github/release-notes/v1.1.2.md b/.github/release-notes/v1.1.2.md new file mode 100644 index 000000000..226d11ae6 --- /dev/null +++ b/.github/release-notes/v1.1.2.md @@ -0,0 +1,39 @@ +# Memmy v1.1.2 + +## Highlights + +- Added a standalone, independently versioned Memory service and browser viewer. The viewer brings Overview, Memories, Tasks, Experiences, World Models, Skills, User Memories, Analytics, Logs, and Settings into one place. +- Moved built-in Cross-Agent discovery, history import, and integration management into Memory itself for Cursor, Claude Code, Codex, OpenCode, OpenClaw, Hermes, DeepSeek Harness, WorkBuddy, Pi, and QwenWork. Scans support startup and scheduled synchronization, incremental resume, progress reporting, pause, and stop controls. +- Added a slash-command menu to the terminal UI, with live Gateway commands, search and filtering, keyboard navigation and completion, plus safe local `/stop`, `/last-compaction`, and `/quit` actions. + +## Agent and CLI improvements + +- `memmy onboard` now opens the interactive setup flow by default. Use `memmy onboard --defaults` for non-interactive initialization or refresh; the terminal documentation has been reorganized around the current CLI and TUI flows. +- Enabled the image-generation tool gate by default while continuing to honor an explicit `enabled: false` setting. +- Memory-backed L3 context is now refreshed before every system-prompt build, so later turns can see updated World Model context while retaining the last successful snapshot across transient read failures. +- Improved compatibility with Claude Sonnet 5 and Claude Opus 4.7/5 endpoints by omitting unsupported temperature fields. + +## Memory improvements + +- Added `memmy-memory install`, `upgrade`, and service lifecycle flows for versioned standalone runtimes. Runtime archives are checksum-verified, activated atomically, registered with the platform user service manager, health-checked, and protected against accidental downgrades. +- Memmy Desktop now leaves the standalone Memory service running after the desktop app exits by default. A new **Stop Memory when quitting** setting opts back into stopping it, and the CLI stop flow verifies the local service before requesting shutdown. +- Long inputs for known OpenAI embedding models are now tokenized and split with a safe 7,500-token ceiling. OpenAI-compatible deployment aliases can opt into the same handling with `memmyMemory.embedding.maxInputTokens` or `MEMMY_EMBEDDING_MAX_INPUT_TOKENS`. +- Improved large Agent-history imports by tracking targeted summary and indexing progress, isolating token-limit failures, avoiding unnecessary deterministic retries, and tolerating Agent Skill links that disappear during a scan. + +## Desktop, setup, and migration reliability + +- The desktop composer now accepts supported pasted document and text files in addition to pasted images. +- Settings can filter local BYOK token usage by model, distinguish duplicate model names by endpoint or preset, and refresh usage when the Token tab regains focus. Usage recording also follows the active `MEMMY_HOME`. +- Model settings now distinguish Memmy Platform local Embedding, cloud Embedding, and custom Embedding assignments. An unconfigured account assignment is no longer silently treated as local. +- First-use reports now hide model planning preambles and internal task-context payloads, strip raw HTML while preserving visible text, and disable unnecessary thinking output for the account report model. +- Windows packaged installs now expose the `.cmd` CLI launchers through the user `PATH`, preserve unrelated and expanded-path entries, and make the launch-at-login setting reflect the effective registered startup command. +- Hardened Windows upgrades and installation-directory moves with source and target validation, junction checks, transactional data relocation, rollback and interrupted-upgrade recovery, and protection against unrelated existing runtimes. +- Desktop packaging now prunes only verified non-runtime dependency files with fail-closed guards, while Linux CLI installs include the workspace dependencies required by packaged migrations. + +## Upgrade notes + +- Memory, its viewer, and `memmy-memory` now have the independent component version `2.1.0`; this is intentionally separate from the Memmy application version `1.1.2`. +- Scripts that previously used `memmy onboard --wizard` must switch to `memmy onboard`. Unattended setup should use `memmy onboard --defaults`. +- Existing configurations whose Memory summary timeout is exactly the legacy default of 45 seconds are migrated to 180 seconds. User-defined timeout values are preserved. +- After upgrading, quitting Memmy Desktop no longer stops Memory unless **Stop Memory when quitting** is enabled. +- Image generation is enabled by default at the tool-gate level. Set `tools.imageGeneration.enabled: false` to retain an explicitly disabled configuration. diff --git a/.github/workflows/memory-release.yml b/.github/workflows/memory-release.yml new file mode 100644 index 000000000..cb20d634b --- /dev/null +++ b/.github/workflows/memory-release.yml @@ -0,0 +1,102 @@ +name: Memory 2.1 Release + +on: + workflow_dispatch: + inputs: + version: + description: Memory version (X.Y.Z) + required: true + default: 2.1.0 + push: + tags: + - "memory-v*" + +permissions: + contents: write + +jobs: + verify: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run typecheck -w @memmy/memory + - run: npm test -w @memmy/memory + - run: npx vitest run App/shell/desktop/tests/packaged-runtime-boundary.test.ts App/shell/desktop/tests/runtime-services.test.ts + + runtime: + needs: verify + strategy: + fail-fast: false + matrix: + include: + - target: darwin-arm64 + os: macos-14 + - target: darwin-x64 + os: macos-15-intel + - target: linux-arm64 + os: ubuntu-24.04-arm + - target: linux-x64 + os: ubuntu-24.04 + - target: windows-arm64 + os: windows-11-arm + - target: windows-x64 + os: windows-2025 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - name: Resolve version + id: version + shell: bash + run: | + version="${{ inputs.version }}" + if [[ -z "$version" ]]; then version="${GITHUB_REF_NAME#memory-v}"; fi + node -e 'if (!/^\d+\.\d+\.\d+$/.test(process.argv[1])) process.exit(1)' "$version" + echo "value=$version" >> "$GITHUB_OUTPUT" + - name: Build self-contained runtime + shell: bash + run: node Memory/src/cli/scripts/build-runtime.mjs --target "${{ matrix.target }}" --version "${{ steps.version.outputs.value }}" --output Memory/dist/release-part + - name: Build CLI launcher + shell: bash + env: + MEMMY_MEMORY_TARGET: ${{ matrix.target }} + MEMMY_MEMORY_VERSION: ${{ steps.version.outputs.value }} + run: bash Memory/src/cli/scripts/build-binary.sh + - uses: actions/upload-artifact@v4 + with: + name: memory-${{ matrix.target }} + path: | + Memory/dist/release-part/*.tar.gz + Memory/src/cli/dist/binaries/*.tar.gz + if-no-files-found: error + + publish: + needs: runtime + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: Memory/dist/release-input + - name: Resolve version + id: version + run: | + version="${{ inputs.version }}" + if [[ -z "$version" ]]; then version="${GITHUB_REF_NAME#memory-v}"; fi + echo "value=$version" >> "$GITHUB_OUTPUT" + - run: node Memory/src/cli/scripts/assemble-release.mjs Memory/dist/release-input Memory/dist/release "${{ steps.version.outputs.value }}" + - uses: softprops/action-gh-release@v2 + with: + tag_name: memory-v${{ steps.version.outputs.value }} + name: Memmy Memory ${{ steps.version.outputs.value }} + generate_release_notes: true + files: Memory/dist/release/* diff --git a/.gitignore b/.gitignore index 85d4c2bcb..4418fbd9c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ sessions/ .env.* !.env.example App/backend/src/adapters/outbound/skill-writer/workspace-bridge/memmy-workspace-bridge.mjs +Memory/src/agent-source/integration/workspace-bridge/memmy-workspace-bridge.mjs diff --git a/AgentSourceCore/package.json b/AgentSourceCore/package.json new file mode 100644 index 000000000..cdd2583db --- /dev/null +++ b/AgentSourceCore/package.json @@ -0,0 +1,13 @@ +{ + "name": "@memmy/agent-source-core", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + } +} diff --git a/AgentSourceCore/src/index.test.ts b/AgentSourceCore/src/index.test.ts new file mode 100644 index 000000000..3b85da8c7 --- /dev/null +++ b/AgentSourceCore/src/index.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { conversationContentHash, orderedTurns, splitTurn, type ConversationMessage } from "./index.js"; + +const message = (id: string, role: ConversationMessage["role"], content: string, createdAt: string): ConversationMessage => ({ + messageId: id, sourceId: "fixture", conversationId: "conversation", role, content, createdAt, + workspacePath: null, gitRoot: null, rawMeta: {} +}); + +describe("agent source core", () => { + it("emits stable turns across page boundaries", async () => { + const pages = (async function*() { + yield message("u1", "user", "hello", "2026-01-01T00:00:00Z"); + yield message("t1", "tool", "tool", "2026-01-01T00:00:01Z"); + yield message("a1", "assistant", "world", "2026-01-01T00:00:02Z"); + yield message("u2", "user", "next", "2026-01-01T00:00:03Z"); + yield message("a2", "assistant", "done", "2026-01-01T00:00:04Z"); + })(); + const turns = []; + for await (const turn of orderedTurns(pages)) turns.push(turn); + expect(turns).toHaveLength(2); + expect(turns.map((turn) => turn.messages[0]?.messageId)).toEqual(["u1", "u2"]); + }); + + it("splits oversized content with unique part hashes", () => { + const turn = { sourceId: "fixture", conversationId: "conversation", turnIndex: 0, messages: [message("u", "user", "x".repeat(30_000), "2026-01-01T00:00:00Z"), message("a", "assistant", "ok", "2026-01-01T00:00:01Z")] }; + const parts = splitTurn(turn, 4000, 1_000_000); + expect(parts.length).toBeGreaterThan(1); + expect(new Set(parts.map((part) => part.contentHash)).size).toBe(parts.length); + expect(parts.every((part) => Buffer.byteLength(part.content) <= 1_000_000)).toBe(true); + expect(conversationContentHash(turn.messages)).toHaveLength(64); + }); +}); diff --git a/AgentSourceCore/src/index.ts b/AgentSourceCore/src/index.ts new file mode 100644 index 000000000..e6f510fdb --- /dev/null +++ b/AgentSourceCore/src/index.ts @@ -0,0 +1,397 @@ +import { createHash } from "node:crypto"; + +export interface ConversationMessage { + messageId: string; + sourceId: string; + conversationId: string; + role: "user" | "assistant" | "tool" | "system"; + content: string; + createdAt: string; + workspacePath: string | null; + gitRoot: string | null; + rawMeta: Readonly>; + ordinal?: number; +} + +export interface SourceDescriptor { + sourceId: string; + displayName: string; + builtin: boolean; + dataPath: string; +} + +export interface ScanProgress { + sourceId: string; + phase: "discover" | "read" | "redact" | "emit" | "scan" | "prepare" | "add" | "summarize" | "done" | "stopped"; + current: number; + total: number; + message?: string; +} + +export interface ScanOptions { + since?: string; + maxMessages?: number; + maxScanTargets?: number; + order?: "source_default" | "recent_first"; + signal?: AbortSignal; + /** Production scanners set this to bypass legacy whole-window buffering. */ + fullHistory?: boolean; + onProgress?: (progress: ScanProgress) => void; +} + +export interface SourceAdapter { + readonly descriptor: SourceDescriptor; + detect(): Promise; + scan(options: ScanOptions): AsyncIterable; +} + +export interface ScanStore { + stage(message: ConversationMessage): boolean; + stageBatch(messages: readonly ConversationMessage[]): number; + messages(sourceId: string, cursor?: MessageCursor, limit?: number): Iterable; + saveScanCursor(sourceId: string, cursor: MessageCursor): void; + getScanCursor(sourceId: string): MessageCursor | null; + saveSourceState(state: ScanSourceState): void; + getSourceState(sourceId: string): ScanSourceState | null; + sourceCount(): number; + count(sourceId?: string): number; + saveCheckpoint(checkpoint: ConversationCheckpoint): void; + getCheckpoint(sourceId: string, conversationId: string): ConversationCheckpoint | null; + saveConversationMeta(meta: PreparedConversation): void; + getConversationMeta(sourceId: string, conversationId: string): PreparedConversation | null; + selectAllConversations(sourceId: string): void; + saveTurnMeta(meta: PreparedTurn): void; + getTurnMeta(sourceId: string, conversationId: string, turnId: string): PreparedTurn | null; + selectInitialTurns(sourceIds: readonly string[], globalLimit: number, absentSourceLimit: number): void; + saveResult(result: ScanStoredResult): void; + resultCount(sourceId?: string): number; + results(sourceId?: string, cursor?: string, limit?: number): Iterable; + close(): void; + remove(): void; +} + +export interface PreparedTurn { + sourceId: string; + conversationId: string; + turnId: string; + firstMessageId: string; + firstCreatedAt: string; + lastMessageId: string; + lastCreatedAt: string; + selected: boolean; +} + +export interface PreparedConversation { + sourceId: string; + conversationId: string; + lastMessageId: string; + lastCreatedAt: string; + contentHash: string; + selected: boolean; +} + +export interface MessageCursor { + conversationId: string; + createdAt: string; + messageId: string; + ordinal: number; +} + +export interface ConversationCheckpoint { + sourceId: string; + conversationId: string; + lastMessageId: string; + lastCreatedAt: string; + contentHash: string; + updatedAt: string; +} + +export interface ScanStoredResult { + sourceId: string; + conversationId: string; + memoryId?: string; + error?: string; + /** Opaque keyset cursor populated when a result is read from a store. */ + cursor?: string; +} + +export type ScanStage = "stage" | "prepare" | "ingest" | "summarize" | "done" | "failed" | "paused" | "canceled"; + +export interface ScanSourceState { + sourceId: string; + mode: string; + phase: ScanStage; + messageCount: number; + resultCount: number; + errorCount: number; + scanStartedAt?: string; + watermarkedSince?: string; + updatedAt: string; + error?: string; +} + +export interface ImportedTurn { + sourceId: string; + conversationId: string; + turnIndex: number; + messages: ConversationMessage[]; +} + +export interface TurnPart extends ImportedTurn { + parentTurnId: string; + partIndex: number; + partCount: number; + content: string; + contentHash: string; +} + +export function compareMessageOrder(left: ConversationMessage, right: ConversationMessage): number { + return left.conversationId.localeCompare(right.conversationId) + || Date.parse(left.createdAt) - Date.parse(right.createdAt) + || left.messageId.localeCompare(right.messageId) + || (left.ordinal ?? 0) - (right.ordinal ?? 0); +} + +export function compareCursor(left: ConversationMessage, right: MessageCursor): number { + return left.conversationId.localeCompare(right.conversationId) + || Date.parse(left.createdAt) - Date.parse(right.createdAt) + || left.messageId.localeCompare(right.messageId) + || (left.ordinal ?? 0) - right.ordinal; +} + +export async function* orderedTurns(messages: AsyncIterable): AsyncIterable { + let current: ConversationMessage[] = []; + let conversationId = ""; + let turnIndex = 0; + for await (const message of messages) { + if (message.conversationId !== conversationId) { + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + current = []; + conversationId = message.conversationId; + turnIndex = 0; + } + if (message.role === "user" && current.length > 0) { + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; + turnIndex += 1; + current = []; + } + current.push(message); + } + if (isCompleteTurn(current)) yield { sourceId: current[0]!.sourceId, conversationId, turnIndex, messages: current }; +} + +export function isCompleteTurn(messages: readonly ConversationMessage[]): boolean { + const first = messages[0]; + const last = messages[messages.length - 1]; + return first?.role === "user" && Boolean(first.content.trim()) + && last?.role === "assistant" && Boolean(last.content.trim()); +} + +export function renderMessageContent(message: ConversationMessage): string { + if (message.role !== "tool" || /^Tool:\s*/im.test(message.content)) return message.content; + const toolName = stringMeta(message.rawMeta, "toolName") ?? stringMeta(message.rawMeta, "hermesToolName"); + const callId = stringMeta(message.rawMeta, "toolCallId") ?? stringMeta(message.rawMeta, "hermesToolCallId"); + return [toolName ? `Tool: ${toolName}` : undefined, callId ? `Call ID: ${callId}` : undefined, message.content] + .filter(Boolean).join("\n\n"); +} + +export function renderTurn(messages: readonly ConversationMessage[]): string { + return messages.map((message) => `## ${message.role}\n\n${renderMessageContent(message)}`).join("\n\n"); +} + +export function conversationContentHash(messages: Iterable): string { + const hash = createHash("sha256"); + hash.update("["); + let first = true; + for (const message of messages) { + if (!first) hash.update(","); + first = false; + hash.update(JSON.stringify({ + messageId: message.messageId, + role: message.role, + content: message.content, + createdAt: message.createdAt, + toolName: hashMetaString(message.rawMeta, "toolName") ?? hashMetaString(message.rawMeta, "hermesToolName"), + toolCallId: hashMetaString(message.rawMeta, "toolCallId") ?? hashMetaString(message.rawMeta, "hermesToolCallId") + })); + } + hash.update("]"); + return hash.digest("hex"); +} + +export function stableTurnIdentity(turn: ImportedTurn): string { + const firstUser = turn.messages.find((message) => message.role === "user"); + if (!firstUser) throw new Error("turn is missing user message"); + return `${turn.sourceId}::${turn.conversationId}::${firstUser.messageId}`; +} + +/** Preserves the pre-staging idempotency key for an unsplit turn. */ +export function legacyTurnRequestId(turn: ImportedTurn): string { + const first = turn.messages[0]; + if (!first) throw new Error("turn is empty"); + return createHash("sha256").update([stableTurnIdentity(turn), first.createdAt, renderTurn(turn.messages)].join("\u0000")).digest("hex"); +} + +/** Preserves the pre-staging stable turn id for an unsplit turn. */ +export function legacyTurnId(turn: ImportedTurn): string { + return `${turn.sourceId}:${createHash("sha256").update(stableTurnIdentity(turn)).digest("hex").slice(0, 24)}`; +} + +export function splitTurn(turn: ImportedTurn, maxTokens = 4000, maxBytes = 1024 * 1024): TurnPart[] { + const chunks: ConversationMessage[][] = []; + let current: ConversationMessage[] = []; + const fits = (candidate: readonly ConversationMessage[]) => { + const content = renderTurn(candidate); + return estimateTokens(content) <= maxTokens && Buffer.byteLength(content) <= maxBytes; + }; + for (const message of turn.messages) { + if (fits([...current, message])) { + current.push(message); + continue; + } + if (current.length > 0) { chunks.push(current); current = []; } + const pieces = splitMessage(message, maxTokens, maxBytes); + if (current.length === 0 && chunks.length > 0) { + // A user prefix followed by an oversized assistant/tool message should + // remain one logical part whenever the prefix can share any content. + const previous = chunks[chunks.length - 1]; + if (previous && !isCompleteTurn(previous)) { + const combined = combinePrefix(previous, pieces[0]!, maxTokens, maxBytes); + if (combined) { + chunks[chunks.length - 1] = combined.messages; + if (combined.remainder) chunks.push(...splitMessage(combined.remainder, maxTokens, maxBytes).map((piece) => [piece])); + for (const piece of pieces.slice(1)) chunks.push([piece]); + continue; + } + } + } + for (const piece of pieces) chunks.push([piece]); + } + if (current.length > 0) chunks.push(current); + if (chunks.length === 0) chunks.push([...turn.messages]); + const parentTurnId = createHash("sha256").update(stableTurnIdentity(turn)).digest("hex").slice(0, 24); + return chunks.map((messages, partIndex) => { + const content = renderTurn(messages); + return { + ...turn, + messages, + parentTurnId, + partIndex, + partCount: chunks.length, + content, + contentHash: createHash("sha256").update(content).digest("hex") + }; + }); +} + +function combinePrefix( + prefix: readonly ConversationMessage[], + piece: ConversationMessage, + maxTokens: number, + maxBytes: number +): { messages: ConversationMessage[]; remainder?: ConversationMessage } | null { + const fits = (content: string) => { + const candidate = [...prefix, { ...piece, content }]; + const rendered = renderTurn(candidate); + return estimateTokens(rendered) <= maxTokens && Buffer.byteLength(rendered) <= maxBytes; + }; + if (fits(piece.content)) return { messages: [...prefix, piece] }; + const characters = Array.from(piece.content); + let low = 0; + let high = characters.length; + let best = 0; + while (low <= high) { + const middle = Math.floor((low + high) / 2); + if (fits(characters.slice(0, middle).join(""))) { + best = middle; + low = middle + 1; + } else { + high = middle - 1; + } + } + if (best === 0) return null; + const content = characters.slice(0, best).join(""); + const remainder = characters.slice(best).join(""); + return { + messages: [...prefix, { ...piece, content }], + ...(remainder ? { remainder: { ...piece, content: remainder } } : {}) + }; +} + +function splitMessage(message: ConversationMessage, maxTokens: number, maxBytes: number): ConversationMessage[] { + const emptyRendered = renderTurn([{ ...message, content: "" }]); + const bodyTokenLimit = Math.max(1, maxTokens - estimateTokens(emptyRendered)); + const bodyByteLimit = Math.max(1, maxBytes - Buffer.byteLength(emptyRendered)); + const fitsContent = (content: string) => { + const rendered = renderTurn([{ ...message, content }]); + return estimateTokens(rendered) <= maxTokens && Buffer.byteLength(rendered) <= maxBytes; + }; + if (fitsContent(message.content)) return [message]; + const pieces: string[] = []; + for (const paragraph of message.content.split(/\n\s*\n/)) { + if (!paragraph) continue; + if (paragraph && estimateTokens(paragraph) <= bodyTokenLimit && Buffer.byteLength(paragraph) <= bodyByteLimit && fitsContent(paragraph)) { + pieces.push(paragraph); + } else { + pieces.push(...splitText(paragraph, bodyTokenLimit, bodyByteLimit)); + } + } + return (pieces.length > 0 ? pieces : [""]).map((content) => ({ ...message, content })); +} + +function splitText(value: string, maxTokens: number, maxBytes: number): string[] { + const maxChars = Math.max(1, maxTokens * 4); + const chunks: string[] = []; + let current = ""; + for (const line of value.split(/\r?\n/u)) { + const candidate = current ? `${current}\n${line}` : line; + if (current && (estimateTokens(candidate) > maxTokens || Buffer.byteLength(candidate) > maxBytes)) { + chunks.push(...splitUtf8(current, maxBytes)); + current = ""; + } + if (line.length > maxChars || Buffer.byteLength(line) > maxBytes) { + if (current) { chunks.push(...splitUtf8(current, maxBytes)); current = ""; } + let part = ""; + for (const character of line) { + if (part && (part.length >= maxChars || Buffer.byteLength(part + character) > maxBytes)) { + chunks.push(part); + part = ""; + } + part += character; + } + if (part) chunks.push(part); + } else { + current = candidate; + } + } + if (current) chunks.push(...splitUtf8(current, maxBytes)); + return chunks.length > 0 ? chunks : [""]; +} + +function splitUtf8(value: string, maxBytes: number): string[] { + const parts: string[] = []; + let current = ""; + for (const character of value) { + const candidate = current + character; + if (current && Buffer.byteLength(candidate) > maxBytes) { + parts.push(current); + current = character; + } else { + current = candidate; + } + } + if (current) parts.push(current); + return parts.length > 0 ? parts : [""]; +} + +export function estimateTokens(value: string): number { return Math.ceil(value.length / 4); } + +function stringMeta(meta: Readonly>, key: string): string | undefined { + const value = meta[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function hashMetaString(meta: Readonly>, key: string): string | undefined { + const value = meta[key]; + return typeof value === "string" ? value : undefined; +} diff --git a/AgentSourceCore/tsconfig.json b/AgentSourceCore/tsconfig.json new file mode 100644 index 000000000..6c185e9e9 --- /dev/null +++ b/AgentSourceCore/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/App/backend/local-api-contracts/src/index.ts b/App/backend/local-api-contracts/src/index.ts index d2c6cadbc..23ff36e2c 100644 --- a/App/backend/local-api-contracts/src/index.ts +++ b/App/backend/local-api-contracts/src/index.ts @@ -78,10 +78,15 @@ export const AppSettingsDtoSchema = z.object({ // Notification sound enabled. notificationSoundEnabled: z.boolean().default(true), // Menu bar icon enabled. - menuBarIconEnabled: z.boolean().default(true) + menuBarIconEnabled: z.boolean().default(true), + // Stop the standalone Memory daemon when Desktop exits. + stopMemoryServiceOnExit: z.boolean().default(false) }); export type AppSettingsDto = z.infer; +export const FirstEncounterReportStatusSchema = z.enum(["pending", "shown", "skipped"]); +export type FirstEncounterReportStatus = z.infer; + export const OnboardingStateDtoSchema = z.object({ // Completed. completed: z.boolean(), @@ -93,6 +98,8 @@ export const OnboardingStateDtoSchema = z.object({ acceptedTermsVersion: z.string().nullable(), // Scan permission. scanPermission: ScanPermissionSchema, + // Installation-local first encounter report state. + firstEncounterReportStatus: FirstEncounterReportStatusSchema.optional(), // Improvement program. improvementProgram: ImprovementProgramSchema, // Completed at. @@ -413,7 +420,12 @@ export type AgentSourceScanInput = z.infer; export const OnboardingInsightReportInputSchema = z.object({ locale: z.enum(["zh-CN", "en-US"]).optional(), - stream: z.boolean().optional() + stream: z.boolean().optional(), + detectedAgents: z.array(z.object({ + sourceId: z.string().min(1), + displayName: z.string().min(1), + recentSessionCount: z.number().int().nonnegative() + })).max(50).optional() }).default({}); export type OnboardingInsightReportInput = z.infer; @@ -523,6 +535,9 @@ export const ScanResultSchema = z.object({ emittedMessages: z.number().int().nonnegative(), skipped: z.number().int().nonnegative(), memoryIds: z.array(z.string().min(1)).optional(), + memoryIdCount: z.number().int().nonnegative().optional(), + errorCount: z.number().int().nonnegative().optional(), + detailsTruncated: z.boolean().optional(), errors: z.array( z.object({ conversationId: z.string().min(1), @@ -532,6 +547,18 @@ export const ScanResultSchema = z.object({ }); export type ScanResult = z.infer; +/** Schema for paged persisted scan details. */ +export const ScanResultPageSchema = z.object({ + items: z.array(z.object({ + sourceId: z.string().min(1), + conversationId: z.string().min(1), + memoryId: z.string().min(1).optional(), + error: z.string().min(1).optional() + })), + nextCursor: z.string().nullable() +}); +export type ScanResultPage = z.infer; + /** Schema for legal agreement locale urls. */ export const LegalAgreementLocaleUrlsSchema = z.object({ "zh-CN": z.string().url(), @@ -598,7 +625,8 @@ export const PatchAppSettingsInputSchema = z defaultLaunchMode: DefaultLaunchModeSchema, taskDoneNotificationEnabled: z.boolean(), notificationSoundEnabled: z.boolean(), - menuBarIconEnabled: z.boolean() + menuBarIconEnabled: z.boolean(), + stopMemoryServiceOnExit: z.boolean() }) .partial(); export type PatchAppSettingsInput = z.infer; @@ -1022,11 +1050,22 @@ export const EffectiveModelCandidatesSchema = z.object({ }); export type EffectiveModelCandidates = z.infer; +/** Runtime ownership switches stored under ~/.memmy/config.yaml#memmyMemory. */ +export const MemoryRuntimeModelSettingsSchema = z.object({ + roleRouting: z.object({ + summary: z.enum(["follow", "fixed"]), + evolution: z.enum(["follow", "fixed"]) + }), + embeddingMode: z.enum(["cloud", "local", "custom"]) +}); +export type MemoryRuntimeModelSettings = z.infer; + /** Schema for model config view. */ export const ModelConfigViewSchema = z.object({ configRevision: z.string().min(1), providers: z.array(TextModelProviderViewSchema), modelAssignments: ModelAssignmentsSchema, + memorySettings: MemoryRuntimeModelSettingsSchema.optional(), effectiveCandidates: EffectiveModelCandidatesSchema, configured: z.boolean(), updatedAt: z.string().datetime() diff --git a/App/backend/local-api-contracts/src/memory-runtime.ts b/App/backend/local-api-contracts/src/memory-runtime.ts index d62f53c9c..003b33675 100644 --- a/App/backend/local-api-contracts/src/memory-runtime.ts +++ b/App/backend/local-api-contracts/src/memory-runtime.ts @@ -301,6 +301,10 @@ export type MemoryModelsStatus = z.infer; /** Schema for memory health snapshot. */ export const MemoryHealthSnapshotSchema = z.object({ ok: z.boolean(), + serviceVersion: NonEmptyStringSchema.optional(), + protocolVersion: z.number().int().positive().optional(), + viewerVersion: NonEmptyStringSchema.optional(), + viewerUrl: z.url().optional(), version: NonEmptyStringSchema, uptimeMs: z.number().nonnegative(), mode: z.enum(["local", "cloud", "dev"]), @@ -314,7 +318,8 @@ export const MemoryHealthSnapshotSchema = z.object({ routes: z.array(z.string()), tools: z.array(z.string()), memoryLayers: z.array(MemoryLayerSchema), - supportsCli: z.boolean() + supportsCli: z.boolean(), + service: z.array(z.string()).optional() }), features: L3WorldModelFeaturesSchema.optional(), models: MemoryModelsStatusSchema, @@ -524,7 +529,8 @@ export const AddMemoryOutputSchema = z.object({ summary: z.string(), tags: z.array(z.string()), createdAt: IsoTimeSchema, - serverTime: IsoTimeSchema + serverTime: IsoTimeSchema, + duplicate: z.boolean().optional() }); export type AddMemoryOutput = z.infer; diff --git a/App/backend/local-api-contracts/src/model-catalog-resolver.ts b/App/backend/local-api-contracts/src/model-catalog-resolver.ts index 1794a07f5..289eadfad 100644 --- a/App/backend/local-api-contracts/src/model-catalog-resolver.ts +++ b/App/backend/local-api-contracts/src/model-catalog-resolver.ts @@ -13,6 +13,8 @@ export interface RuntimeCatalogEndpoint { extraBody?: Record; } +export const BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID = "memmy-builtin-local-embedding"; + export interface RuntimeCatalogProvider { apiKey?: string; extraHeaders?: Record; diff --git a/App/backend/package.json b/App/backend/package.json index bbac18037..235b64f1f 100644 --- a/App/backend/package.json +++ b/App/backend/package.json @@ -14,14 +14,15 @@ "scripts": { "workspace-bridge:build": "node src/adapters/outbound/skill-writer/workspace-bridge/build-runtime.mjs", "workspace-bridge:build:dist": "node src/adapters/outbound/skill-writer/workspace-bridge/build-runtime.mjs --dist", - "build": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node -e \"const fs=require('node:fs'); fs.cpSync('src/infrastructure/app-state-store/migrations', 'dist/src/infrastructure/app-state-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/agent-source-store/migrations', 'dist/src/infrastructure/agent-source-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/idempotency-store/migrations', 'dist/src/infrastructure/idempotency-store/migrations', { recursive: true }); if (fs.existsSync('src/adapters/outbound/agent-adapter/plugins')) fs.cpSync('src/adapters/outbound/agent-adapter/plugins', 'dist/src/adapters/outbound/agent-adapter/plugins', { recursive: true });\" && npm run workspace-bridge:build:dist", + "build": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.json && node -e \"const fs=require('node:fs'); fs.cpSync('src/infrastructure/app-state-store/migrations', 'dist/src/infrastructure/app-state-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/agent-source-store/migrations', 'dist/src/infrastructure/agent-source-store/migrations', { recursive: true }); fs.cpSync('src/infrastructure/idempotency-store/migrations', 'dist/src/infrastructure/idempotency-store/migrations', { recursive: true }); if (fs.existsSync('src/adapters/outbound/agent-adapter/plugins')) fs.cpSync('src/adapters/outbound/agent-adapter/plugins', 'dist/src/adapters/outbound/agent-adapter/plugins', { recursive: true });\" && npm run workspace-bridge:build:dist", "lint": "eslint \"src/**/*.ts\" \"vitest.config.ts\"", - "typecheck": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && tsc -p tsconfig.json --noEmit", - "test": "npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && npm run workspace-bridge:build && vitest run", + "typecheck": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && tsc -p tsconfig.json --noEmit", + "test": "npm run build -w @memmy/agent-source-core && npm run build -w @memmy/migrations && npm run build -w @memmy/local-api-contracts && npm run workspace-bridge:build && vitest run", "test:agent-adapter:coverage": "npm run build -w @memmy/local-api-contracts && vitest run src/adapters/outbound/agent-adapter/tests --coverage", "db:migrate": "tsx src/infrastructure/app-state-store/cli/migrate.ts" }, "dependencies": { + "@memmy/agent-source-core": "0.0.0", "@memmy/local-api-contracts": "0.0.0", "@memmy/migrations": "0.0.0", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts b/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts index 781e2342c..0a20fd3f1 100644 --- a/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts +++ b/App/backend/src/adapters/inbound/local-api/routes/agent-sources.ts @@ -9,6 +9,7 @@ import { AgentSourceScanJobResponseSchema, AgentSourceScanStatusResponseSchema, AgentSourceViewSchema, + ScanResultPageSchema, ManagedAgentSourceImportInputSchema, ManagedAgentSourceImportResultSchema, ManagedAgentSourceUpdateInputSchema, @@ -23,9 +24,12 @@ import { withErrorEnvelope } from "../../../../services/error-envelope.js"; import type { AgentSourceAutoInjectService } from "../../../../services/agent-source-auto-inject-service.js"; import type { AgentSourceService } from "../../../../services/agent-source-service.js"; import { + deleteDurableScanStore, deletePersistedScanResume, + readDurableScanResults, readLatestPersistedScanResume } from "../../../../services/agent-source-scan-journal.js"; +import { migrateLegacyScanJournals } from "../../../../services/agent-source-scan-migration.js"; import type { ProgressBus } from "../../../../services/progress-bus.js"; import { type AgentSourceScanJobState, @@ -67,6 +71,7 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe lastProgress: PipelineProgress & { jobId: string }; resume: RouteScanResumeState | null; }; + if (options.scanProcess?.databasePath) migrateLegacyScanJournals(options.scanProcess.databasePath); const restoredScanJob = toPausedScanJob(readLatestPersistedScanResume(options.scanProcess?.databasePath)); let pausedScanJob: PausedScanJob | null = restoredScanJob; let lastScanProgress: (PipelineProgress & { jobId: string }) | null = restoredScanJob?.lastProgress ?? null; @@ -135,6 +140,15 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe })); }); + app.get("/api/agent-sources/scan/jobs/:jobId/results", { preHandler: options.authenticateRuntimeToken }, async (request, reply) => { + if (!options.scanProcess) return reply.send({ items: [], nextCursor: null }); + const params = request.params as { jobId: string }; + const query = request.query as { cursor?: string; limit?: string }; + const limit = Math.min(500, Math.max(1, Number.parseInt(query.limit ?? "100", 10) || 100)); + const cursor = query.cursor ?? "0"; + return reply.send(ScanResultPageSchema.parse(readDurableScanResults(options.scanProcess.databasePath, params.jobId, cursor, limit))); + }); + app.post("/api/agent-sources/scan", { preHandler: options.authenticateRuntimeToken }, async (request, reply) => { const input = AgentSourceScanInputSchema.parse(request.body); const sourceId = input.sourceId; @@ -156,7 +170,10 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe return reply.send(AgentSourceScanJobResponseSchema.parse({ jobId: activeScanJob.jobId })); } - const pausedJob = pausedScanJob?.resume && pausedScanJob.sourceId === sourceId && pausedScanJob.mode === mode ? pausedScanJob : null; + const pausedJob = pausedScanJob && pausedScanJob.sourceId === sourceId && pausedScanJob.mode === mode + && (pausedScanJob.resume || options.scanProcess) + ? pausedScanJob + : null; if (!pausedJob) { cleanupResumeState(pausedScanJob?.resume ?? null); pausedScanJob = null; @@ -218,8 +235,10 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe activeScanJob = null; abortScanJob(canceledJob); cleanupResumeState(canceledJob.resume); + deleteDurableScanStore(options.scanProcess?.databasePath, canceledJob.jobId); } cleanupResumeState(pausedScanJob?.resume ?? null); + if (pausedScanJob) deleteDurableScanStore(options.scanProcess?.databasePath, pausedScanJob.jobId); pausedScanJob = null; lastScanProgress = null; return reply.send(OkResponseSchema.parse({ ok: true })); @@ -547,6 +566,7 @@ export function registerAgentSourceRoutes(app: FastifyInstance, options: Registe deletePersistedScanResume(options.scanProcess?.databasePath, resume.jobId); } + } function toPausedScanJob( diff --git a/App/backend/src/adapters/inbound/local-api/routes/app-config.ts b/App/backend/src/adapters/inbound/local-api/routes/app-config.ts index ab9b1cd99..0e1f5a9c9 100644 --- a/App/backend/src/adapters/inbound/local-api/routes/app-config.ts +++ b/App/backend/src/adapters/inbound/local-api/routes/app-config.ts @@ -59,6 +59,15 @@ export function registerAppConfigRoutes(app: FastifyInstance, options: RegisterA }) ); + app.get( + "/api/app/scan-preferences", + { preHandler: options.authenticateRuntimeToken }, + withErrorEnvelope(async (_request, reply) => { + const response = ScanPreferencesSchema.parse(await options.appConfig.getScanPreferences()); + return reply.send(response); + }) + ); + app.patch( "/api/app/onboarding", { preHandler: options.authenticateRuntimeToken }, diff --git a/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts b/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts index b358a4d15..d18b5fc42 100644 --- a/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/claude-code/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveClaudeCodeProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverClaudeCodeSessions } from "./project-discovery.js"; @@ -68,13 +68,13 @@ export function createClaudeCodeSourceAdapter(deps: CreateClaudeCodeSourceAdapte message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readClaudeCodeTranscript(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts b/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts index 0bff8b96e..29d88e7e1 100644 --- a/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/claude-code/project-discovery.ts @@ -34,7 +34,7 @@ export async function discoverClaudeCodeSessions( const projectPath = join(options.root, projectEntry.name); const files = await readDirectoryIfExists(projectPath); for (const file of files) { - if (!file.isFile() || !file.name.endsWith(".jsonl")) { + if (!file.isFile() || !file.name.endsWith(".jsonl") || /\.jsonl\.bak-/u.test(file.name)) { continue; } diff --git a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts index 3b9680ae4..62b5dcab9 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCodexSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readCodexRollout, type RawCodexMessage } from "./rollout-reader.js"; @@ -67,13 +67,13 @@ export function createCodexSourceAdapter(deps: CreateCodexSourceAdapterDeps = {} message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readCodexRollout(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, phase: "redact", current: emittedMessages, total: emittedMessages + 1 }); emittedMessages += 1; diff --git a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts index 956d10c4f..6dd193392 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/rollout-reader.ts @@ -2,6 +2,8 @@ import { basename } from "node:path"; import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; +const MAX_TOOL_NAME_ENTRIES = 4096; + export interface RawCodexMessage { /** Message id. */ messageId: string; @@ -73,6 +75,10 @@ function toToolMessage( const name = getString(payload.name) ?? "tool"; if (callId) { toolNamesByCallId.set(callId, name); + if (toolNamesByCallId.size > MAX_TOOL_NAME_ENTRIES) { + const oldest = toolNamesByCallId.keys().next().value; + if (typeof oldest === "string") toolNamesByCallId.delete(oldest); + } } return { messageId: `${rolloutId}:${lineNumber}`, diff --git a/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts index 57d2b41df..5c2cb1a47 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/session-discovery.ts @@ -5,6 +5,8 @@ import { dirname, join } from "node:path"; import { readJsonlObjects } from "../jsonl-lines.js"; import { readDirectoryIfExists } from "../read-directory.js"; +const ROLLOUT_FILE_SUFFIX = ".jsonl"; + export interface CodexSessionFile { /** Session file path. */ sessionFilePath: string; @@ -56,7 +58,7 @@ async function listRolloutFiles( continue; } - if (entry.isFile() && entry.name.startsWith("rollout-") && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && isPrimaryRolloutFile(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } @@ -71,6 +73,12 @@ async function listRolloutFiles( .map((file) => file.path); } +function isPrimaryRolloutFile(name: string): boolean { + return name.startsWith("rollout-") && + name.endsWith(ROLLOUT_FILE_SUFFIX) && + name.indexOf(ROLLOUT_FILE_SUFFIX) === name.length - ROLLOUT_FILE_SUFFIX.length; +} + async function readFirstCwd(filePath: string): Promise { try { for await (const record of readJsonlObjects(filePath)) { diff --git a/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts b/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts index 0a358c51c..bf6d549f7 100644 --- a/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts +++ b/App/backend/src/adapters/outbound/agent-source/codex/tests/adapter.test.ts @@ -7,6 +7,9 @@ import { createCodexSourceAdapter } from "../index.js"; import { readCodexRollout } from "../rollout-reader.js"; import { discoverCodexSessions } from "../session-discovery.js"; +// Generated synthetic data for redaction coverage, never a live credential. +const SYNTHETIC_API_KEY = `sk-${"fixture".repeat(8)}`; + let tempDir: string | undefined; afterEach(() => { @@ -27,7 +30,7 @@ describe("codex source adapter", () => { messageId: "019e72be-500b-7f02-9400-112c5a194e5c:2", conversationId: "019e72be-500b-7f02-9400-112c5a194e5c", role: "user", - content: "Use OPENAI_API_KEY=sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN" + content: `Use OPENAI_API_KEY=${SYNTHETIC_API_KEY}` }), expect.objectContaining({ messageId: "019e72be-500b-7f02-9400-112c5a194e5c:3", @@ -67,6 +70,25 @@ describe("codex source adapter", () => { expect.objectContaining({ sourceId: "codex", role: "tool" }), expect.objectContaining({ sourceId: "codex", role: "assistant" }) ]); + expect(messages.some((message) => message.content.includes(SYNTHETIC_API_KEY))).toBe(false); + }); + + it("ignores backup copies of rollout files", async () => { + const fixture = createFixture(); + const backupPath = `${fixture.rolloutPath}.bak-strip-input-image.jsonl`; + writeFileSync( + backupPath, + JSON.stringify({ + timestamp: "2026-05-29T11:00:00.000Z", + type: "session_meta", + payload: { cwd: join(fixture.workspacePath, "backup-copy") } + }), + "utf8" + ); + + await expect(discoverCodexSessions({ root: fixture.sessionsRoot })).resolves.toEqual([ + expect.objectContaining({ sessionFilePath: fixture.rolloutPath }) + ]); }); it("redacts large image tool outputs without failing the scan", async () => { @@ -157,7 +179,7 @@ function createFixture(): { sessionsRoot: string; workspacePath: string; rollout rolloutPath, [ JSON.stringify({ timestamp: "2026-05-29T10:00:00.000Z", type: "session_meta", payload: { cwd: workspacePath } }), - JSON.stringify({ timestamp: "2026-05-29T10:00:01.000Z", type: "response_item", payload: { type: "message", role: "user", content: [{ type: "input_text", text: "Use OPENAI_API_KEY=sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN" }] } }), + JSON.stringify({ timestamp: "2026-05-29T10:00:01.000Z", type: "response_item", payload: { type: "message", role: "user", content: [{ type: "input_text", text: `Use OPENAI_API_KEY=${SYNTHETIC_API_KEY}` }] } }), JSON.stringify({ timestamp: "2026-05-29T10:00:02.000Z", type: "response_item", payload: { type: "function_call", name: "shell", call_id: "call-shell-1", arguments: "{\"cmd\":\"pwd\"}" } }), JSON.stringify({ timestamp: "2026-05-29T10:00:03.000Z", type: "response_item", payload: { type: "function_call_output", call_id: "call-shell-1", output: "/tmp/project" } }), JSON.stringify({ timestamp: "2026-05-29T10:00:04.000Z", type: "response_item", payload: { type: "message", role: "assistant", content: [{ type: "output_text", text: "Done" }] } }) diff --git a/App/backend/src/adapters/outbound/agent-source/conversation-window.ts b/App/backend/src/adapters/outbound/agent-source/conversation-window.ts index 78b331deb..14cdd8c05 100644 --- a/App/backend/src/adapters/outbound/agent-source/conversation-window.ts +++ b/App/backend/src/adapters/outbound/agent-source/conversation-window.ts @@ -45,6 +45,29 @@ export async function collectConversationWindow included.has(message.conversationId)); } +/** Streams a source target without materializing its complete history. */ +export async function* streamConversationWindow( + input: AsyncIterable, + since?: string, + signal?: AbortSignal, + maxMessages?: number, + fullHistory = false +): AsyncIterable { + if (maxMessages !== undefined && maxMessages <= 0) return; + if (fullHistory) { + let emitted = 0; + for await (const message of input) { + signal?.throwIfAborted(); + if (maxMessages !== undefined && emitted >= maxMessages) break; + emitted += 1; + yield message; + } + return; + } + // Keep the legacy complete-conversation semantics for non-production callers. + for (const message of await collectConversationWindow(input, since, signal, maxMessages)) yield message; +} + export function remainingMessageCapacity(limit: number | undefined, emitted: number): number | undefined { return limit === undefined ? undefined : Math.max(0, limit - emitted); } diff --git a/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts b/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts index 3db120766..f46b4255b 100644 --- a/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/cursor/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveCursorDataPaths } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; +import { readCursorVscdb, streamCursorVscdb, type RawCursorMessage } from "./vscdb-reader.js"; import { discoverCursorWorkspaces, type CursorWorkspace } from "./workspace-discovery.js"; const CURSOR_SOURCE_ID = "cursor"; @@ -84,13 +84,13 @@ export function createCursorSourceAdapter(deps: CreateCursorSourceAdapterDeps = message: target.storageHash }); - const messages = await collectConversationWindow( - readCursorVscdb(target.stateDbPath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamCursorVscdb(target.stateDbPath) : readCursorVscdb(target.stateDbPath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts b/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts index 1f257f5f0..af24e7ebf 100644 --- a/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/cursor/vscdb-reader.ts @@ -3,6 +3,7 @@ import { DatabaseSync } from "node:sqlite"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw cursor message. */ export interface RawCursorMessage { @@ -61,6 +62,36 @@ export async function* readCursorVscdb(path: string): AsyncIterable { + const db = new DatabaseSync(path, { readOnly: true }); + try { + if (hasTable(db, "ItemTable")) { + const statement = db.prepare("SELECT key, value FROM ItemTable WHERE value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + for (const message of extractMessagesFromItemRow(row)) yield message; + } + } + if (hasTable(db, "cursorDiskKV")) { + const statement = db.prepare("SELECT key, value FROM cursorDiskKV WHERE key LIKE 'bubbleId:%' AND value IS NOT NULL ORDER BY key ASC"); + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (Buffer.byteLength(row.value) > MAX_RECORD_BYTES) continue; + const message = extractMessageFromBubbleRow(row); + if (message) yield message; + } + } + } finally { + db.close(); + } +} + /** Reads read item table messages. */ async function readItemTableMessages(db: DatabaseSync): Promise { if (!hasTable(db, "ItemTable")) { @@ -76,7 +107,7 @@ async function readItemTableMessages(db: DatabaseSync): Promise MAX_RECORD_BYTES) continue; const message = extractMessageFromBubbleRow(row); if (message) { messages.push(message); diff --git a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts index 278fdc8f2..62fcee55b 100644 --- a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/adapter.ts @@ -1,11 +1,11 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveDeepseekHarnessHomeDirectory, resolveDeepseekHarnessSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverDeepseekHarnessSessions } from "./session-discovery.js"; -import { readDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; +import { streamDeepseekHarnessSession, type RawDeepseekHarnessMessage } from "./session-reader.js"; const SOURCE_ID = "deepseek_harness"; @@ -61,13 +61,13 @@ export function createDeepseekHarnessSourceAdapter( total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( - toAsyncIterable(await readDeepseekHarnessSession(session.sessionFilePath, options.signal)), + for await (const rawMessage of streamConversationWindow( + streamDeepseekHarnessSession(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { options.signal?.throwIfAborted(); emittedMessages += 1; options.onProgress?.({ sourceId: SOURCE_ID, phase: "emit", current: emittedMessages, total: emittedMessages }); @@ -91,10 +91,6 @@ function toConversationMessage( }; } -async function* toAsyncIterable(values: readonly T[]): AsyncIterable { - for (const value of values) yield value; -} - function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error; } diff --git a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts index 63efdcba2..00b5dae73 100644 --- a/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/deepseek-harness/session-reader.ts @@ -1,6 +1,8 @@ +import { createReadStream } from "node:fs"; import { readFile } from "node:fs/promises"; import { basename } from "node:path"; -import { decompress, ZstdErrorCode } from "fzstd"; +import { decompress, Decompress, ZstdErrorCode } from "fzstd"; +import { readJsonlObjects, type JsonObject } from "../jsonl-lines.js"; const ZSTD_FRAME_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); @@ -25,6 +27,108 @@ export async function readDeepseekHarnessSession( return parseSessionRows(text, filePath, signal); } +/** Streams uncompressed sessions; compressed legacy files use the existing decoder. */ +export async function* streamDeepseekHarnessSession( + filePath: string, + signal?: AbortSignal +): AsyncIterable { + if (filePath.endsWith(".zstd")) { + let conversationId = basename(filePath).replace(/\.jsonl\.zstd$/u, ""); + let workspacePath: string | null = null; + for await (const record of streamZstdJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } + return; + } + let conversationId = basename(filePath).replace(/\.jsonl$/u, ""); + let workspacePath: string | null = null; + for await (const record of readJsonlObjects(filePath, signal)) { + signal?.throwIfAborted(); + if (record.type === "session") { + if (typeof record.id === "string") conversationId = record.id; + if (typeof record.cwd === "string") workspacePath = record.cwd; + continue; + } + const message = toMessage(record, conversationId, workspacePath); + if (message) yield message; + } +} + +/** Streams zstd frames and parses bounded JSONL records without materializing the file. */ +async function* streamZstdJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { + const input = createReadStream(filePath); + const output: Buffer[] = []; + let outputBytes = 0; + let carry: Buffer = Buffer.alloc(0); + let overLimit = false; + const decoder = new Decompress((chunk) => { + outputBytes += chunk.byteLength; + if (outputBytes > 8 * 1024 * 1024) throw new Error("DeepSeek Harness decompressed chunk exceeds 8 MiB staging limit"); + output.push(Buffer.from(chunk)); + }); + try { + for await (const chunk of input) { + signal?.throwIfAborted(); + decoder.push(chunk as Buffer); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length > 64 * 1024 * 1024) continue; + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + if (carry.length > 64 * 1024 * 1024) { carry = Buffer.alloc(0); overLimit = true; } + } + } + decoder.push(new Uint8Array(), true); + while (output.length > 0) { + const data = output.shift()!; + outputBytes -= data.byteLength; + carry = carry.length === 0 ? data : Buffer.concat([carry, data]); + let newline = carry.indexOf(0x0a); + while (newline >= 0) { + const line = carry.subarray(0, newline); + carry = carry.subarray(newline + 1); + newline = carry.indexOf(0x0a); + if (overLimit) { overLimit = false; continue; } + if (line.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(line); + if (parsed) yield parsed; + } + } + } + if (!overLimit && carry.length > 0 && carry.length <= 64 * 1024 * 1024) { + const parsed = parseJsonObject(carry); + if (parsed) yield parsed; + } + } finally { + input.destroy(); + } +} + +function parseJsonObject(line: Buffer): JsonObject | null { + try { + const parsed = JSON.parse(line.toString("utf8").trim()) as unknown; + return isRecord(parsed) ? parsed as JsonObject : null; + } catch { + return null; + } +} + function decompressFrames(bytes: Buffer): string { if (!bytes.subarray(0, ZSTD_FRAME_MAGIC.length).equals(ZSTD_FRAME_MAGIC)) { throw new Error("DeepSeek Harness session has no Zstandard frame header"); diff --git a/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts b/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts index cdbc95155..77e3d1ce9 100644 --- a/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/hermes/adapter.ts @@ -2,7 +2,7 @@ import { access } from "node:fs/promises"; import { join } from "node:path"; import { resolveHermesHomeDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readHermesRollout, type RawHermesRolloutMessage } from "./rollout-reader.js"; @@ -79,13 +79,13 @@ export function createHermesSourceAdapter(deps: CreateHermesSourceAdapterDeps = ? streamJsonlMessages(target.session, options.signal) : streamStateDbMessages(target.stateDbPath); - const messages = await collectConversationWindow( + for await (const message of streamConversationWindow( iterable, options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const message of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts index 36e5b2f57..5153c022e 100644 --- a/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/hermes/session-discovery.ts @@ -58,7 +58,7 @@ async function listJsonlFiles(root: string, order: "path_asc" | "recent_first", continue; } - if (entry.isFile() && entry.name.endsWith(".jsonl")) { + if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { const fileStat = await stat(path); files.push({ path, mtimeMs: fileStat.mtimeMs }); } diff --git a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts index 7d3ef80a6..c0fd7cbd6 100644 --- a/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts +++ b/App/backend/src/adapters/outbound/agent-source/jsonl-lines.ts @@ -1,6 +1,5 @@ /** Jsonl lines module. */ import { createReadStream } from "node:fs"; -import { createInterface } from "node:readline"; export type JsonPrimitive = string | number | boolean | null; export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; @@ -15,34 +14,63 @@ export type JsonObject = { readonly [key: string]: JsonValue }; * @returns The JSON objects parsed line by line. */ export async function* readJsonlObjects(filePath: string, signal?: AbortSignal): AsyncIterable { - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ - input: stream, - crlfDelay: Number.POSITIVE_INFINITY - }); + const stream = createReadStream(filePath); + const maxRecordBytes = 64 * 1024 * 1024; + let segments: Buffer[] = []; + let recordBytes = 0; + let overLimit = false; - try { - for await (const line of lines) { - throwIfAborted(signal, filePath); - if (line.trim().length === 0) { - continue; - } + const append = (segment: Buffer): void => { + if (overLimit || segment.length === 0) return; + recordBytes += segment.length; + if (recordBytes > maxRecordBytes) { + segments = []; + overLimit = true; + return; + } + segments.push(segment); + }; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - continue; - } + const reset = (): void => { + segments = []; + recordBytes = 0; + overLimit = false; + }; - if (!isJsonObject(parsed)) { - continue; - } + const parseSegments = (): JsonObject | null => { + if (overLimit) return null; + const line = segments.length === 1 ? segments[0]! : Buffer.concat(segments, recordBytes); + const text = line.toString("utf8").trim(); + if (!text) return null; + try { + const parsed = JSON.parse(text) as unknown; + return isJsonObject(parsed) ? parsed : null; + } catch { + return null; + } + }; - yield parsed; + try { + for await (const chunk of stream) { + throwIfAborted(signal, filePath); + const buffer = chunk as Buffer; + let start = 0; + while (start <= buffer.length) { + const newline = buffer.indexOf(0x0a, start); + if (newline < 0) { + append(buffer.subarray(start)); + break; + } + append(buffer.subarray(start, newline)); + const parsed = parseSegments(); + reset(); + if (parsed) yield parsed; + start = newline + 1; + } } + const parsed = parseSegments(); + if (parsed) yield parsed; } finally { - lines.close(); stream.destroy(); } } diff --git a/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts b/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts index 2a95d9527..6123da83a 100644 --- a/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts +++ b/App/backend/src/adapters/outbound/agent-source/jsonl-session-files.ts @@ -33,7 +33,7 @@ export async function discoverJsonlSessionFiles( const path = join(directory, entry.name); if (entry.isDirectory()) { directories.push(path); - } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + } else if (entry.isFile() && entry.name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(entry.name)) { files.push({ path, mtimeMs: (await stat(path)).mtimeMs }); } } diff --git a/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts b/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts index cf6259b48..488b3c1c2 100644 --- a/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts +++ b/App/backend/src/adapters/outbound/agent-source/onboarding-insight-samplers.ts @@ -133,7 +133,7 @@ export function createWorkbuddyInsightSampler(input: { root: string }): Onboardi sourceId: "workbuddy", displayName: "WorkBuddy", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: isPotentialWorkbuddyMessageLine, extractMessage: extractWorkbuddySampledMessage }); @@ -144,7 +144,7 @@ export function createPiInsightSampler(input: { root: string }): OnboardingInsig sourceId: "pi", displayName: "Pi", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: (line) => /"type"\s*:\s*"message"/u.test(line), extractMessage: extractPiSampledMessage }); @@ -155,7 +155,7 @@ export function createQwenworkInsightSampler(input: { root: string }): Onboardin sourceId: "qwenwork", displayName: "QwenWork", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: (line) => /"type"\s*:\s*"(?:user|assistant|system)"/u.test(line), extractMessage: extractQwenworkSampledMessage }); @@ -166,7 +166,7 @@ export function createCodexInsightSampler(input: { root: string }): OnboardingIn sourceId: "codex", displayName: "Codex", root: input.root, - matchesFile: (name) => name.startsWith("rollout-") && name.endsWith(".jsonl"), + matchesFile: (name) => name.startsWith("rollout-") && name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), shouldParseLine: isPotentialCodexMessageLine, extractMessage: extractCodexMessage }); @@ -177,7 +177,7 @@ export function createClaudeCodeInsightSampler(input: { root: string }): Onboard sourceId: "claude_code", displayName: "Claude Code", root: input.root, - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), extractMessage: extractClaudeCodeMessage }); } @@ -188,7 +188,7 @@ export function createHermesInsightSampler(input: { root: string }): OnboardingI sourceId: "hermes", displayName: "Hermes", root: join(input.root, "sessions"), - matchesFile: (name) => name.endsWith(".jsonl"), + matchesFile: (name) => name.endsWith(".jsonl") && !/\.jsonl\.bak-/u.test(name), extractMessage: extractGenericJsonlMessage }); diff --git a/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts b/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts index 1e96b4439..23c1c5a1e 100644 --- a/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/openclaw/adapter.ts @@ -1,7 +1,7 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpenclawStateDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { discoverOpenclawDatabases } from "./db-discovery.js"; @@ -65,13 +65,13 @@ export function createOpenclawSourceAdapter(deps: CreateOpenclawSourceAdapterDep message: database.databasePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readOpenclawDatabase(database.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts b/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts index e71486ba2..293fa1e94 100644 --- a/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/opencode/adapter.ts @@ -1,10 +1,10 @@ /** Adapter module. */ import { access } from "node:fs/promises"; import { resolveOpencodeDatabasePath } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; -import { readOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; +import { readOpencodeDatabase, streamOpencodeDatabase, type RawOpencodeDatabaseMessage } from "./db-reader.js"; const OPENCODE_SOURCE_ID = "opencode"; @@ -58,13 +58,13 @@ export function createOpencodeSourceAdapter(deps: CreateOpencodeSourceAdapterDep message: target.databasePath }); - const messages = await collectConversationWindow( - readOpencodeDatabase(target.databasePath), + for await (const rawMessage of streamConversationWindow( + options.fullHistory ? streamOpencodeDatabase(target.databasePath) : readOpencodeDatabase(target.databasePath), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts b/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts index c651dce53..388e68892 100644 --- a/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/opencode/db-reader.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { setImmediate as yieldToEventLoop } from "node:timers/promises"; const SQLITE_ROW_YIELD_INTERVAL = 100; +const MAX_RECORD_BYTES = 64 * 1024 * 1024; /** Contract for raw opencode database message. */ export interface RawOpencodeDatabaseMessage { @@ -58,6 +59,39 @@ export async function* readOpencodeDatabase(path: string): AsyncIterable { + const db = new DatabaseSync(path, { readOnly: true }); + try { + if (!hasTable(db, "message") || !hasTable(db, "part") || !hasTable(db, "session")) return; + const statement = db.prepare(` + SELECT m.id AS message_id, m.session_id AS session_id, m.time_created AS message_time_created, + m.data AS message_data, s.directory AS session_directory, p.id AS part_id, + p.time_created AS part_time_created, p.data AS part_data + FROM message m LEFT JOIN session s ON s.id = m.session_id LEFT JOIN part p ON p.message_id = m.id + ORDER BY m.session_id ASC, m.time_created ASC, m.id ASC, p.time_created ASC, p.id ASC + `); + let current: MessageAccumulator | null = null; + let rows = 0; + for (const row of statement.iterate() as Iterable) { + rows += 1; + if (rows % SQLITE_ROW_YIELD_INTERVAL === 0) await yieldToEventLoop(); + if (!current || current.messageId !== row.message_id) { + if (current?.contentParts.length) yield toDatabaseMessage(current); + current = Buffer.byteLength(row.message_data) <= MAX_RECORD_BYTES ? createAccumulator(row) : null; + } + if (!current || !row.part_data || !row.part_id || Buffer.byteLength(row.part_data) > MAX_RECORD_BYTES) continue; + const text = getPartText(parseJson(row.part_data)); + if (!text) continue; + current.partIds.push(row.part_id); + current.contentParts.push(text); + } + if (current?.contentParts.length) yield toDatabaseMessage(current); + } finally { + db.close(); + } +} + async function readMessages(db: DatabaseSync): Promise { const statement = db.prepare(` SELECT @@ -84,7 +118,7 @@ async function readMessages(db: DatabaseSync): Promise MAX_RECORD_BYTES) { continue; } @@ -117,6 +151,7 @@ function getOrCreateAccumulator( accumulators: Map, row: OpencodePartRow ): MessageAccumulator | null { + if (Buffer.byteLength(row.message_data) > MAX_RECORD_BYTES) return null; const existing = accumulators.get(row.message_id); if (existing) { return existing; @@ -135,7 +170,21 @@ function getOrCreateAccumulator( const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; const explicitRoot = getNestedString(messageData, "path", "root"); const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; - const accumulator: MessageAccumulator = { + const accumulator = createAccumulator(row); + if (!accumulator) return null; + accumulators.set(row.message_id, accumulator); + return accumulator; +} + +function createAccumulator(row: OpencodePartRow): MessageAccumulator | null { + const messageData = parseJson(row.message_data); + if (!isRecord(messageData)) return null; + const role = normalizeRole(messageData.role); + if (!role) return null; + const workspacePath = getNestedString(messageData, "path", "cwd") ?? row.session_directory; + const explicitRoot = getNestedString(messageData, "path", "root"); + const gitRoot = explicitRoot && explicitRoot !== "/" ? explicitRoot : workspacePath ? findGitRoot(workspacePath) : null; + return { messageId: row.message_id, conversationId: row.session_id, role, @@ -145,8 +194,19 @@ function getOrCreateAccumulator( partIds: [], contentParts: [] }; - accumulators.set(row.message_id, accumulator); - return accumulator; +} + +function toDatabaseMessage(message: MessageAccumulator): RawOpencodeDatabaseMessage { + return { + messageId: message.messageId, + conversationId: message.conversationId, + role: message.role, + content: message.contentParts.join("\n"), + createdAt: message.createdAt, + workspacePath: message.workspacePath, + gitRoot: message.gitRoot, + rawMeta: Object.freeze({ opencodePartIds: message.partIds }) + }; } function getPartText(partData: unknown): string | null { diff --git a/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts b/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts index b3ba25852..301455660 100644 --- a/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/pi/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolvePiAgentDirectory, resolvePiSessionsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createPiSourceAdapter(deps: CreatePiSourceAdapterDeps = {}): Sou total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readPiHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts b/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts index 19e12d2d0..2d794c53e 100644 --- a/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/qwenwork/adapter.ts @@ -2,7 +2,7 @@ import { existsSync } from "node:fs"; import { access } from "node:fs/promises"; import { dirname, join } from "node:path"; import { resolveQwenworkHomeDirectory, resolveQwenworkProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { discoverJsonlSessionFiles } from "../jsonl-session-files.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; @@ -59,13 +59,13 @@ export function createQwenworkSourceAdapter(deps: CreateQwenworkSourceAdapterDep total: sessions.length, message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readQwenworkHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { emittedMessages += 1; options.onProgress?.({ sourceId: descriptor.sourceId, phase: "emit", current: emittedMessages, total: emittedMessages }); yield { diff --git a/App/backend/src/adapters/outbound/agent-source/types.ts b/App/backend/src/adapters/outbound/agent-source/types.ts index ce4bce8bb..bc7f26ee5 100644 --- a/App/backend/src/adapters/outbound/agent-source/types.ts +++ b/App/backend/src/adapters/outbound/agent-source/types.ts @@ -1,34 +1,20 @@ -/** Types module. */ +/** Types module. Shared with the standalone runtime. */ +import type { + ConversationMessage as CoreConversationMessage, + SourceDescriptor as CoreSourceDescriptor, + ScanOptions as CoreScanOptions, + ScanProgress as CoreScanProgress, + SourceAdapter as CoreSourceAdapter +} from "@memmy/agent-source-core"; /** Contract for source descriptor. */ -export interface SourceDescriptor { - sourceId: string; - displayName: string; - builtin: boolean; - dataPath: string; -} +export type SourceDescriptor = CoreSourceDescriptor; /** Contract for conversation message. */ -export interface ConversationMessage { - messageId: string; - sourceId: string; - conversationId: string; - role: "user" | "assistant" | "tool" | "system"; - content: string; - createdAt: string; - workspacePath: string | null; - gitRoot: string | null; - rawMeta: Readonly>; -} +export type ConversationMessage = CoreConversationMessage; /** Contract for scan progress. */ -export interface ScanProgress { - sourceId: string; - phase: "discover" | "read" | "redact" | "emit" | "scan" | "add" | "summarize" | "done" | "stopped"; - current: number; - total: number; - message?: string; -} +export type ScanProgress = CoreScanProgress; /** Contract for scan result. */ export interface ScanResult { @@ -40,18 +26,7 @@ export interface ScanResult { } /** Contract for source adapter. */ -export interface SourceAdapter { - readonly descriptor: SourceDescriptor; - detect(): Promise; - scan(options: ScanOptions): AsyncIterable; -} +export type SourceAdapter = CoreSourceAdapter; /** Contract for scan options. */ -export interface ScanOptions { - since?: string; - maxMessages?: number; - maxScanTargets?: number; - order?: "source_default" | "recent_first"; - signal?: AbortSignal; - onProgress?: (progress: ScanProgress) => void; -} +export type ScanOptions = CoreScanOptions; diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts index ae6d07c00..cf6e7bd97 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/adapter.ts @@ -4,7 +4,7 @@ import { resolveWorkbuddyHomeDirectory, resolveWorkbuddyProjectsDirectory } from "../../agent-paths.js"; -import { collectConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; +import { streamConversationWindow, remainingMessageCapacity } from "../conversation-window.js"; import { redactSecrets } from "../secret-redactor.js"; import type { ConversationMessage, ScanOptions, SourceAdapter, SourceDescriptor } from "../types.js"; import { readWorkbuddyHistory, type RawWorkbuddyMessage } from "./history-reader.js"; @@ -73,13 +73,13 @@ export function createWorkbuddySourceAdapter(deps: CreateWorkbuddySourceAdapterD message: session.sessionFilePath }); - const messages = await collectConversationWindow( + for await (const rawMessage of streamConversationWindow( readWorkbuddyHistory(session.sessionFilePath, options.signal), options.since, options.signal, - remainingMessageCapacity(options.maxMessages, emittedMessages) - ); - for (const rawMessage of messages) { + remainingMessageCapacity(options.maxMessages, emittedMessages), + options.fullHistory + )) { throwIfAborted(options.signal); options.onProgress?.({ sourceId: descriptor.sourceId, diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts index 1b69de428..bb26c279b 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/history-reader.ts @@ -1,6 +1,5 @@ -import { createReadStream } from "node:fs"; import { basename } from "node:path"; -import { createInterface } from "node:readline"; +import { readJsonlObjects } from "../jsonl-lines.js"; export interface RawWorkbuddyMessage { messageId: string; @@ -18,27 +17,13 @@ export async function* readWorkbuddyHistory( signal?: AbortSignal ): AsyncIterable { const fallbackConversationId = basename(filePath, ".jsonl"); - const stream = createReadStream(filePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Number.POSITIVE_INFINITY }); let lineNumber = 0; - - try { - for await (const line of lines) { - lineNumber += 1; - throwIfAborted(signal, filePath); - const record = parseRecord(line); - if (!record) { - continue; - } - - const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); - if (message) { - yield message; - } + for await (const record of readJsonlObjects(filePath, signal)) { + lineNumber += 1; + const message = toRawWorkbuddyMessage(record, fallbackConversationId, lineNumber); + if (message) { + yield message; } - } finally { - lines.close(); - stream.destroy(); } } @@ -278,18 +263,6 @@ function formatStructuredValue(value: unknown): string { } } -function parseRecord(line: string): Record | null { - if (!line.trim()) { - return null; - } - try { - const parsed: unknown = JSON.parse(line); - return isRecord(parsed) ? parsed : null; - } catch { - return null; - } -} - function isRoleEvent(value: string): boolean { return ["user", "human", "assistant", "ai", "tool", "function", "system", "developer"].includes(value); } @@ -325,9 +298,3 @@ function recordValue(value: unknown): Record | null { function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } - -function throwIfAborted(signal: AbortSignal | undefined, filePath: string): void { - if (signal?.aborted) { - throw new DOMException(`WorkBuddy history read aborted: ${filePath}`, "AbortError"); - } -} diff --git a/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts b/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts index 03c78af87..f79e6ae2d 100644 --- a/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts +++ b/App/backend/src/adapters/outbound/agent-source/workbuddy/session-discovery.ts @@ -55,7 +55,7 @@ async function listJsonlFiles(root: string): Promise string; -} - -interface LocalMemoryRow { - id: string; - timeline: string; - user_id: string; - conversation_id: string | null; - session_id: string | null; - agent_id: string | null; - app_id: string | null; - memory_type: string; - status: MemoryStatus; - visibility: string; - memory_key: string | null; - memory_value: string; - tags_json: string; - info_json: string; - properties_json: string; - memory_layer: MemoryLayer; - content_hash: string | null; - version: number; - created_at: string; - updated_at: string; - deleted_at: string | null; -} - -interface LocalRawTurnRow { - id: string; - session_id: string | null; - episode_id: string | null; - turn_id: string; - user_id: string; - conversation_id: string | null; - user_text: string | null; - assistant_text: string | null; - reasoning_summary: string | null; - tool_calls_json: string; - tool_results_json: string; - source_memory_ids_json: string; - usage_json: string; - message_payload_json: string; - status: string; - redacted_at: string | null; - deleted_at: string | null; - created_at: string; -} - -interface LocalUserMemoryRow { - id: string; - source_turn_id: string; - user_id: string; - memory_types_json: string; - content: string; - source_turn_refs_json: string; - status: "active" | "archived" | "deleted"; - archived_at: string | null; - archive_reason: string | null; - created_at: string; - updated_at: string; - deleted_at: string | null; -} - -interface LocalEpisodeRow { - id: string; - session_id: string; - status: "open" | "closed" | "processing"; - title?: string | null; - summary?: string | null; - l1_memory_ids_json: string; - raw_turn_ids_json?: string; - skill_memory_ids_json?: string; - turn_count?: number | null; - r_task?: number | null; - reward_detail_json?: string; - pipeline_status?: "idle" | "running" | "succeeded" | "failed" | string | null; - pipeline_error?: string | null; - meta_json?: string; - opened_at: string; - closed_at?: string | null; - updated_at: string; -} - -interface LocalApiLogRow { - source: MemosSqliteSource; - id: number; - tool_name: "memory_add" | "memory_search" | "skill_generate" | "skill_evolve"; - source_agent: string | null; - input_json: string; - output_json: string; - duration_ms: number; - success: number; - called_at: string; -} - -type MemoryRow = { source: MemosSqliteSource; row: LocalMemoryRow }; - -interface LocalDeleteResult { - changeSeq: number; - syncCursor: string; - auditId?: string; - serverTime: string; -} - -/** Handles discover memos sqlite sources. */ -export function discoverMemosSqliteSources(env: NodeJS.ProcessEnv = process.env): MemosSqliteSource[] { - const explicitPath = (env.MEMMY_MEMORY_DB_PATH ?? env.MEMMY_MEMOS_DB_PATH ?? "").trim(); - const dbPath = explicitPath - ? resolve(expandHome(explicitPath)) - : join(resolve(expandHome(env.MEMMY_HOME ?? DEFAULT_MEMORY_HOME)), "memory-service", "memory.sqlite"); - - if (!existsSync(dbPath)) { - return []; - } - - return [{ - id: "memmy-memory", - label: sourceLabelFromPath(dbPath), - dbPath - }]; -} - -/** Creates create memos sqlite memory client. */ -export function createMemosSqliteMemoryClient(options: CreateMemosSqliteMemoryClientOptions): MemoryClient { - const now = options.now ?? (() => new Date().toISOString()); - const sources = options.sources.filter((source) => existsSync(source.dbPath)); - - return { - async health() { - const storageReady = sources.length > 0; - return { - ok: storageReady, - version: "memmy-memory-sqlite", - uptimeMs: 0, - mode: "dev", - storage: { - backend: "sqlite", - schemaVersion: "memory-service", - ready: storageReady - }, - models: { - summary: { - provider: "sqlite-local", - configured: false, - remote: false, - routing: null - }, - evolution: { - provider: "sqlite-local", - configured: false, - remote: false, - routing: null - }, - embedding: { - provider: "sqlite-local", - configured: false, - remote: false, - mode: null - } - }, - capabilities: { - routes: ["/api/v1/memory/search", "/api/v1/memory/:id", "/api/v1/memory/logs", "/api/v1/panel/overview", "/api/v1/panel/analysis", "/api/v1/panel/items"], - tools: ["memory.search", "memory.get", "memory.delete"], - memoryLayers: ["L1", "L2", "L3", "Skill"], - supportsCli: false - }, - serverTime: now() - }; - }, - - async reloadConfig() { - return readOnlyOperationUnavailable(); - }, - - async openSession(_input: OpenSessionInput): Promise { - return readOnlyOperationUnavailable(); - }, - - async closeSession(_input: CloseSessionInput & { sessionId: string }): Promise { - return readOnlyOperationUnavailable(); - }, - - async startTurn(_input: StartTurnInput): Promise { - return readOnlyOperationUnavailable(); - }, - - async completeTurn(_input: CompleteTurnInput & { turnId: string }): Promise { - return readOnlyOperationUnavailable(); - }, - - async search(input): Promise { - const limit = 8; - const hits = listMemoryRows(sources) - .map((row) => ({ row, item: toListItem(row) })) - .filter(({ item }) => itemMatchesPanelInput(item, { q: input.query })) - .slice(0, limit) - .map(({ item }, index): RecallHit => ({ - id: item.id, - kind: item.kind, - memoryLayer: item.memoryLayer, - status: item.status, - title: item.title, - snippet: item.summary, - score: Math.max(0.1, 1 - index * 0.08), - tags: item.tags, - updatedAt: item.updatedAt, - source: item.kind === "skill" ? "skill" : "search" - })); - - const injectedContext = { - markdown: hits.map((hit) => `- ${hit.title ?? hit.id}: ${hit.snippet}`).join("\n"), - sections: hits.map((hit) => ({ - id: hit.id, - title: hit.title ?? hit.id, - kind: hit.kind, - memoryLayer: hit.memoryLayer, - memoryIds: [hit.id], - content: hit.snippet - })) - }; - if (input.verbose !== true) { - return { injectedContext: injectedContext.markdown }; - } - return { - injectedContext: injectedContext.markdown, - debug: { - searchEventId: `sqlite-search-${Date.now()}`, - hits, - sourceMemoryIds: hits.map((hit) => hit.id), - status: [], - sections: injectedContext.sections, - serverTime: now() - } - }; - }, - - async getMemory(input): Promise { - const row = findMemoryRow(sources, input.memoryId); - if (!row) { - throw new MemoryLayerError("not_found", 404, `memory not found: ${input.memoryId}`); - } - - const detail = toDetailItem(row, sources); - return { item: detail.item, version: detail.version, etag: detail.etag }; - }, - - async addMemory(_input: AddMemoryInput): Promise { - return readOnlyOperationUnavailable(); - }, - - async deleteMemory(input): Promise { - const userMemory = findWritableUserMemoryRow(sources, input.memoryId); - if (userMemory) { - const deleted = softDeleteUserMemoryRow(userMemory, now()); - return { - ok: true, - id: encodeId(userMemory.source, userMemory.row.id), - kind: "user_memory", - status: "deleted", - changeSeq: deleted.changeSeq, - syncCursor: deleted.syncCursor, - serverTime: deleted.serverTime - }; - } - const target = findWritableMemoryRow(sources, input.memoryId); - if (!target) { - throw new MemoryLayerError("not_found", 404, `memory not found: ${input.memoryId}`); - } - - const kind = kindForRow(target.row); - const deleted = hardDeleteMemoryRow(target, now()); - return { - ok: true, - id: encodeId(target.source, target.row.id), - kind, - status: "deleted", - changeSeq: deleted.changeSeq, - syncCursor: deleted.syncCursor, - auditId: deleted.auditId, - serverTime: deleted.serverTime - }; - }, - - async recallEvidence(queryId): Promise { - throw new MemoryLayerError("not_found", 404, `recall event not found: ${queryId}`); - }, - - async enqueueImportSummaries() { - return readOnlyOperationUnavailable(); - }, - - async getMemoryProcessingStatus() { - return readOnlyOperationUnavailable(); - }, - - async retryMemoryProcessing() { - return readOnlyOperationUnavailable(); - }, - - async runWorker() { - return readOnlyOperationUnavailable(); - }, - - async panelOverview(): Promise { - const rows = listMemoryRows(sources); - const dates = lastDateKeys(now(), PANEL_DAILY_ACTIVITY_DAYS); - - return { - counts: { - memories: rows.filter((item) => item.row.memory_layer === "L1").length, - userMemories: 0, - skills: rows.filter((item) => item.row.memory_layer === "Skill").length, - experiences: rows.filter((item) => item.row.memory_layer === "L2").length, - worldModels: rows.filter((item) => item.row.memory_layer === "L3").length - }, - dailyActivity: countRowsByDate(rows, dates, (item) => item.row.created_at), - sourceDistribution: buildSourceDistribution(rows) - }; - }, - - async panelAnalysis(): Promise { - const rows = listMemoryRows(sources); - const dates = lastSevenDateKeys(now()); - const logs = listApiLogRows(sources, {}, 10_000) - .filter((row) => dates.includes(dateKey(row.called_at))); - const skillRows = rows.filter((item) => item.row.memory_layer === "Skill"); - const recallScores = logs - .filter((row) => row.tool_name === "memory_search") - .map((row) => recallScoreFromLog(row)) - .filter((score): score is number => score !== undefined); - const durations = logs.map((row) => nonNegativeInt(row.duration_ms, 0)); - - return { - metrics: { - avgRecallScore: roundDecimal(average(recallScores) ?? 0, 2), - recallEvents: logs.filter((row) => row.tool_name === "memory_search").length, - activeSkills: skillRows.filter((item) => item.row.status === "activated").length, - recentlyUsedSkills: skillRows.filter((item) => dates.includes(dateKey(item.row.updated_at))).length, - avgToolLatencyMs: roundInt(average(durations) ?? 0), - p95ToolLatencyMs: percentile95(durations) - }, - dailyMemoryWrites: countRowsByDate(rows, dates, (item) => item.row.created_at), - dailySkillEvolutions: countRowsByDate(skillRows, dates, (item) => item.row.updated_at), - toolLatency: buildToolLatency(logs, dates) - }; - }, - - async panelItems(input: PanelItemsInput): Promise { - const pageSize = 20; - const rows = input.layer === "UserMemory" - ? listUserMemoryRows(sources).map((row) => ({ item: toUserMemoryListItem(row), sourceAgent: undefined })) - : listMemoryRows(sources).map((row) => ({ item: toListItem(row), sourceAgent: sourceAgentForRow(row) })); - const filtered = rows - .filter(({ item, sourceAgent }) => itemMatchesPanelInput(item, input, sourceAgent)) - .map(({ item }) => item) - .sort((a, b) => - b.createdAt.localeCompare(a.createdAt) || - b.updatedAt.localeCompare(a.updatedAt) || - b.id.localeCompare(a.id) - ); - const total = filtered.length; - const totalPages = Math.max(1, Math.ceil(total / pageSize)); - const page = Math.min(normalizePage(input.page), totalPages); - const offset = (page - 1) * pageSize; - const items = filtered.slice(offset, offset + pageSize); - - return { - items, - page, - pageSize, - total, - totalPages, - hasNext: page < totalPages, - hasPrev: page > 1, - serverTime: now() - }; - }, - - async panelTasks(input: PanelTasksInput): Promise { - const query = input.q?.trim().toLowerCase() ?? ""; - const rows = listEpisodes(sources) - .map(({ source, row }) => ({ source, row, turns: listRawTurnsForEpisode(source, row.id) })) - .filter(({ row, turns }) => episodeMatchesQuery(row, turns, query)) - .sort((a, b) => - normalizeIsoTime(b.row.opened_at ?? b.row.updated_at ?? "").localeCompare(normalizeIsoTime(a.row.opened_at ?? a.row.updated_at ?? "")) || - normalizeIsoTime(b.row.updated_at ?? b.row.opened_at ?? "").localeCompare(normalizeIsoTime(a.row.updated_at ?? a.row.opened_at ?? "")) || - b.row.id.localeCompare(a.row.id) - ); - const pageSize = 20; - const total = rows.length; - const totalPages = Math.max(1, Math.ceil(total / pageSize)); - const page = Math.min(normalizePage(input.page), totalPages); - const pageRows = rows.slice((page - 1) * pageSize, page * pageSize); - - return { - tasks: pageRows.map(({ source, row, turns }) => ({ - id: encodeId(source, row.id), - episode: { - ...episodeDetailForRow(row), - id: encodeId(source, row.id) - } as PanelTasksOutput["tasks"][number]["episode"], - memoryIds: prefixIds(source, readJsonArray(row.l1_memory_ids_json)), - turns: turns.map((turn) => rawTurnSummaryForRow(source, row.id, turn)), - updatedAt: normalizeIsoTime(row.updated_at ?? row.opened_at ?? now()) - })), - page, - pageSize, - total, - totalPages, - hasNext: page < totalPages, - hasPrev: page > 1, - serverTime: now() - }; - }, - - async deletePanelTask(taskId: string): Promise { - return hardDeletePanelTask(sources, taskId, now()); - }, - - async memoryApiLogs(input: MemoryApiLogsInput): Promise { - const limit = normalizeLimit(input.limit); - const offset = normalizeOffset(input.offset); - const rows = listApiLogRows(sources, input, limit + offset); - - return { - logs: rows.slice(offset, offset + limit).map((row) => ({ - id: row.id, - toolName: row.tool_name, - ...(row.source_agent ? { sourceAgent: row.source_agent } : {}), - inputJson: row.input_json, - outputJson: apiLogOutputWithCurrentTraceSummary(row), - durationMs: nonNegativeInt(row.duration_ms, 0), - success: row.success !== 0, - calledAt: normalizeIsoTime(row.called_at) - })), - total: countApiLogRows(sources, input), - limit, - offset, - nextOffset: rows.length > offset + limit ? offset + limit : undefined, - serverTime: now() - }; - } - }; -} - -/** - * Throws the unified error for write operations not supported by the local SQLite data source. - */ -function readOnlyOperationUnavailable(): never { - throw new MemoryLayerError("memory_layer_unavailable", 503, "local sqlite memory source does not support this write operation"); -} - -function listMemoryRows(sources: readonly MemosSqliteSource[]): MemoryRow[] { - return sources.flatMap((source) => withDb(source, (db) => { - if (!tableExists(db, "memories")) { - return []; - } - - return db - .prepare("select * from memories where deleted_at is null and status != 'deleted'") - .all() - .map((row) => ({ source, row: row as unknown as LocalMemoryRow })); - })); -} - -type UserMemoryRow = { source: MemosSqliteSource; row: LocalUserMemoryRow }; - -function listUserMemoryRows(sources: readonly MemosSqliteSource[]): UserMemoryRow[] { - return sources.flatMap((source) => withDb(source, (db) => { - if (!tableExists(db, "user_memories")) return []; - return db.prepare( - "select * from user_memories where deleted_at is null and status != 'deleted'" - ).all().map((row) => ({ source, row: row as unknown as LocalUserMemoryRow })); - })); -} - -/** - * Reads Memory API log rows from local SQLite data sources. - * - * @param sources the list of SQLite data sources. - * @param input the log filter conditions. - * @param maxRows the maximum number of rows to prefetch for cross-source merge sorting. - * @returns log rows sorted by call time in descending order. - */ -function listApiLogRows( - sources: readonly MemosSqliteSource[], - input: MemoryApiLogsInput, - maxRows: number -): LocalApiLogRow[] { - const tools = normalizeApiLogTools(input.tools); - const placeholders = tools.map(() => "?").join(", "); - const agentFilter = apiLogSourceAgentFilter(input); - return sources - .flatMap((source) => withDb(source, (db) => { - if (!tableExists(db, "api_logs")) { - return []; - } - - return db - .prepare( - `SELECT id, tool_name, source_agent, input_json, output_json, duration_ms, success, called_at - FROM api_logs - WHERE tool_name IN (${placeholders}) - ${agentFilter.sql} - ORDER BY called_at DESC, id DESC - LIMIT ?` - ) - .all(...tools, ...agentFilter.parameters, maxRows) - .map((row) => ({ ...row as unknown as Omit, source })); - })) - .sort((a, b) => b.called_at.localeCompare(a.called_at) || b.id - a.id) - .slice(0, maxRows); -} - -function apiLogOutputWithCurrentTraceSummary(row: LocalApiLogRow): string { - if (row.tool_name !== "memory_add") return row.output_json; - - try { - const output = readJsonObject(row.output_json); - const details = output.details; - if (!Array.isArray(details)) return row.output_json; - - let changed = false; - const nextDetails = details.map((detail) => { - const record = objectAt(detail, []); - const role = stringValue(record.role); - if (role !== "trace" && role !== "span") return detail; - const memoryId = stringValue(role === "span" ? record.spanId : record.traceId) ?? stringValue(record.traceId); - if (!memoryId) return detail; - - const memory = withDb(row.source, (db) => { - if (!tableExists(db, "memories")) return undefined; - return db.prepare("SELECT * FROM memories WHERE id = ?").get(memoryId) as LocalMemoryRow | undefined; - }); - const value = memory - ? role === "span" - ? spanGoalFromParsed(parsedRow(memory)) - : summaryFromParsed(memory, parsedRow(memory)) - : undefined; - const key = role === "span" ? "spanGoal" : "summary"; - if (!value || record[key] === value) return detail; - changed = true; - return { ...record, [key]: value }; - }); - - return changed ? JSON.stringify({ ...output, details: nextDetails }) : row.output_json; - } catch { - return row.output_json; - } -} - -/** - * Counts the Memory API logs in local SQLite data sources. - * - * @param sources the list of SQLite data sources. - * @param input the log filter conditions. - * @returns the total number of logs matching the filter conditions. - */ -function countApiLogRows(sources: readonly MemosSqliteSource[], input: MemoryApiLogsInput): number { - const tools = normalizeApiLogTools(input.tools); - const placeholders = tools.map(() => "?").join(", "); - const agentFilter = apiLogSourceAgentFilter(input); - return sources.reduce((total, source) => total + withDb(source, (db) => { - if (!tableExists(db, "api_logs")) { - return 0; - } - - const row = db - .prepare(`SELECT COUNT(*) AS count FROM api_logs WHERE tool_name IN (${placeholders}) ${agentFilter.sql}`) - .get(...tools, ...agentFilter.parameters) as { count: number }; - return nonNegativeInt(row.count, 0); - }), 0); -} - -function apiLogSourceAgentFilter(input: MemoryApiLogsInput): { sql: string; parameters: string[] } { - const sourceAgent = input.sourceAgent?.trim(); - const excludedSourceAgents = uniqueStrings( - (input.excludedSourceAgents ?? []).map(normalizeSourceAgentKey).filter(Boolean) - ); - const excludedPlaceholders = excludedSourceAgents.map(() => "?").join(", "); - if (sourceAgent) { - const normalizedSourceAgent = normalizeSourceAgentKey(sourceAgent); - return { - sql: `AND lower(replace(replace(TRIM(source_agent), '-', '_'), ' ', '_')) = ?`, - parameters: [normalizedSourceAgent] - }; - } - if (excludedSourceAgents.length > 0) { - return { - sql: `AND ( - NULLIF(TRIM(source_agent), '') IS NULL - OR lower(replace(replace(TRIM(source_agent), '-', '_'), ' ', '_')) NOT IN (${excludedPlaceholders}) - )`, - parameters: excludedSourceAgents - }; - } - return { sql: "", parameters: [] }; -} - -function buildSourceDistribution(rows: MemoryRow[]): PanelOverviewOutput["sourceDistribution"] { - const counts = new Map(); - for (const row of rows) { - const source = sourceLabelForRow(row); - counts.set(source, (counts.get(source) ?? 0) + 1); - } - - const total = rows.length; - return Array.from(counts.entries()) - .map(([source, count]) => ({ - source, - count, - percentage: total > 0 ? roundDecimal((count / total) * 100, 1) : 0 - })) - .sort((a, b) => b.count - a.count || a.source.localeCompare(b.source)); -} - -function countRowsByDate( - rows: T[], - dates: string[], - getTime: (row: T) => string | null | undefined -): Array<{ date: string; count: number }> { - const counts = new Map(dates.map((date) => [date, 0])); - for (const row of rows) { - const key = dateKey(getTime(row)); - if (counts.has(key)) { - counts.set(key, (counts.get(key) ?? 0) + 1); - } - } - - return dates.map((date) => ({ date, count: counts.get(date) ?? 0 })); -} - -function buildToolLatency(logs: LocalApiLogRow[], dates: string[]): PanelAnalysisOutput["toolLatency"] { - const byTool = new Map(); - for (const row of logs) { - const rows = byTool.get(row.tool_name) ?? []; - rows.push(row); - byTool.set(row.tool_name, rows); - } - - const tools = Array.from(byTool.entries()) - .map(([name, rows]) => { - const durations = rows.map((row) => nonNegativeInt(row.duration_ms, 0)); - return { - name, - calls: rows.length, - avgMs: roundInt(average(durations) ?? 0), - p95Ms: percentile95(durations) - }; - }) - .sort((a, b) => b.calls - a.calls || a.name.localeCompare(b.name)); - - return { - tools, - series: tools.map((tool) => { - const rows = byTool.get(tool.name as LocalApiLogRow["tool_name"]) ?? []; - return { - name: tool.name, - points: dates.map((date) => { - const durations = rows - .filter((row) => dateKey(row.called_at) === date) - .map((row) => nonNegativeInt(row.duration_ms, 0)); - return { date, avgMs: roundInt(average(durations) ?? 0) }; - }) - }; - }) - }; -} - -function recallScoreFromLog(row: LocalApiLogRow): number | undefined { - const output = readJsonObject(row.output_json); - const score = numberValue(objectAt(output, ["stats"]).topRelevance); - return score === undefined ? undefined : Math.max(0, score); -} - -function lastSevenDateKeys(nowIso: string): string[] { - return lastDateKeys(nowIso, 7); -} - -function lastDateKeys(nowIso: string, days: number): string[] { - const parsed = Date.parse(nowIso); - const end = Number.isFinite(parsed) ? new Date(parsed) : new Date(); - return Array.from({ length: days }, (_item, index) => { - const day = new Date(end); - day.setUTCDate(end.getUTCDate() - (days - 1 - index)); - return day.toISOString().slice(0, 10); - }); -} - -function dateKey(value: string | null | undefined): string { - const parsed = Date.parse(value ?? ""); - return Number.isFinite(parsed) ? new Date(parsed).toISOString().slice(0, 10) : ""; -} - -function roundDecimal(value: number, decimals: number): number { - return Number(value.toFixed(decimals)); -} - -function roundInt(value: number): number { - return Math.max(0, Math.round(value)); -} - -function percentile95(values: number[]): number { - if (values.length === 0) { - return 0; - } - - const sorted = [...values].sort((a, b) => a - b); - const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * 0.95) - 1)); - return roundInt(sorted[index] ?? 0); -} - -function listEpisodes(sources: readonly MemosSqliteSource[]): Array<{ source: MemosSqliteSource; row: LocalEpisodeRow }> { - return sources.flatMap((source) => withDb(source, (db) => { - if (tableExists(db, "episodes")) { - return db.prepare("select * from episodes").all().map((row) => ({ source, row: row as unknown as LocalEpisodeRow })); - } - - if (!tableExists(db, "cloud_episodes")) { - return []; - } - - return db.prepare("select * from cloud_episodes").all().map((row) => ({ source, row: row as unknown as LocalEpisodeRow })); - })); -} - -function listRawTurnsForEpisode(source: MemosSqliteSource, episodeId: string): LocalRawTurnRow[] { - return withDb(source, (db) => { - if (!tableExists(db, "raw_turns")) { - return []; - } - - return db - .prepare( - `select * from raw_turns - where episode_id = ? and redacted_at is null and deleted_at is null - order by created_at asc, id asc` - ) - .all(episodeId) as unknown as LocalRawTurnRow[]; - }); -} - -function episodeMatchesQuery(row: LocalEpisodeRow, turns: readonly LocalRawTurnRow[], query: string): boolean { - if (!query) { - return true; - } - - return [ - row.id, - row.title, - row.summary, - ...turns.flatMap((turn) => [turn.user_text, turn.assistant_text, turn.reasoning_summary]) - ].some((value) => value?.toLowerCase().includes(query)); -} - -function rawTurnSummaryForRow( - source: MemosSqliteSource, - episodeId: string, - turn: LocalRawTurnRow -): PanelTasksOutput["tasks"][number]["turns"][number] { - const toolResults = readJson(turn.tool_results_json); - return removeUndefined({ - rawTurnId: encodeId(source, turn.id), - episodeId: encodeId(source, episodeId), - turnId: turn.turn_id, - userText: turn.user_text ?? undefined, - assistantText: turn.assistant_text ?? undefined, - reasoningSummary: turn.reasoning_summary ?? undefined, - toolCalls: readToolCalls(turn.tool_calls_json), - toolResults: Array.isArray(toolResults) ? toolResults : [], - createdAt: normalizeIsoTime(turn.created_at) - }) as PanelTasksOutput["tasks"][number]["turns"][number]; -} - -function hardDeletePanelTask( - sources: readonly MemosSqliteSource[], - encodedId: string, - serverTime: string -): DeletePanelTaskOutput { - const decoded = decodeId(encodedId); - const candidates = decoded.sourceId ? sources.filter((source) => source.id === decoded.sourceId) : sources; - const target = listEpisodes(candidates).find(({ source, row }) => - row.id === decoded.rawId || encodeId(source, row.id) === encodedId - ); - if (!target) { - throw new MemoryLayerError("not_found", 404, `task not found: ${encodedId}`); - } - - return withWritableDb(target.source, (db) => { - db.exec("PRAGMA foreign_keys = ON"); - db.exec("BEGIN IMMEDIATE"); - try { - const deletedMemoryIds: string[] = []; - for (const memoryId of readJsonArray(target.row.l1_memory_ids_json)) { - const memory = db - .prepare("select * from memories where id = ? and deleted_at is null and status != 'deleted' limit 1") - .get(memoryId) as unknown as LocalMemoryRow | undefined; - if (!memory) { - continue; - } - - const memoryTarget = { source: target.source, row: memory }; - deleteMemoryAuxiliaryRows(db, memoryId); - db.prepare("delete from memories where id = ?").run(memoryId); - appendDeleteChangeLog(db, memoryTarget, serverTime); - deletedMemoryIds.push(encodeId(target.source, memoryId)); - } - - const result = db.prepare("delete from episodes where id = ?").run(target.row.id) as { changes?: number | bigint }; - if (Number(result.changes ?? 0) !== 1) { - throw new MemoryLayerError("not_found", 404, `task not found: ${encodedId}`); - } - - db.exec("COMMIT"); - return { - ok: true, - id: encodeId(target.source, target.row.id), - deletedMemoryIds, - serverTime - }; - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - }); -} - -function findMemoryRow(sources: readonly MemosSqliteSource[], encodedId: string, kind?: MemoryKind): MemoryRow | null { - const decoded = decodeId(encodedId); - const candidates = decoded.sourceId ? sources.filter((source) => source.id === decoded.sourceId) : sources; - - return ( - listMemoryRows(candidates).find((row) => { - if (kind && kindForRow(row.row) !== kind) { - return false; - } - - return row.row.id === decoded.rawId || encodeId(row.source, row.row.id) === encodedId; - }) ?? null - ); -} - -function findWritableMemoryRow(sources: readonly MemosSqliteSource[], encodedId: string): MemoryRow | null { - const decoded = decodeId(encodedId); - const candidates = decoded.sourceId ? sources.filter((source) => source.id === decoded.sourceId) : sources; - - for (const source of candidates) { - const row = withDb(source, (db) => { - if (!tableExists(db, "memories")) { - return null; - } - - return db - .prepare("select * from memories where id = ? and deleted_at is null and status != 'deleted' limit 1") - .get(decoded.rawId) as unknown as LocalMemoryRow | undefined; - }); - - if (row) { - return { source, row }; - } - } - - return null; -} - -function findWritableUserMemoryRow( - sources: readonly MemosSqliteSource[], - encodedId: string -): UserMemoryRow | null { - const decoded = decodeId(encodedId); - const candidates = decoded.sourceId ? sources.filter((source) => source.id === decoded.sourceId) : sources; - for (const source of candidates) { - const row = withDb(source, (db) => { - if (!tableExists(db, "user_memories")) return undefined; - return db.prepare( - "select * from user_memories where id = ? and deleted_at is null and status != 'deleted' limit 1" - ).get(decoded.rawId) as unknown as LocalUserMemoryRow | undefined; - }); - if (row) return { source, row }; - } - return null; -} - -function softDeleteUserMemoryRow(target: UserMemoryRow, serverTime: string): LocalDeleteResult { - return withWritableDb(target.source, (db) => { - db.exec("BEGIN IMMEDIATE"); - try { - if (tableExists(db, "user_memories_fts")) { - db.prepare("delete from user_memories_fts where id = ?").run(target.row.id); - } - const result = db.prepare( - `update user_memories - set memory_types_json = '[]', content = '[DELETED]', source_turn_refs_json = '[]', - status = 'deleted', embedding_json = null, embedding_model = null, - embedding_provider = null, updated_at = ?, deleted_at = ? - where id = ? and deleted_at is null and status != 'deleted'` - ).run(serverTime, serverTime, target.row.id) as { changes?: number | bigint }; - if (Number(result.changes ?? 0) !== 1) { - throw new MemoryLayerError("not_found", 404, `memory not found: ${encodeId(target.source, target.row.id)}`); - } - db.exec("COMMIT"); - return { - changeSeq: 0, - syncCursor: `sqlite-delete:${target.source.id}:0`, - serverTime - }; - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - }); -} - -function hardDeleteMemoryRow(target: MemoryRow, serverTime: string): LocalDeleteResult { - return withWritableDb(target.source, (db) => { - db.exec("BEGIN IMMEDIATE"); - try { - deleteMemoryAuxiliaryRows(db, target.row.id); - const result = db - .prepare("delete from memories where id = ? and deleted_at is null and status != 'deleted'") - .run(target.row.id) as { changes?: number | bigint }; - if (Number(result.changes ?? 0) !== 1) { - throw new MemoryLayerError("not_found", 404, `memory not found: ${encodeId(target.source, target.row.id)}`); - } - - const changeSeq = appendDeleteChangeLog(db, target, serverTime); - db.exec("COMMIT"); - return { - changeSeq, - syncCursor: `sqlite-delete:${target.source.id}:${changeSeq}`, - serverTime - }; - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - }); -} - -function deleteMemoryAuxiliaryRows(db: DatabaseSync, memoryId: string): void { - if (tableExists(db, "memories_fts")) { - db.prepare("delete from memories_fts where id = ?").run(memoryId); - } - - if (tableExists(db, "memory_vector_entries")) { - const vectors = db - .prepare("select id, embedding_dim from memory_vector_entries where memory_id = ?") - .all(memoryId) as Array<{ id: number; embedding_dim: number }>; - for (const vector of vectors) { - if (!Number.isSafeInteger(vector.embedding_dim) || vector.embedding_dim <= 0) continue; - const table = `memory_vec_${vector.embedding_dim}`; - if (tableExists(db, table)) { - db.prepare(`delete from ${table} where rowid = ?`).run(BigInt(vector.id)); - } - } - db.prepare("delete from memory_vector_entries where memory_id = ?").run(memoryId); - } - - if (tableExists(db, "embedding_retry_queue")) { - db.prepare("delete from embedding_retry_queue where target_id = ?").run(memoryId); - } -} - -function appendDeleteChangeLog(db: DatabaseSync, target: MemoryRow, createdAt: string): number { - if (!tableExists(db, "memory_change_log")) { - return nonNegativeInt(target.row.version, 0) + 1; - } - - const result = db - .prepare( - `insert into memory_change_log ( - memory_id, namespace_id, kind, op, entity_id, user_id, - change_type, version, before_json, after_json, source, created_at - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - .run( - target.row.id, - target.source.id, - kindForRow(target.row), - "deleted", - target.row.id, - target.row.user_id, - "delete", - nonNegativeInt(target.row.version, 0) + 1, - JSON.stringify(target.row), - null, - "panel.delete", - createdAt - ) as { lastInsertRowid?: number | bigint }; - - return Number(result.lastInsertRowid ?? 0); -} - -function toListItem(row: MemoryRow): MemoryListItem { - const parsed = parsedRow(row.row); - const source = sourceLabelForRow(row, parsed); - const spanGoal = spanGoalFromParsed(parsed); - return { - id: encodeId(row.source, row.row.id), - kind: kindForRow(row.row), - memoryLayer: row.row.memory_layer, - status: row.row.status, - title: truncate(firstNonEmpty(titleFromParsed(row.row, parsed), firstLine(row.row.memory_value), row.row.id), 80), - summary: firstNonEmpty(summaryFromParsed(row.row, parsed), row.row.memory_value), - tags: withSourceTag(source, tagsForRow(row.row, parsed)), - metrics: metricsForRow(parsed), - metadata: { source, ...(spanGoal ? { spanGoal } : {}) }, - createdAt: normalizeIsoTime(row.row.created_at), - updatedAt: normalizeIsoTime(row.row.updated_at), - version: nonNegativeInt(row.row.version, 1) - }; -} - -function toUserMemoryListItem(row: UserMemoryRow): MemoryListItem { - const memoryTypes = readJsonArray(row.row.memory_types_json); - const sourceTurnRefs = readJsonArray(row.row.source_turn_refs_json); - return { - id: encodeId(row.source, row.row.id), - kind: "user_memory", - memoryLayer: "UserMemory", - status: row.row.status === "active" ? "activated" : row.row.status, - title: truncate(firstLine(row.row.content) || row.row.id, 80), - summary: row.row.content, - tags: memoryTypes, - metadata: { - source: row.source.label, - sourceTurnId: row.row.source_turn_id, - sourceTurnRefs, - memoryTypes, - archivedAt: row.row.archived_at, - archiveReason: row.row.archive_reason - }, - createdAt: normalizeIsoTime(row.row.created_at), - updatedAt: normalizeIsoTime(row.row.updated_at), - version: Math.max(1, sourceTurnRefs.length) - }; -} - -function spanGoalFromParsed(parsed: ParsedRow): string | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - if (stringValue(internalInfo.memory_kind) !== "span") return undefined; - const goal = stringValue(objectAt(internalInfo, ["span"]).span_goal)?.trim(); - return goal || undefined; -} - -function toDetailItem(row: MemoryRow, sources?: readonly MemosSqliteSource[]): GetMemoryOutput { - const item = toListItem(row); - const parsed = parsedRow(row.row); - return { - item: { - ...item, - body: row.row.memory_value, - createdAt: normalizeIsoTime(row.row.created_at), - sourceMemoryIds: sourceMemoryIds(row), - metadata: metadataForRow(row, parsed, sources) - }, - version: item.version, - etag: `${item.id}-${item.version}` - }; -} - -function itemMatchesPanelInput(item: MemoryListItem, input: PanelItemsInput, sourceAgent?: string): boolean { - if (input.layer && item.memoryLayer !== input.layer) return false; - if (input.status && item.status !== input.status) return false; - const selectedSourceAgent = input.sourceAgent?.trim(); - if (selectedSourceAgent && normalizeSourceAgentKey(sourceAgent) !== normalizeSourceAgentKey(selectedSourceAgent)) return false; - const excludedSourceAgents = new Set((input.excludedSourceAgents ?? []).map(normalizeSourceAgentKey).filter(Boolean)); - if (!selectedSourceAgent && excludedSourceAgents.has(normalizeSourceAgentKey(sourceAgent))) return false; - return itemMatchesQueryAndTags(item, input.q); -} - -function itemMatchesQueryAndTags(item: Pick, query?: string, tags?: readonly string[]): boolean { - const normalizedQuery = query?.trim().toLowerCase(); - if (normalizedQuery) { - const haystack = `${item.id} ${item.title} ${item.summary} ${item.tags.join(" ")}`.toLowerCase(); - if (!haystack.includes(normalizedQuery)) { - return false; - } - } - - if (tags && tags.length > 0) { - const itemTags = new Set(item.tags); - if (!tags.every((tag) => itemTags.has(tag))) { - return false; - } - } - - return true; -} - -function kindForRow(row: LocalMemoryRow): MemoryKind { - const parsedKind = stringValue(objectAt(readJsonObject(row.properties_json), ["internal_info"]).memory_kind); - if ( - parsedKind === "trace" || - parsedKind === "span" || - parsedKind === "policy" || - parsedKind === "world_model" || - parsedKind === "skill" - ) { - return parsedKind; - } - - if (row.memory_layer === "L2") return "policy"; - if (row.memory_layer === "L3") return "world_model"; - if (row.memory_layer === "Skill") return "skill"; - return "trace"; -} - -function titleFromParsed(row: LocalMemoryRow, parsed: ParsedRow): string | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const policy = objectAt(internalInfo, ["policy"]); - const worldModel = objectAt(internalInfo, ["world_model"]); - const skill = objectAt(internalInfo, ["skill"]); - - return firstDefinedString( - stringValue(internalInfo.title), - stringValue(policy.title), - stringValue(worldModel.title), - stringValue(skill.title), - stringValue(parsed.info.title), - firstReadableMemoryValueLine(row.memory_value), - humanizeIdentifier(stringValue(skill.name)), - isInternalMemoryKey(row.memory_key ?? undefined) ? undefined : row.memory_key ?? undefined - ); -} - -function summaryFromParsed(row: LocalMemoryRow, parsed: ParsedRow): string | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const policy = objectAt(internalInfo, ["policy"]); - const worldModel = objectAt(internalInfo, ["world_model"]); - const skill = objectAt(internalInfo, ["skill"]); - return firstDefinedString( - stringValue(parsed.info.summary), - stringValue(internalInfo.summary), - stringValue(policy.trigger), - stringValue(policy.procedure), - stringValue(worldModel.summary), - stringValue(worldModel.body), - stringValue(skill.invocation_guide), - stringValue(skill.invocationGuide), - firstReadableMemoryValueLine(row.memory_value), - row.memory_value - ); -} - -interface ParsedRow { - info: Record; - properties: Record; -} - -function parsedRow(row: LocalMemoryRow): ParsedRow { - return { - info: readJsonObject(row.info_json), - properties: readJsonObject(row.properties_json) - }; -} - -function tagsForRow(row: LocalMemoryRow, parsed: ParsedRow): string[] { - return uniqueStrings([ - ...readJsonArray(row.tags_json), - ...stringArray(parsed.info.tags), - ...stringArray(parsed.properties.tags) - ]); -} - -function metricsForRow(parsed: ParsedRow): MemoryMetrics | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const trace = objectAt(internalInfo, ["trace"]); - const value = numberValue(internalInfo.value) ?? numberValue(trace.value); - const alpha = numberValue(internalInfo.alpha) ?? numberValue(trace.alpha); - const reflection = firstDefinedString( - stringValue(internalInfo.reflection), - stringValue(trace.reflection) - ); - if (value === undefined && alpha === undefined && reflection === undefined) { - return undefined; - } - - return { - value, - alpha, - reflectionDone: Boolean(reflection) - }; -} - -function sourceMemoryIds(row: MemoryRow): string[] { - const parsed = parsedRow(row.row); - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - return prefixIds(row.source, uniqueStrings([ - ...stringArray(parsed.info.source_memory_ids), - ...stringArray(internalInfo.source_memory_ids), - ...stringArray(internalInfo.source_l1_memory_ids), - ...stringArray(internalInfo.source_trace_ids) - ])); -} - -function sourceLabelForRow(row: MemoryRow, parsed: ParsedRow = parsedRow(row.row)): string { - return sourceAgentForRow(row, parsed) ?? row.source.label; -} - -function sourceAgentForRow(row: MemoryRow, parsed: ParsedRow = parsedRow(row.row)): string | undefined { - return [ - sourceLabelFromParsed(parsed), - sourceLabelFromSessionId(row.row.session_id), - sourceLabelFromSessionId(row.row.conversation_id), - row.row.agent_id?.trim() || undefined, - row.row.app_id?.trim() || undefined - ].find((value): value is string => Boolean(value)); -} - -function normalizeSourceAgentKey(value: string | undefined): string { - return value?.trim().toLowerCase().replace(/[\s-]+/gu, "_") ?? ""; -} - -function sourceLabelFromParsed(parsed: ParsedRow): string | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - return normalizedAgentSource(stringValue(parsed.info.source)) - ?? normalizedAgentSource(stringValue(internalInfo.source)); -} - -function sourceLabelFromSessionId(value: string | null): string | undefined { - const normalized = value?.trim().toLowerCase(); - if (!normalized) return undefined; - if (normalized === "claude" || normalized.startsWith("claude-")) return "claude-code"; - if (normalized === "open-code" || normalized.startsWith("open-code-")) return "opencode"; - for (const source of ["deepseek-harness", "hermes", "openclaw", "codex", "cursor", "claude-code", "opencode", "workbuddy", "pi", "qwenwork"]) { - if (normalized === source || normalized.startsWith(`${source}-`)) return source; - } - return undefined; -} - -function normalizedAgentSource(value: string | undefined): string | undefined { - const normalized = value?.trim().toLowerCase(); - if (normalized === "claude") return "claude-code"; - if (normalized === "open-code") return "opencode"; - if (normalized === "deepseek_harness") return "deepseek-harness"; - return ["deepseek-harness", "hermes", "openclaw", "codex", "cursor", "claude-code", "opencode", "workbuddy", "pi", "qwenwork"].includes(normalized ?? "") - ? normalized - : undefined; -} - -function withSourceTag(sourceLabel: string, tags: string[]): string[] { - return uniqueStrings([sourceLabel, ...tags.filter(Boolean)]); -} - -function metadataForRow(row: MemoryRow, parsed: ParsedRow, sources?: readonly MemosSqliteSource[]): Record { - const kind = kindForRow(row.row); - return removeUndefined({ - traceDetail: kind === "trace" ? traceDetailForRow(row, parsed, sources) : undefined, - spanDetail: kind === "span" ? spanDetailForRow(row, parsed) : undefined, - source: sourceLabelForRow(row, parsed), - sourceId: row.source.id, - dbPath: row.source.dbPath, - info: sanitizeMetadataValue(parsed.info), - properties: sanitizeMetadataValue(parsed.properties), - raw: sanitizeMetadataValue({ - ...row.row, - embedding: undefined, - info_json: undefined, - properties_json: undefined - }) - }); -} - -function spanDetailForRow(row: MemoryRow, parsed: ParsedRow): Record | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const span = objectAt(internalInfo, ["span"]); - const rawTurnId = stringValue(span.raw_turn_id); - const rawTurn = readRawTurn(row.source, rawTurnId, undefined); - const toolCallStart = numberValue(span.tool_call_start); - const toolCallEnd = numberValue(span.tool_call_end); - if (!rawTurn || toolCallStart === undefined || toolCallEnd === undefined) { - return undefined; - } - - return removeUndefined({ - toolCallStart, - toolCallEnd, - toolCalls: readToolCalls(rawTurn.tool_calls_json).slice(toolCallStart, toolCallEnd + 1) - }); -} - -function traceDetailForRow(row: MemoryRow, parsed: ParsedRow, sources?: readonly MemosSqliteSource[]): Record | undefined { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const selectedTrace = traceObject(parsed); - const turnId = firstDefinedString( - stringValue(parsed.info.turn_id), - stringValue(selectedTrace.turn_id), - row.row.conversation_id ?? undefined - ); - const rawTurnId = firstDefinedString( - stringValue(parsed.info.raw_turn_id), - stringValue(internalInfo.raw_turn_id), - stringValue(internalInfo.source_raw_turn_id), - stringValue(selectedTrace.raw_turn_id) - ); - const rows = siblingTraceRows(row, parsed, sources ?? [row.source], turnId, rawTurnId); - const parsedRows = rows.map((candidate) => ({ row: candidate, parsed: parsedRow(candidate.row) })); - const rawTurn = readRawTurn(row.source, rawTurnId, turnId); - const episodeId = firstDefinedString( - rawTurn?.episode_id ?? undefined, - stringValue(parsed.info.episode_id), - stringValue(selectedTrace.episode_id) - ); - const episode = readEpisode(row.source, episodeId); - const traceRows = parsedRows.length > 0 ? parsedRows : [{ row, parsed }]; - const steps = traceRows.map(({ row: candidate, parsed: candidateParsed }) => traceStepForRow(candidate, candidateParsed)); - const values = steps.map((step) => numberValue(step.value)).filter((value): value is number => value !== undefined); - const alphas = steps.map((step) => numberValue(step.alpha)).filter((value): value is number => value !== undefined); - const priorities = steps.map((step) => numberValue(step.priority)).filter((value): value is number => value !== undefined); - const selectedStep = traceStepForRow(row, parsed); - const agentText = firstDefinedString( - rawTurn?.assistant_text ?? undefined, - stringValue(selectedTrace.agent_text), - firstAgentSpanSummary(traceRows) - ); - const parsedAgentText = parseBracketToolBlocks(agentText); - const storedToolCalls = rawTurn - ? readToolCalls(rawTurn.tool_calls_json) - : uniqueToolCalls(steps.flatMap((step) => Array.isArray(step.toolCalls) ? step.toolCalls.filter(isRecordValue) : [])); - const toolCalls = storedToolCalls.length > 0 ? storedToolCalls : parsedAgentText.toolCalls; - const summary = firstNonEmpty( - stringValue(parsed.info.summary), - stringValue(selectedTrace.summary), - stringValue(internalInfo.summary), - itemSummaryFallback(traceRows) - ); - - return removeUndefined({ - episodeId, - turnId, - rawTurnId, - episode: episode ? episodeDetailForRow(episode) : undefined, - turn: rawTurn ? removeUndefined({ - id: rawTurn.id, - turnId: rawTurn.turn_id, - createdAt: normalizeIsoTime(rawTurn.created_at), - userText: rawTurn.user_text ?? undefined, - assistantText: rawTurn.assistant_text ?? undefined, - toolCalls: readToolCalls(rawTurn.tool_calls_json) - }) : undefined, - capturedAt: firstDefinedString(rawTurn ? normalizeIsoTime(rawTurn.created_at) : undefined, traceTimestamp(selectedTrace), normalizeIsoTime(row.row.created_at)), - value: average(values) ?? numberValue(selectedStep.value), - alpha: average(alphas) ?? numberValue(selectedStep.alpha), - priority: priorities.length > 0 ? Math.max(...priorities) : numberValue(selectedStep.priority), - rHuman: numberValue(parsed.info.r_human) ?? numberValue(internalInfo.r_human), - summary, - userQuery: firstDefinedString(rawTurn?.user_text ?? undefined, stringValue(selectedTrace.user_text), firstUserSpanSummary(traceRows)), - finalResponse: firstDefinedString(parsedAgentText.text, agentText), - toolCalls, - steps - }); -} - -function episodeDetailForRow(episode: LocalEpisodeRow): Record { - const rewardDetail = readJsonObject(episode.reward_detail_json ?? "{}"); - const meta = readJsonObject(episode.meta_json ?? "{}"); - const skillMemoryIds = readJsonArray(episode.skill_memory_ids_json ?? "[]"); - - return removeUndefined({ - id: episode.id, - sessionId: stringValue(episode.session_id), - title: stringValue(episode.title), - summary: stringValue(episode.summary), - status: episode.status, - startedAt: optionalIsoTime(episode.opened_at), - endedAt: optionalIsoTime(episode.closed_at ?? undefined), - turnCount: nonNegativeOptionalInt(episode.turn_count), - rTask: numberValue(episode.r_task), - rewardSkipped: booleanValue(rewardDetail.skipped), - rewardReason: stringValue(rewardDetail.reason), - closeReason: stringValue(meta.closeReason), - topicState: stringValue(meta.topicState), - abandonReason: stringValue(meta.abandonReason), - pipelineStatus: stringValue(episode.pipeline_status), - pipelineError: stringValue(episode.pipeline_error), - skillMemoryIds, - linkedSkillId: skillMemoryIds[0], - skillStatus: skillStatusForEpisode(episode), - skillReason: skillReasonForEpisode(episode) - }); -} - -function skillStatusForEpisode(episode: LocalEpisodeRow): string { - const rewardDetail = readJsonObject(episode.reward_detail_json ?? "{}"); - const meta = readJsonObject(episode.meta_json ?? "{}"); - const rTask = numberValue(episode.r_task); - - if (jsonArrayLength(episode.skill_memory_ids_json) > 0) { - return "succeeded"; - } - - if (episode.pipeline_status === "running") { - return "running"; - } - - if (episode.pipeline_status === "failed") { - return "failed"; - } - - if (rTask !== undefined && rTask <= -0.5) { - return "skipped"; - } - - if ( - booleanValue(rewardDetail.skipped) === true || - stringValue(meta.closeReason) === "abandoned" || - (rTask !== undefined && rTask < 0.3) - ) { - return "skipped"; - } - - return "queued"; -} - -function skillReasonForEpisode(episode: LocalEpisodeRow): string | undefined { - const rewardDetail = readJsonObject(episode.reward_detail_json ?? "{}"); - const meta = readJsonObject(episode.meta_json ?? "{}"); - const rTask = numberValue(episode.r_task); - - if (jsonArrayLength(episode.skill_memory_ids_json) > 0) { - return "已从该任务沉淀出可复用技能。"; - } - - if (episode.pipeline_error && episode.pipeline_error.trim()) { - return `技能沉淀失败：${episode.pipeline_error.trim()}`; - } - - if (rTask !== undefined && rTask <= -0.5) { - return `任务评分 ${rTask.toFixed(2)}，被视为反例；不会沉淀出新的经验或技能。`; - } - - if (booleanValue(rewardDetail.skipped) === true) { - const turnCount = nonNegativeOptionalInt(episode.turn_count) ?? 0; - if (turnCount < 2) { - return "对话轮次不足，需要至少 2 轮完整问答才能生成摘要或技能。"; - } - - return "Reward 评分被跳过，暂不生成技能。"; - } - - if (stringValue(meta.closeReason) === "abandoned") { - return "任务在完成打分前结束，暂不生成技能。"; - } - - if (rTask !== undefined && rTask < 0.3) { - return `任务评分 ${rTask.toFixed(2)} 未达到沉淀阈值，暂不生成技能。`; - } - - if (episode.pipeline_status === "running") { - return "正在沉淀技能。"; - } - - if (episode.pipeline_status === "succeeded" && jsonArrayLength(episode.skill_memory_ids_json) === 0) { - return "本任务未产出可复用技能。"; - } - - if (episode.status === "open") { - return "任务仍在进行中，暂未启动技能沉淀。"; - } - - if (episode.pipeline_status === "idle" || !episode.pipeline_status) { - return "等待评分完成后判断是否沉淀技能。"; - } - - return undefined; -} - -function jsonArrayLength(raw: string | null | undefined): number { - if (!raw) { - return 0; - } - - const parsed = readJson(raw); - return Array.isArray(parsed) ? parsed.length : 0; -} - -function readEpisode(source: MemosSqliteSource, episodeId: string | undefined): LocalEpisodeRow | null { - if (!episodeId) { - return null; - } - - return withDb(source, (db) => { - if (!tableExists(db, "episodes")) { - return null; - } - - const row = db.prepare("select * from episodes where id = ? limit 1").get(episodeId); - return row ? (row as unknown as LocalEpisodeRow) : null; - }); -} - -function siblingTraceRows( - row: MemoryRow, - parsed: ParsedRow, - sources: readonly MemosSqliteSource[], - turnId: string | undefined, - rawTurnId: string | undefined -): MemoryRow[] { - const candidates = listMemoryRows(sources.filter((source) => source.id === row.source.id)); - const rows = candidates.filter((candidate) => { - if (kindForRow(candidate.row) !== "trace") { - return false; - } - - const candidateParsed = candidate.row.id === row.row.id ? parsed : parsedRow(candidate.row); - const candidateInternalInfo = objectAt(candidateParsed.properties, ["internal_info"]); - const candidateTrace = traceObject(candidateParsed); - const candidateTurnId = firstDefinedString( - stringValue(candidateParsed.info.turn_id), - stringValue(candidateTrace.turn_id), - candidate.row.conversation_id ?? undefined - ); - const candidateRawTurnId = firstDefinedString( - stringValue(candidateParsed.info.raw_turn_id), - stringValue(candidateInternalInfo.raw_turn_id), - stringValue(candidateInternalInfo.source_raw_turn_id), - stringValue(candidateTrace.raw_turn_id) - ); - - return ( - (turnId !== undefined && candidateTurnId === turnId) || - (rawTurnId !== undefined && candidateRawTurnId === rawTurnId) || - candidate.row.id === row.row.id - ); - }); - - return rows.sort((a, b) => { - const aTrace = traceObject(parsedRow(a.row)); - const bTrace = traceObject(parsedRow(b.row)); - const aStep = numberValue(aTrace.step_index) ?? 0; - const bStep = numberValue(bTrace.step_index) ?? 0; - if (aStep !== bStep) { - return aStep - bStep; - } - - return normalizeIsoTime(a.row.created_at).localeCompare(normalizeIsoTime(b.row.created_at)); - }); -} - -function traceStepForRow(row: MemoryRow, parsed: ParsedRow): Record { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const trace = traceObject(parsed); - const rawSpan = rawSpanForParsed(parsed); - const toolCalls = toolCallsFromTrace(trace); - const role = toolCalls.length > 0 ? "tool" : rawSpan.user_text === true ? "user" : rawSpan.agent_text === true ? "assistant" : "assistant"; - - return removeUndefined({ - id: encodeId(row.source, row.row.id), - stepIndex: numberValue(trace.step_index) ?? numberValue(internalInfo.step_index), - role, - capturedAt: firstDefinedString(traceTimestamp(trace), normalizeIsoTime(row.row.created_at)), - summary: firstNonEmpty( - stringValue(trace.summary), - stringValue(internalInfo.summary), - stringValue(parsed.info.summary), - row.row.memory_value - ), - reflection: firstDefinedString(stringValue(trace.reflection), stringValue(internalInfo.reflection)), - value: numberValue(trace.value) ?? numberValue(internalInfo.value) ?? numberValue(parsed.info.value), - alpha: numberValue(trace.alpha) ?? numberValue(internalInfo.alpha) ?? numberValue(parsed.info.alpha), - priority: numberValue(trace.priority) ?? numberValue(internalInfo.priority) ?? numberValue(parsed.info.priority), - toolCalls, - rawSpan: removeUndefined({ - userText: rawSpan.user_text === true, - agentText: rawSpan.agent_text === true, - toolCallCount: numberValue(rawSpan.tool_call_count) - }) - }); -} - -function traceObject(parsed: ParsedRow): Record { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - return objectAt(internalInfo, ["trace"]); -} - -function rawSpanForParsed(parsed: ParsedRow): Record { - const internalInfo = objectAt(parsed.properties, ["internal_info"]); - const trace = traceObject(parsed); - return firstRecord(trace.raw_span, internalInfo.raw_span); -} - -function readRawTurn(source: MemosSqliteSource, rawTurnId: string | undefined, turnId: string | undefined): LocalRawTurnRow | null { - if (!rawTurnId && !turnId) { - return null; - } - - return withDb(source, (db) => { - if (!tableExists(db, "raw_turns")) { - return null; - } - - if (rawTurnId) { - const row = db.prepare("select * from raw_turns where id = ? limit 1").get(rawTurnId); - if (row) { - return row as unknown as LocalRawTurnRow; - } - } - - if (turnId) { - const row = db.prepare("select * from raw_turns where turn_id = ? limit 1").get(turnId); - if (row) { - return row as unknown as LocalRawTurnRow; - } - } - - return null; - }); -} - -function readToolCalls(raw: string): Array> { - const parsed = readJson(raw); - return Array.isArray(parsed) - ? parsed - .filter((call): call is Record => Boolean(call) && typeof call === "object" && !Array.isArray(call)) - .map(normalizeToolCall) - : []; -} - -function toolCallsFromTrace(trace: Record): Array> { - const calls = trace.tool_calls; - return Array.isArray(calls) - ? calls - .filter((call): call is Record => Boolean(call) && typeof call === "object" && !Array.isArray(call)) - .map(normalizeToolCall) - : []; -} - -function parseBracketToolBlocks(value: string | undefined): { text?: string; toolCalls: Array> } { - if (!value || !/^\[tool\]\s*$/im.test(value)) { - return { text: value, toolCalls: [] }; - } - - const lines = value.split(/\r?\n/); - const textLines: string[] = []; - const toolBlocks: string[] = []; - - for (let index = 0; index < lines.length;) { - const line = lines[index] ?? ""; - if (/^\[tool\]\s*$/i.test(line.trim())) { - index += 1; - const blockLines: string[] = []; - let sawToolField = false; - while (index < lines.length && !/^\[(user|assistant|tool|system)\]\s*$/i.test((lines[index] ?? "").trim())) { - const currentLine = lines[index] ?? ""; - const nextMeaningfulLine = nextNonEmptyLine(lines, index + 1); - if ( - sawToolField && - currentLine.trim() === "" && - nextMeaningfulLine && - !isToolFieldLine(nextMeaningfulLine) && - !/^\[(user|assistant|tool|system)\]\s*$/i.test(nextMeaningfulLine.trim()) - ) { - break; - } - - blockLines.push(currentLine); - if (isToolFieldLine(currentLine)) { - sawToolField = true; - } - index += 1; - } - const block = blockLines.join("\n").trim(); - if (block) { - toolBlocks.push(block); - } - continue; - } - - textLines.push(line); - index += 1; - } - - return { - text: cleanBracketToolText(textLines.join("\n")), - toolCalls: toolBlocks.map(parseBracketToolBlock).map(normalizeToolCall) - }; -} - -function nextNonEmptyLine(lines: readonly string[], start: number): string | undefined { - for (let index = start; index < lines.length; index += 1) { - const line = lines[index]; - if (line?.trim()) { - return line; - } - } - return undefined; -} - -function isToolFieldLine(line: string): boolean { - return /^(Tool|Call ID|Status|Input|Output|Error):\s*/i.test(line.trim()); -} - -function parseBracketToolBlock(text: string): Record { - const fallbackOutput = toolBlockValue(text, "Input") === undefined && toolBlockValue(text, "Output") === undefined - ? stripToolHeaderLines(text).trim() - : ""; - const status = firstToolLineValue(text, "Status"); - const error = firstToolLineValue(text, "Error"); - return removeUndefined({ - id: firstToolLineValue(text, "Call ID"), - name: firstToolLineValue(text, "Tool") ?? "tool", - input: toolBlockValue(text, "Input"), - output: toolBlockValue(text, "Output") ?? (fallbackOutput ? fallbackOutput : undefined), - error, - success: error ? false : successFromToolStatus(status) - }); -} - -function normalizeToolCall(call: Record): Record { - return removeUndefined({ - id: stringValue(call.id), - name: firstDefinedString(stringValue(call.name), stringValue(call.tool), stringValue(call.tool_name), "tool"), - input: sanitizeMetadataValue(call.input ?? call.args ?? call.arguments), - output: sanitizeMetadataValue(call.output ?? call.result), - error: stringValue(call.error) ?? stringValue(call.errorCode) ?? stringValue(call.error_code), - success: typeof call.success === "boolean" ? call.success : undefined, - startedAt: normalizeToolTime(call.startedAt ?? call.started_at), - endedAt: normalizeToolTime(call.endedAt ?? call.ended_at) - }); -} - -function firstToolLineValue(text: string, label: string): string | undefined { - const match = text.match(new RegExp(`^${escapeRegExp(label)}:\\s*(.+)$`, "im")); - return match?.[1]?.trim() || undefined; -} - -function toolBlockValue(text: string, label: string): unknown { - const lines = text.split(/\r?\n/); - const labelPattern = new RegExp(`^${escapeRegExp(label)}:[\\t ]*(.*)$`, "i"); - const start = lines.findIndex((line) => labelPattern.test(line)); - if (start < 0) { - return undefined; - } - - const inlineValue = lines[start]?.match(labelPattern)?.[1]?.trim(); - const nextFieldOffset = lines.slice(start + 1).findIndex((line, offset) => - lines[start + offset]?.trim() === "" && isToolFieldLine(line) - ); - const end = nextFieldOffset < 0 ? lines.length : start + 1 + nextFieldOffset; - const value = inlineValue || lines.slice(start + 1, end).join("\n").trim(); - if (!value) { - return undefined; - } - - try { - return JSON.parse(value); - } catch { - return value; - } -} - -function stripToolHeaderLines(text: string): string { - return text - .split(/\r?\n/) - .filter((line) => !/^(Tool|Call ID|Status|Error):\s*/i.test(line.trim())) - .join("\n"); -} - -function successFromToolStatus(status: string | undefined): boolean | undefined { - if (!status) { - return undefined; - } - return !/(error|fail|cancel|timeout)/i.test(status); -} - -function cleanBracketToolText(value: string): string | undefined { - const text = value.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); - return text || undefined; -} - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function normalizeToolTime(value: unknown): string | number | undefined { - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - - if (typeof value === "string" && value.trim()) { - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : value; - } - - return undefined; -} - -function traceTimestamp(trace: Record): string | undefined { - const ts = trace.ts; - if (typeof ts === "number" && Number.isFinite(ts)) { - const value = ts > 10_000_000_000 ? ts : ts * 1000; - return new Date(value).toISOString(); - } - - return optionalIsoTime(stringValue(ts)); -} - -function firstUserSpanSummary(rows: Array<{ parsed: ParsedRow; row: MemoryRow }>): string | undefined { - return rows.find(({ parsed }) => rawSpanForParsed(parsed).user_text === true)?.row.row.memory_value; -} - -function firstAgentSpanSummary(rows: Array<{ parsed: ParsedRow; row: MemoryRow }>): string | undefined { - return rows.find(({ parsed }) => rawSpanForParsed(parsed).agent_text === true)?.row.row.memory_value; -} - -function itemSummaryFallback(rows: Array<{ parsed: ParsedRow; row: MemoryRow }>): string | undefined { - return rows.map(({ parsed, row }) => summaryFromParsed(row.row, parsed)).find((summary) => summary && summary.trim()); -} - -function average(values: number[]): number | undefined { - return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : undefined; -} - -function uniqueToolCalls(calls: Array>): Array> { - const seen = new Set(); - return calls.filter((call) => { - const key = firstDefinedString(stringValue(call.id), `${stringValue(call.name) ?? "tool"}:${JSON.stringify(call.input ?? {})}`) ?? "tool"; - if (seen.has(key)) { - return false; - } - - seen.add(key); - return true; - }); -} - -function firstRecord(...values: unknown[]): Record { - return values.find((value): value is Record => Boolean(value) && typeof value === "object" && !Array.isArray(value)) ?? {}; -} - -function isRecordValue(value: unknown): value is Record { - return Boolean(value) && typeof value === "object" && !Array.isArray(value); -} - -function removeUndefined>(value: T): T { - return Object.fromEntries(Object.entries(value).filter(([, entryValue]) => entryValue !== undefined)) as T; -} - -function tableExists(db: DatabaseSync, tableName: string): boolean { - const row = db.prepare("select name from sqlite_master where type = 'table' and name = ?").get(tableName); - return Boolean(row); -} - -function withDb(source: MemosSqliteSource, read: (db: DatabaseSync) => T): T { - const db = new DatabaseSync(source.dbPath, { readOnly: true }); - try { - return read(db); - } finally { - db.close(); - } -} - -function withWritableDb(source: MemosSqliteSource, write: (db: DatabaseSync) => T): T { - const db = new DatabaseSync(source.dbPath, { allowExtension: true }); - try { - const extensionPath = getSqliteVecLoadablePath(); - const unpackedPath = extensionPath.replace(/app\.asar([\\/])/, "app.asar.unpacked$1"); - db.loadExtension(existsSync(unpackedPath) ? unpackedPath : extensionPath); - return write(db); - } finally { - db.close(); - } -} - -function encodeId(source: MemosSqliteSource, rawId: string): string { - return `${source.id}${SOURCE_ID_SEPARATOR}${rawId}`; -} - -function decodeId(id: string): { sourceId?: string; rawId: string } { - const index = id.indexOf(SOURCE_ID_SEPARATOR); - if (index <= 0) { - return { rawId: id }; - } - - return { sourceId: id.slice(0, index), rawId: id.slice(index + SOURCE_ID_SEPARATOR.length) }; -} - -function prefixIds(source: MemosSqliteSource, ids: string[]): string[] { - return ids.map((id) => (id.includes(SOURCE_ID_SEPARATOR) ? id : encodeId(source, id))); -} - -function readJson(raw: string): unknown { - try { - return JSON.parse(raw); - } catch { - return null; - } -} - -function readJsonArray(raw: string): string[] { - return stringArray(readJson(raw)); -} - -function readJsonObject(raw: string): Record { - const parsed = readJson(raw); - return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record) : {}; -} - -function objectAt(value: unknown, keys: string[]): Record { - let current = value; - for (const key of keys) { - if (!current || typeof current !== "object" || Array.isArray(current)) { - return {}; - } - - current = (current as Record)[key]; - } - - return current && typeof current === "object" && !Array.isArray(current) ? (current as Record) : {}; -} - -function stringArray(value: unknown): string[] { - return Array.isArray(value) ? value.map(String).filter(Boolean) : []; -} - -function uniqueStrings(values: string[]): string[] { - return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean))); -} - -function firstNonEmpty(...values: Array): string { - return values.find((value) => value && value.trim().length > 0)?.trim() ?? "Untitled memory"; -} - -function firstDefinedString(...values: Array): string | undefined { - return values - .map((value) => value?.trim()) - .find((value): value is string => Boolean(value && !isWorldSectionHeading(value) && !isInternalMemoryKey(value))); -} - -function stringValue(value: unknown): string | undefined { - return typeof value === "string" ? value : undefined; -} - -function numberValue(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) ? value : undefined; -} - -function booleanValue(value: unknown): boolean | undefined { - return typeof value === "boolean" ? value : undefined; -} - -function nonNegativeInt(value: unknown, fallback: number): number { - return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : fallback; -} - -function nonNegativeOptionalInt(value: unknown): number | undefined { - return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; -} - -function truncate(value: string, maxLength: number): string { - return value.length > maxLength ? `${value.slice(0, maxLength - 3)}...` : value; -} - -function firstLine(value: string): string { - return value.split(/\r?\n/, 1)[0]?.trim() ?? ""; -} - -function firstReadableMemoryValueLine(value: string): string | undefined { - return value - .split(/\r?\n/) - .map((line) => line.replace(/^\s*#{1,6}\s+/, "").replace(/^\s*[-*]\s+/, "").replace(/\*\*([^*]+)\*\*/g, "$1").trim()) - .find((line) => line && !isWorldSectionHeading(line) && !isInternalMemoryKey(line)); -} - -function humanizeIdentifier(value: string | undefined): string | undefined { - if (!value) return undefined; - const cleaned = value.trim(); - if (!/^[a-z0-9_:-]+$/i.test(cleaned)) return cleaned; - return cleaned - .replace(/^(skill|policy|trace|world)[:_]/i, "") - .split(/[_:-]+/) - .filter(Boolean) - .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()) - .join(" ") || undefined; -} - -function isWorldSectionHeading(value: string): boolean { - return /^(Environment|Inference|Constraints|Environment Knowledge|环境|环境拓扑|行为规律|约束禁忌|结构化认知)$/i.test(value.trim()); -} - -function isInternalMemoryKey(value: string | undefined): boolean { - return Boolean(value && /^(trace|policy|world|world_model|skill)[:_]/i.test(value.trim())); -} - -function normalizeIsoTime(value: string | null | undefined): string { - if (!value) { - return new Date(0).toISOString(); - } - - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date(0).toISOString(); -} - -function optionalIsoTime(value: string | undefined): string | undefined { - return value ? normalizeIsoTime(value) : undefined; -} - -/** - * Normalizes the log tool filter conditions. - * - * @param tools the tool-name list provided by the user. - * @returns a tool-name list containing at least the default displayable tools. - */ -function normalizeApiLogTools(tools: MemoryApiLogsInput["tools"]): Array { - return tools?.length ? tools : ["memory_add", "memory_search"]; -} - -/** - * Normalizes the log pagination count. - * - * @param limit the limit provided by the user. - * @returns a pagination count between 1 and 500. - */ -function normalizeLimit(limit: number | undefined): number { - return typeof limit === "number" && Number.isInteger(limit) && limit > 0 ? Math.min(limit, 500) : 50; -} - -/** - * Normalizes the log pagination offset. - * - * @param offset the offset provided by the user. - * @returns a non-negative integer offset. - */ -function normalizeOffset(offset: number | undefined): number { - return typeof offset === "number" && Number.isInteger(offset) && offset >= 0 ? offset : 0; -} - -function normalizePage(page: number | undefined): number { - return Number.isFinite(page) && page! > 0 ? Math.floor(page!) : 1; -} - -function sourceLabelFromPath(dbPath: string): string { - const homeName = basename(resolve(dbPath, "..", "..")); - return homeName && homeName !== "." ? homeName : "Memmy"; -} - -function expandHome(value: string): string { - return value === "~" || value.startsWith("~/") ? join(homedir(), value.slice(2)) : value; -} - -function sanitizeMetadataValue(value: unknown, key = ""): unknown { - if (key === "embedding" || key === "vec" || key === "vec_summary" || key === "vec_action") { - return undefined; - } - - if (value instanceof Uint8Array) { - return undefined; - } - - if (Array.isArray(value)) { - return value - .map((item) => sanitizeMetadataValue(item)) - .filter((item) => item !== undefined); - } - - if (value && typeof value === "object") { - const result: Record = {}; - for (const [entryKey, entryValue] of Object.entries(value as Record)) { - const sanitized = sanitizeMetadataValue(entryValue, entryKey); - if (sanitized !== undefined) { - result[entryKey] = sanitized; - } - } - - return result; - } - - return value; -} diff --git a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts index dd2825a88..75cb6e228 100644 --- a/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts +++ b/App/backend/src/adapters/outbound/memory-client/tests/http-memory-client.test.ts @@ -16,6 +16,8 @@ describe("HttpMemoryClient", () => { expect(Object.values(MEMORY_LAYER_PATHS)).toEqual([ "/api/v1/health", "/api/v1/admin/reload-config", + "/api/v1/admin/export", + "/api/v1/admin/data", "/api/v1/sessions/open", "/api/v1/sessions/:sessionId/close", "/api/v1/turns/start", @@ -79,6 +81,8 @@ describe("HttpMemoryClient", () => { summary: { routing: "fixed" } } }); + await expect(client.exportBundle!()).resolves.toMatchObject({ manifest: { service: "memmy-memory-service" } }); + await expect(client.clearAllData!()).resolves.toMatchObject({ ok: true, cleared: {} }); await expect(client.openSession(openSessionInput())).resolves.toMatchObject({ status: "open" }); await expect(client.closeSession(closeSessionInput())).resolves.toMatchObject({ status: "closed" }); await expect(client.startTurn(startTurnInput())).resolves.toMatchObject({ status: [] }); @@ -109,6 +113,8 @@ describe("HttpMemoryClient", () => { expect(requests.map((request) => `${request.method} ${request.path}`)).toEqual([ "GET /api/v1/health", "POST /api/v1/admin/reload-config", + "GET /api/v1/admin/export", + "DELETE /api/v1/admin/data", "POST /api/v1/sessions/open", "POST /api/v1/sessions/session-1/close", "POST /api/v1/turns/start", @@ -222,6 +228,19 @@ describe("HttpMemoryClient", () => { }); }); + it("does not replay a worker request after a server failure", async () => { + let calls = 0; + const baseUrl = await startServer(async (_request, response) => { + calls += 1; + response.writeHead(500, { "content-type": "application/json" }); + response.end("{}"); + }); + const client = createHttpMemoryClient({ baseUrl, token: "", timeoutMs: 500, maxRetries: 3 }); + + await expect(client.runWorker({ limit: 20 })).rejects.toThrow("memory layer 5xx"); + expect(calls).toBe(1); + }); + it("retries 5xx responses and succeeds before max retries is exhausted", async () => { let calls = 0; const baseUrl = await startServer(async (_request, response) => { @@ -409,6 +428,12 @@ function requestBodySource(body: unknown): string | undefined { function fixtureFor(method: string, path: string, body: unknown): unknown { if (method === "GET" && path === "/api/v1/health") return healthOutput(); if (method === "POST" && path === "/api/v1/admin/reload-config") return reloadConfigOutput(); + if (method === "GET" && path === "/api/v1/admin/export") { + return { manifest: { service: "memmy-memory-service" }, tables: {} }; + } + if (method === "DELETE" && path === "/api/v1/admin/data") { + return { ok: true, cleared: {}, clearedAt: now(), serverTime: now() }; + } if (method === "POST" && path === "/api/v1/sessions/open") return openSessionOutput(); if (method === "POST" && path === "/api/v1/sessions/session-1/close") return closeSessionOutput(); if (method === "POST" && path === "/api/v1/turns/start") return startTurnOutput(body); diff --git a/App/backend/src/adapters/outbound/memory-client/tests/memos-sqlite-memory-client.test.ts b/App/backend/src/adapters/outbound/memory-client/tests/memos-sqlite-memory-client.test.ts deleted file mode 100644 index e87d3d77b..000000000 --- a/App/backend/src/adapters/outbound/memory-client/tests/memos-sqlite-memory-client.test.ts +++ /dev/null @@ -1,817 +0,0 @@ -/** Memos sqlite memory client tests. */ -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import { getLoadablePath as getSqliteVecLoadablePath } from "sqlite-vec"; -import { afterEach, describe, expect, it } from "vitest"; -import { createMemosSqliteMemoryClient } from "../memos-sqlite-memory-client.js"; - -const NOW = "2026-06-08T10:00:00.000Z"; - -let tempDir: string | undefined; - -afterEach(() => { - if (tempDir) { - rmSync(tempDir, { recursive: true, force: true }); - tempDir = undefined; - } -}); - -describe("createMemosSqliteMemoryClient", () => { - it("preserves Span memory kinds in panel responses", async () => { - const dbPath = createMemoryDatabase({ - id: "span_sqlite_1", - sessionId: "codex-session-span", - agentId: "codex", - tagsJson: JSON.stringify(["span"]), - infoJson: JSON.stringify({ source: "worker.span_big_turn.v1" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_layer: "L1", - memory_kind: "span", - source: "worker.span_big_turn.v1", - span: { span_goal: "Inspect the local span data" } - } - }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.panelItems({ layer: "L1", page: 1 })).resolves.toMatchObject({ - items: [{ - id: "memmy-memory::span_sqlite_1", - kind: "span", - metadata: { spanGoal: "Inspect the local span data" } - }] - }); - }); - - it("lists and deletes User Memory through the sqlite fallback", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_user_memory_seed", - sessionId: "codex-user-memory", - agentId: "codex", - tagsJson: "[]", - infoJson: "{}", - propertiesJson: JSON.stringify({ internal_info: { memory_layer: "L1" } }) - }); - insertUserMemory(dbPath, "user_memory_sqlite_1", "我最喜欢的水果是苹果"); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.panelItems({ layer: "UserMemory", q: "苹果", page: 1 })).resolves.toMatchObject({ - total: 1, - items: [{ - id: "memmy-memory::user_memory_sqlite_1", - kind: "user_memory", - memoryLayer: "UserMemory", - tags: ["User Preference"] - }] - }); - await expect(client.deleteMemory({ memoryId: "memmy-memory::user_memory_sqlite_1" })).resolves.toMatchObject({ - kind: "user_memory", - status: "deleted" - }); - await expect(client.panelItems({ layer: "UserMemory", page: 1 })).resolves.toMatchObject({ total: 0, items: [] }); - }); - - it("exposes only the span's raw-turn tool-call range in detail metadata", async () => { - const dbPath = createMemoryDatabase({ - id: "span_sqlite_steps", - sessionId: "codex-session-span", - agentId: "codex", - tagsJson: JSON.stringify(["span"]), - infoJson: JSON.stringify({ raw_turn_id: "raw-span-1" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_layer: "L1", - memory_kind: "span", - span: { raw_turn_id: "raw-span-1", tool_call_start: 1, tool_call_end: 2 } - } - }), - rawTurn: { - id: "raw-span-1", - toolCalls: [ - { id: "tool-0", name: "read_file" }, - { id: "tool-1", name: "rg" }, - { id: "tool-2", name: "npm_test" }, - { id: "tool-3", name: "git_diff" } - ] - } - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const detail = await client.getMemory({ memoryId: "memmy-memory::span_sqlite_steps" }); - - expect(detail.item.metadata.spanDetail).toEqual({ - toolCallStart: 1, - toolCallEnd: 2, - toolCalls: [ - { id: "tool-1", name: "rg" }, - { id: "tool-2", name: "npm_test" } - ] - }); - }); - - it("derives Hermes source from the session id when the row agent is the default", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_hermes_1", - sessionId: "hermes-20260608_165922_f6cf51", - agentId: "codex", - tagsJson: JSON.stringify(["trace"]), - infoJson: "{}", - propertiesJson: JSON.stringify({ internal_info: { source: "turn.complete", value: 0.42, alpha: 0.8, reflection: "Useful turn." } }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const list = await client.panelItems({ layer: "L1", page: 1 }); - expect(list.items[0]?.tags).toEqual(["hermes", "trace"]); - expect(list.items[0]?.metadata?.source).toBe("hermes"); - expect(list.items[0]?.metrics).toEqual({ value: 0.42, alpha: 0.8, reflectionDone: true }); - await expect(client.panelItems({ layer: "L1", sourceAgent: "hermes", page: 1 })) - .resolves.toMatchObject({ total: 1, items: [{ id: expect.stringContaining("trace_hermes_1") }] }); - await expect(client.panelItems({ layer: "L1", sourceAgent: "codex", page: 1 })) - .resolves.toMatchObject({ total: 0, items: [] }); - - const detail = await client.getMemory({ memoryId: "memmy-memory::trace_hermes_1" }); - expect(detail.item.metadata.source).toBe("hermes"); - expect(detail.item.metrics).toEqual({ value: 0.42, alpha: 0.8, reflectionDone: true }); - }); - - it("filters custom L1 panel item sources as other", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_other_1", - sessionId: "test-agent-session", - agentId: "test_agent", - tagsJson: JSON.stringify(["trace"]), - infoJson: "{}", - propertiesJson: JSON.stringify({ internal_info: { source: "memory.add" } }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.panelItems({ - layer: "L1", - excludedSourceAgents: ["memmy-agent", "cursor", "claude_code", "codex", "opencode", "openclaw", "hermes"], - page: 1 - })).resolves.toMatchObject({ - total: 1, - items: [{ id: expect.stringContaining("trace_other_1"), metadata: { source: "test_agent" } }] - }); - await expect(client.panelItems({ layer: "L1", sourceAgent: "memmy-agent", page: 1 })) - .resolves.toMatchObject({ total: 0, items: [] }); - }); - - it("parses bracket tool blocks from imported trace agent text", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_codex_1", - sessionId: "codex-session-1", - agentId: "codex", - memoryValue: "Imported Codex trace.", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_layer: "L1", - memory_kind: "trace", - source: "codex", - trace: { - turn_id: "codex-session-1:1", - user_text: "检查当前目录", - agent_text: [ - "我先看一下当前目录。", - "", - "[tool]", - "Tool: exec_command", - "Call ID: call-shell", - "Input:", - "{\"cmd\":\"pwd\"}", - "", - "Output:", - "/tmp/project", - "", - "目录确认完成。" - ].join("\n"), - raw_span: { user_text: true, agent_text: true, tool_call_count: 0 }, - tool_calls: [] - } - } - }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const detail = await client.getMemory({ memoryId: "memmy-memory::trace_codex_1" }); - const traceDetail = detail.item.metadata.traceDetail as { - userQuery?: string; - finalResponse?: string; - toolCalls?: Array<{ id?: string; name?: string; input?: unknown; output?: unknown }>; - }; - - expect(traceDetail.userQuery).toBe("检查当前目录"); - expect(traceDetail.finalResponse).toBe("我先看一下当前目录。\n\n目录确认完成。"); - expect(traceDetail.toolCalls).toEqual([ - { - id: "call-shell", - name: "exec_command", - input: { cmd: "pwd" }, - output: "/tmp/project" - } - ]); - }); - - it("preserves multiline bracket tool payloads through CRLF and the block end", async () => { - const prettyInput = JSON.stringify({ - search_query: [ - { q: "memory parser regression" }, - { q: "tool payload boundaries" } - ], - response_length: "long" - }, null, 2); - const prettyOutput = JSON.stringify([ - { title: "first result", score: 0.9 }, - { title: "second result", score: 0.8 } - ], null, 2); - const dbPath = createMemoryDatabase({ - id: "trace_codex_multiline", - sessionId: "codex-session-multiline", - agentId: "codex", - memoryValue: "Imported Codex multiline trace.", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_layer: "L1", - memory_kind: "trace", - source: "codex", - trace: { - turn_id: "codex-session-multiline:1", - user_text: "检查多行工具载荷", - agent_text: [ - "我会检查工具载荷。", - "", - "[tool]", - "Tool: web_search", - "Call ID: call-search", - "Input:", - prettyInput, - "", - "Output:", - prettyOutput, - "", - "[tool]", - "Tool: exec_command", - "Call ID: call-exec", - "Input:", - "printf 'first line\\nsecond line'", - "", - "Output:", - "first line", - "second line" - ].join("\r\n"), - raw_span: { user_text: true, agent_text: true, tool_call_count: 0 }, - tool_calls: [] - } - } - }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const detail = await client.getMemory({ memoryId: "memmy-memory::trace_codex_multiline" }); - const traceDetail = detail.item.metadata.traceDetail as { - finalResponse?: string; - toolCalls?: Array<{ id?: string; name?: string; input?: unknown; output?: unknown }>; - }; - - expect(traceDetail.finalResponse).toBe("我会检查工具载荷。"); - expect(traceDetail.toolCalls).toEqual([ - { - id: "call-search", - name: "web_search", - input: JSON.parse(prettyInput), - output: JSON.parse(prettyOutput) - }, - { - id: "call-exec", - name: "exec_command", - input: "printf 'first line\\nsecond line'", - output: "first line\nsecond line" - } - ]); - }); - - it("exposes generated skill status from linked episodes", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_skill_1", - sessionId: "codex-session-skill", - agentId: "codex", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex", episode_id: "episode-skill-1" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_layer: "L1", - memory_kind: "trace", - source: "codex", - trace: { - episode_id: "episode-skill-1", - turn_id: "turn-skill-1", - user_text: "沉淀一个技能", - agent_text: "已沉淀。", - tool_calls: [] - } - } - }), - episode: { - id: "episode-skill-1", - sessionId: "codex-session-skill", - skillMemoryIds: ["skill_sqlite_1"] - } - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const detail = await client.getMemory({ memoryId: "memmy-memory::trace_skill_1" }); - const traceDetail = detail.item.metadata.traceDetail as { - episode?: { - skillStatus?: string; - skillReason?: string; - skillMemoryIds?: string[]; - linkedSkillId?: string; - }; - }; - - expect(traceDetail.episode).toMatchObject({ - skillStatus: "succeeded", - skillReason: "已从该任务沉淀出可复用技能。", - skillMemoryIds: ["skill_sqlite_1"], - linkedSkillId: "skill_sqlite_1" - }); - }); - - it("matches panel item searches by memory id", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_sqlite_panel_id", - sessionId: "codex-session-search-id", - agentId: "codex", - memoryValue: "Plain SQLite memory body.", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex" }), - propertiesJson: JSON.stringify({ internal_info: { memory_layer: "L1", memory_kind: "trace", source: "codex" } }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - const list = await client.panelItems({ layer: "L1", q: "trace_sqlite_panel_id", page: 1 }); - - expect(list.items.map((item) => item.id)).toEqual(["memmy-memory::trace_sqlite_panel_id"]); - expect(list.items[0]?.metadata?.source).toBe("codex"); - }); - - it("filters memory_add and memory_search logs by exact and other source Agent", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_log_filter", - sessionId: "codex-session-log-filter", - agentId: "codex", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex" }), - propertiesJson: JSON.stringify({ internal_info: { source: "codex", memory_kind: "trace" } }) - }); - seedApiLogs(dbPath); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.memoryApiLogs({ - tools: ["memory_add", "memory_search"], - sourceAgent: "openclaw", - limit: 20, - offset: 0 - })).resolves.toMatchObject({ - total: 2, - logs: [ - { toolName: "memory_add", sourceAgent: "openclaw", outputJson: expect.stringContaining("OpenClaw") }, - { toolName: "memory_search", sourceAgent: "openclaw", inputJson: expect.stringContaining("session_openclaw") } - ] - }); - const otherLogs = await client.memoryApiLogs({ - tools: ["memory_add", "memory_search"], - excludedSourceAgents: ["memmy-agent", "cursor", "claude_code", "codex", "opencode", "openclaw", "hermes"], - limit: 20, - offset: 0 - }); - expect(otherLogs).toMatchObject({ - total: 4, - logs: [ - { toolName: "memory_add", sourceAgent: "test_agent", outputJson: expect.stringContaining("custom Agent") }, - { toolName: "memory_add", outputJson: expect.stringContaining("CLI") }, - { toolName: "memory_search", sourceAgent: "test_agent", inputJson: expect.stringContaining("session_test_agent") }, - { toolName: "memory_search" } - ] - }); - expect(otherLogs.logs.map((log) => log.sourceAgent)).toEqual(["test_agent", undefined, "test_agent", undefined]); - - await expect(client.memoryApiLogs({ - tools: ["memory_search"], - sourceAgent: "openclaw", - limit: 20, - offset: 0 - })).resolves.toMatchObject({ total: 1, logs: [{ toolName: "memory_search" }] }); - }); - - it("uses the current span goal when reading memory_add logs", async () => { - const dbPath = createMemoryDatabase({ - id: "span_log_goal", - sessionId: "codex-session-log-summary", - agentId: "codex", - memoryValue: "Goal: Current goal from the span", - tagsJson: JSON.stringify(["span"]), - infoJson: JSON.stringify({ span_goal: "Current goal from the span" }), - propertiesJson: JSON.stringify({ - internal_info: { - memory_kind: "span", - span: { span_goal: "Current goal from the span" } - } - }) - }); - seedApiLogs(dbPath); - const db = new DatabaseSync(dbPath); - db.prepare(` - INSERT INTO api_logs (tool_name, source_agent, input_json, output_json, duration_ms, success, called_at) - VALUES (?, ?, ?, ?, ?, ?, ?) - `).run( - "memory_add", - "codex", - "{}", - JSON.stringify({ details: [{ role: "span", traceId: "span_log_goal" }] }), - 1, - 1, - "2026-06-08T09:04:00.000Z" - ); - db.close(); - - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.memoryApiLogs({ - tools: ["memory_add"], sourceAgent: "codex", limit: 20, offset: 0 - })).resolves.toMatchObject({ - logs: [{ outputJson: expect.stringContaining("Current goal from the span") }] - }); - }); - - it("deletes local SQLite memories so list, search, and detail cannot read them", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_delete_1", - sessionId: "codex-session-delete", - agentId: "codex", - memoryValue: "Delete this exact SQLite memory.", - tagsJson: JSON.stringify(["trace", "codex", "delete-me"]), - infoJson: JSON.stringify({ source: "codex" }), - propertiesJson: JSON.stringify({ internal_info: { source: "codex", memory_kind: "trace" } }) - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.deleteMemory({ memoryId: "memmy-memory::trace_delete_1" })).resolves.toMatchObject({ - ok: true, - id: "memmy-memory::trace_delete_1", - kind: "trace", - status: "deleted" - }); - await expect(client.panelItems({ layer: "L1", page: 1 })).resolves.toMatchObject({ items: [] }); - await expect(client.search({ query: "Delete this exact SQLite memory.", verbose: true })).resolves.toMatchObject({ - debug: { hits: [] } - }); - await expect(client.getMemory({ memoryId: "memmy-memory::trace_delete_1" })).rejects.toMatchObject({ - code: "not_found", - status: 404 - }); - - expect(readMemoryRowCount(dbPath, "trace_delete_1")).toBe(0); - expect(readVectorRowCount(dbPath)).toBe(0); - }); - - it("lists and atomically deletes tasks independently from memory pagination", async () => { - const dbPath = createMemoryDatabase({ - id: "trace_task_1", - sessionId: "codex-session-task", - agentId: "codex", - memoryValue: "Task-owned memory.", - tagsJson: JSON.stringify(["trace", "codex"]), - infoJson: JSON.stringify({ source: "codex", episode_id: "episode-task-1" }), - propertiesJson: JSON.stringify({ internal_info: { source: "codex", memory_kind: "trace" } }), - episode: { id: "episode-task-1", sessionId: "codex-session-task", skillMemoryIds: [] } - }); - const client = createMemosSqliteMemoryClient({ - sources: [{ id: "memmy-memory", label: "memmy", dbPath }], - now: () => NOW - }); - - await expect(client.panelTasks({ q: "episode-task-1", page: 99 })).resolves.toMatchObject({ - tasks: [{ id: "memmy-memory::episode-task-1", memoryIds: ["memmy-memory::trace_task_1"] }], - page: 1, - total: 1, - totalPages: 1 - }); - await expect(client.deletePanelTask("memmy-memory::episode-task-1")).resolves.toMatchObject({ - ok: true, - id: "memmy-memory::episode-task-1", - deletedMemoryIds: ["memmy-memory::trace_task_1"] - }); - await expect(client.panelTasks({ page: 1 })).resolves.toMatchObject({ tasks: [], total: 0, page: 1 }); - expect(readMemoryRowCount(dbPath, "trace_task_1")).toBe(0); - }); -}); - -function createMemoryDatabase(row: { - id: string; - sessionId: string | null; - agentId: string | null; - memoryValue?: string; - tagsJson: string; - infoJson: string; - propertiesJson: string; - episode?: { - id: string; - sessionId: string; - skillMemoryIds: string[]; - }; - rawTurn?: { - id: string; - toolCalls: Array>; - }; -}): string { - tempDir = mkdtempSync(join(tmpdir(), "memmy-sqlite-client-")); - const dbPath = join(tempDir, "memory.sqlite"); - const db = new DatabaseSync(dbPath, { allowExtension: true }); - db.loadExtension(getSqliteVecLoadablePath()); - db.exec(` - CREATE TABLE memories ( - id TEXT PRIMARY KEY, - timeline TEXT NOT NULL, - user_id TEXT NOT NULL, - conversation_id TEXT, - session_id TEXT, - agent_id TEXT, - app_id TEXT, - memory_type TEXT NOT NULL, - status TEXT NOT NULL, - visibility TEXT NOT NULL, - memory_key TEXT, - memory_value TEXT NOT NULL, - tags_json TEXT NOT NULL, - info_json TEXT NOT NULL, - properties_json TEXT NOT NULL, - memory_layer TEXT NOT NULL, - content_hash TEXT, - version INTEGER NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - deleted_at TEXT - ) - `); - db.prepare(` - INSERT INTO memories ( - id, timeline, user_id, conversation_id, session_id, agent_id, app_id, - memory_type, status, visibility, memory_key, memory_value, - tags_json, info_json, properties_json, memory_layer, content_hash, - version, created_at, updated_at, deleted_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - row.id, - "default", - "local-user", - null, - row.sessionId, - row.agentId, - null, - "LongTermMemory", - "activated", - "private", - row.id, - row.memoryValue ?? "Hermes wrote this turn.", - row.tagsJson, - row.infoJson, - row.propertiesJson, - "L1", - null, - 1, - NOW, - NOW, - null - ); - db.exec(` - CREATE TABLE memory_vector_entries ( - id INTEGER PRIMARY KEY, - memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE, - vector_field TEXT NOT NULL, - embedding_model TEXT, - embedding_provider TEXT, - embedding_dim INTEGER NOT NULL, - updated_at TEXT NOT NULL, - UNIQUE (memory_id, vector_field) - ); - CREATE VIRTUAL TABLE memory_vec_3 USING vec0(embedding float[3] distance_metric=cosine); - `); - db.prepare(` - INSERT INTO memory_vector_entries ( - id, memory_id, vector_field, embedding_model, embedding_provider, embedding_dim, updated_at - ) VALUES (1, ?, 'vec_summary', 'test', 'openai_compatible', 3, ?) - `).run(row.id, NOW); - db.prepare(`INSERT INTO memory_vec_3 (rowid, embedding) VALUES (?, ?)`) - .run(1n, Buffer.from(new Float32Array([1, 0, 0]).buffer)); - if (row.rawTurn) { - db.exec(` - CREATE TABLE raw_turns ( - id TEXT PRIMARY KEY, - session_id TEXT, - episode_id TEXT, - turn_id TEXT, - user_id TEXT, - conversation_id TEXT, - user_text TEXT, - assistant_text TEXT, - reasoning_summary TEXT, - tool_calls_json TEXT, - tool_results_json TEXT, - source_memory_ids_json TEXT, - usage_json TEXT, - message_payload_json TEXT, - status TEXT, - redacted_at TEXT, - deleted_at TEXT, - created_at TEXT - ) - `); - db.prepare(` - INSERT INTO raw_turns ( - id, session_id, episode_id, turn_id, user_id, conversation_id, user_text, - assistant_text, reasoning_summary, tool_calls_json, tool_results_json, - source_memory_ids_json, usage_json, message_payload_json, status, - redacted_at, deleted_at, created_at - ) VALUES (?, ?, NULL, ?, ?, NULL, NULL, NULL, NULL, ?, '[]', '[]', '{}', '{}', 'succeeded', NULL, NULL, ?) - `).run(row.rawTurn.id, row.sessionId, row.rawTurn.id, "local-user", JSON.stringify(row.rawTurn.toolCalls), NOW); - } - if (row.episode) { - db.exec(` - CREATE TABLE episodes ( - id TEXT PRIMARY KEY, - session_id TEXT, - status TEXT NOT NULL, - title TEXT, - summary TEXT, - l1_memory_ids_json TEXT NOT NULL DEFAULT '[]', - raw_turn_ids_json TEXT, - skill_memory_ids_json TEXT, - turn_count INTEGER, - r_task REAL, - reward_detail_json TEXT, - pipeline_status TEXT, - pipeline_error TEXT, - meta_json TEXT, - opened_at TEXT, - closed_at TEXT, - updated_at TEXT - ) - `); - db.prepare(` - INSERT INTO episodes ( - id, session_id, status, title, summary, l1_memory_ids_json, raw_turn_ids_json, - skill_memory_ids_json, turn_count, r_task, reward_detail_json, - pipeline_status, pipeline_error, meta_json, opened_at, closed_at, updated_at - ) VALUES (?, ?, 'closed', NULL, NULL, ?, '[]', ?, 1, 0.8, '{}', 'idle', NULL, '{}', ?, ?, ?) - `).run( - row.episode.id, - row.episode.sessionId, - JSON.stringify([row.id]), - JSON.stringify(row.episode.skillMemoryIds), - NOW, - NOW, - NOW - ); - } - db.close(); - return dbPath; -} - -function readMemoryRowCount(dbPath: string, memoryId: string): number { - const db = new DatabaseSync(dbPath, { readOnly: true }); - try { - const row = db.prepare("select count(*) as count from memories where id = ?").get(memoryId) as { count: number }; - return row.count; - } finally { - db.close(); - } -} - -function insertUserMemory(dbPath: string, id: string, content: string): void { - const db = new DatabaseSync(dbPath); - try { - db.exec(` - CREATE TABLE user_memories ( - id TEXT PRIMARY KEY, - source_turn_id TEXT NOT NULL, - user_id TEXT NOT NULL, - memory_types_json TEXT NOT NULL, - content TEXT NOT NULL, - normalized_user_text_hash TEXT NOT NULL, - source_turn_refs_json TEXT NOT NULL, - status TEXT NOT NULL, - replaces_memory_id TEXT, - replaced_by_memory_id TEXT, - archived_at TEXT, - archive_reason TEXT, - embedding_json TEXT, - embedding_model TEXT, - embedding_provider TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - deleted_at TEXT - ) - `); - db.prepare(` - INSERT INTO user_memories ( - id, source_turn_id, user_id, memory_types_json, content, - normalized_user_text_hash, source_turn_refs_json, status, - created_at, updated_at - ) VALUES (?, 'turn-user-memory', 'local-user', '["User Preference"]', ?, 'hash', '["turn-user-memory"]', 'active', ?, ?) - `).run(id, content, NOW, NOW); - } finally { - db.close(); - } -} - -function seedApiLogs(dbPath: string): void { - const db = new DatabaseSync(dbPath); - try { - db.exec(` - CREATE TABLE api_logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tool_name TEXT NOT NULL, - source_agent TEXT, - input_json TEXT NOT NULL, - output_json TEXT NOT NULL, - duration_ms INTEGER NOT NULL, - success INTEGER NOT NULL, - called_at TEXT NOT NULL - ) - `); - const insert = db.prepare(` - INSERT INTO api_logs ( - tool_name, source_agent, input_json, output_json, duration_ms, success, called_at - ) VALUES (?, ?, ?, ?, 1, 1, ?) - `); - insert.run("memory_add", "openclaw", "{}", JSON.stringify({ - details: [{ sourceAgent: "openclaw", summary: "Stored by OpenClaw" }] - }), "2026-06-08T09:03:00.000Z"); - insert.run("memory_add", "test_agent", "{}", JSON.stringify({ - details: [{ sourceAgent: "test_agent", summary: "Stored by custom Agent" }] - }), "2026-06-08T09:02:30.000Z"); - insert.run("memory_add", null, "{}", JSON.stringify({ - details: [{ summary: "Stored directly through CLI" }] - }), "2026-06-08T09:02:00.000Z"); - insert.run("memory_search", "openclaw", JSON.stringify({ sessionId: "session_openclaw" }), JSON.stringify({ candidates: [] }), "2026-06-08T09:01:00.000Z"); - insert.run("memory_search", "test_agent", JSON.stringify({ sessionId: "session_test_agent" }), JSON.stringify({ candidates: [] }), "2026-06-08T09:00:30.000Z"); - insert.run("memory_search", null, "{}", JSON.stringify({ candidates: [] }), "2026-06-08T09:00:00.000Z"); - } finally { - db.close(); - } -} - -function readVectorRowCount(dbPath: string): number { - const db = new DatabaseSync(dbPath, { readOnly: true, allowExtension: true }); - try { - db.loadExtension(getSqliteVecLoadablePath()); - const row = db.prepare("select count(*) as count from memory_vec_3").get() as { count: number }; - return row.count; - } finally { - db.close(); - } -} diff --git a/App/backend/src/adapters/outbound/memory-client/types.ts b/App/backend/src/adapters/outbound/memory-client/types.ts index 8b4a43a26..40040dd8a 100644 --- a/App/backend/src/adapters/outbound/memory-client/types.ts +++ b/App/backend/src/adapters/outbound/memory-client/types.ts @@ -43,6 +43,8 @@ export interface MemoryRequestContext { export interface MemoryClient { health(): Promise; reloadConfig(input?: MemoryReloadConfigInput): Promise; + exportBundle?(): Promise>; + clearAllData?(): Promise<{ ok: true; clearedAt: string; cleared: Record }>; openSession(input: OpenSessionInput, context?: MemoryRequestContext): Promise; closeSession(input: CloseSessionInput & { sessionId: string }, context?: MemoryRequestContext): Promise; diff --git a/App/backend/src/adapters/outbound/skill-writer/deepseek-harness/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/deepseek-harness/tests/target.test.ts index b206152bc..ad341a563 100644 --- a/App/backend/src/adapters/outbound/skill-writer/deepseek-harness/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/deepseek-harness/tests/target.test.ts @@ -124,8 +124,14 @@ describe("DeepSeek Harness skill target", () => { expect(handoff?.id).toBe("@memmy/memmy-memory"); let definition: Record | undefined; - handoff?.factory().apply({ - conversationEvents: { register: (value: Record) => { definition = value; } } + const client = handoff?.factory(); + expect(client?.inject).toEqual([]); + client?.apply({ + get(name: string) { + return name === "conversationEvents" + ? { register: (value: Record) => { definition = value; } } + : undefined; + } }); const message = { id: "user-1", @@ -164,6 +170,51 @@ describe("DeepSeek Harness skill target", () => { }); }); + it("prefers the uiConversation event registry when both APIs are available", async () => { + const rootDirectory = createRoot(); + const target = createDeepseekHarnessSkillTarget({ rootDirectory }); + await target.installPlugin?.("deepseek_harness"); + const clientPath = join(installedPluginDirectory(rootDirectory), "client.js"); + let handoff: { id: string; factory(): Record } | undefined; + runInNewContext(readFileSync(clientPath, "utf8"), { + window: { __ModuleLoader__: { load: (value: typeof handoff) => { handoff = value; } } } + }); + + let modernRegistrations = 0; + let legacyRegistrations = 0; + const client = handoff?.factory(); + client?.apply({ + get(name: string) { + if (name === "uiConversation") { + return { events: { register: () => { modernRegistrations += 1; } } }; + } + if (name === "conversationEvents") { + return { register: () => { legacyRegistrations += 1; } }; + } + return undefined; + } + }); + + expect(modernRegistrations).toBe(1); + expect(legacyRegistrations).toBe(0); + }); + + it("fails clearly when neither conversation event API is available", async () => { + const rootDirectory = createRoot(); + const target = createDeepseekHarnessSkillTarget({ rootDirectory }); + await target.installPlugin?.("deepseek_harness"); + const clientPath = join(installedPluginDirectory(rootDirectory), "client.js"); + let handoff: { id: string; factory(): Record } | undefined; + runInNewContext(readFileSync(clientPath, "utf8"), { + window: { __ModuleLoader__: { load: (value: typeof handoff) => { handoff = value; } } } + }); + + const client = handoff?.factory(); + expect(() => client?.apply({ get: () => undefined })).toThrow( + "memmy-memory requires uiConversation.events or conversationEvents" + ); + }); + it("replaces legacy versioned patch markers", async () => { const rootDirectory = createRoot(); const patchPath = join(rootDirectory, "cordis.patch.yml"); diff --git a/App/backend/src/adapters/outbound/skill-writer/templates/memmy-deepseek-harness-plugin.ts b/App/backend/src/adapters/outbound/skill-writer/templates/memmy-deepseek-harness-plugin.ts index 5d1cad190..7e8264299 100644 --- a/App/backend/src/adapters/outbound/skill-writer/templates/memmy-deepseek-harness-plugin.ts +++ b/App/backend/src/adapters/outbound/skill-writer/templates/memmy-deepseek-harness-plugin.ts @@ -527,10 +527,18 @@ export const DEEPSEEK_HARNESS_PLUGIN_CLIENT = String.raw`window.__ModuleLoader__ const exports = module.exports; const name = "memmy-memory-client"; - const inject = ["conversationEvents"]; + const inject = []; + + function resolveConversationEventRegistry(ctx) { + const uiConversation = ctx.get("uiConversation"); + if (uiConversation && uiConversation.events) return uiConversation.events; + const conversationEvents = ctx.get("conversationEvents"); + if (conversationEvents) return conversationEvents; + throw new Error("memmy-memory requires uiConversation.events or conversationEvents"); + } function apply(ctx) { - ctx.conversationEvents.register({ + resolveConversationEventRegistry(ctx).register({ kind: "memmy-optimistic-user", target: "chat", match(event) { diff --git a/App/backend/src/index.ts b/App/backend/src/index.ts index 733ecfa18..9833f50df 100644 --- a/App/backend/src/index.ts +++ b/App/backend/src/index.ts @@ -7,8 +7,6 @@ import { createAppStateStore } from "./infrastructure/app-state-store/index.js"; import { createHttpCloudClient, type CloudClient } from "./adapters/outbound/cloud-client/index.js"; import { createHttpMemoryClient, - createMemosSqliteMemoryClient, - discoverMemosSqliteSources, type MemoryClient, type MemoryLayerConfig } from "./adapters/outbound/memory-client/index.js"; @@ -18,14 +16,13 @@ import { readConfiguredAgentTimeZone, readAgentGatewayBootstrapSecret } from "./infrastructure/memmy-config/index.js"; +import { + createMemoryScanPreferencesStore, + ensureMemoryScanPreferences +} from "./infrastructure/memmy-config/agent-access.js"; import { createPermissionManager } from "./permission/index.js"; import { createLocalApiServer } from "./adapters/inbound/local-api/server.js"; import { createBackendServices, type BootstrapScenario } from "./services/index.js"; -import { - createAgentSourceAutoScanService, - DEFAULT_AGENT_SOURCE_AUTO_SCAN_INTERVAL_MS, - type AgentSourceAutoScanService -} from "./services/agent-source-auto-scan-service.js"; import { resolveCloudClientConfig, type CloudClientConfig } from "./config/service-urls.js"; import { resetAccountRuntimeForDesktopInstallChange } from "./services/desktop-install-state-service.js"; import { @@ -63,10 +60,6 @@ export interface CreateLocalBackendOptions { desktopInstallFingerprint?: string; /** Login channel supported by the current desktop package. */ accountChannel?: AccountChannel; - /** Agent source auto scan interval in ms. Defaults to one hour. */ - agentSourceAutoScanIntervalMs?: number; - /** Agent source startup scan delay in ms. Defaults to five minutes. */ - agentSourceAutoScanInitialDelayMs?: number; /** Running Agent Gateway client; when present, refreshes MCP after startup config writes. */ memmyAgentAdminClient?: MemmyAgentAdminClient; } @@ -88,7 +81,6 @@ export async function createLocalBackend(options: CreateLocalBackendOptions): Pr } const appStateStore = createAppStateStore({ databasePath: options.databasePath }); let server: Awaited> | null = null; - let autoScan: AgentSourceAutoScanService | null = null; try { if (options.desktopInstallFingerprint) { @@ -104,6 +96,11 @@ export async function createLocalBackend(options: CreateLocalBackendOptions): Pr memmyConfigPath, accountChannel: options.accountChannel }); + await ensureMemoryScanPreferences( + memmyConfigPath, + appStateStore.repositories.bootstrap.getScanPreferences() + ); + const scanPreferencesStore = createMemoryScanPreferencesStore(memmyConfigPath); const permissionManager = createPermissionManager({ appStateStore, @@ -140,6 +137,7 @@ export async function createLocalBackend(options: CreateLocalBackendOptions): Pr bootstrapScenario: options.bootstrapScenario, memmyConfigWriter, memmyConfigPath, + scanPreferencesStore, accountChannel: options.accountChannel, memmyAgentAdminClient: options.memmyAgentAdminClient, memmyAgentAdminBootstrapSecret: await readAgentGatewayBootstrapSecret(memmyConfigPath) @@ -184,17 +182,7 @@ export async function createLocalBackend(options: CreateLocalBackendOptions): Pr memory: options.memoryBaseUrl ? { baseUrl: options.memoryBaseUrl } : undefined }); await writeRuntimeConfigFile(runtimeConfig, options.runtimeConfigPath ?? resolveDefaultRuntimeConfigPath()); - autoScan = createAgentSourceAutoScanService({ - baseUrl: runtimeConfig.baseUrl, - localToken, - intervalMs: options.agentSourceAutoScanIntervalMs ?? DEFAULT_AGENT_SOURCE_AUTO_SCAN_INTERVAL_MS, - initialDelayMs: options.agentSourceAutoScanInitialDelayMs, - getScanPreferences: () => appStateStore.repositories.bootstrap.getScanPreferences() - }); - autoScan.start(); - const boundServer = server; - const boundAutoScan = autoScan; return { runtimeConfig, getAppSettings() { @@ -204,13 +192,11 @@ export async function createLocalBackend(options: CreateLocalBackendOptions): Pr return appStateStore.repositories.bootstrap.recordLastLaunchMode(mode); }, async close() { - boundAutoScan.close(); await boundServer.close(); appStateStore.close(); } }; } catch (error) { - autoScan?.close(); await server?.close().catch(() => undefined); appStateStore.close(); throw error; @@ -256,10 +242,8 @@ export function readMemoryLayerConfig(env: NodeJS.ProcessEnv): MemoryLayerConfig /** * Creates the default MemoryClient. * - * Priority: - * 1. The standard HTTP memory layer pointed to by MEMMY_MEMORY_LAYER_URL. - * 2. A read-only client over this project's MemoryService SQLite database. - * Fails outright when no real data source is available, to avoid the desktop app silently showing fake data. + * Memory is a process boundary: Desktop always talks to it over HTTP and never + * reads the service-owned SQLite database. */ function createDefaultMemoryClient(env: NodeJS.ProcessEnv): MemoryClient { const memoryLayerConfig = readMemoryLayerConfig(env); @@ -267,12 +251,5 @@ function createDefaultMemoryClient(env: NodeJS.ProcessEnv): MemoryClient { return createHttpMemoryClient(memoryLayerConfig); } - if (env.MEMMY_DISABLE_MEMOS_SQLITE !== "1") { - const sources = discoverMemosSqliteSources(env); - if (sources.length > 0) { - return createMemosSqliteMemoryClient({ sources }); - } - } - - throw new Error("MEMMY_MEMORY_LAYER_URL or a local Memmy memory SQLite source is required"); + throw new Error("MEMMY_MEMORY_LAYER_URL is required"); } diff --git a/App/backend/src/infrastructure/agent-source-scan-store/index.ts b/App/backend/src/infrastructure/agent-source-scan-store/index.ts new file mode 100644 index 000000000..5a44dfef2 --- /dev/null +++ b/App/backend/src/infrastructure/agent-source-scan-store/index.ts @@ -0,0 +1,219 @@ +import { mkdirSync, rmSync } from "node:fs"; +import { dirname } from "node:path"; +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import type { + ConversationCheckpoint, + ConversationMessage, + MessageCursor, + PreparedConversation, + PreparedTurn, + ScanSourceState, + ScanStore, + ScanStoredResult +} from "@memmy/agent-source-core"; + +const MAX_RECORD_BYTES = 64 * 1024 * 1024; +const MAX_PAGE_BYTES = 8 * 1024 * 1024; + +export interface AppScanJobMeta { + jobId: string; + sourceId: string; + mode: string; + phase: string; + createdAt: string; + updatedAt: string; + error?: string; +} + +export interface AppAgentSourceScanStore extends ScanStore { + readonly path: string; + saveMeta(meta: AppScanJobMeta): void; + getMeta(): AppScanJobMeta | null; + clearMeta(): void; + conversationCount(sourceId: string): number; +} + +export function openAppAgentSourceScanStore(path: string, job: AppScanJobMeta): AppAgentSourceScanStore { + mkdirSync(dirname(path), { recursive: true }); + const db = new DatabaseSync(path); + db.exec(` + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA busy_timeout = 5000; + CREATE TABLE IF NOT EXISTS schema_meta (version INTEGER NOT NULL); + INSERT INTO schema_meta(version) SELECT 2 WHERE NOT EXISTS (SELECT 1 FROM schema_meta); + UPDATE schema_meta SET version = 2 WHERE version < 2; + CREATE TABLE IF NOT EXISTS scan_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + job_id TEXT NOT NULL, + source_id TEXT NOT NULL, + mode TEXT NOT NULL, + phase TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + error TEXT + ); + CREATE TABLE IF NOT EXISTS staged_messages ( + job_id TEXT NOT NULL, + source_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + message_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + created_at TEXT NOT NULL, + workspace_path TEXT, + git_root TEXT, + raw_meta_json TEXT NOT NULL, + ordinal INTEGER NOT NULL, + PRIMARY KEY (job_id, source_id, message_id) + ); + CREATE TABLE IF NOT EXISTS scan_source_state ( + source_id TEXT PRIMARY KEY, + mode TEXT NOT NULL, + phase TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0, + result_count INTEGER NOT NULL DEFAULT 0, + error_count INTEGER NOT NULL DEFAULT 0, + scan_started_at TEXT, + watermarked_since TEXT, + updated_at TEXT NOT NULL, + error TEXT + ); + `); + // Older stores created before the job_id column are upgraded in place. + try { db.exec("ALTER TABLE staged_messages ADD COLUMN job_id TEXT NOT NULL DEFAULT ''"); } catch { /* already present */ } + db.exec("CREATE INDEX IF NOT EXISTS staged_order ON staged_messages(job_id, source_id, conversation_id, created_at, message_id, ordinal)"); + db.exec("CREATE TABLE IF NOT EXISTS scan_cursors (source_id TEXT PRIMARY KEY, conversation_id TEXT NOT NULL, created_at TEXT NOT NULL, message_id TEXT NOT NULL, ordinal INTEGER NOT NULL)"); + db.exec("CREATE TABLE IF NOT EXISTS checkpoints (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, content_hash TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY(source_id, conversation_id))"); + db.exec("CREATE TABLE IF NOT EXISTS conversation_meta (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, content_hash TEXT NOT NULL, selected INTEGER NOT NULL, PRIMARY KEY(source_id, conversation_id))"); + db.exec("CREATE TABLE IF NOT EXISTS turn_meta (source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, turn_id TEXT NOT NULL, first_message_id TEXT NOT NULL, first_created_at TEXT NOT NULL, last_message_id TEXT NOT NULL, last_created_at TEXT NOT NULL, selected INTEGER NOT NULL, PRIMARY KEY(source_id, conversation_id, turn_id))"); + db.exec("CREATE INDEX IF NOT EXISTS turn_selection_order ON turn_meta(first_created_at DESC, source_id, conversation_id, first_message_id, turn_id)"); + db.exec("CREATE TABLE IF NOT EXISTS scan_results (id INTEGER PRIMARY KEY AUTOINCREMENT, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, memory_id TEXT, error TEXT)"); + db.exec("CREATE INDEX IF NOT EXISTS scan_result_identity ON scan_results(source_id,conversation_id,memory_id,error)"); + const meta = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, created_at AS createdAt, updated_at AS updatedAt, error FROM scan_meta WHERE id=1").get() as AppScanJobMeta | undefined; + if (!meta) { + db.prepare("INSERT INTO scan_meta(id,job_id,source_id,mode,phase,created_at,updated_at,error) VALUES(1,?,?,?,?,?,?,?)").run(job.jobId, job.sourceId, job.mode, job.phase, job.createdAt, job.updatedAt, job.error ?? null); + } + let ordinal = Number((db.prepare("SELECT COALESCE(MAX(ordinal), -1) AS value FROM staged_messages WHERE job_id=?").get(job.jobId) as { value: number }).value) + 1; + const insert = db.prepare("INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) VALUES(?,?,?,?,?,?,?,?,?,?,?)"); + const store: AppAgentSourceScanStore = { + path, + stage(message) { + const bytes = Buffer.byteLength(JSON.stringify(message)); + if (bytes > MAX_RECORD_BYTES) throw new Error(`scan record exceeds 64 MiB limit (${bytes} bytes)`); + const result = insert.run(job.jobId, message.sourceId, message.conversationId, message.messageId, message.role, message.content, message.createdAt, message.workspacePath, message.gitRoot, JSON.stringify(message.rawMeta), ordinal++); + return Number(result.changes) > 0; + }, + stageBatch(messages) { + let inserted = 0; + db.exec("BEGIN IMMEDIATE"); + try { for (const message of messages) if (store.stage(message)) inserted += 1; db.exec("COMMIT"); } + catch (error) { db.exec("ROLLBACK"); throw error; } + return inserted; + }, + messages(sourceId, cursor, limit = 500) { + limit = Number.isFinite(limit) ? Math.min(500, Math.max(1, Math.floor(limit))) : 500; + const parameters: SQLInputValue[] = [job.jobId, sourceId]; + let where = "job_id=? AND source_id=?"; + if (cursor) { + where += " AND ((conversation_id > ?) OR (conversation_id = ? AND (created_at > ? OR (created_at = ? AND (message_id > ? OR (message_id = ? AND ordinal > ?))))))"; + parameters.push(cursor.conversationId, cursor.conversationId, cursor.createdAt, cursor.createdAt, cursor.messageId, cursor.messageId, cursor.ordinal); + } + const iterator = db.prepare(`SELECT source_id AS sourceId, conversation_id AS conversationId, message_id AS messageId, role, content, created_at AS createdAt, workspace_path AS workspacePath, git_root AS gitRoot, raw_meta_json AS rawMetaJson, ordinal FROM staged_messages WHERE ${where} ORDER BY conversation_id, created_at, message_id, ordinal LIMIT ?`).iterate(...parameters, limit) as Iterable>; + return (function*() { + let bytes = 0; + let count = 0; + for (const row of iterator) { + const message = rowToMessage(row); + yield message; + count += 1; + bytes += Buffer.byteLength(JSON.stringify(message)); + if (count >= 500 || bytes >= MAX_PAGE_BYTES) break; + } + })(); + }, + saveScanCursor(sourceId, cursor) { db.prepare("INSERT INTO scan_cursors(source_id,conversation_id,created_at,message_id,ordinal) VALUES(?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET conversation_id=excluded.conversation_id,created_at=excluded.created_at,message_id=excluded.message_id,ordinal=excluded.ordinal").run(sourceId, cursor.conversationId, cursor.createdAt, cursor.messageId, cursor.ordinal); }, + getScanCursor(sourceId) { const row = db.prepare("SELECT conversation_id AS conversationId,created_at AS createdAt,message_id AS messageId,ordinal FROM scan_cursors WHERE source_id=?").get(sourceId) as MessageCursor|undefined; return row ?? null; }, + saveSourceState(state) { + db.prepare(`INSERT INTO scan_source_state(source_id,mode,phase,message_count,result_count,error_count,scan_started_at,watermarked_since,updated_at,error) + VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id) DO UPDATE SET mode=excluded.mode,phase=excluded.phase,message_count=excluded.message_count,result_count=excluded.result_count,error_count=excluded.error_count,scan_started_at=excluded.scan_started_at,watermarked_since=excluded.watermarked_since,updated_at=excluded.updated_at,error=excluded.error`) + .run(state.sourceId, state.mode, state.phase, state.messageCount, state.resultCount, state.errorCount, state.scanStartedAt ?? null, state.watermarkedSince ?? null, state.updatedAt, state.error ?? null); + }, + getSourceState(sourceId) { + const row = db.prepare("SELECT source_id AS sourceId,mode,phase,message_count AS messageCount,result_count AS resultCount,error_count AS errorCount,scan_started_at AS scanStartedAt,watermarked_since AS watermarkedSince,updated_at AS updatedAt,error FROM scan_source_state WHERE source_id=?").get(sourceId) as ScanSourceState | undefined; + return row ?? null; + }, + sourceCount() { return Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count); }, + count(sourceId) { + const row = db.prepare(`SELECT COUNT(*) AS count FROM staged_messages WHERE job_id=?${sourceId ? " AND source_id=?" : ""}`).get(job.jobId, ...(sourceId ? [sourceId] : [])) as { count: number }; + return Number(row.count); + }, + conversationCount(sourceId) { + const row = db.prepare("SELECT COUNT(DISTINCT conversation_id) AS count FROM staged_messages WHERE job_id=? AND source_id=?").get(job.jobId, sourceId) as { count: number }; + return Number(row.count); + }, + saveCheckpoint(checkpoint) { db.prepare("INSERT INTO checkpoints(source_id,conversation_id,last_message_id,last_created_at,content_hash,updated_at) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,updated_at=excluded.updated_at").run(checkpoint.sourceId, checkpoint.conversationId, checkpoint.lastMessageId, checkpoint.lastCreatedAt, checkpoint.contentHash, checkpoint.updatedAt); }, + getCheckpoint(sourceId, conversationId) { return (db.prepare("SELECT source_id AS sourceId, conversation_id AS conversationId, last_message_id AS lastMessageId, last_created_at AS lastCreatedAt, content_hash AS contentHash, updated_at AS updatedAt FROM checkpoints WHERE source_id=? AND conversation_id=?").get(sourceId, conversationId) as ConversationCheckpoint | undefined) ?? null; }, + saveConversationMeta(meta) { db.prepare("INSERT INTO conversation_meta(source_id,conversation_id,last_message_id,last_created_at,content_hash,selected) VALUES(?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id) DO UPDATE SET last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,content_hash=excluded.content_hash,selected=excluded.selected").run(meta.sourceId,meta.conversationId,meta.lastMessageId,meta.lastCreatedAt,meta.contentHash,meta.selected?1:0); }, + getConversationMeta(sourceId, conversationId) { const row = db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,content_hash AS contentHash,selected FROM conversation_meta WHERE source_id=? AND conversation_id=?").get(sourceId,conversationId) as (Omit & {selected:number})|undefined; return row ? {...row, selected: row.selected === 1} : null; }, + selectAllConversations(sourceId) { db.prepare("UPDATE conversation_meta SET selected=1 WHERE source_id=?").run(sourceId); }, + saveTurnMeta(meta) { db.prepare("INSERT INTO turn_meta(source_id,conversation_id,turn_id,first_message_id,first_created_at,last_message_id,last_created_at,selected) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,conversation_id,turn_id) DO UPDATE SET first_message_id=excluded.first_message_id,first_created_at=excluded.first_created_at,last_message_id=excluded.last_message_id,last_created_at=excluded.last_created_at,selected=excluded.selected").run(meta.sourceId,meta.conversationId,meta.turnId,meta.firstMessageId,meta.firstCreatedAt,meta.lastMessageId,meta.lastCreatedAt,meta.selected?1:0); }, + getTurnMeta(sourceId, conversationId, turnId) { const row = db.prepare("SELECT source_id AS sourceId,conversation_id AS conversationId,turn_id AS turnId,first_message_id AS firstMessageId,first_created_at AS firstCreatedAt,last_message_id AS lastMessageId,last_created_at AS lastCreatedAt,selected FROM turn_meta WHERE source_id=? AND conversation_id=? AND turn_id=?").get(sourceId,conversationId,turnId) as (Omit & {selected:number})|undefined; return row ? {...row,selected:row.selected===1} : null; }, + selectInitialTurns(sourceIds, globalLimit, absentSourceLimit) { + if (sourceIds.length === 0) return; + const placeholders = sourceIds.map(() => "?").join(","); + db.exec("BEGIN IMMEDIATE"); + try { + db.prepare(`UPDATE turn_meta SET selected=0 WHERE source_id IN (${placeholders})`).run(...sourceIds); + db.prepare(`UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT rowid FROM turn_meta WHERE source_id IN (${placeholders}) ORDER BY first_created_at DESC,source_id,conversation_id,first_message_id,turn_id LIMIT ?)`).run(...sourceIds, globalLimit); + db.prepare(`WITH ranked AS (SELECT source_id,turn_id,ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY first_created_at DESC,conversation_id,first_message_id,turn_id) AS rank FROM turn_meta WHERE source_id IN (${placeholders})), absent AS (SELECT source_id FROM turn_meta WHERE source_id IN (${placeholders}) GROUP BY source_id HAVING MAX(selected)=0) UPDATE turn_meta SET selected=1 WHERE rowid IN (SELECT t.rowid FROM turn_meta t JOIN ranked r ON r.source_id=t.source_id AND r.turn_id=t.turn_id JOIN absent a ON a.source_id=t.source_id WHERE r.rank <= ?)`).run(...sourceIds, ...sourceIds, absentSourceLimit); + db.exec("COMMIT"); + } catch (error) { db.exec("ROLLBACK"); throw error; } + }, + saveResult(result) { + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT ?,?,?,? WHERE NOT EXISTS ( + SELECT 1 FROM scan_results WHERE source_id=? AND conversation_id=? AND memory_id IS ? AND error IS ? + )`).run(result.sourceId, result.conversationId, result.memoryId ?? null, result.error ?? null, result.sourceId, result.conversationId, result.memoryId ?? null, result.error ?? null); + }, + resultCount(sourceId) { + const row = db.prepare(`SELECT COUNT(*) AS count FROM scan_results${sourceId ? " WHERE source_id=?" : ""}`).get(...(sourceId ? [sourceId] : [])) as { count: number }; + return Number(row.count); + }, + results(sourceId, cursor, limit = 100) { + const safeLimit = Math.min(500, Math.max(1, Math.floor(limit))); + const numericCursor = Number(cursor); + const safeCursor = Number.isFinite(numericCursor) && numericCursor >= 0 ? Math.floor(numericCursor) : 0; + const parameters: SQLInputValue[] = [sourceId ?? "%", safeCursor]; + const iterator = db.prepare("SELECT id, source_id AS sourceId, conversation_id AS conversationId, memory_id AS memoryId, error FROM scan_results WHERE source_id LIKE ? AND id > ? ORDER BY id LIMIT ?").iterate(...parameters, safeLimit) as Iterable>; + return (function*() { + for (const row of iterator) { + const result = { sourceId: String(row.sourceId), conversationId: String(row.conversationId), ...(row.memoryId ? { memoryId: String(row.memoryId) } : {}), ...(row.error ? { error: String(row.error) } : {}) } as ScanStoredResult; + Object.defineProperty(result, "cursor", { value: String(row.id), enumerable: false }); + yield result; + } + })(); + }, + saveMeta(meta) { db.prepare("UPDATE scan_meta SET job_id=?,source_id=?,mode=?,phase=?,created_at=?,updated_at=?,error=? WHERE id=1").run(meta.jobId, meta.sourceId, meta.mode, meta.phase, meta.createdAt, meta.updatedAt, meta.error ?? null); }, + getMeta() { return (db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, created_at AS createdAt, updated_at AS updatedAt, error FROM scan_meta WHERE id=1").get() as AppScanJobMeta | undefined) ?? null; }, + clearMeta() { db.prepare("DELETE FROM scan_meta WHERE id=1").run(); }, + close() { db.close(); }, + remove() { db.close(); rmSync(path, { force: true }); rmSync(`${path}-wal`, { force: true }); rmSync(`${path}-shm`, { force: true }); } + }; + return store; +} + +export function removeAppAgentSourceScanStore(path: string): void { + rmSync(path, { force: true }); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); +} + +function rowToMessage(row: Record): ConversationMessage { + return { + sourceId: String(row.sourceId), conversationId: String(row.conversationId), messageId: String(row.messageId), + role: row.role as ConversationMessage["role"], content: String(row.content), createdAt: String(row.createdAt), + workspacePath: row.workspacePath == null ? null : String(row.workspacePath), gitRoot: row.gitRoot == null ? null : String(row.gitRoot), + rawMeta: JSON.parse(String(row.rawMetaJson)) as Record, ordinal: Number(row.ordinal) + }; +} diff --git a/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts new file mode 100644 index 000000000..1c3076700 --- /dev/null +++ b/App/backend/src/infrastructure/agent-source-scan-store/tests/repository.test.ts @@ -0,0 +1,45 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openAppAgentSourceScanStore } from "../index.js"; + +let directory: string | undefined; +afterEach(() => { if (directory) rmSync(directory, { recursive: true, force: true }); directory = undefined; }); + +describe("durable scan store", () => { + it("deduplicates staged rows and reads keyset pages", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-store-")); + const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "fixture", mode: "full", phase: "stage", createdAt: "2026-01-01", updatedAt: "2026-01-01" }); + const message = { messageId: "m1", sourceId: "fixture", conversationId: "c1", role: "user" as const, content: "hello", createdAt: "2026-01-01T00:00:00Z", workspacePath: null, gitRoot: null, rawMeta: {} }; + expect(store.stageBatch([message, message])).toBe(1); + store.saveSourceState({ sourceId: "fixture", mode: "full", phase: "stage", messageCount: 1, resultCount: 0, errorCount: 0, updatedAt: "2026-01-01" }); + expect(store.sourceCount()).toBe(1); + expect([...store.messages("fixture", undefined, 1)]).toHaveLength(1); + store.saveResult({ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }); + store.saveResult({ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }); + expect([...store.results("fixture", "0", 1)]).toEqual([{ sourceId: "fixture", conversationId: "c1", memoryId: "memory-1" }]); + store.remove(); + }); + + it("selects global recent turns and keeps an absent source fallback", () => { + directory = mkdtempSync(join(tmpdir(), "memmy-scan-store-")); + const store = openAppAgentSourceScanStore(join(directory, "job.sqlite"), { jobId: "job", sourceId: "all", mode: "initial_subset", phase: "prepare", createdAt: "2026-01-01", updatedAt: "2026-01-01" }); + const addTurn = (sourceId: string, index: number, day: string) => store.saveTurnMeta({ + sourceId, + conversationId: `conversation-${sourceId}-${index}`, + turnId: `${sourceId}::conversation-${sourceId}-${index}::user-${index}`, + firstMessageId: `user-${index}`, + firstCreatedAt: `2026-01-${day}T00:00:00Z`, + lastMessageId: `assistant-${index}`, + lastCreatedAt: `2026-01-${day}T00:01:00Z`, + selected: true + }); + addTurn("source-a", 1, "01"); + addTurn("source-b", 1, "02"); + store.selectInitialTurns(["source-a", "source-b"], 1, 1); + expect(store.getTurnMeta("source-b", "conversation-source-b-1", "source-b::conversation-source-b-1::user-1")?.selected).toBe(true); + expect(store.getTurnMeta("source-a", "conversation-source-a-1", "source-a::conversation-source-a-1::user-1")?.selected).toBe(true); + store.remove(); + }); +}); diff --git a/App/backend/src/infrastructure/app-state-store/local-data-store.ts b/App/backend/src/infrastructure/app-state-store/local-data-store.ts index a05a4bcf2..f33d1953a 100644 --- a/App/backend/src/infrastructure/app-state-store/local-data-store.ts +++ b/App/backend/src/infrastructure/app-state-store/local-data-store.ts @@ -1,20 +1,18 @@ /** Local data store module. */ import { spawn } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; import type { DatabaseSync } from "node:sqlite"; -import { DatabaseSync as SqliteDatabaseSync } from "node:sqlite"; import type { ExportLocalDataInput, LocalDataExportResponse } from "@memmy/local-api-contracts"; -import { getLoadablePath as getSqliteVecLoadablePath } from "sqlite-vec"; import YAML from "yaml"; import type { SecretStore } from "./secret-store.js"; export interface LocalDataStore { getDataPath(): string; revealDataPath(dataPath: string): void; - exportData(input: ExportLocalDataInput): LocalDataExportResponse; - clearMemoryDatabase(clearedAt: string): void; + exportData(input: ExportLocalDataInput, bundle: Record): LocalDataExportResponse; + clearImportState(): void; } export interface CreateFilesystemLocalDataStoreOptions { @@ -28,31 +26,6 @@ export interface CreateFilesystemLocalDataStoreOptions { } const DEFAULT_MEMORY_HOME = join(homedir(), ".memmy"); -const MEMORY_DATA_TABLES = [ - "memories_fts", - "user_memories_fts", - "memory_vector_entries", - "memory_processing_state", - "trace_policy_links", - "skill_trials", - "feedback", - "decision_repairs", - "raw_turns", - "episodes", - "sessions", - "recall_events", - "l2_candidate_pool", - "evolution_jobs", - "embedding_retry_queue", - "artifacts", - "audit_logs", - "api_logs", - "memory_change_log", - "idempotency_keys", - "user_memories", - "memories" -] as const; - /** Creates create filesystem local data store. */ export function createFilesystemLocalDataStore(options: CreateFilesystemLocalDataStoreOptions): LocalDataStore { const memoryDatabasePath = resolveMemoryDatabasePath(options); @@ -67,14 +40,11 @@ export function createFilesystemLocalDataStore(options: CreateFilesystemLocalDat (options.revealPath ?? revealPathInFileManager)(dataPath); }, - exportData(input) { + exportData(input, bundle) { const exportRoot = resolveExportRoot(input.targetPath, memoryDataPath); const exportPath = join(exportRoot, `memmy-export-${toExportTimestamp(new Date())}`); mkdirSync(exportPath, { recursive: true }); - - copyIfExists(memoryDatabasePath, join(exportPath, "memory.sqlite")); - copyIfExists(`${memoryDatabasePath}-wal`, join(exportPath, "memory.sqlite-wal")); - copyIfExists(`${memoryDatabasePath}-shm`, join(exportPath, "memory.sqlite-shm")); + writeFileSync(join(exportPath, "memory.json"), `${JSON.stringify(bundle, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); return { exportPath, @@ -82,8 +52,7 @@ export function createFilesystemLocalDataStore(options: CreateFilesystemLocalDat }; }, - clearMemoryDatabase(_clearedAt) { - clearSqliteMemoryTables(memoryDatabasePath); + clearImportState() { options.db.exec(` DELETE FROM account_ingestion_seen; DELETE FROM account_agent_source_watermarks; @@ -93,72 +62,6 @@ export function createFilesystemLocalDataStore(options: CreateFilesystemLocalDat }; } -function clearSqliteMemoryTables(databasePath: string): void { - if (!existsSync(databasePath)) { - return; - } - - const db = new SqliteDatabaseSync(databasePath, { allowExtension: true }); - try { - const extensionPath = getSqliteVecLoadablePath(); - const unpackedPath = extensionPath.replace(/app\.asar([\\/])/, "app.asar.unpacked$1"); - db.loadExtension(existsSync(unpackedPath) ? unpackedPath : extensionPath); - db.exec("PRAGMA busy_timeout = 5000; PRAGMA foreign_keys = OFF"); - db.exec("BEGIN IMMEDIATE"); - try { - for (const table of sqliteVectorTables(db)) { - deleteTableRowsIfExists(db, table); - } - for (const table of MEMORY_DATA_TABLES) { - deleteTableRowsIfExists(db, table); - } - deleteSqliteSequenceRows(db); - db.exec("COMMIT"); - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - try { - db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); - } catch { - // The cleanup data has already been committed; a WAL truncation failure should not make the user think the cleanup failed. - } - } finally { - db.close(); - } -} - -function sqliteVectorTables(db: DatabaseSync): string[] { - const rows = db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' - AND name GLOB 'memory_vec_[0-9]*' - AND sql LIKE 'CREATE VIRTUAL TABLE%USING vec0%'` - ) - .all() as Array<{ name: string }>; - return rows.map((row) => row.name).filter((name) => /^memory_vec_\d+$/.test(name)); -} - -function deleteTableRowsIfExists(db: DatabaseSync, table: string): void { - if (tableExists(db, table)) { - db.prepare(`DELETE FROM ${table}`).run(); - } -} - -function deleteSqliteSequenceRows(db: DatabaseSync): void { - if (!tableExists(db, "sqlite_sequence")) { - return; - } - db.prepare("DELETE FROM sqlite_sequence WHERE name IN (?, ?)").run("api_logs", "memory_change_log"); -} - -function tableExists(db: DatabaseSync, table: string): boolean { - const row = db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?").get(table); - return Boolean(row); -} - function resolveMemoryDatabasePath(options: CreateFilesystemLocalDataStoreOptions): string { if (options.memoryDatabasePath) { return resolve(expandHome(options.memoryDatabasePath)); @@ -251,18 +154,6 @@ function hasParentTraversal(targetPath: string): boolean { return targetPath.split(/[\\/]+/).includes(".."); } -/** - * Copies the file if it exists. - * - * @param source the source path. - * @param target the target path. - */ -function copyIfExists(source: string, target: string): void { - if (existsSync(source)) { - copyFileSync(source, target); - } -} - /** * Counts the total byte size of files in a directory. * diff --git a/App/backend/src/infrastructure/app-state-store/migrations/0026-stop-memory-service-on-exit.sql b/App/backend/src/infrastructure/app-state-store/migrations/0026-stop-memory-service-on-exit.sql new file mode 100644 index 000000000..e4e17c463 --- /dev/null +++ b/App/backend/src/infrastructure/app-state-store/migrations/0026-stop-memory-service-on-exit.sql @@ -0,0 +1,3 @@ +ALTER TABLE app_settings + ADD COLUMN stop_memory_service_on_exit INTEGER NOT NULL DEFAULT 0 + CHECK (stop_memory_service_on_exit IN (0, 1)); diff --git a/App/backend/src/infrastructure/app-state-store/migrations/0027-first-encounter-report-status.sql b/App/backend/src/infrastructure/app-state-store/migrations/0027-first-encounter-report-status.sql new file mode 100644 index 000000000..2e7d966d1 --- /dev/null +++ b/App/backend/src/infrastructure/app-state-store/migrations/0027-first-encounter-report-status.sql @@ -0,0 +1,12 @@ +ALTER TABLE account_onboarding_state + ADD COLUMN first_encounter_report_status TEXT NOT NULL DEFAULT 'pending' + CHECK (first_encounter_report_status IN ('pending', 'shown', 'skipped')); + +UPDATE account_onboarding_state +SET first_encounter_report_status = CASE + WHEN scan_permission = 'none' THEN 'skipped' + WHEN scan_permission IN ('scan_only', 'scan_and_write_skill') THEN 'shown' + ELSE 'pending' +END, +updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') +WHERE uuid = 'local-agent-sources'; diff --git a/App/backend/src/infrastructure/app-state-store/repositories/bootstrap-repo.ts b/App/backend/src/infrastructure/app-state-store/repositories/bootstrap-repo.ts index 89f72d2d6..f8b523852 100644 --- a/App/backend/src/infrastructure/app-state-store/repositories/bootstrap-repo.ts +++ b/App/backend/src/infrastructure/app-state-store/repositories/bootstrap-repo.ts @@ -37,6 +37,7 @@ interface AppSettingsRow { task_done_notification_enabled: number; notification_sound_enabled: number; menu_bar_icon_enabled: number; + stop_memory_service_on_exit: number; auto_scan_known_agents: number; watch_file_changes: number; auto_inject_skill: number; @@ -48,6 +49,7 @@ interface OnboardingStateRow { has_accepted_terms: number; accepted_terms_version: string | null; scan_permission: string; + first_encounter_report_status: string; improvement_program: string; completed_at: string | null; } @@ -117,6 +119,7 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository task_done_notification_enabled, notification_sound_enabled, menu_bar_icon_enabled, + stop_memory_service_on_exit, auto_scan_known_agents, watch_file_changes, auto_inject_skill @@ -135,7 +138,8 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository skinId: row.skin, taskDoneNotificationEnabled: toBoolean(row.task_done_notification_enabled), notificationSoundEnabled: toBoolean(row.notification_sound_enabled), - menuBarIconEnabled: toBoolean(row.menu_bar_icon_enabled) + menuBarIconEnabled: toBoolean(row.menu_bar_icon_enabled), + stopMemoryServiceOnExit: toBoolean(row.stop_memory_service_on_exit) }); }, @@ -163,7 +167,8 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository defaultLaunchMode: { column: "default_launch_mode" }, taskDoneNotificationEnabled: { column: "task_done_notification_enabled", serialize: toInteger }, notificationSoundEnabled: { column: "notification_sound_enabled", serialize: toInteger }, - menuBarIconEnabled: { column: "menu_bar_icon_enabled", serialize: toInteger } + menuBarIconEnabled: { column: "menu_bar_icon_enabled", serialize: toInteger }, + stopMemoryServiceOnExit: { column: "stop_memory_service_on_exit", serialize: toInteger } }, patch ); @@ -185,9 +190,9 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository getOnboardingState() { const uuid = resolveOnboardingUuidWithDefaults(db); - const installationScanPermission = getRequiredRow>( + const installationState = getRequiredRow>( db, - "SELECT scan_permission FROM account_onboarding_state WHERE uuid = ?", + "SELECT scan_permission, first_encounter_report_status FROM account_onboarding_state WHERE uuid = ?", [INSTALLATION_SCAN_SCOPE_UUID] ); const row = getRequiredRow( @@ -210,7 +215,8 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository currentStep: row.current_step, hasAcceptedTerms: toBoolean(row.has_accepted_terms), acceptedTermsVersion: row.accepted_terms_version, - scanPermission: installationScanPermission.scan_permission, + scanPermission: installationState.scan_permission, + firstEncounterReportStatus: installationState.first_encounter_report_status, improvementProgram: row.improvement_program, completedAt: row.completed_at }); @@ -218,7 +224,7 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository updateOnboarding(patch) { const uuid = resolveOnboardingUuidWithDefaults(db); - const { scanPermission, ...accountPatch } = patch; + const { scanPermission, firstEncounterReportStatus, ...accountPatch } = patch; applyPatch( db, "account_onboarding_state", @@ -233,12 +239,15 @@ export function createBootstrapRepository(db: DatabaseSync): BootstrapRepository accountPatch, { column: "uuid", value: uuid } ); - if (scanPermission !== undefined) { + if (scanPermission !== undefined || firstEncounterReportStatus !== undefined) { applyPatch( db, "account_onboarding_state", - { scanPermission: { column: "scan_permission" } }, - { scanPermission }, + { + scanPermission: { column: "scan_permission" }, + firstEncounterReportStatus: { column: "first_encounter_report_status" } + }, + { scanPermission, firstEncounterReportStatus }, { column: "uuid", value: INSTALLATION_SCAN_SCOPE_UUID } ); } diff --git a/App/backend/src/infrastructure/app-state-store/tests/index.test.ts b/App/backend/src/infrastructure/app-state-store/tests/index.test.ts index 771fd2a90..283543b58 100644 --- a/App/backend/src/infrastructure/app-state-store/tests/index.test.ts +++ b/App/backend/src/infrastructure/app-state-store/tests/index.test.ts @@ -124,13 +124,15 @@ describe("app state store migrations", () => { expect(onboarding).toMatchObject({ completed: false, currentStep: "scan_permission_required", - scanPermission: "unset" + scanPermission: "unset", + firstEncounterReportStatus: "pending" }); expect(settings.userMode).toBe("unset"); expect(settings.menuBarIconEnabled).toBe(true); + expect(settings.stopMemoryServiceOnExit).toBe(false); expect(agentSources).toEqual([]); - expect(firstMigrationCount).toBe(30); - expect(secondMigrationCount).toBe(30); + expect(firstMigrationCount).toBe(32); + expect(secondMigrationCount).toBe(32); }); it("preserves the authenticated account when upgrading the legacy 0007 database", () => { @@ -1809,7 +1811,8 @@ describe("app state store migrations", () => { "auto_scan_known_agents", "watch_file_changes", "auto_inject_skill", - "installation_id" + "installation_id", + "stop_memory_service_on_exit" ]); expect(settings).toMatchObject({ defaultLaunchMode: "last", @@ -1818,7 +1821,8 @@ describe("app state store migrations", () => { skinId: "default", taskDoneNotificationEnabled: true, notificationSoundEnabled: true, - menuBarIconEnabled: true + menuBarIconEnabled: true, + stopMemoryServiceOnExit: false }); expect(cloudAccountColumns).toEqual([ "uuid", @@ -2216,6 +2220,49 @@ describe("bootstrap repository writes", () => { expect(accountAPrivacy).toMatchObject({ localOnlyMode: true, allowMemoryImprovementUpload: false }); expect(accountATokenUsage).toMatchObject({ planName: "Account A Plan", remainingTokens: 60 }); }); + + it("shares the first encounter report state across accounts, BYOK, and database reopen", () => { + tempDir = mkdtempSync(join(tmpdir(), "memmy-app-state-")); + const databasePath = join(tempDir, "app.sqlite"); + const first = createAppStateStore({ databasePath }); + + first.repositories.accountSession.upsert({ + profile: accountProfile("user-a", "a@example.com", "Account A"), + uuid: "cloud-account-a" + }); + first.repositories.bootstrap.updateOnboarding({ firstEncounterReportStatus: "shown" }); + + first.repositories.accountSession.upsert({ + profile: accountProfile("user-b", "b@example.com", "Account B"), + uuid: "cloud-account-b" + }); + expect(first.repositories.bootstrap.getOnboardingState().firstEncounterReportStatus).toBe("shown"); + + first.repositories.bootstrap.updateAppSettings({ userMode: "byok" }); + expect(first.repositories.bootstrap.getOnboardingState().firstEncounterReportStatus).toBe("shown"); + first.close(); + + const reopened = createAppStateStore({ databasePath }); + expect(reopened.repositories.bootstrap.getOnboardingState().firstEncounterReportStatus).toBe("shown"); + reopened.close(); + }); + + it("keeps a denied first encounter report skipped after scan permission changes", () => { + tempDir = mkdtempSync(join(tmpdir(), "memmy-app-state-")); + const store = createAppStateStore({ databasePath: join(tempDir, "app.sqlite") }); + + store.repositories.bootstrap.updateOnboarding({ + scanPermission: "none", + firstEncounterReportStatus: "skipped" + }); + store.repositories.bootstrap.updateOnboarding({ scanPermission: "scan_only" }); + + expect(store.repositories.bootstrap.getOnboardingState()).toMatchObject({ + scanPermission: "scan_only", + firstEncounterReportStatus: "skipped" + }); + store.close(); + }); }); function getMigrationCount(db: { prepare(sql: string): { get(): unknown } }): number { diff --git a/App/backend/src/infrastructure/app-state-store/tests/local-data-store.test.ts b/App/backend/src/infrastructure/app-state-store/tests/local-data-store.test.ts index f9e5d465c..c871caaaa 100644 --- a/App/backend/src/infrastructure/app-state-store/tests/local-data-store.test.ts +++ b/App/backend/src/infrastructure/app-state-store/tests/local-data-store.test.ts @@ -1,9 +1,7 @@ /** Local data store tests. */ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import { getLoadablePath as getSqliteVecLoadablePath } from "sqlite-vec"; import { afterEach, describe, expect, it } from "vitest"; import { createAppStateStore } from "../index.js"; import { createFilesystemLocalDataStore } from "../local-data-store.js"; @@ -18,20 +16,25 @@ afterEach(() => { }); describe("filesystem local data store", () => { - it("exports the memory database as a directory copy", () => { + it("writes the service export bundle without reading the Memory database", () => { tempDir = mkdtempSync(join(tmpdir(), "memmy-local-data-")); const databasePath = join(tempDir, "app.sqlite"); const memoryDatabasePath = join(tempDir, "memory.sqlite"); - writeFileSync(memoryDatabasePath, "memory-db"); const store = createAppStateStore({ databasePath }); const localData = createFilesystemLocalDataStore({ databasePath, db: store.db, secretStore: store.secretStore, memoryDatabasePath }); - const result = localData.exportData({ targetPath: join(tempDir, "exports") }); + const result = localData.exportData( + { targetPath: join(tempDir, "exports") }, + { manifest: { service: "memmy-memory-service" }, tables: { memories: [] } } + ); store.close(); expect(result.bytes).toBeGreaterThan(0); expect(result.exportPath).toContain("memmy-export-"); - expect(existsSync(join(result.exportPath, "memory.sqlite"))).toBe(true); + expect(existsSync(join(result.exportPath, "memory.json"))).toBe(true); + expect(JSON.parse(readFileSync(join(result.exportPath, "memory.json"), "utf8"))).toMatchObject({ + manifest: { service: "memmy-memory-service" } + }); }); it("rejects traversal-like export targets", () => { @@ -45,16 +48,15 @@ describe("filesystem local data store", () => { memoryDatabasePath: join(tempDir, "memory.sqlite") }); - expect(() => localData.exportData({ targetPath: "../escape" })).toThrow("targetPath must not contain .."); + expect(() => localData.exportData({ targetPath: "../escape" }, {})).toThrow("targetPath must not contain .."); store.close(); }); - it("clears memory database rows without clearing app configuration", () => { + it("clears Desktop import state without opening the Memory database", () => { tempDir = mkdtempSync(join(tmpdir(), "memmy-local-data-")); const databasePath = join(tempDir, "app.sqlite"); const memoryDatabasePath = join(tempDir, "memory.sqlite"); const store = createAppStateStore({ databasePath }); - createMemoryDatabase(memoryDatabasePath); const localData = createFilesystemLocalDataStore({ databasePath, db: store.db, secretStore: store.secretStore, memoryDatabasePath }); store.repositories.bootstrap.updateAppSettings({ language: "zh-CN", theme: "dark" }); @@ -90,7 +92,7 @@ describe("filesystem local data store", () => { }); store.repositories.agentSources.markSeen("dedup-key-1", "cursor"); - localData.clearMemoryDatabase("2026-06-02T10:00:00.000Z"); + localData.clearImportState(); const settings = store.repositories.bootstrap.getAppSettings(); const session = store.repositories.accountSession.get(); const active = store.db.prepare("SELECT active_uuid FROM app_settings WHERE id = 'default'").get() as { active_uuid: string | null }; @@ -106,16 +108,6 @@ describe("filesystem local data store", () => { }; const seenCount = store.db.prepare("SELECT COUNT(*) AS count FROM account_ingestion_seen").get() as { count: number }; const watermarkCount = store.db.prepare("SELECT COUNT(*) AS count FROM account_agent_source_watermarks").get() as { count: number }; - const memoryDb = new DatabaseSync(memoryDatabasePath, { readOnly: true, allowExtension: true }); - memoryDb.loadExtension(getSqliteVecLoadablePath()); - const memoryCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM memories").get() as { count: number }; - const userMemoryCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM user_memories").get() as { count: number }; - const userMemoryFtsCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM user_memories_fts").get() as { count: number }; - const processingCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM memory_processing_state").get() as { count: number }; - const vectorCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM memory_vec_3").get() as { count: number }; - const apiLogCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM api_logs").get() as { count: number }; - const migrationCount = memoryDb.prepare("SELECT COUNT(*) AS count FROM schema_migrations").get() as { count: number }; - memoryDb.close(); store.close(); expect(settings).toMatchObject({ @@ -130,43 +122,5 @@ describe("filesystem local data store", () => { expect(lastScannedCount.count).toBe(0); expect(seenCount.count).toBe(0); expect(watermarkCount.count).toBe(0); - expect(memoryCount.count).toBe(0); - expect(userMemoryCount.count).toBe(0); - expect(userMemoryFtsCount.count).toBe(0); - expect(processingCount.count).toBe(0); - expect(vectorCount.count).toBe(0); - expect(apiLogCount.count).toBe(0); - expect(migrationCount.count).toBe(1); }); }); - -function createMemoryDatabase(databasePath: string): void { - const db = new DatabaseSync(databasePath, { allowExtension: true }); - db.loadExtension(getSqliteVecLoadablePath()); - db.exec(` - CREATE TABLE schema_migrations (id TEXT PRIMARY KEY); - CREATE TABLE memories (id TEXT PRIMARY KEY, memory_value TEXT NOT NULL); - CREATE TABLE user_memories (id TEXT PRIMARY KEY, content TEXT NOT NULL); - CREATE VIRTUAL TABLE user_memories_fts USING fts5(id, content); - CREATE TABLE memory_processing_state (memory_id TEXT PRIMARY KEY, state TEXT NOT NULL); - CREATE TABLE memory_vector_entries ( - id INTEGER PRIMARY KEY, - memory_id TEXT NOT NULL, - vector_field TEXT NOT NULL, - embedding_dim INTEGER NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE VIRTUAL TABLE memory_vec_3 USING vec0(embedding float[3] distance_metric=cosine); - CREATE TABLE api_logs (id INTEGER PRIMARY KEY AUTOINCREMENT, tool_name TEXT NOT NULL); - INSERT INTO schema_migrations (id) VALUES ('001_runtime_schema'); - INSERT INTO memories (id, memory_value) VALUES ('memory-1', 'remember this'); - INSERT INTO user_memories (id, content) VALUES ('user-memory-1', 'prefers concise code'); - INSERT INTO user_memories_fts (id, content) VALUES ('user-memory-1', 'prefers concise code'); - INSERT INTO memory_processing_state (memory_id, state) VALUES ('memory-1', 'summarizing'); - INSERT INTO memory_vector_entries VALUES (1, 'memory-1', 'vec_summary', 3, '2026-01-01'); - INSERT INTO api_logs (tool_name) VALUES ('memory_add'); - `); - db.prepare(`INSERT INTO memory_vec_3 (rowid, embedding) VALUES (?, ?)`) - .run(1n, Buffer.from(new Float32Array([1, 0, 0]).buffer)); - db.close(); -} diff --git a/App/backend/src/infrastructure/memmy-config/agent-access.ts b/App/backend/src/infrastructure/memmy-config/agent-access.ts new file mode 100644 index 000000000..d2fd9abc5 --- /dev/null +++ b/App/backend/src/infrastructure/memmy-config/agent-access.ts @@ -0,0 +1,98 @@ +import { readFileSync } from "node:fs"; +import type { PatchScanPreferencesInput, ScanPreferences } from "@memmy/local-api-contracts"; +import { ScanPreferencesSchema } from "@memmy/local-api-contracts"; +import { mutateRuntimeConfig } from "@memmy/migrations"; +import YAML from "yaml"; + +export interface ScanPreferencesStore { + getScanPreferences(): ScanPreferences; + updateScanPreferences(patch: PatchScanPreferencesInput): Promise; +} + +export const DEFAULT_MEMORY_SCAN_PREFERENCES: ScanPreferences = { + autoScanKnownAgents: true, + watchFileChanges: true, + autoInjectSkill: false +}; + +export async function ensureMemoryScanPreferences( + configPath: string, + legacyPreferences: ScanPreferences +): Promise { + await mutateRuntimeConfig(configPath, (root) => { + const memory = record(root.memmyMemory); + if (isCompletePreferences(memory.agentAccess)) return; + root.memmyMemory = { + ...memory, + agentAccess: { + ...legacyPreferences, + ...record(memory.agentAccess) + } + }; + }); +} + +export function createMemoryScanPreferencesStore(configPath: string): ScanPreferencesStore { + return { + getScanPreferences() { + return readMemoryScanPreferences(configPath); + }, + + async updateScanPreferences(patch) { + await mutateRuntimeConfig(configPath, (root) => { + const memory = record(root.memmyMemory); + root.memmyMemory = { + ...memory, + agentAccess: { + ...readPreferencesRecord(memory.agentAccess), + ...patch + } + }; + }); + return readMemoryScanPreferences(configPath); + } + }; +} + +export function readMemoryScanPreferences(configPath: string): ScanPreferences { + try { + const parsed = YAML.parse(readFileSync(configPath, "utf8")) as unknown; + return ScanPreferencesSchema.parse({ + ...DEFAULT_MEMORY_SCAN_PREFERENCES, + ...readPreferencesRecord(record(record(parsed).memmyMemory).agentAccess) + }); + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + return { ...DEFAULT_MEMORY_SCAN_PREFERENCES }; + } + throw error; + } +} + +function readPreferencesRecord(value: unknown): Partial { + const input = record(value); + return { + ...(typeof input.autoScanKnownAgents === "boolean" + ? { autoScanKnownAgents: input.autoScanKnownAgents } + : {}), + ...(typeof input.watchFileChanges === "boolean" + ? { watchFileChanges: input.watchFileChanges } + : {}), + ...(typeof input.autoInjectSkill === "boolean" + ? { autoInjectSkill: input.autoInjectSkill } + : {}) + }; +} + +function isCompletePreferences(value: unknown): boolean { + const input = record(value); + return typeof input.autoScanKnownAgents === "boolean" + && typeof input.watchFileChanges === "boolean" + && typeof input.autoInjectSkill === "boolean"; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : {}; +} diff --git a/App/backend/src/infrastructure/memmy-config/index.ts b/App/backend/src/infrastructure/memmy-config/index.ts index 43a410866..e46ec75f8 100644 --- a/App/backend/src/infrastructure/memmy-config/index.ts +++ b/App/backend/src/infrastructure/memmy-config/index.ts @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; import { + BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, resolveAssignedModel as resolveCatalogAssignment, resolveCloudServiceBaseUrl, type ActualModelContext, @@ -750,6 +751,7 @@ function updateAccountAssignment( ): void { const assignments = isRecord(config.modelAssignments) ? { ...config.modelAssignments } : {}; const existing = isRecord(assignments.account) ? { ...assignments.account } : {}; + const sameOwner = existingString(existing.ownerAccountId) === ownerAccountId; const presets = isRecord(config.modelPresets) ? config.modelPresets : {}; const agent = isRecord(existing.agent) ? { ...existing.agent } : {}; const currentCandidates = Array.isArray(agent.candidates) @@ -780,9 +782,14 @@ function updateAccountAssignment( const next: Record = { ...existing, ownerAccountId, agent }; for (const [field, capability] of Object.entries(singles) as Array<[keyof typeof singles, AccountCapability]>) { const current = existingString(existing[field]); - next[field] = current && assignmentPresetIsUsable(presets, current, capability, ownerAccountId) + const keepBuiltInLocalEmbedding = field === "embedding" + && sameOwner + && current === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID; + next[field] = keepBuiltInLocalEmbedding ? current - : presetIds[capability]; + : current && assignmentPresetIsUsable(presets, current, capability, ownerAccountId) + ? current + : presetIds[capability]; } assignments.account = next; config.modelAssignments = assignments; diff --git a/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts b/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts index aa4b3a536..289e83bdb 100644 --- a/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts +++ b/App/backend/src/infrastructure/memmy-config/model-config-catalog.ts @@ -1,4 +1,5 @@ import { + BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, type CatalogEndpointInput, type CatalogProviderId, type ModelAssignment, @@ -166,10 +167,166 @@ function mergeModelConfig(config: ConfigRecord, input: ModelConfigInput): Config modelPresets: nextPresets, modelAssignments }; + projectMemoryConfig(next, modelAssignments, config, existingAssignments); patchCompatibilityDefault(next, modelAssignments); return next; } +function projectMemoryConfig( + config: ConfigRecord, + assignments: ModelAssignments, + previousConfig: ConfigRecord, + previousAssignments: ModelAssignments +): void { + const mode = record(config.app).userMode === "account" ? "account" : "byok"; + const assignment = assignments[mode]; + const memory = { ...record(config.memmyMemory) }; + const routing = { ...record(memory.roleRouting) }; + + const previousModeAssignment = previousAssignments[mode]; + const previousRouting = record(record(previousConfig.memmyMemory).roleRouting); + projectMemoryRole( + config, + memory, + routing, + "evolution", + assignment.memoryEvolution, + assignment.agent.default, + previousModeAssignment.memoryEvolution, + previousRouting.evolution + ); + projectMemoryRole( + config, + memory, + routing, + "summary", + assignment.memorySummary, + assignment.memoryEvolution ?? assignment.agent.default, + previousModeAssignment.memorySummary, + previousRouting.summary + ); + memory.roleRouting = routing; + memory.embedding = projectedMemoryEmbedding( + config, + record(memory.embedding), + assignment.embedding, + previousModeAssignment.embedding + ); + config.memmyMemory = memory; + + function projectMemoryRole( + root: ConfigRecord, + target: ConfigRecord, + roleRouting: ConfigRecord, + role: "summary" | "evolution", + presetId: string | null, + inheritedPresetId: string | null, + previousPresetId: string | null, + previousRoute: unknown + ): void { + const preservesFixedRoute = previousRoute === "fixed" && presetId === previousPresetId; + const followsInheritedModel = !preservesFixedRoute && (!presetId || presetId === inheritedPresetId); + roleRouting[role] = followsInheritedModel ? "follow" : "fixed"; + if (followsInheritedModel) return; + const connection = memoryConnection(root, presetId!); + if (connection) target[role] = mergeMemoryConnection(record(target[role]), connection); + } +} + +function projectedMemoryEmbedding( + config: ConfigRecord, + previous: ConfigRecord, + presetId: string | null, + previousPresetId: string | null +): ConfigRecord { + if (presetId === previousPresetId && previous.mode === "custom") { + const connection = presetId ? memoryConnection(config, presetId) : null; + return connection + ? { ...mergeMemoryConnection(previous, connection), mode: "custom" } + : previous; + } + if (presetId === previousPresetId && previous.mode === "local") { + return { + ...withoutMemoryConnection(previous), + mode: "local", + provider: "local" + }; + } + if (!presetId) { + return { + ...withoutMemoryConnection(previous), + mode: "local", + provider: "local" + }; + } + const preset = record(record(config.modelPresets)[presetId]); + if (preset.source === "account") { + return { + ...withoutMemoryConnection(previous), + mode: "cloud" + }; + } + const connection = memoryConnection(config, presetId); + return connection + ? { + ...mergeMemoryConnection(previous, connection), + mode: "custom", + provider: "openai_compatible" + } + : { + ...withoutMemoryConnection(previous), + mode: "local", + provider: "local" + }; +} + +function memoryConnection(config: ConfigRecord, presetId: string): ConfigRecord | null { + const preset = record(record(config.modelPresets)[presetId]); + const providerId = stringValue(preset.provider); + const endpointId = stringValue(preset.endpoint); + const model = stringValue(preset.model); + if (!providerId || !endpointId || !model) return null; + const provider = record(record(config.providers)[providerId]); + const endpoint = record(record(provider.endpoints)[endpointId]); + const apiBase = stringValue(endpoint.apiBase); + if (!apiBase) return null; + const apiKey = stringValue(endpoint.apiKey) ?? stringValue(provider.apiKey); + const extraHeaders = { ...record(provider.extraHeaders), ...record(endpoint.extraHeaders) }; + const extraBody = { ...record(provider.extraBody), ...record(endpoint.extraBody) }; + return { + provider: memoryProvider(providerId), + sourceProvider: providerId, + endpoint: apiBase, + model, + ...(apiKey ? { apiKey } : {}), + ...(Object.keys(extraHeaders).length ? { extraHeaders } : {}), + ...(Object.keys(extraBody).length ? { extraBody } : {}) + }; +} + +function memoryProvider(providerId: string): string { + if (providerId === "anthropic") return "anthropic"; + if (providerId === "gemini") return "gemini"; + return "openai_compatible"; +} + +function mergeMemoryConnection(previous: ConfigRecord, connection: ConfigRecord): ConfigRecord { + return { + ...withoutMemoryConnection(previous), + ...connection + }; +} + +function withoutMemoryConnection(value: ConfigRecord): ConfigRecord { + const next = { ...value }; + for (const key of [ + "provider", "sourceProvider", "vendor", "endpoint", "apiBase", "baseUrl", + "model", "modelId", "apiKey", "extraHeaders", "extraBody", "custom", + "actualModelContext", "selectionError" + ]) delete next[key]; + return next; +} + function normalizeProviderInput(input: TextModelProviderInput): TextModelProviderInput { return { ...input, @@ -334,6 +491,9 @@ function resolvePresetId( function validateUniqueModels(presets: ConfigRecord): void { const combinations = new Set(); for (const [presetId, value] of Object.entries(presets)) { + if (presetId === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID) { + throw new InvalidModelConfigError(`Preset ID is reserved: ${presetId}`); + } const preset = record(value); const provider = stringValue(preset.provider); const endpoint = stringValue(preset.endpoint); @@ -390,6 +550,15 @@ function validateAssignmentReference( presets: ConfigRecord, previous: ModelAssignment ): void { + if (presetId === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID) { + if (capability !== "embedding") { + throw new InvalidModelConfigError("The built-in local Embedding assignment is only valid for embedding"); + } + if (namespace === "account" && !assignment.ownerAccountId) { + throw new InvalidModelConfigError("The account built-in local Embedding assignment requires an account owner"); + } + return; + } const preset = record(presets[presetId]); if (!Object.keys(preset).length) { if (namespace === "account" && stableJson(assignment) === stableJson(previous)) return; @@ -449,6 +618,7 @@ function buildModelConfigView( configRevision, providers: providerViews, modelAssignments: assignments, + memorySettings: memorySettings(config), effectiveCandidates, configured: Boolean(defaultId && byId.get(defaultId)?.available), updatedAt: updatedAtValue @@ -569,10 +739,30 @@ function revisionFor(config: ConfigRecord): string { providers: config.providers ?? null, modelPresets: config.modelPresets ?? null, modelAssignments: config.modelAssignments ?? null, + memmyMemory: config.memmyMemory ?? null, agents: { defaults: record(config.agents).defaults ?? null } })).digest("hex"); } +function memorySettings(config: ConfigRecord): { + roleRouting: { summary: "follow" | "fixed"; evolution: "follow" | "fixed" }; + embeddingMode: "cloud" | "local" | "custom"; +} { + const memory = record(config.memmyMemory); + const routing = record(memory.roleRouting); + const embedding = record(memory.embedding); + const appMode = record(config.app).userMode === "account" ? "account" : "byok"; + return { + roleRouting: { + summary: routing.summary === "fixed" ? "fixed" : "follow", + evolution: routing.evolution === "fixed" ? "fixed" : "follow" + }, + embeddingMode: embedding.mode === "cloud" || embedding.mode === "custom" || embedding.mode === "local" + ? embedding.mode + : appMode === "account" ? "cloud" : "local" + }; +} + function stableJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; if (isRecord(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`; diff --git a/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts b/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts index 77158b8a9..14fc88ec7 100644 --- a/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts +++ b/App/backend/src/infrastructure/memmy-config/tests/account-model-projection.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import YAML from "yaml"; import { afterEach, describe, expect, it } from "vitest"; +import { BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID } from "@memmy/local-api-contracts"; import { clearAccountModelProjectionFromMemmyConfig, readModelConfigCatalog, @@ -172,6 +173,22 @@ describe("account model projection current catalog", () => { }); }); + it("preserves built-in local Embedding for the same account and restores cloud for a new owner", async () => { + const file = await configFile(currentByokCatalog()); + await writeAccountModelProjectionToMemmyConfig({ cloudUuid: "token-a", userId: "owner-a" }, file); + const local = await readConfig(file); + local.modelAssignments.account.embedding = BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID; + await writeFile(file, YAML.stringify(local), "utf8"); + + await writeAccountModelProjectionToMemmyConfig({ cloudUuid: "token-a", userId: "owner-a" }, file); + expect((await readConfig(file)).modelAssignments.account.embedding) + .toBe(BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID); + + await writeAccountModelProjectionToMemmyConfig({ cloudUuid: "token-b", userId: "owner-b" }, file); + expect((await readConfig(file)).modelAssignments.account.embedding) + .toBe(accountId("owner-b", "embedding")); + }); + it("switches owners without reviving the previous owner's platform definitions", async () => { const file = await configFile(currentByokCatalog()); await writeAccountModelProjectionToMemmyConfig({ cloudUuid: "token-a", userId: "owner-a" }, file); diff --git a/App/backend/src/infrastructure/memmy-config/tests/agent-access.test.ts b/App/backend/src/infrastructure/memmy-config/tests/agent-access.test.ts new file mode 100644 index 000000000..53191ec46 --- /dev/null +++ b/App/backend/src/infrastructure/memmy-config/tests/agent-access.test.ts @@ -0,0 +1,66 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createMemoryScanPreferencesStore, + ensureMemoryScanPreferences, + readMemoryScanPreferences +} from "../agent-access.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("memmyMemory agent access preferences", () => { + it("migrates legacy Desktop preferences without replacing existing Memory fields", async () => { + const path = fixture({ + memmyMemory: { + summary: { model: "keep-me" }, + agentAccess: { autoScanKnownAgents: false } + } + }); + await ensureMemoryScanPreferences(path, { + autoScanKnownAgents: true, + watchFileChanges: false, + autoInjectSkill: true + }); + + const raw = YAML.parse(readFileSync(path, "utf8")) as any; + expect(raw.memmyMemory.summary.model).toBe("keep-me"); + expect(raw.memmyMemory.agentAccess).toEqual({ + autoScanKnownAgents: false, + watchFileChanges: false, + autoInjectSkill: true + }); + }); + + it("reads and patches the same preferences used by the Viewer", async () => { + const path = fixture({ memmyMemory: {} }); + await ensureMemoryScanPreferences(path, { + autoScanKnownAgents: true, + watchFileChanges: true, + autoInjectSkill: false + }); + const store = createMemoryScanPreferencesStore(path); + await store.updateScanPreferences({ watchFileChanges: false, autoInjectSkill: true }); + + expect(store.getScanPreferences()).toEqual({ + autoScanKnownAgents: true, + watchFileChanges: false, + autoInjectSkill: true + }); + expect(readMemoryScanPreferences(path)).toEqual(store.getScanPreferences()); + }); +}); + +function fixture(content: unknown): string { + const root = mkdtempSync(join(tmpdir(), "memmy-agent-access-")); + roots.push(root); + const path = join(root, "config.yaml"); + writeFileSync(path, YAML.stringify(content)); + return path; +} diff --git a/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts b/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts index 3faadf25a..5bf114bbc 100644 --- a/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts +++ b/App/backend/src/infrastructure/memmy-config/tests/model-config-catalog.test.ts @@ -3,7 +3,11 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import YAML from "yaml"; import { afterEach, describe, expect, it } from "vitest"; -import type { ModelAssignments, ModelConfigInput } from "@memmy/local-api-contracts"; +import { + BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, + type ModelAssignments, + type ModelConfigInput +} from "@memmy/local-api-contracts"; import { InvalidModelConfigError, ModelConfigChangedError, @@ -65,6 +69,76 @@ function openAiInput(revision: string, presetId?: string): ModelConfigInput { } describe("model config catalog", () => { + it("persists the reserved built-in local Embedding assignment without a preset", async () => { + const file = fixture({ modelAssignments: emptyAssignments() }); + const current = await readModelConfigCatalog(file); + const assignments = emptyAssignments(); + assignments.byok.embedding = BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID; + + const saved = await writeModelConfigCatalog(file, { + configRevision: current.configRevision, + providers: [], + modelAssignments: assignments + }); + + expect(saved.modelAssignments.byok.embedding).toBe(BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID); + }); + + it("rejects an ownerless account built-in local Embedding assignment", async () => { + const file = fixture({ modelAssignments: emptyAssignments() }); + const current = await readModelConfigCatalog(file); + const assignments = emptyAssignments(); + assignments.account.embedding = BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID; + + await expect(writeModelConfigCatalog(file, { + configRevision: current.configRevision, + providers: [], + modelAssignments: assignments + })).rejects.toThrow(/requires an account owner/); + }); + + it("rejects the built-in local Embedding identifier outside the Embedding assignment", async () => { + const file = fixture({ modelAssignments: emptyAssignments() }); + const current = await readModelConfigCatalog(file); + const assignments = emptyAssignments(); + assignments.byok.memorySummary = BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID; + + await expect(writeModelConfigCatalog(file, { + configRevision: current.configRevision, + providers: [], + modelAssignments: assignments + })).rejects.toThrow(/only valid for embedding/); + }); + + it("reserves the built-in local Embedding identifier against preset collisions", async () => { + const file = fixture({ + providers: { + openai: { + apiKey: "sk-existing", + endpoints: { + chat: { apiBase: "https://api.example.test/v1", protocol: "openai-chat-completions" } + } + } + }, + modelPresets: { + [BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID]: { + provider: "openai", + endpoint: "chat", + model: "gpt-5", + source: "byok", + capabilities: ["agent"] + } + }, + modelAssignments: emptyAssignments() + }); + const current = await readModelConfigCatalog(file); + + await expect(writeModelConfigCatalog( + file, + openAiInput(current.configRevision, BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID) + )).rejects.toThrow(/Preset ID is reserved/); + }); + it("creates unique server preset IDs, masks all credentials, and never persists labels", async () => { const file = fixture({ futureSection: { keepMe: true } }); const current = await readModelConfigCatalog(file); @@ -264,6 +338,112 @@ describe("model config catalog", () => { expect(accountSaved.modelAssignments.account).not.toEqual(accountBefore); }); + it("projects Desktop memory selections into the authoritative memmyMemory section", async () => { + const file = fixture({ app: { userMode: "byok" } }); + const revision = (await readModelConfigCatalog(file)).configRevision; + const definitions: ModelConfigInput = { + configRevision: revision, + providers: [{ + provider: "openai", + apiKey: "sk-memory", + endpoints: [ + { + endpointId: "chat", + apiBase: "https://models.example/v1", + protocol: "openai-chat-completions" + }, + { + endpointId: "embedding", + apiBase: "https://models.example/v1", + protocol: "openai-embeddings" + } + ], + models: [ + { + endpointId: "chat", + model: "agent-model", + source: "byok", + capabilities: ["agent", "memory_summary", "memory_evolution"] + }, + { + endpointId: "chat", + model: "memory-model", + source: "byok", + capabilities: ["memory_summary", "memory_evolution"] + }, + { + endpointId: "embedding", + model: "embedding-model", + source: "byok", + capabilities: ["embedding"] + } + ] + }], + modelAssignments: emptyAssignments() + }; + const created = await writeModelConfigCatalog(file, definitions); + const models = created.providers[0]!.models; + const agentId = models.find((model) => model.model === "agent-model")!.presetId; + const memoryId = models.find((model) => model.model === "memory-model")!.presetId; + const embeddingId = models.find((model) => model.model === "embedding-model")!.presetId; + const assigned: ModelConfigInput = { + ...definitions, + configRevision: created.configRevision, + providers: [{ + ...definitions.providers[0]!, + models: definitions.providers[0]!.models.map((model) => ({ + ...model, + presetId: model.model === "agent-model" + ? agentId + : model.model === "memory-model" + ? memoryId + : embeddingId + })) + }], + modelAssignments: { + ...emptyAssignments(), + byok: { + ...emptyAssignment(), + agent: { candidates: [agentId], default: agentId }, + memorySummary: memoryId, + memoryEvolution: memoryId, + embedding: embeddingId + } + } + }; + const saved = await writeModelConfigCatalog(file, assigned); + const raw = YAML.parse(readFileSync(file, "utf8")) as any; + expect(raw.memmyMemory).toMatchObject({ + roleRouting: { summary: "follow", evolution: "fixed" }, + evolution: { + provider: "openai_compatible", + endpoint: "https://models.example/v1", + model: "memory-model", + apiKey: "sk-memory" + }, + embedding: { + mode: "custom", + provider: "openai_compatible", + endpoint: "https://models.example/v1", + model: "embedding-model", + apiKey: "sk-memory" + } + }); + expect(saved.memorySettings).toEqual({ + roleRouting: { summary: "follow", evolution: "fixed" }, + embeddingMode: "custom" + }); + + const followInput = structuredClone(assigned); + followInput.configRevision = saved.configRevision; + followInput.modelAssignments.byok.memorySummary = agentId; + followInput.modelAssignments.byok.memoryEvolution = agentId; + const followed = await writeModelConfigCatalog(file, followInput); + expect(followed.memorySettings?.roleRouting.summary).toBe("follow"); + expect(followed.memorySettings?.roleRouting.evolution).toBe("follow"); + expect((YAML.parse(readFileSync(file, "utf8")) as any).memmyMemory.roleRouting.summary).toBe("follow"); + }); + it("rejects duplicate endpoint definitions, invalid protocol capabilities, and duplicate models", async () => { const file = fixture(); const revision = (await readModelConfigCatalog(file)).configRevision; diff --git a/App/backend/src/project-version.ts b/App/backend/src/project-version.ts index 45441c7e5..d244ec383 100644 --- a/App/backend/src/project-version.ts +++ b/App/backend/src/project-version.ts @@ -1,2 +1,2 @@ /** Generated from the root package.json by scripts/sync-project-version.mjs. */ -export const MEMMY_VERSION = "1.1.1"; +export const MEMMY_VERSION = "1.1.2"; diff --git a/App/backend/src/services/agent-source-auto-scan-service.ts b/App/backend/src/services/agent-source-auto-scan-service.ts deleted file mode 100644 index 7ce6b1f91..000000000 --- a/App/backend/src/services/agent-source-auto-scan-service.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** Agent source auto scan service module. */ -import type { ScanPreferences } from "@memmy/local-api-contracts"; - -export const DEFAULT_AGENT_SOURCE_AUTO_SCAN_INTERVAL_MS = 60 * 60 * 1000; -export const DEFAULT_AGENT_SOURCE_AUTO_SCAN_INITIAL_DELAY_MS = 5 * 60 * 1000; - -type Timer = ReturnType; -type ScanTrigger = "startup" | "recurring"; - -export interface AgentSourceAutoScanService { - start(): void; - close(): void; -} - -export interface CreateAgentSourceAutoScanServiceOptions { - baseUrl: string; - localToken: string; - intervalMs?: number; - initialDelayMs?: number; - fetchFn?: typeof fetch; - getScanPreferences: () => ScanPreferences; -} - -/** Creates create agent source auto scan service. */ -export function createAgentSourceAutoScanService( - options: CreateAgentSourceAutoScanServiceOptions -): AgentSourceAutoScanService { - const intervalMs = options.intervalMs ?? DEFAULT_AGENT_SOURCE_AUTO_SCAN_INTERVAL_MS; - const initialDelayMs = options.initialDelayMs ?? DEFAULT_AGENT_SOURCE_AUTO_SCAN_INITIAL_DELAY_MS; - const fetchFn = options.fetchFn ?? fetch; - let timer: Timer | null = null; - let abortController: AbortController | null = null; - let closed = false; - let running = false; - - const schedule = (delayMs: number, trigger: ScanTrigger) => { - if (closed) { - return; - } - - timer = setTimeout(() => { - timer = null; - void runScan(trigger).finally(() => schedule(intervalMs, "recurring")); - }, delayMs); - timer.unref?.(); - }; - - const runScan = async (trigger: ScanTrigger) => { - if (running || closed) { - return; - } - - running = true; - try { - const preferences = options.getScanPreferences(); - const enabled = trigger === "startup" - ? preferences.autoScanKnownAgents - : preferences.watchFileChanges; - if (!enabled) { - return; - } - - abortController = new AbortController(); - await fetchFn(`${options.baseUrl}/api/agent-sources/scan`, { - method: "POST", - headers: { - "x-memmy-local-token": options.localToken - }, - signal: abortController.signal - }); - } catch { - // Auto scan is best-effort. Manual scans and the next scheduled tick remain available. - } finally { - running = false; - abortController = null; - } - }; - - return { - start() { - if (timer || closed) { - return; - } - - const startupScanEnabled = options.getScanPreferences().autoScanKnownAgents; - schedule( - startupScanEnabled ? initialDelayMs : intervalMs, - startupScanEnabled ? "startup" : "recurring" - ); - }, - - close() { - closed = true; - if (timer) { - clearTimeout(timer); - timer = null; - } - abortController?.abort(); - abortController = null; - } - }; -} diff --git a/App/backend/src/services/agent-source-scan-journal.ts b/App/backend/src/services/agent-source-scan-journal.ts index e441db02d..4dc15c2f0 100644 --- a/App/backend/src/services/agent-source-scan-journal.ts +++ b/App/backend/src/services/agent-source-scan-journal.ts @@ -1,8 +1,13 @@ /** Agent source scan journal service helpers. */ import { DatabaseSync } from "node:sqlite"; +import { readdirSync, existsSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; import { createAgentSourceScanJournal } from "../infrastructure/agent-source-scan-journal/index.js"; +import { openAppAgentSourceScanStore } from "../infrastructure/agent-source-scan-store/index.js"; import type { ScanResumeStateReference } from "./agent-source-scan-runner.js"; +const COMPLETED_DETAILS_RETENTION_MS = 60 * 60 * 1000; + export interface PersistedScanResume { jobId: string; sourceId: string; @@ -20,7 +25,9 @@ export function readLatestPersistedScanResume(databasePath: string | undefined): PRAGMA busy_timeout = 5000; `); const job = createAgentSourceScanJournal(db).findLatestJob(); - if (!job) return null; + if (!job) { + return readLatestDurableScanResume(databasePath); + } return { jobId: job.jobId, sourceId: job.sourceId, @@ -47,6 +54,43 @@ export function readLatestPersistedScanResume(databasePath: string | undefined): } } +function readLatestDurableScanResume(databasePath: string): PersistedScanResume | null { + const directory = join(dirname(databasePath), "agent-source-scans"); + if (!existsSync(directory)) return null; + const files = readdirSync(directory).filter((file) => file.endsWith(".sqlite")); + let latest: PersistedScanResume | null = null; + let latestUpdatedAt = Number.NEGATIVE_INFINITY; + for (const file of files) { + const path = join(directory, file); + try { + const db = new DatabaseSync(path); + const row = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, mode, phase, updated_at AS updatedAt FROM scan_meta WHERE id=1").get() as { jobId: string; sourceId: string; mode?: "initial_subset" | "incremental" | "full"; phase: string; updatedAt: string } | undefined; + const count = Number((db.prepare("SELECT COUNT(*) AS count FROM staged_messages").get() as { count: number }).count); + let sourceCount = 1; + try { + sourceCount = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count) || 1; + } catch { + // Stores created by the first staging build did not have source state. + } + const resultCount = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count); + db.close(); + if (!row) continue; + const updatedAt = Date.parse(row.updatedAt); + if (row.phase === "done") { + if (Number.isFinite(updatedAt) && Date.now() - updatedAt > COMPLETED_DETAILS_RETENTION_MS) deleteDurableScanStore(databasePath, row.jobId); + continue; + } + if (updatedAt >= latestUpdatedAt) { + latestUpdatedAt = updatedAt; + latest = row.phase === "summarize" + ? { jobId: row.jobId, sourceId: row.sourceId, mode: row.mode, resume: { storage: "sqlite", phase: "summarize", jobId: row.jobId, sourceId: row.sourceId, resultCount } } + : { jobId: row.jobId, sourceId: row.sourceId, mode: row.mode, resume: { storage: "sqlite", phase: "add", jobId: row.jobId, sourceId: row.sourceId, messageCount: count, sourceCount: sourceCount || 1 } }; + } + } catch { /* corrupt stores are retained for diagnostics */ } + } + return latest; +} + /** Deletes persisted scan resume state for one job. */ export function deletePersistedScanResume(databasePath: string | undefined, jobId: string): void { if (!databasePath) { @@ -64,3 +108,31 @@ export function deletePersistedScanResume(databasePath: string | undefined, jobI db.close(); } } + +export function deleteDurableScanStore(databasePath: string | undefined, jobId: string): void { + if (!databasePath) return; + const path = join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`); + rmSync(path, { force: true }); + rmSync(`${path}-wal`, { force: true }); + rmSync(`${path}-shm`, { force: true }); +} + +export function readDurableScanResults(databasePath: string | undefined, jobId: string, cursor = "0", limit = 100): { items: unknown[]; nextCursor: string | null } { + if (!databasePath) return { items: [], nextCursor: null }; + const path = join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`); + if (!existsSync(path)) return { items: [], nextCursor: null }; + const store = openAppAgentSourceScanStore(path, { jobId, sourceId: "all", mode: "incremental", phase: "ingest", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }); + let removeAfterRead = false; + try { + const safeLimit = Math.min(500, Math.max(1, limit)); + const safeCursor = Number.isFinite(Number(cursor)) && Number(cursor) >= 0 ? String(Math.floor(Number(cursor))) : "0"; + const rows = [...store.results(undefined, safeCursor, safeLimit)]; + const items = rows.map(({ cursor: _cursor, ...item }) => item); + const nextCursor = rows.length < safeLimit ? null : rows.at(-1)?.cursor ?? null; + removeAfterRead = nextCursor === null && store.getMeta()?.phase === "done"; + return { items, nextCursor }; + } finally { + store.close(); + if (removeAfterRead) deleteDurableScanStore(databasePath, jobId); + } +} diff --git a/App/backend/src/services/agent-source-scan-migration.ts b/App/backend/src/services/agent-source-scan-migration.ts new file mode 100644 index 000000000..5c7e4c08d --- /dev/null +++ b/App/backend/src/services/agent-source-scan-migration.ts @@ -0,0 +1,91 @@ +import { existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +/** Migrates one legacy app-state journal into the durable per-job store. */ +export function migrateLegacyScanJournal(databasePath: string, storePath: string, jobId: string): boolean { + if (!existsSync(databasePath)) return false; + mkdirSync(dirname(storePath), { recursive: true }); + const db = new DatabaseSync(storePath); + try { + db.exec(` + PRAGMA busy_timeout = 5000; + CREATE TABLE IF NOT EXISTS schema_meta (version INTEGER NOT NULL); + INSERT INTO schema_meta(version) SELECT 2 WHERE NOT EXISTS (SELECT 1 FROM schema_meta); + UPDATE schema_meta SET version = 2 WHERE version < 2; + CREATE TABLE IF NOT EXISTS scan_meta (id INTEGER PRIMARY KEY CHECK (id = 1), job_id TEXT NOT NULL, source_id TEXT NOT NULL, mode TEXT NOT NULL, phase TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS staged_messages (job_id TEXT NOT NULL, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, message_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL, created_at TEXT NOT NULL, workspace_path TEXT, git_root TEXT, raw_meta_json TEXT NOT NULL, ordinal INTEGER NOT NULL, PRIMARY KEY(job_id,source_id,message_id)); + CREATE TABLE IF NOT EXISTS scan_source_state (source_id TEXT PRIMARY KEY, mode TEXT NOT NULL, phase TEXT NOT NULL, message_count INTEGER NOT NULL DEFAULT 0, result_count INTEGER NOT NULL DEFAULT 0, error_count INTEGER NOT NULL DEFAULT 0, scan_started_at TEXT, watermarked_since TEXT, updated_at TEXT NOT NULL, error TEXT); + CREATE TABLE IF NOT EXISTS scan_results (id INTEGER PRIMARY KEY AUTOINCREMENT, source_id TEXT NOT NULL, conversation_id TEXT NOT NULL, memory_id TEXT, error TEXT); + CREATE INDEX IF NOT EXISTS scan_result_identity ON scan_results(source_id,conversation_id,memory_id,error); + `); + db.exec(`ATTACH DATABASE '${databasePath.replaceAll("'", "''")}' AS legacy`); + const oldMessages = Number((db.prepare("SELECT COUNT(*) AS count FROM legacy.account_agent_source_scan_messages WHERE job_id=?").get(jobId) as { count: number }).count); + const oldSources = Number((db.prepare("SELECT COUNT(*) AS count FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").get(jobId) as { count: number }).count); + const oldResults = Number((db.prepare("SELECT COALESCE(SUM(COALESCE(json_array_length(memory_ids_json),0) + COALESCE(json_array_length(errors_json),0)),0) AS count FROM legacy.account_agent_source_scan_results WHERE job_id=?").get(jobId) as { count: number }).count); + const oldSourceErrors = Number((db.prepare("SELECT COALESCE(SUM(COALESCE(json_array_length(errors_json),0)),0) AS count FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").get(jobId) as { count: number }).count); + const oldJob = db.prepare("SELECT job_id AS jobId, source_id AS sourceId, COALESCE(mode,'incremental') AS mode, phase, created_at AS createdAt, updated_at AS updatedAt FROM legacy.account_agent_source_scan_jobs WHERE job_id=?").get(jobId) as { jobId: string; sourceId: string; mode: string; phase: string; createdAt: string; updatedAt: string } | undefined; + if (!oldJob) return false; + db.prepare("INSERT OR IGNORE INTO scan_meta(id,job_id,source_id,mode,phase,created_at,updated_at,error) VALUES(1,?,?,?,?,?,?,NULL)").run(oldJob.jobId, oldJob.sourceId, oldJob.mode, oldJob.phase, oldJob.createdAt, oldJob.updatedAt); + const existingResults = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count); + db.exec("BEGIN"); + db.prepare(`INSERT OR IGNORE INTO staged_messages(job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,ordinal) + SELECT job_id,source_id,conversation_id,message_id,role,content,created_at,workspace_path,git_root,raw_meta_json,message_order + FROM legacy.account_agent_source_scan_messages WHERE job_id=?`).run(jobId); + db.prepare(`INSERT OR IGNORE INTO scan_source_state(source_id,mode,phase,message_count,result_count,error_count,scan_started_at,watermarked_since,updated_at,error) + SELECT s.source_id, COALESCE(s.scan_mode, ?), ?, + (SELECT COUNT(*) FROM legacy.account_agent_source_scan_messages m WHERE m.job_id=s.job_id AND m.source_id=s.source_id), + (SELECT COUNT(*) FROM legacy.account_agent_source_scan_results r WHERE r.job_id=s.job_id AND r.source_id=s.source_id), + COALESCE(json_array_length(s.errors_json),0), s.scan_started_at, s.watermarked_since, s.updated_at, NULL + FROM legacy.account_agent_source_scan_source_state s WHERE s.job_id=?`).run(oldJob.mode, oldJob.phase, jobId); + // json_each keeps legacy result arrays out of JavaScript memory. + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT r.source_id, 'scan', json_each.value, NULL + FROM legacy.account_agent_source_scan_results r, json_each(r.memory_ids_json) WHERE r.job_id=?`).run(jobId); + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT r.source_id, json_extract(json_each.value,'$.conversationId'), NULL, json_extract(json_each.value,'$.reason') + FROM legacy.account_agent_source_scan_results r, json_each(r.errors_json) WHERE r.job_id=?`).run(jobId); + db.prepare(`INSERT INTO scan_results(source_id,conversation_id,memory_id,error) + SELECT s.source_id, json_extract(json_each.value,'$.conversationId'), NULL, json_extract(json_each.value,'$.reason') + FROM legacy.account_agent_source_scan_source_state s, json_each(s.errors_json) WHERE s.job_id=?`).run(jobId); + const newMessages = Number((db.prepare("SELECT COUNT(*) AS count FROM staged_messages WHERE job_id=?").get(jobId) as { count: number }).count); + const newSources = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_source_state").get() as { count: number }).count); + const newResults = Number((db.prepare("SELECT COUNT(*) AS count FROM scan_results").get() as { count: number }).count) - existingResults; + if (newMessages < oldMessages || newSources < oldSources || newResults < oldResults + oldSourceErrors) { db.exec("ROLLBACK"); return false; } + db.prepare("DELETE FROM legacy.account_agent_source_scan_messages WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_source_state WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_results WHERE job_id=?").run(jobId); + db.prepare("DELETE FROM legacy.account_agent_source_scan_jobs WHERE job_id=?").run(jobId); + db.exec("COMMIT"); + return true; + } catch { + try { db.exec("ROLLBACK"); } catch { /* preserve legacy journal */ } + return false; + } finally { + try { db.exec("DETACH DATABASE legacy"); } catch { /* no attachment */ } + db.close(); + } +} + +/** Migrates every legacy scan job during App startup without loading journal arrays. */ +export function migrateLegacyScanJournals(databasePath: string): number { + if (!existsSync(databasePath)) return 0; + let jobIds: string[] = []; + try { + const db = new DatabaseSync(databasePath); + try { + jobIds = (db.prepare("SELECT job_id AS jobId FROM account_agent_source_scan_jobs ORDER BY job_id").all() as Array<{ jobId: string }>) + .map((row) => row.jobId) + .filter((jobId) => typeof jobId === "string" && jobId.length > 0); + } finally { + db.close(); + } + } catch { + return 0; + } + let migrated = 0; + for (const jobId of jobIds) { + if (migrateLegacyScanJournal(databasePath, join(dirname(databasePath), "agent-source-scans", `${jobId}.sqlite`), jobId)) migrated += 1; + } + return migrated; +} diff --git a/App/backend/src/services/agent-source-scan-process.ts b/App/backend/src/services/agent-source-scan-process.ts index f5a9570f2..7dd0898f0 100644 --- a/App/backend/src/services/agent-source-scan-process.ts +++ b/App/backend/src/services/agent-source-scan-process.ts @@ -1,7 +1,5 @@ import { createHttpMemoryClient, - createMemosSqliteMemoryClient, - discoverMemosSqliteSources, type MemoryClient, type MemoryLayerConfig } from "../adapters/outbound/memory-client/index.js"; @@ -11,21 +9,14 @@ import { } from "../analytics/agent-source-analytics.js"; import { createMemoryDesktopAddAnalytics } from "../analytics/memory-add-analytics.js"; import { createAppStateStore, type AppStateStore } from "../infrastructure/app-state-store/index.js"; -import { createAgentSourceScanJournal, type AgentSourceScanJournal } from "../infrastructure/agent-source-scan-journal/index.js"; import { createAgentSourceService } from "./agent-source-service.js"; import { createBuiltinAgentSourceRegistry } from "./builtin-agent-source-registry.js"; import { createBuiltinSkillTargetRegistry } from "./builtin-skill-target-registry.js"; import { createIngestionService } from "./ingestion-service.js"; import { createSkillDistributionService } from "./skill-distribution-service.js"; -import { - type AgentSourceScanProcessCommand, - type AgentSourceScanProcessData, - type AgentSourceScanProcessMessage, - isScanResumeStateReference, - type ScanResumeState, - type ScanResumeStateReference, - runAgentSourceScanJob -} from "./agent-source-scan-runner.js"; +import { isScanResumeStateReference, type AgentSourceScanProcessCommand, type AgentSourceScanProcessData, type AgentSourceScanProcessMessage, runAgentSourceScanJob } from "./agent-source-scan-runner.js"; +import { dirname, join } from "node:path"; +import { migrateLegacyScanJournal } from "./agent-source-scan-migration.js"; const DEFAULT_MEMORY_LAYER_TIMEOUT_MS = 20_000; @@ -64,13 +55,21 @@ async function runProcess(data: AgentSourceScanProcessData): Promise { let appStateStore: AppStateStore | null = null; try { appStateStore = createAppStateStore({ databasePath: data.databasePath }); - const scanJournal = createAgentSourceScanJournal(appStateStore.db); + const legacyMigrationSucceeded = migrateLegacyScanJournal( + data.databasePath, + join(dirname(data.databasePath), "agent-source-scans", `${data.job.jobId}.sqlite`), + data.job.jobId + ); + const preservedResume = data.job.resume && isScanResumeStateReference(data.job.resume) ? data.job.resume : null; + const preserveLegacyResume = Boolean(preservedResume && !legacyMigrationSucceeded); const memoryClient = createDefaultMemoryClient(process.env); const agentSources = createAgentSources(appStateStore, memoryClient); await runAgentSourceScanJob( { ...data.job, - resume: readResumeState(scanJournal, data.job.resume), + // The durable per-job store is the source of truth. Do not hydrate the + // legacy journal arrays into JavaScript while resuming an old job. + resume: null, controller }, agentSources, @@ -79,7 +78,7 @@ async function runProcess(data: AgentSourceScanProcessData): Promise { postProcessMessage({ type: "progress", progress }); }, onResumeChanged(resume) { - postProcessMessage({ type: "resume", resume: resume ? writeResumeState(scanJournal, data, resume) : null }); + postProcessMessage({ type: "resume", resume: preserveLegacyResume ? preservedResume : null }); }, onCompleted(results) { postProcessMessage({ type: "completed", results }); @@ -123,6 +122,7 @@ function createAgentSources(appStateStore: AppStateStore, memoryClient: MemoryCl skillDistributionService: createSkillDistributionService({ targetRegistry: createBuiltinSkillTargetRegistry() }), + scanStoreDirectory: `${dataDirectoryForScanStore(appStateStore)}`, agentSourceAnalytics: createAgentSourceLifecycleAnalytics({ getUserId: resolveAnalyticsUserId, getUserMode: resolveAnalyticsUserMode, @@ -130,20 +130,17 @@ function createAgentSources(appStateStore: AppStateStore, memoryClient: MemoryCl }); } +function dataDirectoryForScanStore(appStateStore: AppStateStore): string { + return join(dirname(appStateStore.databasePath), "agent-source-scans"); +} + function createDefaultMemoryClient(env: NodeJS.ProcessEnv): MemoryClient { const memoryLayerConfig = readMemoryLayerConfig(env); if (memoryLayerConfig) { return createHttpMemoryClient(memoryLayerConfig); } - if (env.MEMMY_DISABLE_MEMOS_SQLITE !== "1") { - const sources = discoverMemosSqliteSources(env); - if (sources.length > 0) { - return createMemosSqliteMemoryClient({ sources }); - } - } - - throw new Error("MEMMY_MEMORY_LAYER_URL or a local Memmy memory SQLite source is required"); + throw new Error("MEMMY_MEMORY_LAYER_URL is required"); } function readMemoryLayerConfig(env: NodeJS.ProcessEnv): MemoryLayerConfig | null { @@ -160,50 +157,6 @@ function readMemoryLayerConfig(env: NodeJS.ProcessEnv): MemoryLayerConfig | null }; } -function readResumeState(scanJournal: AgentSourceScanJournal, resume: AgentSourceScanProcessData["job"]["resume"]): ScanResumeState | null { - if (!resume) { - return null; - } - - if (!isScanResumeStateReference(resume)) { - return resume; - } - - return scanJournal.readResume(resume.jobId); -} - -function writeResumeState( - scanJournal: AgentSourceScanJournal, - data: AgentSourceScanProcessData, - resume: ScanResumeState -): ScanResumeStateReference { - scanJournal.writeResume({ - jobId: data.job.jobId, - sourceId: data.job.sourceId, - mode: data.job.mode, - resume - }); - - if (resume.phase === "add") { - return { - storage: "sqlite", - phase: "add", - jobId: data.job.jobId, - sourceId: resume.collected[0]?.sourceId ?? data.job.sourceId, - messageCount: resume.collected.reduce((sum, source) => sum + source.messages.length, 0), - sourceCount: resume.collected.length - }; - } - - return { - storage: "sqlite", - phase: "summarize", - jobId: data.job.jobId, - sourceId: data.job.sourceId, - resultCount: resume.results.length - }; -} - function postProcessMessage(message: AgentSourceScanProcessMessage): void { if (process.connected) process.send?.(message); } diff --git a/App/backend/src/services/agent-source-scan-runner.ts b/App/backend/src/services/agent-source-scan-runner.ts index e5fc95c7b..1f3ce842d 100644 --- a/App/backend/src/services/agent-source-scan-runner.ts +++ b/App/backend/src/services/agent-source-scan-runner.ts @@ -120,39 +120,26 @@ export async function runAgentSourceScanJob( }; let results: ScanResult[]; - if (job.resume?.phase === "summarize") { - results = job.resume.results; + let legacyPipeline = false; + if (agentSources.supportsPersistentScan) { + results = job.sourceId === "all" + ? await agentSources.scanAll({ ...scanOptions, scanJobId: job.jobId }) + : [await agentSources.scanOne(job.sourceId, { ...scanOptions, scanJobId: job.jobId })]; + callbacks.onResumeChanged(null); } else { + legacyPipeline = true; const collected = job.resume?.phase === "add" ? job.resume.collected - : job.sourceId === "all" - ? await agentSources.collectAll(scanOptions) - : [await agentSources.collectOne(job.sourceId, scanOptions)]; - if (job.controller.signal.aborted) { - return; - } + : job.sourceId === "all" ? await agentSources.collectAll(scanOptions) : [await agentSources.collectOne(job.sourceId, scanOptions)]; callbacks.onResumeChanged({ phase: "add", collected }); results = await agentSources.ingestCollected(collected, scanOptions); - if (job.controller.signal.aborted) { - return; - } - } - - callbacks.onResumeChanged({ phase: "summarize", results }); - const failures = await agentSources.processImportSummaries( - results.flatMap((result) => result.memoryIds ?? []), - { ...scanOptions, progressSourceId: job.sourceId } - ); - const resultByMemoryId = new Map(); - for (const result of results) { - for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result); } - for (const failure of failures) { - const result = resultByMemoryId.get(failure.memoryId); - result?.errors.push({ - conversationId: failure.memoryId, - reason: failure.reason - }); + if (legacyPipeline) { + callbacks.onResumeChanged({ phase: "summarize", results }); + const failures = await agentSources.processImportSummaries(results.flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: job.sourceId }); + const resultByMemoryId = new Map(); + for (const result of results) for (const memoryId of result.memoryIds ?? []) resultByMemoryId.set(memoryId, result); + for (const failure of failures) resultByMemoryId.get(failure.memoryId)?.errors.push({ conversationId: failure.memoryId, reason: failure.reason }); } if (job.controller.signal.aborted) { return; diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index dc3d6a7f1..f9cce1191 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -21,7 +21,8 @@ import type { import type { ConversationMessage, ScanOptions, - ScanProgress + ScanProgress, + SourceAdapter } from "../adapters/outbound/agent-source/types.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; import type { SourceRegistry } from "../adapters/outbound/agent-source/source-registry.js"; @@ -43,13 +44,25 @@ import { extractManagedAgentHistory, selectIncrementalManagedMessages } from "./managed-agent-history.js"; +import { + orderedTurns, + splitTurn, + stableTurnIdentity, + isCompleteTurn, + legacyTurnId, + legacyTurnRequestId +} from "@memmy/agent-source-core"; +import { openAppAgentSourceScanStore, type AppAgentSourceScanStore } from "../infrastructure/agent-source-scan-store/index.js"; export type { ScanProgress } from "../adapters/outbound/agent-source/types.js"; const SCAN_MESSAGE_YIELD_INTERVAL = 100; -const IMPORT_WORKER_BATCH_SIZE = 20; +const IMPORT_WORKER_BATCH_SIZE = 1; const IMPORT_PROCESSING_COHORT_SIZE = 100; -const IMPORT_WORKER_TIMEOUT_MS = 600_000; +// A targeted run leases one job so the timeout never depends on queue ordering. +// One summary can make three content attempts, each with four 180s HTTP attempts +// and backoff; keep a safety margin without replaying the worker request. +const IMPORT_WORKER_TIMEOUT_MS = 2_400_000; const IMPORT_PROGRESS_POLL_INTERVAL_MS = 250; const INITIAL_GLOBAL_MEMORY_LIMIT = 1_000; const INITIAL_ABSENT_SOURCE_MEMORY_LIMIT = 200; @@ -57,6 +70,7 @@ const INITIAL_SOURCE_MEMORY_LIMIT = 1_000; /** Contract for agent source service. */ export interface AgentSourceService { + readonly supportsPersistentScan?: true; list(): Promise; scanAll(options?: AgentSourceScanOptions): Promise; scanOne(sourceId: string, options?: AgentSourceScanOptions): Promise; @@ -92,6 +106,7 @@ export interface AgentSourceScanOptions { signal?: AbortSignal; onProgress?: (progress: ScanProgress) => void; progressSourceId?: string; + scanJobId?: string; } /** Contract for create agent source service options. */ @@ -105,6 +120,7 @@ export interface CreateAgentSourceServiceOptions { getScanPermission?: () => Promise; now?: () => string; createId?: () => string; + scanStoreDirectory?: string; } /** Creates create agent source service. */ @@ -114,19 +130,20 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption const agentSourceAnalytics = options.agentSourceAnalytics ?? createAgentSourceLifecycleAnalytics(); return { + supportsPersistentScan: true, async list() { return await listSources(options); }, async scanAll(scanOptions = {}) { - const collected = await this.collectAll(scanOptions); - const results = await this.ingestCollected(collected, scanOptions); - const failures = await this.processImportSummaries( - results.flatMap((result) => result.memoryIds ?? []), - { ...scanOptions, progressSourceId: "all" } - ); - appendProcessingFailuresToResults(results, failures); - return results; + if (!scanOptions.scanJobId && !options.scanStoreDirectory) { + const collected = await this.collectAll(scanOptions); + const results = await this.ingestCollected(collected, scanOptions); + const failures = await this.processImportSummaries(results.flatMap((result) => result.memoryIds ?? []), { ...scanOptions, progressSourceId: "all" }); + appendProcessingFailuresToResults(results, failures); + return results; + } + return scanPersistent(options, "all", scanOptions, now); }, async collectAll(scanOptions = {}) { @@ -157,15 +174,21 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption }, async scanOne(sourceId, scanOptions = {}) { - const collected = await this.collectOne(sourceId, scanOptions); - const result = await ingestCollectedSource(options, collected, scanOptions, now); - const failures = await processPendingImportSummaries( - options, - result.memoryIds ?? [], - { ...scanOptions, progressSourceId: sourceId } - ); - appendProcessingFailures(result, failures); - return result; + if (!scanOptions.scanJobId && !options.scanStoreDirectory) { + const collected = await this.collectOne(sourceId, scanOptions); + const result = await ingestCollectedSource(options, collected, scanOptions, now); + const failures = await processPendingImportSummaries(options, result.memoryIds ?? [], { ...scanOptions, progressSourceId: sourceId }); + appendProcessingFailures(result, failures); + return result; + } + const results = await scanPersistent(options, sourceId, scanOptions, now); + return results[0] ?? { + sourceId, + discoveredConversations: 0, + emittedMessages: 0, + skipped: 0, + errors: [{ conversationId: "scan", reason: "No result" }] + }; }, async addManual(input) { @@ -510,6 +533,406 @@ async function detectAvailableSourceAdapters(options: CreateAgentSourceServiceOp return detected.filter((entry) => entry.available).map((entry) => entry.adapter); } +interface PersistentSourceStage { + adapter: SourceAdapter; + sourceId: string; + mode: AgentSourceScanMode; + since?: string; + errors: Array<{ conversationId: string; reason: string }>; + scanErrorCount: number; +} + +/** Runs the production scan through a durable, bounded staging store. */ +async function scanPersistent( + options: CreateAgentSourceServiceOptions, + requestedSourceId: string, + scanOptions: AgentSourceScanOptions, + now: () => string +): Promise { + const adapters = requestedSourceId === "all" + ? options.sourceRegistry.list() + : [options.sourceRegistry.require(requestedSourceId)]; + const jobId = scanOptions.scanJobId ?? randomUUID(); + const directory = options.scanStoreDirectory ?? `${process.cwd()}/agent-source-scans`; + const store = openAppAgentSourceScanStore(`${directory}/${jobId}.sqlite`, { + jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: "stage", createdAt: now(), updatedAt: now() + }); + const results: ScanResult[] = []; + let completed = false; + try { + const available: SourceAdapter[] = []; + for (const adapter of adapters) { + scanOptions.signal?.throwIfAborted(); + if (await adapter.detect()) available.push(adapter); + else if (requestedSourceId !== "all") throw new AgentSourceUnavailableError(adapter.descriptor.displayName); + } + const globalInitial = requestedSourceId === "all" && available.length > 0 && + (scanOptions.mode === "initial_subset" || (scanOptions.mode === undefined && available.every((adapter) => !options.agentSourceRepository.getScanWatermark(adapter.descriptor.sourceId)))); + const stages: PersistentSourceStage[] = []; + for (const adapter of available) { + scanOptions.signal?.throwIfAborted(); + const sourceId = adapter.descriptor.sourceId; + const watermark = options.agentSourceRepository.getScanWatermark(sourceId); + const mode = scanOptions.mode ?? (watermark ? "incremental" : "initial_subset"); + const since = scanOptions.since ?? (mode === "incremental" ? watermarkCursor(watermark) : undefined); + stages.push(await stagePersistentSource(options, store, adapter, mode, since, scanOptions, now)); + } + if (globalInitial) { + for (const stage of stages) { + scanOptions.signal?.throwIfAborted(); + store.saveMeta({ jobId: store.getMeta()?.jobId ?? jobId, sourceId: store.getMeta()?.sourceId ?? requestedSourceId, mode: stage.mode, phase: "prepare", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const sourceState = store.getSourceState(stage.sourceId); + store.saveSourceState({ ...(sourceState ?? { sourceId: stage.sourceId, mode: stage.mode, messageCount: store.count(stage.sourceId), resultCount: store.resultCount(stage.sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "prepare", updatedAt: now() }); + await preparePersistentSource(options, store, stage.sourceId, stage.mode); + } + store.selectInitialTurns(stages.map((stage) => stage.sourceId), INITIAL_GLOBAL_MEMORY_LIMIT, INITIAL_ABSENT_SOURCE_MEMORY_LIMIT); + } + for (const stage of stages) { + const result = await ingestPersistentStagedSource(options, store, stage, scanOptions, now, globalInitial); + results.push(result); + store.saveSourceState({ + sourceId: stage.sourceId, + mode: stage.mode, + phase: result.errorCount && result.errorCount > 0 ? "failed" : "done", + messageCount: result.emittedMessages, + resultCount: store.resultCount(stage.sourceId), + errorCount: result.errorCount ?? result.errors.length, + updatedAt: now(), + ...(stage.since ? { watermarkedSince: stage.since } : {}) + }); + } + const hasErrors = results.some((result) => (result.errorCount ?? result.errors.length) > 0); + store.saveMeta({ jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: hasErrors ? "failed" : "done", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now(), ...(hasErrors ? { error: "Agent source scan completed with errors" } : {}) }); + // Keep large result details available for the paged results endpoint until + // the client consumes the final page. Small jobs can release their store + // immediately as before. + completed = !hasErrors && !results.some((result) => result.detailsTruncated); + return results; + } catch (error) { + store.saveMeta({ ...(store.getMeta() ?? { jobId, sourceId: requestedSourceId, mode: scanOptions.mode ?? "incremental", phase: "stage", createdAt: now(), updatedAt: now() }), phase: "failed", updatedAt: now(), error: error instanceof Error ? error.message : "Agent source scan failed" }); + throw error; + } finally { + if (completed) store.remove(); + else store.close(); + } +} + +async function stagePersistentSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + adapter: SourceAdapter, + mode: AgentSourceScanMode, + since: string | undefined, + scanOptions: AgentSourceScanOptions, + now: () => string +): Promise { + const sourceId = adapter.descriptor.sourceId; + options.agentSourceRepository.upsertSource({ sourceId, displayName: adapter.descriptor.displayName, dataPath: adapter.descriptor.dataPath, builtin: adapter.descriptor.builtin }); + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "stage", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + store.saveSourceState({ sourceId, mode, phase: "stage", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: 0, updatedAt: now(), ...(since ? { watermarkedSince: since } : {}) }); + const errors: Array<{ conversationId: string; reason: string }> = []; + let scanErrorCount = 0; + let batch: ConversationMessage[] = []; + let bytes = 0; + let emittedOrdinal = 0; + try { + for await (const message of adapter.scan({ + since, + order: scanOptions.order ?? (mode === "initial_subset" ? "recent_first" : "source_default"), + fullHistory: true, + signal: scanOptions.signal, + onProgress: (progress) => emitProgress(scanOptions, { ...progress, phase: "scan" }) + })) { + scanOptions.signal?.throwIfAborted(); + const messageBytes = Buffer.byteLength(JSON.stringify(message)); + if (messageBytes > 64 * 1024 * 1024) { + scanErrorCount += 1; + if (errors.length < 1000) errors.push({ conversationId: message.conversationId, reason: "scan record exceeds 64 MiB limit" }); + store.saveResult({ sourceId, conversationId: message.conversationId, error: "scan record exceeds 64 MiB limit" }); + continue; + } + if (batch.length > 0 && (batch.length >= 500 || bytes + messageBytes > 8 * 1024 * 1024)) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch = []; + bytes = 0; + } + batch.push({ ...message, ordinal: emittedOrdinal++ }); + bytes += messageBytes; + if (batch.length >= 500 || bytes >= 8 * 1024 * 1024) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + batch = []; + bytes = 0; + } + } + if (batch.length > 0) { + store.stageBatch(batch); + const last = batch[batch.length - 1]!; + store.saveScanCursor(sourceId, { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }); + } + } catch (error) { + if (scanOptions.signal?.aborted) throw error; + scanErrorCount += 1; + const reason = error instanceof Error ? error.message : "Agent source scan failed"; + if (errors.length < 1000) errors.push({ conversationId: "scan", reason }); + store.saveResult({ sourceId, conversationId: "scan", error: reason }); + } + store.saveSourceState({ sourceId, mode, phase: scanErrorCount > 0 ? "failed" : "stage", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: scanErrorCount, updatedAt: now(), ...(since ? { watermarkedSince: since } : {}) }); + return { adapter, sourceId, mode, since, errors, scanErrorCount }; +} + +async function ingestPersistentStagedSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + stage: PersistentSourceStage, + scanOptions: AgentSourceScanOptions, + now: () => string, + globalInitial: boolean +): Promise { + const { sourceId, mode, since } = stage; + if (!globalInitial) { + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "prepare", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const sourceState = store.getSourceState(sourceId); + store.saveSourceState({ ...(sourceState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "prepare", updatedAt: now() }); + await preparePersistentSource(options, store, sourceId, mode); + if (mode === "initial_subset") store.selectInitialTurns([sourceId], INITIAL_SOURCE_MEMORY_LIMIT, 0); + } + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "ingest", createdAt: store.getMeta()?.createdAt ?? now(), updatedAt: now() }); + const preparedState = store.getSourceState(sourceId); + store.saveSourceState({ ...(preparedState ?? { sourceId, mode, messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount: stage.scanErrorCount, updatedAt: now() }), phase: "ingest", updatedAt: now() }); + const ingestion = await ingestPersistentSource(options, store, sourceId, scanOptions, []); + const scannedAt = now(); + options.agentSourceRepository.setLastScannedAt(sourceId, scannedAt); + const skillResult = await ingestSourceSkills(options, sourceId, scanOptions, store); + if (stage.errors.length === 0 && ingestion.errors.length === 0 && skillResult.errorCount === 0) updatePersistentWatermark(options, sourceId, mode, ingestion.latestSeenAt, scannedAt, since); + const allErrors = [...stage.errors, ...ingestion.errors, ...skillResult.errors]; + const errorCount = stage.scanErrorCount + ingestion.errorCount + skillResult.errorCount; + const memoryIdCount = ingestion.memoryIdCount + skillResult.memoryIdCount; + store.saveMeta({ jobId: store.getMeta()?.jobId ?? scanOptions.scanJobId ?? "", sourceId: store.getMeta()?.sourceId ?? sourceId, mode, phase: "summarize", createdAt: store.getMeta()?.createdAt ?? scannedAt, updatedAt: scannedAt }); + store.saveSourceState({ sourceId, mode, phase: "summarize", messageCount: store.count(sourceId), resultCount: store.resultCount(sourceId), errorCount, updatedAt: scannedAt, ...(since ? { watermarkedSince: since } : {}) }); + return { + sourceId, + discoveredConversations: store.conversationCount(sourceId), + emittedMessages: store.count(sourceId), + skipped: ingestion.deduped, + memoryIds: ingestion.memoryIds, + memoryIdCount, + errorCount, + detailsTruncated: errorCount > 1000 || memoryIdCount > 1000, + errors: allErrors.slice(0, 1000) + }; +} + +async function preparePersistentSource(options: CreateAgentSourceServiceOptions, store: AppAgentSourceScanStore, sourceId: string, mode: AgentSourceScanMode): Promise { + let cursor: { conversationId: string; createdAt: string; messageId: string; ordinal: number } | undefined; + let currentId: string | null = null; + let currentTurn: ConversationMessage[] = []; + let turnIndex = 0; + let hash = createHash("sha256"); + let first = true; + let latest: ConversationMessage | null = null; + const flushTurn = () => { + if (!currentTurn.length || !isCompleteTurn(currentTurn)) return; + const firstMessage = currentTurn[0]!; + const lastMessage = currentTurn[currentTurn.length - 1]!; + const turn = { sourceId, conversationId: firstMessage.conversationId, turnIndex, messages: currentTurn }; + store.saveTurnMeta({ + sourceId, + conversationId: firstMessage.conversationId, + turnId: stableTurnIdentity(turn), + firstMessageId: firstMessage.messageId, + firstCreatedAt: firstMessage.createdAt, + lastMessageId: lastMessage.messageId, + lastCreatedAt: lastMessage.createdAt, + selected: true + }); + turnIndex += 1; + }; + const flush = () => { + if (!currentId || !latest) return; + hash.update("]"); + const contentHash = hash.digest("hex"); + const checkpoint = options.agentSourceRepository.getConversationCheckpoint(sourceId, currentId); + const selected = mode === "full" || mode === "initial_subset" || !checkpoint + || Date.parse(latest.createdAt) > Date.parse(checkpoint.lastCreatedAt) + || (Date.parse(latest.createdAt) === Date.parse(checkpoint.lastCreatedAt) && latest.messageId.localeCompare(checkpoint.lastMessageId) > 0) + || checkpoint.contentHash !== contentHash; + store.saveConversationMeta({ sourceId, conversationId: currentId, lastMessageId: latest.messageId, lastCreatedAt: latest.createdAt, contentHash, selected }); + }; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + if (message.conversationId !== currentId) { + flushTurn(); + flush(); + currentId = message.conversationId; + currentTurn = []; + turnIndex = 0; + hash = createHash("sha256"); + hash.update("["); + first = true; + } + if (message.role === "user" && currentTurn.length > 0) { + flushTurn(); + currentTurn = []; + } + currentTurn.push(message); + if (!first) hash.update(","); + first = false; + hash.update(JSON.stringify({ messageId: message.messageId, role: message.role, content: message.content, createdAt: message.createdAt, toolName: hashMetaString(message, "toolName") ?? hashMetaString(message, "hermesToolName"), toolCallId: hashMetaString(message, "toolCallId") ?? hashMetaString(message, "hermesToolCallId") })); + latest = message; + } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; + } + flushTurn(); + if (currentId && latest) flush(); +} + +function hashMetaString(message: ConversationMessage, key: string): string | undefined { + const value = message.rawMeta[key]; + return typeof value === "string" ? value : undefined; +} + +async function ingestPersistentSource( + options: CreateAgentSourceServiceOptions, + store: AppAgentSourceScanStore, + sourceId: string, + scanOptions: AgentSourceScanOptions, + initialErrors: readonly { conversationId: string; reason: string }[] +): Promise<{ memoryIds: string[]; memoryIdCount: number; deduped: number; errorCount: number; errors: Array<{ conversationId: string; reason: string }>; latestSeenAt: string | null }> { + const memoryIds: string[] = []; + let memoryIdCount = 0; + const pendingIds: string[] = []; + const errors = initialErrors.slice(0, 1000); + let errorCount = initialErrors.length; + let deduped = 0; + let latestSeenAt: string | null = null; + let activeConversationId: string | null = null; + let activeConversationFailed = false; + const commitConversation = () => { + if (!activeConversationId || activeConversationFailed) return; + const meta = store.getConversationMeta(sourceId, activeConversationId); + if (!meta) return; + const updatedAt = new Date().toISOString(); + const checkpoint = { sourceId, conversationId: activeConversationId, lastMessageId: meta.lastMessageId, lastCreatedAt: meta.lastCreatedAt, contentHash: meta.contentHash, updatedAt }; + store.saveCheckpoint(checkpoint); + options.agentSourceRepository.upsertConversationCheckpoint(checkpoint); + }; + const pages = (async function*() { + let cursor: { conversationId: string; createdAt: string; messageId: string; ordinal: number } | undefined; + while (true) { + const page = readScanPage(store, sourceId, cursor); + if (page.length === 0) break; + for (const message of page) { + yield message; + } + const last = page[page.length - 1]!; + cursor = { conversationId: last.conversationId, createdAt: last.createdAt, messageId: last.messageId, ordinal: last.ordinal ?? 0 }; + } + })(); + for await (const turn of orderedTurns(pages)) { + scanOptions.signal?.throwIfAborted(); + const turnLatest = turn.messages[turn.messages.length - 1]?.createdAt ?? null; + if (turnLatest && (!latestSeenAt || Date.parse(turnLatest) > Date.parse(latestSeenAt))) latestSeenAt = turnLatest; + if (turn.conversationId !== activeConversationId) { + commitConversation(); + activeConversationId = turn.conversationId; + activeConversationFailed = false; + } + const conversationMeta = store.getConversationMeta(sourceId, turn.conversationId); + if (conversationMeta?.selected === false) continue; + const selectedTurn = store.getTurnMeta(sourceId, turn.conversationId, stableTurnIdentity(turn)); + if (selectedTurn && !selectedTurn.selected) continue; + let turnSucceeded = true; + // Leave ample room for JSON escaping and the add-memory envelope while + // keeping every request below the 1 MiB wire limit. + const parts = splitTurn(turn, 4000, 512 * 1024); + for (const part of parts) { + const contentHash = createHash("sha256").update(part.content).digest("hex"); + const requestId = parts.length === 1 + ? legacyTurnRequestId(turn) + : createHash("sha256").update([stableTurnIdentity(turn), String(part.partIndex), contentHash].join("\u0000")).digest("hex"); + const turnId = parts.length === 1 ? legacyTurnId(turn) : `${sourceId}:${part.parentTurnId}:${part.partIndex}`; + try { + const added = await options.memoryClient.addMemory({ + requestId, + adapterId: `agent-source:${sourceId}`, + content: part.content, + layer: "L1", + title: firstTurnLine(part.messages) ?? `${sourceId} conversation`, + tags: ["agent-source", sourceId], + source: sourceId, + turnId, + createdAt: part.messages[0]!.createdAt, + deferProcessing: true + }); + if (added.duplicate) deduped += part.messages.length; + else { + memoryIdCount += 1; + if (memoryIds.length < 1000) memoryIds.push(added.id); + pendingIds.push(added.id); + if (pendingIds.length >= IMPORT_PROCESSING_COHORT_SIZE) { + const cohort = pendingIds.splice(0, pendingIds.length); + const failures = await processPendingImportSummaries(options, cohort, { ...scanOptions, progressSourceId: sourceId }); + if (failures.length > 0) activeConversationFailed = true; + const mapped = failures.map((failure) => ({ conversationId: failure.memoryId, reason: failure.reason })); + errorCount += mapped.length; + errors.push(...mapped.slice(0, Math.max(0, 1000 - errors.length))); + for (const failure of failures) store.saveResult({ sourceId, conversationId: failure.memoryId, error: failure.reason }); + } + } + store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId: added.id }); + } catch (error) { + turnSucceeded = false; + activeConversationFailed = true; + const reason = error instanceof Error ? error.message : "Agent source ingestion failed"; + errorCount += 1; + if (errors.length < 1000) errors.push({ conversationId: turn.conversationId, reason }); + store.saveResult({ sourceId, conversationId: turn.conversationId, error: reason }); + } + } + if (!turnSucceeded) activeConversationFailed = true; + emitProgress(scanOptions, { sourceId, phase: "add", current: memoryIds.length + deduped, total: store.count(sourceId), message: "Adding raw memories" }); + } + if (pendingIds.length > 0) { + const failures = await processPendingImportSummaries(options, pendingIds, { ...scanOptions, progressSourceId: sourceId }); + if (failures.length > 0) activeConversationFailed = true; + const mapped = failures.map((failure) => ({ conversationId: failure.memoryId, reason: failure.reason })); + errorCount += mapped.length; + errors.push(...mapped.slice(0, Math.max(0, 1000 - errors.length))); + for (const failure of failures) store.saveResult({ sourceId, conversationId: failure.memoryId, error: failure.reason }); + } + commitConversation(); + return { memoryIds, memoryIdCount, deduped, errorCount, errors, latestSeenAt }; +} + +function readScanPage(store: AppAgentSourceScanStore, sourceId: string, cursor?: { conversationId: string; createdAt: string; messageId: string; ordinal: number }): ConversationMessage[] { + const page: ConversationMessage[] = []; + let bytes = 0; + for (const message of store.messages(sourceId, cursor, 500)) { + page.push(message); + bytes += Buffer.byteLength(JSON.stringify(message)); + if (page.length >= 500 || bytes >= 8 * 1024 * 1024) break; + } + return page; +} + +function firstTurnLine(messages: readonly ConversationMessage[]): string | undefined { + const value = messages.find((message) => message.role === "user")?.content; + const line = value?.split(/\r?\n/).map((part) => part.trim()).find(Boolean); + return line ? (line.length <= 120 ? line : `${line.slice(0, 117)}...`) : undefined; +} + +function updatePersistentWatermark(options: CreateAgentSourceServiceOptions, sourceId: string, mode: AgentSourceScanMode, latestSeenAt: string | null, scannedAt: string, since?: string): void { + const existing = options.agentSourceRepository.getScanWatermark(sourceId); + options.agentSourceRepository.upsertScanWatermark({ sourceId, mode, baselineAt: existing?.baselineAt ?? since ?? scannedAt, latestSeenCreatedAt: maxIso(existing?.latestSeenCreatedAt ?? null, latestSeenAt), updatedAt: scannedAt }); +} + export interface CollectedSourceScan { sourceId: string; scanMode?: AgentSourceScanMode; @@ -557,6 +980,7 @@ async function collectSourceMessages( messages: [], errors: [] }; + const conversationIds = new Set(); emitProgress(scanOptions, { sourceId, @@ -585,7 +1009,8 @@ async function collectSourceMessages( })) { scanOptions.signal?.throwIfAborted(); collected.messages.push(message); - if (!collected.conversationIds.includes(message.conversationId)) { + if (!conversationIds.has(message.conversationId)) { + conversationIds.add(message.conversationId); collected.conversationIds.push(message.conversationId); } if (collected.messages.length % SCAN_MESSAGE_YIELD_INTERVAL === 0) { @@ -689,7 +1114,7 @@ async function ingestCollectedSource( ) { updateScanWatermark(options, collected, scanOptions, scannedAt); } - errors.push(...await ingestSourceSkills(options, collected.sourceId, scanOptions)); + errors.push(...(await ingestSourceSkills(options, collected.sourceId, scanOptions)).errors); return { sourceId: collected.sourceId, discoveredConversations: collected.conversationIds.length, @@ -700,28 +1125,39 @@ async function ingestCollectedSource( }; } +interface SkillScanOutcome { + errors: Array<{ conversationId: string; reason: string }>; + errorCount: number; + memoryIdCount: number; +} + async function ingestSourceSkills( options: CreateAgentSourceServiceOptions, sourceId: string, - scanOptions: AgentSourceScanOptions -): Promise> { - if (!options.skillDistributionService.listSkills) return []; + scanOptions: AgentSourceScanOptions, + store?: AppAgentSourceScanStore +): Promise { + if (!options.skillDistributionService.listSkills) return { errors: [], errorCount: 0, memoryIdCount: 0 }; const errors: Array<{ conversationId: string; reason: string }> = []; + let errorCount = 0; + let memoryIdCount = 0; let skills; try { skills = await options.skillDistributionService.listSkills(sourceId); } catch (error) { - return [{ + const detail = { conversationId: "skills", reason: error instanceof Error ? error.message : "Agent Skill scan failed" - }]; + }; + store?.saveResult({ sourceId, conversationId: detail.conversationId, error: detail.reason }); + return { errors: [detail], errorCount: 1, memoryIdCount: 0 }; } for (const skill of skills) { scanOptions.signal?.throwIfAborted(); try { - await options.memoryClient.addMemory({ + const added = await options.memoryClient.addMemory({ requestId: `agent-source-skill:${sourceId}:${skill.sourceSkillId}:${skill.sourceContentHash}`, adapterId: `agent-source:${sourceId}`, content: skill.content, @@ -737,14 +1173,19 @@ async function ingestSourceSkills( sourceSkillVersion: skill.sourceSkillVersion, sourceContentHash: skill.sourceContentHash }); + memoryIdCount += 1; + store?.saveResult({ sourceId, conversationId: `skill:${skill.sourceSkillId}`, memoryId: added.id }); } catch (error) { - errors.push({ + errorCount += 1; + const detail = { conversationId: `skill:${skill.sourceSkillId}`, reason: error instanceof Error ? error.message : "Agent Skill import failed" - }); + }; + store?.saveResult({ sourceId, conversationId: detail.conversationId, error: detail.reason }); + if (errors.length < 1000) errors.push(detail); } } - return errors; + return { errors, errorCount, memoryIdCount }; } function filterCheckpointedConversations( @@ -1077,44 +1518,36 @@ async function processPendingImportSummaries( while (pendingMemoryIds.size > 0) { scanOptions.signal?.throwIfAborted(); const targets = [...pendingMemoryIds]; - const result = await options.memoryClient.runWorker({ + const workerOutcome = options.memoryClient.runWorker({ limit: IMPORT_WORKER_BATCH_SIZE, targetMemoryIds: targets, priorityCohortOnly: true, signal: scanOptions.signal, timeoutMs: IMPORT_WORKER_TIMEOUT_MS - }); - - const refreshed = await options.memoryClient.getMemoryProcessingStatus(targets); - const processingByMemoryId = new Map(refreshed.items.map((item) => [item.memoryId, item])); - const activeMemoryIds = new Set(refreshed.items - .filter((item) => item.state === "summary_pending" || item.state === "summarizing" || - item.state === "embedding_pending" || item.state === "embedding") - .map((item) => item.memoryId)); - const previousPending = pendingMemoryIds.size; - for (const memoryId of pendingMemoryIds) { - if (activeMemoryIds.has(memoryId)) continue; - const processing = processingByMemoryId.get(memoryId); - if (!processing) { - failures.push({ memoryId, reason: "Memory processing state is missing" }); - } else if (processing.state === "failed") { - failures.push({ - memoryId, - reason: processing.errorMessage || "Memory processing failed" - }); + }).then((result) => ({ kind: "worker" as const, result })); + let result: Awaited> | undefined; + + while (!result) { + const outcome = await Promise.race([ + workerOutcome, + waitForWorkerProgress(IMPORT_PROGRESS_POLL_INTERVAL_MS, undefined, { signal: scanOptions.signal }) + .then(() => ({ kind: "poll" as const })) + ]); + if (outcome.kind === "worker") result = outcome.result; + const previousPending = pendingMemoryIds.size; + await reconcileImportProcessing(options.memoryClient, pendingMemoryIds, failures); + if (pendingMemoryIds.size < previousPending) lastProgressAt = Date.now(); + emitProgress(scanOptions, { + sourceId: progressSourceId, + phase: "summarize", + current: completedMemoryCount + cohort.length - pendingMemoryIds.size, + total: ownedMemoryIds.length, + message: "Summarizing and indexing latest memories" + }); + if (pendingMemoryIds.size === 0 && !result) { + result = (await workerOutcome).result; } - pendingMemoryIds.delete(memoryId); - } - if (pendingMemoryIds.size < previousPending) { - lastProgressAt = Date.now(); } - emitProgress(scanOptions, { - sourceId: progressSourceId, - phase: "summarize", - current: completedMemoryCount + cohort.length - pendingMemoryIds.size, - total: ownedMemoryIds.length, - message: "Summarizing and indexing latest memories" - }); if (pendingMemoryIds.size === 0) break; if (Date.now() - lastProgressAt >= IMPORT_WORKER_TIMEOUT_MS) { @@ -1130,6 +1563,29 @@ async function processPendingImportSummaries( return failures; } +async function reconcileImportProcessing( + memoryClient: Pick, + pendingMemoryIds: Set, + failures: ProcessingFailure[] +): Promise { + const refreshed = await memoryClient.getMemoryProcessingStatus([...pendingMemoryIds]); + const processingByMemoryId = new Map(refreshed.items.map((item) => [item.memoryId, item])); + const activeMemoryIds = new Set(refreshed.items + .filter((item) => item.state === "summary_pending" || item.state === "summarizing" || + item.state === "embedding_pending" || item.state === "embedding") + .map((item) => item.memoryId)); + for (const memoryId of pendingMemoryIds) { + if (activeMemoryIds.has(memoryId)) continue; + const processing = processingByMemoryId.get(memoryId); + if (!processing) { + failures.push({ memoryId, reason: "Memory processing state is missing" }); + } else if (processing.state === "failed") { + failures.push({ memoryId, reason: processing.errorMessage || "Memory processing failed" }); + } + pendingMemoryIds.delete(memoryId); + } +} + function appendProcessingFailures(result: ScanResult, failures: readonly ProcessingFailure[]): void { result.errors.push(...failures.map((failure) => ({ conversationId: failure.memoryId, diff --git a/App/backend/src/services/app-config-service.ts b/App/backend/src/services/app-config-service.ts index 4228e9529..3566b78fd 100644 --- a/App/backend/src/services/app-config-service.ts +++ b/App/backend/src/services/app-config-service.ts @@ -28,12 +28,14 @@ import type { CloudClient } from "../adapters/outbound/cloud-client/index.js"; import type { AccountSessionRepository } from "../infrastructure/app-state-store/repositories/account-session-repo.js"; import type { BootstrapRepository } from "../infrastructure/app-state-store/repositories/bootstrap-repo.js"; import type { MemmyConfigWriter } from "../infrastructure/memmy-config/index.js"; +import type { ScanPreferencesStore } from "../infrastructure/memmy-config/agent-access.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; import { createHttpModelConfigTester, type ModelConfigTester } from "./model-config-tester.js"; export interface AppConfigService { updateSettings(input: PatchAppSettingsInput): Promise; updatePrivacy(input: PatchPrivacyInput): Promise; + getScanPreferences(): Promise; updateScanPreferences(input: PatchScanPreferencesInput): Promise; updateOnboarding(input: PatchOnboardingInput): Promise; setImprovementProgram(input: SetImprovementProgramInput): Promise; @@ -52,6 +54,7 @@ export interface CreateAppConfigServiceOptions { | "updateAppSettings" | "getAppSettings" | "getOnboardingState" + | "getScanPreferences" | "updatePrivacy" | "updateScanPreferences" | "updateOnboarding" @@ -63,6 +66,7 @@ export interface CreateAppConfigServiceOptions { accountSessionRepository?: Pick; memmyConfigWriter?: MemmyConfigWriter; memoryClient?: Pick; + scanPreferencesStore?: ScanPreferencesStore; } const BUILT_IN_AVATARS = AvatarOptionSchema.array().parse([ @@ -108,8 +112,15 @@ export function createAppConfigService(options: CreateAppConfigServiceOptions): return options.bootstrapRepository.updatePrivacy(input); }, + async getScanPreferences() { + return options.scanPreferencesStore?.getScanPreferences() + ?? options.bootstrapRepository.getScanPreferences(); + }, + async updateScanPreferences(input) { - return options.bootstrapRepository.updateScanPreferences(input); + return options.scanPreferencesStore + ? options.scanPreferencesStore.updateScanPreferences(input) + : options.bootstrapRepository.updateScanPreferences(input); }, async updateOnboarding(input) { diff --git a/App/backend/src/services/bootstrap-service.ts b/App/backend/src/services/bootstrap-service.ts index 9859d6b51..2d21fd9b3 100644 --- a/App/backend/src/services/bootstrap-service.ts +++ b/App/backend/src/services/bootstrap-service.ts @@ -12,6 +12,7 @@ import { import type { AppStateStore } from "../infrastructure/app-state-store/index.js"; import type { CloudClient, CloudHealth } from "../adapters/outbound/cloud-client/index.js"; import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; +import type { ScanPreferencesStore } from "../infrastructure/memmy-config/agent-access.js"; export type BootstrapScenario = "onboarding" | "completed"; @@ -24,6 +25,7 @@ export interface CreateBootstrapServiceOptions { memoryClient: MemoryClient; cloudClient: CloudClient; bootstrapScenario?: BootstrapScenario; + scanPreferencesStore?: Pick; } export function createBootstrapService(options: CreateBootstrapServiceOptions): BootstrapService { @@ -52,7 +54,7 @@ export function createBootstrapService(options: CreateBootstrapServiceOptions): } : onboarding, privacy: bootstrap.getPrivacySettings(), - scanPreferences: bootstrap.getScanPreferences(), + scanPreferences: options.scanPreferencesStore?.getScanPreferences() ?? bootstrap.getScanPreferences(), tokenUsage: tokenUsage ?? createTokenUsagePlaceholder(promotions.agentChatTokenTotal), health: { localApi: "ok", diff --git a/App/backend/src/services/index.ts b/App/backend/src/services/index.ts index 23a997272..89eb59e35 100644 --- a/App/backend/src/services/index.ts +++ b/App/backend/src/services/index.ts @@ -1,6 +1,8 @@ import type { AccountChannel } from "@memmy/local-api-contracts"; +import { dirname, join } from "node:path"; import type { AppStateStore } from "../infrastructure/app-state-store/index.js"; import { type MemmyConfigWriter } from "../infrastructure/memmy-config/index.js"; +import type { ScanPreferencesStore } from "../infrastructure/memmy-config/agent-access.js"; import type { AgentAdapterRegistry } from "../adapters/outbound/agent-adapter/index.js"; import { createBuiltinOnboardingInsightSamplers, @@ -106,6 +108,7 @@ export interface CreateBackendServicesOptions { memmyAgentAdminBootstrapSecret?: string | null; /** Verification channel supported by the current desktop package. */ accountChannel?: AccountChannel; + scanPreferencesStore?: ScanPreferencesStore; } export function createBackendServices(options: CreateBackendServicesOptions): BackendServices { @@ -163,6 +166,7 @@ export function createBackendServices(options: CreateBackendServicesOptions): Ba getUserId: resolveAnalyticsUserId, getUserMode: resolveAnalyticsUserMode, }), + scanStoreDirectory: join(dirname(options.appStateStore.databasePath), "agent-source-scans"), }); const toolConnectionAnalytics = createToolConnectionAnalytics({ getUserId: resolveAnalyticsUserId, @@ -172,13 +176,17 @@ export function createBackendServices(options: CreateBackendServicesOptions): Ba return { memoryClient: options.memoryClient, agentAdapterRegistry: options.agentAdapterRegistry, - bootstrap: createBootstrapService(options), + bootstrap: createBootstrapService({ + ...options, + scanPreferencesStore: options.scanPreferencesStore + }), appConfig: createAppConfigService({ bootstrapRepository: options.appStateStore.repositories.bootstrap, cloudClient: options.cloudClient, accountSessionRepository: options.appStateStore.repositories.accountSession, memmyConfigWriter: options.memmyConfigWriter, - memoryClient: options.memoryClient + memoryClient: options.memoryClient, + scanPreferencesStore: options.scanPreferencesStore }), account: createAccountService({ cloudClient: options.cloudClient, @@ -199,14 +207,18 @@ export function createBackendServices(options: CreateBackendServicesOptions): Ba toolConnectionAnalytics, }), localData: createLocalDataService({ - localDataStore: options.appStateStore.localDataStore + localDataStore: options.appStateStore.localDataStore, + memoryClient: options.memoryClient }), agentSources, agentSourceAutoInject: createAgentSourceAutoInjectService({ agentSources, permissionManager: options.permissionManager, - getScanPreferences: () => options.appStateStore.repositories.bootstrap.getScanPreferences() + getScanPreferences: () => options.scanPreferencesStore?.getScanPreferences() + ?? options.appStateStore.repositories.bootstrap.getScanPreferences() }), + // First-report sampling stays inside Desktop: it reads a small recent-history + // window for onboarding and is separate from Memory's persistent Agent scan. onboardingInsight: createOnboardingInsightService({ samplers: createBuiltinOnboardingInsightSamplers(), conversationWindowReader: createSourceRegistryOnboardingConversationWindowReader(sourceRegistry), diff --git a/App/backend/src/services/ingestion-service.ts b/App/backend/src/services/ingestion-service.ts index fdd4db6ed..bc27f43fb 100644 --- a/App/backend/src/services/ingestion-service.ts +++ b/App/backend/src/services/ingestion-service.ts @@ -232,13 +232,18 @@ async function processConversation( try { const added = await options.memoryClient.addMemory(request); - stats.written += turn.messages.length; - stats.writtenMemories += 1; - stats.memoryIds.push(added.id); + if (added.duplicate) { + stats.deduped += turn.messages.length; + stats.dedupedMemories += 1; + } else { + stats.written += turn.messages.length; + stats.writtenMemories += 1; + stats.memoryIds.push(added.id); + } options.memoryAddAnalytics?.trackAddSucceeded({ ...addAnalyticsBase, durationMs: Date.now() - addStartedAt, - storedCount: 1 + storedCount: added.duplicate ? 0 : 1 }); for (const dedupKey of dedupKeys) { diff --git a/App/backend/src/services/local-data-service.ts b/App/backend/src/services/local-data-service.ts index 52c7b4c6a..31e594e5a 100644 --- a/App/backend/src/services/local-data-service.ts +++ b/App/backend/src/services/local-data-service.ts @@ -10,6 +10,7 @@ import { type LocalDataRevealResponse } from "@memmy/local-api-contracts"; import type { LocalDataStore } from "../infrastructure/app-state-store/local-data-store.js"; +import type { MemoryClient } from "../adapters/outbound/memory-client/index.js"; export interface LocalDataService { getPath(): Promise; @@ -20,12 +21,11 @@ export interface LocalDataService { export interface CreateLocalDataServiceOptions { localDataStore: LocalDataStore; - now?: () => Date; + memoryClient: MemoryClient; } /** Creates create local data service. */ export function createLocalDataService(options: CreateLocalDataServiceOptions): LocalDataService { - const now = options.now ?? (() => new Date()); const getPathResponse = (): LocalDataRevealResponse => LocalDataRevealResponseSchema.parse({ ok: true, dataPath: options.localDataStore.getDataPath() @@ -43,15 +43,18 @@ export function createLocalDataService(options: CreateLocalDataServiceOptions): }, async export(input) { - return LocalDataExportResponseSchema.parse(options.localDataStore.exportData(input)); + if (!options.memoryClient.exportBundle) throw new Error("Memory export API is unavailable"); + const bundle = await options.memoryClient.exportBundle(); + return LocalDataExportResponseSchema.parse(options.localDataStore.exportData(input, bundle)); }, async clear(_input) { - const clearedAt = now().toISOString(); - options.localDataStore.clearMemoryDatabase(clearedAt); + if (!options.memoryClient.clearAllData) throw new Error("Memory clear API is unavailable"); + const result = await options.memoryClient.clearAllData(); + options.localDataStore.clearImportState(); return LocalDataClearResponseSchema.parse({ ok: true, - clearedAt + clearedAt: result.clearedAt }); } }; diff --git a/App/backend/src/services/onboarding-insight-service.ts b/App/backend/src/services/onboarding-insight-service.ts index cc7fdc12c..1f9bf3224 100644 --- a/App/backend/src/services/onboarding-insight-service.ts +++ b/App/backend/src/services/onboarding-insight-service.ts @@ -32,7 +32,6 @@ const MAX_BALANCED_QUERIES = 84; const MAX_PREFERENCE_LLM_QUERIES = 24; const DEFAULT_LLM_TIMEOUT_MS = 90_000; const DEFAULT_LLM_MAX_TOKENS = 2_000; -const MEMMY_ACCOUNT_AGENT_CHAT_THINKING_BUDGET = 500; const MAX_GENERATED_OUTPUT_CHARS = 12_000; const GENERATED_REPORT_OPEN = ""; const GENERATED_REPORT_CLOSE = ""; @@ -273,7 +272,10 @@ export function createOnboardingInsightService(options: CreateOnboardingInsightS return { async generateReport(input = {}, signal) { const startedAt = now(); - const sample = await sampleRecentQueries(options.samplers, options.conversationWindowReader, signal, now); + const sample = mergeDetectedAgents( + await sampleRecentQueries(options.samplers, options.conversationWindowReader, signal, now), + input.detectedAgents + ); const profile = buildProfileSignals(sample); const locale = profile.preferredResponseLanguage ?? input.locale ?? inferLocale(sample.queries); const response = await buildReportResponse({ @@ -290,7 +292,10 @@ export function createOnboardingInsightService(options: CreateOnboardingInsightS }, async *streamReport(input = {}, signal) { const startedAt = now(); - const sample = await sampleRecentQueries(options.samplers, options.conversationWindowReader, signal, now); + const sample = mergeDetectedAgents( + await sampleRecentQueries(options.samplers, options.conversationWindowReader, signal, now), + input.detectedAgents + ); const profile = buildProfileSignals(sample); const locale = profile.preferredResponseLanguage ?? input.locale ?? inferLocale(sample.queries); yield { @@ -561,6 +566,38 @@ async function sampleRecentQueries( }; } +function mergeDetectedAgents( + sample: SampleBundle, + detectedAgents: OnboardingInsightReportInput["detectedAgents"] +): SampleBundle { + if (!detectedAgents?.length) { + return sample; + } + + const detectedBySource = new Map(detectedAgents.map((agent) => [agent.sourceId, agent])); + const discovered = sample.discovered.map((result) => { + const detected = detectedBySource.get(result.sourceId); + detectedBySource.delete(result.sourceId); + return detected + ? { ...result, recentSessionCount: Math.max(result.recentSessionCount, detected.recentSessionCount) } + : result; + }); + + for (const detected of detectedBySource.values()) { + discovered.push({ + sourceId: detected.sourceId, + displayName: detected.displayName, + recentSessionCount: detected.recentSessionCount, + latestActivityAt: null, + queries: [], + recentMessages: [], + errors: [] + }); + } + + return { ...sample, discovered }; +} + function resolveLatestConversationReference( results: readonly OnboardingSampleResult[] ): OnboardingConversationReference | null { @@ -736,7 +773,7 @@ async function buildReportResponse(input: { if (input.sample.queries.length === 0) { return { status: "ready", - reportMarkdown: renderEmptyHistoryReport(input.locale), + reportMarkdown: renderEmptyHistoryReport(input.locale, input.sample), diagnostics: diagnostics(input.sample, false, Math.max(0, input.now() - input.startedAt), input.locale) }; } @@ -750,7 +787,13 @@ async function buildReportResponse(input: { const generatedReport = await generateReportSafely(input.reportGenerator, generationInput); const reportMarkdown = generatedReport?.reportMarkdown ?? renderFallbackReport(input.profile, input.sample, input.locale); const taskContext = generatedReport?.taskContext ?? buildFallbackTaskContext(generationInput); - await persistFirstReportMemory(input.memoryWriter, input.sample, input.locale, reportMarkdown, taskContext); + persistFirstReportMemoryInBackground( + input.memoryWriter, + input.sample, + input.locale, + reportMarkdown, + taskContext + ); return { status: "ready", @@ -775,7 +818,7 @@ async function* streamReportResponse(input: { type: "done", response: { status: "ready", - reportMarkdown: renderEmptyHistoryReport(input.locale), + reportMarkdown: renderEmptyHistoryReport(input.locale, input.sample), diagnostics: diagnostics(input.sample, false, input.elapsedMs, input.locale) } }; @@ -819,7 +862,13 @@ async function* streamReportResponse(input: { : await generateReportSafely(input.reportGenerator, generationInput); const reportMarkdown = generatedReport?.reportMarkdown ?? renderFallbackReport(input.profile, input.sample, input.locale); const taskContext = generatedReport?.taskContext ?? buildFallbackTaskContext(generationInput); - await persistFirstReportMemory(input.memoryWriter, input.sample, input.locale, reportMarkdown, taskContext); + persistFirstReportMemoryInBackground( + input.memoryWriter, + input.sample, + input.locale, + reportMarkdown, + taskContext + ); yield { type: "done", @@ -839,7 +888,19 @@ function renderFallbackReport( return locale === "en-US" ? renderEnglishReport(profile, sample) : renderChineseReport(profile, sample); } -function renderEmptyHistoryReport(locale: "zh-CN" | "en-US"): string { +function renderEmptyHistoryReport(locale: "zh-CN" | "en-US", sample: SampleBundle): string { + const agentNames = sample.discovered.map((agent) => agent.displayName); + if (agentNames.length > 0) { + const names = agentNames.join(", "); + return locale === "en-US" ? [ + `Memmy found ${names} on this device, but the quick first scan did not return readable conversation history.`, + "Once you use Memmy with a real task, it will preserve the useful background, decisions, and next step for future conversations and other Agents." + ].join("\n\n") : [ + `Memmy 已识别到这台设备上的 ${names}，但首次轻量扫描暂时没有读到可用的对话历史。`, + "之后用 Memmy 处理真实任务时，它会记住有用的背景、决策和下一步，方便新对话或其他 Agent 继续。" + ].join("\n\n"); + } + return locale === "en-US" ? [ "There is no readable Agent history on this device yet, so there is nothing useful to pretend I already know.", "Tell Memmy about one real task. It will preserve the useful background, decisions, and next step so a new conversation—or another Agent such as Cursor or Codex—can continue without making you explain it again." @@ -887,6 +948,21 @@ async function persistFirstReportMemory( }); } +function persistFirstReportMemoryInBackground( + memoryWriter: OnboardingFirstReportMemoryWriter | null | undefined, + sample: SampleBundle, + locale: "zh-CN" | "en-US", + reportMarkdown: string, + taskContext: OnboardingTaskContextSummary +): void { + void persistFirstReportMemory(memoryWriter, sample, locale, reportMarkdown, taskContext) + .catch((error) => { + console.warn( + `[onboarding-insight] First-report memory persistence failed: ${error instanceof Error ? error.message : String(error)}` + ); + }); +} + function normalizeGeneratedOutput(output: string | null): string | null { const trimmed = (output ?? "").trim(); return trimmed ? trimmed.slice(0, MAX_GENERATED_OUTPUT_CHARS) : null; @@ -926,10 +1002,19 @@ function parseGeneratedFirstReport( } function findGeneratedReportOpen(output: string): { index: number; marker: string } | null { - if (output.startsWith(GENERATED_REPORT_ALIAS_OPEN)) { - return { index: 0, marker: GENERATED_REPORT_ALIAS_OPEN }; + let first: { index: number; marker: string } | null = null; + for (const marker of GENERATED_REPORT_OPEN_MARKERS) { + let index = output.indexOf(marker); + while (index >= 0) { + const lineStart = output.lastIndexOf("\n", index - 1) + 1; + if (!output.slice(lineStart, index).trim() && (!first || index < first.index)) { + first = { index, marker }; + break; + } + index = output.indexOf(marker, index + marker.length); + } } - return findFirstGeneratedMarker(output, [GENERATED_REPORT_OPEN]); + return first; } function findGeneratedTaskContext( @@ -1160,8 +1245,10 @@ function renderFallbackTrajectory(input: { } class FirstReportStreamParser { - private mode: "prefix" | "report" | "hidden" | "plain" = "prefix"; + private mode: "prefix" | "report" | "hidden" = "prefix"; private buffer = ""; + private visibleSource = ""; + private emittedVisibleChars = 0; push(delta: string): string[] { if (this.mode === "hidden") { @@ -1170,48 +1257,39 @@ class FirstReportStreamParser { this.buffer += delta; if (this.mode === "prefix") { const candidate = this.buffer.trimStart(); - const reportOpen = findLeadingGeneratedMarker(candidate, GENERATED_REPORT_OPEN_MARKERS); - if (!candidate || (!reportOpen && isGeneratedMarkerPrefix(candidate, GENERATED_REPORT_OPEN_MARKERS))) { + if (!candidate) { return []; } + const reportOpen = findGeneratedReportOpen(candidate); if (!reportOpen) { - this.mode = "plain"; - return this.drainVisibleText([ - GENERATED_TASK_CONTEXT_OPEN, - ...GENERATED_REPORT_CLOSE_MARKERS, - GENERATED_JSON_FENCE_OPEN, - GENERATED_NAKED_JSON_OPEN - ]); + return []; } this.mode = "report"; - this.buffer = candidate.slice(reportOpen.length); + this.buffer = candidate.slice(reportOpen.index + reportOpen.marker.length); } - return this.mode === "plain" - ? this.drainVisibleText([ - GENERATED_TASK_CONTEXT_OPEN, - ...GENERATED_REPORT_CLOSE_MARKERS, - GENERATED_JSON_FENCE_OPEN, - GENERATED_NAKED_JSON_OPEN - ]) - : this.drainVisibleText([ - ...GENERATED_REPORT_CLOSE_MARKERS, - GENERATED_TASK_CONTEXT_OPEN, - GENERATED_JSON_FENCE_OPEN, - GENERATED_NAKED_JSON_OPEN - ]); + return this.drainVisibleText([ + ...GENERATED_REPORT_CLOSE_MARKERS, + GENERATED_TASK_CONTEXT_OPEN, + GENERATED_JSON_FENCE_OPEN, + GENERATED_NAKED_JSON_OPEN + ]); } finish(): string[] { - if (this.mode === "prefix" || this.mode === "report" || this.mode === "plain") { + if (this.mode === "prefix") { + this.buffer = ""; + this.visibleSource = ""; + return []; + } + if (this.mode === "report") { const remainder = this.buffer; this.buffer = ""; const aliasBoundary = findGeneratedTaskContextAliasBoundary(remainder); if (aliasBoundary) { const report = remainder.slice(0, aliasBoundary.index); - return report ? [report] : []; + return this.visibleText(report, true); } const internalMarkers = [ - ...(this.mode === "prefix" ? GENERATED_REPORT_OPEN_MARKERS : []), ...GENERATED_REPORT_CLOSE_MARKERS, GENERATED_TASK_CONTEXT_OPEN, GENERATED_TASK_CONTEXT_ALIAS_OPEN, @@ -1219,7 +1297,7 @@ class FirstReportStreamParser { GENERATED_NAKED_JSON_OPEN ]; const isPartialInternalMarker = isGeneratedMarkerPrefix(remainder, internalMarkers); - return remainder && !isPartialInternalMarker ? [remainder] : []; + return remainder && !isPartialInternalMarker ? this.visibleText(remainder, true) : []; } return []; } @@ -1237,18 +1315,29 @@ class FirstReportStreamParser { const report = this.buffer.slice(0, boundary.index); this.buffer = ""; this.mode = "hidden"; - return report ? [report] : []; + return this.visibleText(report, true); } if (boundary) { const report = this.buffer.slice(0, boundary.index); this.buffer = this.buffer.slice(boundary.index); - return report ? [report] : []; + return this.visibleText(report); } const retainedMarkers = [...delimiters, GENERATED_TASK_CONTEXT_ALIAS_OPEN]; const retainedChars = Math.max(...retainedMarkers.map((marker) => matchingDelimiterSuffixLength(this.buffer, marker))); const report = this.buffer.slice(0, this.buffer.length - retainedChars); this.buffer = this.buffer.slice(this.buffer.length - retainedChars); - return report ? [report] : []; + return this.visibleText(report); + } + + private visibleText(text: string, finished = false): string[] { + this.visibleSource += text; + const sanitized = stripRawHtmlTags(this.visibleSource, true); + const delta = sanitized.slice(this.emittedVisibleChars); + this.emittedVisibleChars = sanitized.length; + if (finished) { + this.visibleSource = ""; + } + return delta ? [delta] : []; } } @@ -1294,10 +1383,6 @@ function findFirstGeneratedMarker( return first; } -function findLeadingGeneratedMarker(value: string, markers: readonly string[]): string | null { - return markers.find((marker) => value.startsWith(marker)) ?? null; -} - function isGeneratedMarkerPrefix(value: string, markers: readonly string[]): boolean { return markers.some((marker) => marker.startsWith(value)); } @@ -1851,6 +1936,7 @@ function buildLlmMessages(input: OnboardingInsightGenerationInput): Array<{ role "『最近项目记忆』说明最新会话来自哪个 Agent、用户目标、已做事项、已验证结果、当前状态、仍待处理内容。workspacePath 有值时必须写清项目具体路径。只写当前有效结论，不展开冗长历史。", "『接下来可以做』只列证据支持且尚未完成的 0-3 条待办，按执行顺序排列。第一条应是当前最小且可立即执行的下一步；任务已完成或没有明确待办时，直接说明暂时没有明确待办，不要补通用建议。", "正文长度要求：中文 300-500 字，英文 180-300 words。重点是准确提炼最近一个项目现场，不要扩展成跨项目年度总结。", + "报告正文只允许使用 Markdown，不得包含任何原始 HTML 标签或样式。不要输出思考过程、执行计划、要求确认、Prompt 复述或起草说明。", "你必须一次输出两个区块，严格使用以下顺序和标签；标签前后不要添加其他文字：", `${GENERATED_REPORT_OPEN}\n这里放给用户看的 Markdown 报告正文\n${GENERATED_REPORT_CLOSE}`, `${GENERATED_TASK_CONTEXT_OPEN}\n这里放一个合法 JSON 对象\n${GENERATED_TASK_CONTEXT_CLOSE}`, @@ -2012,10 +2098,7 @@ function openAiCompatibleThinkingControlFields( provider === "memmy_account" && model.includes("agent_chat") ) { - return { - enable_thinking: true, - thinking_budget: MEMMY_ACCOUNT_AGENT_CHAT_THINKING_BUDGET - }; + return { enable_thinking: false }; } if (provider === "dashscope" || baseUrl.includes("dashscope") || model.includes("qwen")) { @@ -2246,10 +2329,60 @@ function extractLlmDelta(body: unknown): string | null { } function sanitizeGeneratedReport(report: string | null): string | null { - const trimmed = stripActionCopyFromReport(report ?? "").trim(); + const trimmed = stripActionCopyFromReport(stripRawHtmlTags(report ?? "")).trim(); return trimmed ? trimmed.slice(0, 4_000) : null; } +function stripRawHtmlTags(value: string, dropTrailingPartial = false): string { + let output = ""; + let codeTicks = 0; + for (let index = 0; index < value.length;) { + if (value[index] === "`") { + let end = index + 1; + while (value[end] === "`") { + end += 1; + } + const ticks = end - index; + if (!codeTicks) { + codeTicks = ticks; + } else if (ticks >= codeTicks) { + codeTicks = 0; + } + output += value.slice(index, end); + index = end; + continue; + } + if (codeTicks || value[index] !== "<") { + output += value[index]; + index += 1; + continue; + } + if (value.startsWith("", index + 4); + if (commentEnd < 0) { + return dropTrailingPartial ? output : `${output}${value.slice(index)}`; + } + index = commentEnd + 3; + continue; + } + const tag = /^<\/?[A-Za-z][A-Za-z0-9-]*(?:\s[^>\n]*|\/?)>/.exec(value.slice(index)); + if (tag) { + index += tag[0].length; + continue; + } + const remainder = value.slice(index); + if (dropTrailingPartial && ( + remainder === "<" || remainder === "\n]*)?$/.test(remainder) + )) { + return output; + } + output += "<"; + index += 1; + } + return output; +} + function stripActionCopyFromReport(report: string): string { const reportBody = report.split(/\[\s*MEMMY_ACTIONS_JSON\s*\]/i, 1)[0] ?? report; const paragraphs = reportBody diff --git a/App/backend/src/services/skill-distribution-service.ts b/App/backend/src/services/skill-distribution-service.ts index aca5a4b9e..a849bc27e 100644 --- a/App/backend/src/services/skill-distribution-service.ts +++ b/App/backend/src/services/skill-distribution-service.ts @@ -134,7 +134,13 @@ async function findSkillFiles(skillsRoot: string): Promise { if (entry.isFile() && entry.name.toLowerCase() === "skill.md") { files.push(entryPath); } else if (depth < 2 && (entry.isDirectory() || entry.isSymbolicLink())) { - const entryStat = await stat(entryPath); + let entryStat; + try { + entryStat = await stat(entryPath); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") continue; + throw error; + } if (entryStat.isDirectory()) await visit(entryPath, depth + 1); } } diff --git a/App/backend/src/services/tests/account-service.test.ts b/App/backend/src/services/tests/account-service.test.ts index f45af8992..76d38026d 100644 --- a/App/backend/src/services/tests/account-service.test.ts +++ b/App/backend/src/services/tests/account-service.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { LOCAL_BYOK_ACCOUNT_UUID } from "../../infrastructure/app-state-store/account-context.js"; import { createAppStateStore } from "../../infrastructure/app-state-store/index.js"; +import { INSTALLATION_SCAN_SCOPE_UUID } from "../../infrastructure/installation-scan-scope.js"; import { createAccountService as createAccountServiceImplementation, type CreateAccountServiceOptions @@ -698,9 +699,16 @@ describe("AccountService", () => { hasAcceptedTerms: true, acceptedTermsVersion: "2026-06-01", scanPermission: "scan_only", + firstEncounterReportStatus: "shown", improvementProgram: "accepted", completedAt: "2026-06-20T12:00:00.000Z" }); + const readInstallationOnboarding = () => store!.db.prepare( + `SELECT scan_permission, first_encounter_report_status, updated_at + FROM account_onboarding_state + WHERE uuid = ?` + ).get(INSTALLATION_SCAN_SCOPE_UUID); + const installationBeforeLogout = readInstallationOnboarding(); const service = createAccountService({ cloudClient: createCloudClientStub(), @@ -709,6 +717,7 @@ describe("AccountService", () => { }); await expect(service.logout()).resolves.toEqual({ ok: true }); + expect(readInstallationOnboarding()).toEqual(installationBeforeLogout); expect(store.repositories.accountSession.get()).toEqual({ authenticated: false }); expect(store.repositories.bootstrap.getOnboardingState()).toMatchObject({ completed: true, @@ -716,6 +725,7 @@ describe("AccountService", () => { hasAcceptedTerms: true, acceptedTermsVersion: "2026-06-01", scanPermission: "scan_only", + firstEncounterReportStatus: "shown", improvementProgram: "not_applicable", completedAt: "2026-06-20T12:00:00.000Z" }); @@ -742,6 +752,7 @@ describe("AccountService", () => { hasAcceptedTerms: true, acceptedTermsVersion: "2026-06-01", scanPermission: "scan_only", + firstEncounterReportStatus: "shown", improvementProgram: "not_applicable", completedAt: "2026-06-20T12:00:00.000Z" }); diff --git a/App/backend/src/services/tests/agent-source-auto-scan-service.test.ts b/App/backend/src/services/tests/agent-source-auto-scan-service.test.ts deleted file mode 100644 index 94ec723e7..000000000 --- a/App/backend/src/services/tests/agent-source-auto-scan-service.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -/** Agent source auto scan service tests. */ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { - createAgentSourceAutoScanService, - DEFAULT_AGENT_SOURCE_AUTO_SCAN_INITIAL_DELAY_MS -} from "../agent-source-auto-scan-service.js"; -import type { ScanPreferences } from "@memmy/local-api-contracts"; - -const enabledPreferences: ScanPreferences = { - autoScanKnownAgents: true, - watchFileChanges: true, - autoInjectSkill: false -}; - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("agent source auto scan service", () => { - it("runs the startup scan after the default five-minute delay", async () => { - vi.useFakeTimers(); - const fetchFn = vi.fn(async () => ({} as Response)); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 1_000, - fetchFn, - getScanPreferences: () => enabledPreferences - }); - - service.start(); - await vi.advanceTimersByTimeAsync(DEFAULT_AGENT_SOURCE_AUTO_SCAN_INITIAL_DELAY_MS - 1); - expect(fetchFn).not.toHaveBeenCalled(); - - await vi.advanceTimersByTimeAsync(1); - - expect(fetchFn).toHaveBeenCalledWith("http://127.0.0.1:19001/api/agent-sources/scan", { - method: "POST", - headers: { - "x-memmy-local-token": "test-token" - }, - signal: expect.any(AbortSignal) - }); - service.close(); - }); - - it("runs one startup scan when hourly incremental sync is disabled", async () => { - vi.useFakeTimers(); - const fetchFn = vi.fn(async () => ({} as Response)); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 1_000, - initialDelayMs: 100, - fetchFn, - getScanPreferences: () => ({ ...enabledPreferences, watchFileChanges: false }) - }); - - service.start(); - await vi.advanceTimersByTimeAsync(100); - expect(fetchFn).toHaveBeenCalledTimes(1); - - await vi.advanceTimersByTimeAsync(5_000); - expect(fetchFn).toHaveBeenCalledTimes(1); - service.close(); - }); - - it("waits for the hourly interval when startup scanning is disabled", async () => { - vi.useFakeTimers(); - const fetchFn = vi.fn(async () => ({} as Response)); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 1_000, - initialDelayMs: 100, - fetchFn, - getScanPreferences: () => ({ ...enabledPreferences, autoScanKnownAgents: false }) - }); - - service.start(); - await vi.advanceTimersByTimeAsync(999); - expect(fetchFn).not.toHaveBeenCalled(); - - await vi.advanceTimersByTimeAsync(1); - expect(fetchFn).toHaveBeenCalledTimes(1); - service.close(); - }); - - it("does not scan when both automatic scan preferences are disabled", async () => { - vi.useFakeTimers(); - const fetchFn = vi.fn(async () => ({} as Response)); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 100, - initialDelayMs: 10, - fetchFn, - getScanPreferences: () => ({ - ...enabledPreferences, - autoScanKnownAgents: false, - watchFileChanges: false - }) - }); - - service.start(); - await vi.advanceTimersByTimeAsync(1_000); - expect(fetchFn).not.toHaveBeenCalled(); - service.close(); - }); - - it("does not overlap auto scan requests", async () => { - vi.useFakeTimers(); - let resolveFetch: (response: Response) => void = () => undefined; - const fetchFn = vi.fn(() => new Promise((resolve) => { - resolveFetch = resolve; - })); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 100, - initialDelayMs: 100, - fetchFn, - getScanPreferences: () => enabledPreferences - }); - - service.start(); - await vi.advanceTimersByTimeAsync(100); - await vi.advanceTimersByTimeAsync(1_000); - - expect(fetchFn).toHaveBeenCalledTimes(1); - - resolveFetch({} as Response); - await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(100); - - expect(fetchFn).toHaveBeenCalledTimes(2); - service.close(); - }); - - it("clears a pending auto scan when closed", async () => { - vi.useFakeTimers(); - const fetchFn = vi.fn(async () => ({} as Response)); - const service = createAgentSourceAutoScanService({ - baseUrl: "http://127.0.0.1:19001", - localToken: "test-token", - intervalMs: 100, - initialDelayMs: 100, - fetchFn, - getScanPreferences: () => enabledPreferences - }); - - service.start(); - service.close(); - await vi.advanceTimersByTimeAsync(100); - - expect(fetchFn).not.toHaveBeenCalled(); - }); -}); diff --git a/App/backend/src/services/tests/agent-source-service.test.ts b/App/backend/src/services/tests/agent-source-service.test.ts index ab80615a7..2bbd25cd2 100644 --- a/App/backend/src/services/tests/agent-source-service.test.ts +++ b/App/backend/src/services/tests/agent-source-service.test.ts @@ -4,7 +4,7 @@ import { MANAGED_AGENT_DISCOVERY_PENDING_DATA_PATH } from "@memmy/local-api-cont import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { createSourceRegistry } from "../../adapters/outbound/agent-source/source-registry.js"; import type { ConversationMessage, @@ -610,9 +610,10 @@ describe("agent source service", () => { expect(enqueueCalls).toEqual([["memory-cursor", "memory-custom"]]); expect(workerCalls).toEqual([ expect.objectContaining({ - limit: 20, + limit: 1, targetMemoryIds: ["memory-cursor", "memory-custom"], - priorityCohortOnly: true + priorityCohortOnly: true, + timeoutMs: 2_400_000 }) ]); }); @@ -671,7 +672,7 @@ describe("agent source service", () => { })).resolves.toEqual([]); expect(workerTargets).toEqual([["memory-a", "memory-b"]]); - expect(workerLimits).toEqual([20]); + expect(workerLimits).toEqual([1]); expect(workerPriorityCohorts).toEqual([true]); expect(progress).toEqual([ { current: 0, total: 2 }, @@ -679,6 +680,60 @@ describe("agent source service", () => { ]); }); + it("emits persisted summary progress while the worker call remains pending", async () => { + const baseMemoryClient = createMockMemoryClient(); + let releaseWorker = () => undefined; + let workerSettled = false; + const workerGate = new Promise((resolve) => { + releaseWorker = resolve; + }); + const service = createService({ + memoryClient: { + ...baseMemoryClient, + async enqueueImportSummaries(memoryIds) { + return { enqueued: memoryIds?.length ?? 0, memoryIds: memoryIds ?? [], serverTime: "2026-05-28T10:00:00.000Z" }; + }, + async runWorker(input) { + await workerGate; + workerSettled = true; + return baseMemoryClient.runWorker(input); + }, + async getMemoryProcessingStatus(memoryIds) { + return { + items: memoryIds.map((memoryId) => ({ + memoryId, + state: "ready" as const, + stage: null, + activeJobId: null, + attemptCount: 1, + manualRetryCount: 0, + retryAction: "retry" as const, + errorCode: null, + errorMessage: null, + failedAt: null, + updatedAt: "2026-05-28T10:00:00.000Z" + })), + serverTime: "2026-05-28T10:00:00.000Z" + }; + } + } + }); + const progress: number[] = []; + const processing = service.processImportSummaries(["memory-a"], { + onProgress(event) { + if (event.phase === "summarize") progress.push(event.current); + } + }); + + try { + await vi.waitFor(() => expect(progress).toContain(1), { timeout: 1_000, interval: 25 }); + expect(workerSettled).toBe(false); + } finally { + releaseWorker(); + await processing; + } + }); + it("finishes an empty owned-memory batch without starting the worker", async () => { const baseMemoryClient = createMockMemoryClient(); const enqueued: string[][] = []; diff --git a/App/backend/src/services/tests/ingestion-service.test.ts b/App/backend/src/services/tests/ingestion-service.test.ts index 97ab7a1f9..3a6d9b62f 100644 --- a/App/backend/src/services/tests/ingestion-service.test.ts +++ b/App/backend/src/services/tests/ingestion-service.test.ts @@ -447,6 +447,55 @@ describe("ingestion service", () => { }); }); + it("counts a QA duplicate returned by memory.add as deduped and marks its source messages seen", async () => { + const markSeen = vi.fn(() => true); + const succeeded: Array> = []; + const service = createService( + { + async addMemory(input) { + return { + id: "hook-memory", + kind: "trace", + memoryLayer: input.layer ?? "L1", + status: "activated", + title: input.title ?? "Hook memory", + summary: input.content, + tags: input.tags ?? [], + createdAt: now(), + serverTime: now(), + duplicate: true + }; + } + }, + { hasSeen: () => false, markSeen }, + undefined, + { + trackAddStarted() {}, + trackAddSucceeded(input) { + succeeded.push({ ...input }); + }, + trackAddFailed() {} + } + ); + + const stats = await service.ingest( + toAsyncIterable([createMessage("conv-a", 1), createMessage("conv-a", 2)]), + { sourceId: "codex" } + ); + + expect(markSeen).toHaveBeenCalledTimes(2); + expect(stats).toMatchObject({ + written: 0, + deduped: 2, + writtenMemories: 0, + dedupedMemories: 1, + memoryIds: [] + }); + expect(succeeded).toEqual([ + expect.objectContaining({ storedCount: 0 }) + ]); + }); + it("does not import user-only or assistant-only turns as memories", async () => { const calls: string[] = []; const service = createService({ diff --git a/App/backend/src/services/tests/local-data-service.test.ts b/App/backend/src/services/tests/local-data-service.test.ts index 532a045a6..abbcdd258 100644 --- a/App/backend/src/services/tests/local-data-service.test.ts +++ b/App/backend/src/services/tests/local-data-service.test.ts @@ -1,11 +1,13 @@ /** Local data service tests. */ import { describe, expect, it } from "vitest"; import { createLocalDataService } from "../local-data-service.js"; +import type { MemoryClient } from "../../adapters/outbound/memory-client/index.js"; describe("LocalDataService", () => { it("returns the local data path without revealing it", async () => { const calls: string[] = []; const service = createLocalDataService({ + memoryClient: {} as MemoryClient, localDataStore: { getDataPath() { calls.push("path"); @@ -17,7 +19,7 @@ describe("LocalDataService", () => { exportData() { return { exportPath: "/tmp/export/memmy-export-1", bytes: 128 }; }, - clearMemoryDatabase() { + clearImportState() { calls.push("clear"); } } @@ -33,7 +35,16 @@ describe("LocalDataService", () => { it("reveals, exports, and clears through the local data store", async () => { const calls: string[] = []; const service = createLocalDataService({ - now: () => new Date("2026-06-02T10:00:00.000Z"), + memoryClient: { + async exportBundle() { + calls.push("memory:export"); + return { manifest: { service: "memmy-memory-service" } }; + }, + async clearAllData() { + calls.push("memory:clear"); + return { ok: true, clearedAt: "2026-06-02T10:00:00.000Z", cleared: {} }; + } + } as MemoryClient, localDataStore: { getDataPath() { calls.push("path"); @@ -42,12 +53,13 @@ describe("LocalDataService", () => { revealDataPath(dataPath) { calls.push(`reveal:${dataPath}`); }, - exportData(input) { + exportData(input, bundle) { calls.push(`export:${input.targetPath}`); + expect(bundle).toMatchObject({ manifest: { service: "memmy-memory-service" } }); return { exportPath: "/tmp/export/memmy-export-1", bytes: 128 }; }, - clearMemoryDatabase(clearedAt) { - calls.push(`clear:${clearedAt}`); + clearImportState() { + calls.push("clear-import-state"); } } }); @@ -61,6 +73,13 @@ describe("LocalDataService", () => { ok: true, clearedAt: "2026-06-02T10:00:00.000Z" }); - expect(calls).toEqual(["path", "reveal:/tmp/memmy-data", "export:/tmp/export", "clear:2026-06-02T10:00:00.000Z"]); + expect(calls).toEqual([ + "path", + "reveal:/tmp/memmy-data", + "memory:export", + "export:/tmp/export", + "memory:clear", + "clear-import-state" + ]); }); }); diff --git a/App/backend/src/services/tests/onboarding-insight-service.test.ts b/App/backend/src/services/tests/onboarding-insight-service.test.ts index df809e835..df7d25c9e 100644 --- a/App/backend/src/services/tests/onboarding-insight-service.test.ts +++ b/App/backend/src/services/tests/onboarding-insight-service.test.ts @@ -81,7 +81,7 @@ describe("onboarding insight service", () => { expect(report.reportMarkdown).toContain("Hi"); }); - it("returns a fixed Memmy introduction when agents have no sampled memory", async () => { + it("acknowledges detected agents when they have no sampled memory", async () => { const generateReport = vi.fn(async () => "should not be used"); const write = vi.fn(async () => undefined); const service = createOnboardingInsightService({ @@ -97,8 +97,8 @@ describe("onboarding insight service", () => { expect(report.status).toBe("ready"); expect(report.reportMarkdown).toBe([ - "这台设备上还没有可读取的 Agent 历史，所以我不会假装已经了解你。", - "先告诉 Memmy 一件你正在做的真实任务。它会记住有用的背景、决策和下一步；之后新开对话，或换到 Cursor、Codex，也不用再从头解释。" + "Memmy 已识别到这台设备上的 Codex，但首次轻量扫描暂时没有读到可用的对话历史。", + "之后用 Memmy 处理真实任务时，它会记住有用的背景、决策和下一步，方便新对话或其他 Agent 继续。" ].join("\n\n")); expect(report.reportMarkdown).not.toContain("not enough recent user messages"); expect(report.diagnostics).toMatchObject({ @@ -461,9 +461,9 @@ describe("onboarding insight service", () => { throw new Error("generateReport not used"); }, async *streamReport() { - yield "Hi，"; + yield "Hi，"; yield "我已经开始读你的最近任务。\r\n"; - yield "## 接下来可以做\n1. 先验证记忆已完成摘要和索引。"; + yield "## 接下来可以做\n1. 先验证记忆已完成摘要和索引。"; } }, memoryWriter: { write }, @@ -506,6 +506,94 @@ describe("onboarding insight service", () => { })); }); + it("drops model planning text before the report envelope from the stream and final report", async () => { + const reportText = "Hi Jiang，\n\n## 你的偏好\n- 使用中文。"; + const service = createOnboardingInsightService({ + samplers: [sampler("codex", "Codex", [query("codex", "1", "生成初见报告")])], + reportGenerator: { + async generateReport() { + throw new Error("generateReport not used"); + }, + async *streamReport() { + yield "好的，我会严格按照你的要求，不暴露 homePathName。\n"; + yield `${reportText}`; + yield ""; + } + }, + now: () => 100 + }); + + const events = await collectStreamEvents(service.streamReport({ locale: "zh-CN" })); + const visibleText = events + .filter((event): event is { type: "chunk"; delta: string } => + Boolean(event && typeof event === "object" && (event as { type?: unknown }).type === "chunk")) + .map((event) => event.delta) + .join(""); + const done = events.find((event) => + event && typeof event === "object" && (event as { type?: unknown }).type === "done" + ) as { response: { reportMarkdown: string } } | undefined; + + expect(visibleText).toBe(reportText); + expect(visibleText).not.toContain("严格按照你的要求"); + expect(visibleText).not.toContain("homePathName"); + expect(done?.response.reportMarkdown).toBe(reportText); + }); + + it("removes raw HTML split across streamed report chunks while preserving its text", async () => { + const service = createOnboardingInsightService({ + samplers: [sampler("codex", "Codex", [query("codex", "1", "生成初见报告")])], + reportGenerator: { + async generateReport() { + throw new Error("generateReport not used"); + }, + async *streamReport() { + yield "Hi Jiang，\n\n以上内容依据现有证据整理"; + yield "\n\n## 接下来可以做\n暂时没有明确待办。"; + } + }, + now: () => 100 + }); + + const events = await collectStreamEvents(service.streamReport({ locale: "zh-CN" })); + const visibleText = events + .filter((event): event is { type: "chunk"; delta: string } => + Boolean(event && typeof event === "object" && (event as { type?: unknown }).type === "chunk")) + .map((event) => event.delta) + .join(""); + const done = events.find((event) => + event && typeof event === "object" && (event as { type?: unknown }).type === "done" + ) as { response: { reportMarkdown: string } } | undefined; + + expect(visibleText).toContain("以上内容依据现有证据整理"); + expect(visibleText).not.toContain(" { + let finishWrite = () => undefined; + const write = vi.fn(() => new Promise((resolve) => { + finishWrite = resolve; + })); + const service = createOnboardingInsightService({ + samplers: [ + sampler("codex", "Codex", [ + query("codex", "1", "直接读取最近任务并快速生成初见报告") + ]) + ], + reportGenerator: null, + memoryWriter: { write }, + now: () => 100 + }); + + const report = await service.generateReport({ locale: "zh-CN" }); + + expect(report.status).toBe("ready"); + expect(write).toHaveBeenCalledTimes(1); + finishWrite(); + }); + it("keeps task context hidden even when the model omits the report closing tag", async () => { const write = vi.fn(async () => undefined); const service = createOnboardingInsightService({ @@ -684,7 +772,7 @@ describe("onboarding insight service", () => { ) as { response: { reportMarkdown: string } } | undefined; expect(report.reportMarkdown).toBe(reportText); - expect(visibleText).toBe(reportText); + expect(visibleText).toBe(""); expect(done?.response.reportMarkdown).toBe(reportText); } ); @@ -716,7 +804,7 @@ describe("onboarding insight service", () => { event && typeof event === "object" && (event as { type?: unknown }).type === "done" ) as { response: { reportMarkdown: string } } | undefined; - expect(visibleText).toBe(reportText); + expect(visibleText).toBe(""); expect(done?.response.reportMarkdown).toBe(reportText); }); @@ -772,7 +860,7 @@ describe("onboarding insight service", () => { throw new Error("generateReport not used"); }, async *streamReport() { - yield "## 最近项目记忆\n正文先展示。"; + yield "## 最近项目记忆\n正文先展示。"; yield "\n{"; yield `${JSON.stringify(taskContext).slice(1)}`; } @@ -809,9 +897,9 @@ describe("onboarding insight service", () => { throw new Error("generateReport not used"); }, async *streamReport() { - yield "报告包含["; + yield "报告包含["; yield "普通说明]，"; - yield "仍然应该正常显示。"; + yield "仍然应该正常显示。"; } }, now: () => 100 @@ -853,15 +941,21 @@ describe("onboarding insight service", () => { now: () => Date.now() }); - const eventsPromise = collectStreamEvents(service.streamReport({ locale: "zh-CN" })); + const eventsPromise = collectStreamEvents(service.streamReport({ + locale: "zh-CN", + detectedAgents: [{ sourceId: "slow_agent", displayName: "Slow Agent", recentSessionCount: 7 }] + })); await vi.advanceTimersByTimeAsync(3_000); const events = await eventsPromise; expect(events[0]).toMatchObject({ type: "sampled", diagnostics: { - discoveredAgentCount: 1, - sampledQueryCount: 1 + discoveredAgentCount: 2, + sampledQueryCount: 1, + agents: expect.arrayContaining([ + expect.objectContaining({ sourceId: "slow_agent", recentSessionCount: 7 }) + ]) } }); expect(events.at(-1)).toMatchObject({ @@ -869,8 +963,11 @@ describe("onboarding insight service", () => { response: { status: "ready", diagnostics: { - discoveredAgentCount: 1, - sampledQueryCount: 1 + discoveredAgentCount: 2, + sampledQueryCount: 1, + agents: expect.arrayContaining([ + expect.objectContaining({ sourceId: "slow_agent", recentSessionCount: 7 }) + ]) } } }); @@ -963,8 +1060,8 @@ describe("onboarding insight service", () => { const body = JSON.parse(String(fetchImpl.mock.calls[0]?.[1]?.body)); expect(body.model).toBe("agent_chat"); expect(body.max_tokens).toBe(2000); - expect(body.enable_thinking).toBe(true); - expect(body.thinking_budget).toBe(500); + expect(body.enable_thinking).toBe(false); + expect(body).not.toHaveProperty("thinking_budget"); expect(body).not.toHaveProperty("reasoning_effort"); expect(body.messages[0].content).not.toContain("保持 4-6 个短段落"); expect(body.messages[0].content).toContain("latestConversation 是所有已扫描 Agent 中时间最新的一个会话"); @@ -978,6 +1075,8 @@ describe("onboarding insight service", () => { expect(body.messages[0].content).toContain("不得把名字替换成“这个线索”"); expect(body.messages[0].content).toContain("有值时要自然说明用户最近更常用中文还是英文"); expect(body.messages[0].content).toContain("不要生成按钮、行动卡片、CTA"); + expect(body.messages[0].content).toContain("不得包含任何原始 HTML 标签或样式"); + expect(body.messages[0].content).toContain("不要输出思考过程、执行计划、要求确认、Prompt 复述或起草说明"); expect(body.messages[0].content).not.toContain("[MEMMY_ACTIONS_JSON]"); const userPayload = JSON.parse(String(body.messages[1].content)); expect(userPayload.reportGoal.primary).toBe("user_preferences_latest_project_memory_and_actionable_todos"); diff --git a/App/backend/src/services/tests/skill-distribution-service.test.ts b/App/backend/src/services/tests/skill-distribution-service.test.ts index 2ddaf12e0..b5abf7768 100644 --- a/App/backend/src/services/tests/skill-distribution-service.test.ts +++ b/App/backend/src/services/tests/skill-distribution-service.test.ts @@ -1,5 +1,5 @@ /** Skill distribution service tests. */ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -40,6 +40,34 @@ describe("skill distribution service", () => { } }); + it("skips broken directory links while scanning valid sibling skills", async () => { + const rootDirectory = mkdtempSync(join(tmpdir(), "memmy-agent-broken-skill-link-")); + try { + mkdirSync(join(rootDirectory, "skills", "valid-skill"), { recursive: true }); + writeFileSync( + join(rootDirectory, "skills", "valid-skill", "SKILL.md"), + "---\nname: valid-skill\n---\nStill discoverable.\n", + "utf8" + ); + symlinkSync( + join(rootDirectory, "missing-skill-target"), + join(rootDirectory, "skills", "broken-skill"), + "junction" + ); + const service = createSkillDistributionService({ + targetRegistry: createSkillTargetRegistry([ + createFakeTarget({ resolveRootDirectory: () => rootDirectory }) + ]) + }); + + await expect(service.listSkills?.("cursor")).resolves.toEqual([ + expect.objectContaining({ sourceSkillId: "valid-skill" }) + ]); + } finally { + rmSync(rootDirectory, { recursive: true, force: true }); + } + }); + it("renders and installs the fixed Memmy skill manifest", async () => { let installed: SkillManifest | undefined; const service = createSkillDistributionService({ diff --git a/App/backend/src/tests/index.test.ts b/App/backend/src/tests/index.test.ts index 314571939..5f5e83bba 100644 --- a/App/backend/src/tests/index.test.ts +++ b/App/backend/src/tests/index.test.ts @@ -239,7 +239,7 @@ describe("local api", () => { } }); - it("fails fast when no real Memory Layer or local SQLite memory source is configured", async () => { + it("fails fast when no HTTP Memory Layer is configured", async () => { const previousMemoryLayerUrl = process.env.MEMMY_MEMORY_LAYER_URL; const previousMemoryDbPath = process.env.MEMMY_MEMORY_DB_PATH; const previousMemosDbPath = process.env.MEMMY_MEMOS_DB_PATH; @@ -259,7 +259,7 @@ describe("local api", () => { cloudClient: createMockCloudClient(), memmyConfigPath: join(tempDir, "config.yaml") }) - ).rejects.toThrow("MEMMY_MEMORY_LAYER_URL or a local Memmy memory SQLite source is required"); + ).rejects.toThrow("MEMMY_MEMORY_LAYER_URL is required"); } finally { restoreOptionalEnv("MEMMY_MEMORY_LAYER_URL", previousMemoryLayerUrl); restoreOptionalEnv("MEMMY_MEMORY_DB_PATH", previousMemoryDbPath); diff --git a/App/frontend/desktop/src/analytics/gtag-init.ts b/App/frontend/desktop/src/analytics/gtag-init.ts index 630cbbc23..ee1a570a4 100644 --- a/App/frontend/desktop/src/analytics/gtag-init.ts +++ b/App/frontend/desktop/src/analytics/gtag-init.ts @@ -16,15 +16,15 @@ const MEASUREMENT_ID = (import.meta.env.MEMMY_GA4_MEASUREMENT_ID as string | und let initialized = false; -export function initGtag(): void { +export function initGtag(measurementId = MEASUREMENT_ID): void { if (initialized) return; void initializeDesktopAnalyticsContext(); - if (!MEASUREMENT_ID) { + if (!measurementId) { console.log("[analytics] initGtag skipped: MEMMY_GA4_MEASUREMENT_ID not set"); return; } initialized = true; - console.log("[analytics] initGtag starting, MEASUREMENT_ID:", MEASUREMENT_ID); + console.log("[analytics] initGtag starting, MEASUREMENT_ID:", measurementId); window.dataLayer = window.dataLayer || []; // eslint-disable-next-line prefer-rest-params @@ -32,12 +32,12 @@ export function initGtag(): void { window.gtag("js", new Date()); const configOptions = resolveGtagConfigOptions(); - window.gtag("config", MEASUREMENT_ID, configOptions); + window.gtag("config", measurementId, configOptions); console.log("[analytics] gtag config:", configOptions); const script = document.createElement("script"); script.async = true; - script.src = `https://www.googletagmanager.com/gtag/js?id=${MEASUREMENT_ID}`; + script.src = `https://www.googletagmanager.com/gtag/js?id=${measurementId}`; document.head.appendChild(script); console.log("[analytics] gtag.js script injection started:", script.src); @@ -48,7 +48,12 @@ export function initGtag(): void { // After the script finishes loading, obtain the client_id and pass it to the main process for later use script.onload = () => { console.log("[analytics] gtag.js script loaded successfully"); - window.gtag("get", MEASUREMENT_ID, "client_id", (clientId: unknown) => { + // This browser-side event opens the GA4 session and enables the automatic + // session_start / first_visit events. Product app_launch goes through cloud. + window.gtag("event", "app_init"); + console.log("[analytics] app_init sent via gtag"); + + window.gtag("get", measurementId, "client_id", (clientId: unknown) => { if (typeof clientId === "string" && clientId) { // Memory gate for Desktop → cloud UI events (do not read shared file here). setDesktopAnalyticsClientId(clientId); @@ -57,13 +62,10 @@ export function initGtag(): void { appEnv: resolveAnalyticsAppEnv(), appEdition: resolveAnalyticsAppEdition() }); - console.log("[analytics] gtag client_id ready:", clientId); } }); - // app_launch stays on gtag so GA4 can auto-collect session_start/first_visit. - window.gtag("event", "app_launch"); - console.log("[analytics] app_launch sent via gtag"); + trackCloudAnalyticsEvent("app_launch"); }; } @@ -87,7 +89,7 @@ async function initializeDesktopAnalyticsContext(): Promise { /** * Desktop UI events go through cloud `/api/analytics/events`. - * Kept as `gtagEvent` for call-site compatibility; only `app_launch` uses gtag directly. + * Kept as `gtagEvent` for call-site compatibility; app_init is emitted directly during setup. */ export function gtagEvent( name: string, diff --git a/App/frontend/desktop/src/analytics/tests/gtag-init.test.ts b/App/frontend/desktop/src/analytics/tests/gtag-init.test.ts new file mode 100644 index 000000000..9a2a74ea3 --- /dev/null +++ b/App/frontend/desktop/src/analytics/tests/gtag-init.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment happy-dom + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const analyticsMocks = vi.hoisted(() => ({ + setDesktopAnalyticsClientId: vi.fn(), + setDesktopAnalyticsContext: vi.fn(), + trackCloudAnalyticsEvent: vi.fn(), +})); + +vi.mock("../cloud-analytics.js", () => analyticsMocks); + +describe("gtag initialization", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + document.head.innerHTML = ""; + window.dataLayer = []; + Reflect.deleteProperty(window, "gtag"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("uses app_init to open the GA session and sends app_launch through cloud without a session id", async () => { + const { initGtag } = await import("../gtag-init.js"); + const appendChild = vi + .spyOn(document.head, "appendChild") + .mockImplementation((node) => node); + + initGtag("G-TEST"); + + const injectedScript = appendChild.mock.calls[0]?.[0] as HTMLScriptElement | undefined; + expect(injectedScript?.src).toBe("https://www.googletagmanager.com/gtag/js?id=G-TEST"); + + injectedScript?.dispatchEvent(new Event("load")); + + const commands = window.dataLayer.map((entry) => Array.from(entry)); + expect(commands).toContainEqual(["event", "app_init"]); + expect(commands).not.toContainEqual(["event", "app_launch"]); + + const clientIdCommand = commands.find( + ([command, , field]) => command === "get" && field === "client_id" + ); + const sessionIdCommand = commands.find( + ([command, , field]) => command === "get" && field === "session_id" + ); + expect(clientIdCommand).toBeDefined(); + expect(sessionIdCommand).toBeUndefined(); + + const clientIdCallback = clientIdCommand?.[3] as ((value: unknown) => void) | undefined; + clientIdCallback?.("client-123"); + + expect(analyticsMocks.setDesktopAnalyticsClientId).toHaveBeenCalledWith("client-123"); + expect(analyticsMocks.trackCloudAnalyticsEvent).toHaveBeenCalledWith("app_launch"); + }); +}); diff --git a/App/frontend/desktop/src/api/agent-source-client.ts b/App/frontend/desktop/src/api/agent-source-client.ts index d3a747388..5bae7edbf 100644 --- a/App/frontend/desktop/src/api/agent-source-client.ts +++ b/App/frontend/desktop/src/api/agent-source-client.ts @@ -4,6 +4,7 @@ import { AgentSourceScanJobResponseSchema, AgentSourceScanInputSchema, AgentSourceScanStatusResponseSchema, + ScanResultPageSchema, AgentSourceViewSchema, ManagedAgentSourceImportResultSchema, OkResponseSchema, @@ -15,6 +16,7 @@ import { type AgentSourceScanInput, type AgentSourceScanStatusResponse, type AgentSourceView, + type ScanResultPage, type ManagedAgentSourceImportResult, type RuntimeConfig } from "@memmy/local-api-contracts"; @@ -24,6 +26,7 @@ export interface AgentSourceClient { listSources(): Promise; startScan(input?: AgentSourceScanInput): Promise; getScanStatus(): Promise; + getScanResults(jobId: string, cursor?: string, limit?: number): Promise; stopScan(): Promise; cancelScan(): Promise; addManualSource(input: AddManualInput): Promise; @@ -63,6 +66,14 @@ export function createHttpAgentSourceClient(config: RuntimeConfig): AgentSourceC }); }, + async getScanResults(jobId, cursor = "0", limit = 100) { + return requestJson({ + config, + path: `/api/agent-sources/scan/jobs/${encodeURIComponent(jobId)}/results?cursor=${encodeURIComponent(cursor)}&limit=${encodeURIComponent(String(limit))}`, + schema: ScanResultPageSchema + }); + }, + async getMemoryPluginConflicts() { const response = await requestJson({ config, diff --git a/App/frontend/desktop/src/api/config-client.ts b/App/frontend/desktop/src/api/config-client.ts index f1ecd2c49..00c147c71 100644 --- a/App/frontend/desktop/src/api/config-client.ts +++ b/App/frontend/desktop/src/api/config-client.ts @@ -135,6 +135,7 @@ export interface ConfigClient { setImprovementProgram(accepted: boolean): Promise; getTokenUsage(): Promise; updateScanPermission(permission: ScanPermission): Promise>; + getScanPreferences(): Promise; updateScanPreferences(preferences: Partial): Promise; getModelConfig(): Promise; saveModelCatalog(config: ModelConfigInput | ModelConfigView): Promise; @@ -216,6 +217,14 @@ export function createHttpConfigClient(config: RuntimeConfig): ConfigClient { }); }, + async getScanPreferences() { + return requestJson({ + config, + path: "/api/app/scan-preferences", + schema: ScanPreferencesSchema + }); + }, + async getModelConfig() { const response = await requestJson({ config, @@ -754,17 +763,10 @@ function fromModelConfigView(view: ModelConfigView): ModelProviderConfig { apiKey: selectedEndpoint?.apiKey ?? "", apiKeyMasked: selectedEndpoint?.apiKeyMasked ?? "", configured: view.configured, - embedding: embeddingPreset && embeddingEndpoint ? { - mode: "custom", - endpoint: embeddingEndpoint.apiBase, - model: embeddingPreset.model, - apiKey: embeddingEndpoint.apiKey, - apiKeyMasked: embeddingEndpoint.apiKeyMasked, - configured: embeddingPreset.available - } : null, + embedding: memoryEmbeddingFromView(view, embeddingPreset, embeddingEndpoint), memmyMemory: { - summary: fromPresetRole(view, summaryPreset, selected), - evolution: fromPresetRole(view, evolutionPreset, selected) + summary: fromPresetRole(view, summaryPreset, selected, view.memorySettings?.roleRouting.summary), + evolution: fromPresetRole(view, evolutionPreset, selected, view.memorySettings?.roleRouting.evolution) }, asr: asrPreset ? fromOptionalPreset(view, asrPreset) : null, imageGen: imagePreset ? fromOptionalPreset(view, imagePreset) : null @@ -774,12 +776,13 @@ function fromModelConfigView(view: ModelConfigView): ModelProviderConfig { function fromPresetRole( view: ModelConfigView, preset: ModelConfigView["providers"][number]["models"][number] | null, - primary: ModelConfigView["providers"][number]["models"][number] | null + primary: ModelConfigView["providers"][number]["models"][number] | null, + routing?: "follow" | "fixed" ): RoleModelProviderConfig { const selected = preset ?? primary; const endpoint = selected ? findEndpoint(view, selected) : null; return { - mode: preset ? "fixed" : "follow", + mode: routing ?? (preset ? "fixed" : "follow"), provider: selected?.provider ?? "openai", endpoint: endpoint?.apiBase ?? "", model: selected?.model ?? "", @@ -789,6 +792,22 @@ function fromPresetRole( }; } +function memoryEmbeddingFromView( + view: ModelConfigView, + preset: ModelConfigView["providers"][number]["models"][number] | null, + endpoint: ModelConfigView["providers"][number]["endpoints"][number] | null +): EmbeddingProviderConfig { + const mode = view.memorySettings?.embeddingMode ?? (preset ? "custom" : "local"); + return { + mode, + endpoint: endpoint?.apiBase ?? "", + model: preset?.model ?? "", + apiKey: endpoint?.apiKey ?? "", + apiKeyMasked: endpoint?.apiKeyMasked ?? "", + configured: mode === "local" || Boolean(preset?.available) + }; +} + function fromOptionalPreset(view: ModelConfigView, preset: ModelConfigView["providers"][number]["models"][number]) { const endpoint = findEndpoint(view, preset); return { diff --git a/App/frontend/desktop/src/app/routes.ts b/App/frontend/desktop/src/app/routes.ts index 0986de53a..d91a76676 100644 --- a/App/frontend/desktop/src/app/routes.ts +++ b/App/frontend/desktop/src/app/routes.ts @@ -134,7 +134,8 @@ export function resolveInitialView(input: ResolveInitialViewInput): AppRoutePath return "/welcome"; } - if (hasCompletedAccountGuide(input.accountSession) || input.guidanceCompleted) { + if (!shouldShowFirstEncounterReport(input.bootstrap.onboarding) && + (hasCompletedAccountGuide(input.accountSession) || input.guidanceCompleted)) { return input.preferredMode === "pet" ? "/pet" : "/main"; } @@ -164,6 +165,16 @@ function hasCompletedAccountGuide(session: AccountSessionView | undefined): bool /** Handles reconcile initial onboarding. */ export function reconcileInitialOnboarding(input: ReconcileInitialOnboardingInput): AppBootstrapResponse { + const firstEncounterPending = shouldShowFirstEncounterReport(input.bootstrap.onboarding); + if (firstEncounterPending && input.bootstrap.onboarding.completed) { + const onboarding = input.bootstrap.app.userMode === "byok" + ? buildByokOnboardingGuidePatch(input.bootstrap.onboarding) + : input.bootstrap.app.userMode === "account" && input.accountSession?.authenticated + ? buildAccountOnboardingStartPatch(input.bootstrap.onboarding) + : null; + return onboarding ? { ...input.bootstrap, onboarding } : input.bootstrap; + } + if ( input.bootstrap.app.userMode !== "account" || !input.accountSession?.authenticated || @@ -177,7 +188,7 @@ export function reconcileInitialOnboarding(input: ReconcileInitialOnboardingInpu ...input.bootstrap, onboarding: { ...input.bootstrap.onboarding, - ...buildAccountOnboardingStartPatch() + ...buildAccountOnboardingStartPatch(input.bootstrap.onboarding) } }; } @@ -221,7 +232,7 @@ export function resolveByokModelCompletion(input: ResolveByokModelCompletionInpu } return { - onboardingPatch: buildByokOnboardingGuidePatch(), + onboardingPatch: buildByokOnboardingGuidePatch(input.onboarding), nextRoute: "/onboarding" }; } @@ -250,7 +261,7 @@ export function resolveByokEntry(input: ResolveByokEntryInput): ResolveByokEntry } return { - onboardingPatch: buildByokOnboardingSetupPatch(), + onboardingPatch: buildByokOnboardingSetupPatch(input.onboarding), nextRoute: "/api-key" }; } @@ -676,13 +687,18 @@ export function buildOnboardingCompletionPatch(completedAt: string): Partial +): OnboardingStateDto { return { completed: false, currentStep: "scan_permission_required", hasAcceptedTerms: true, acceptedTermsVersion: null, - scanPermission: "unset", + scanPermission: installationState?.scanPermission ?? "unset", + ...(installationState?.firstEncounterReportStatus + ? { firstEncounterReportStatus: installationState.firstEncounterReportStatus } + : {}), improvementProgram: "unset", completedAt: null }; @@ -693,13 +709,18 @@ export function buildAccountOnboardingStartPatch(): OnboardingStateDto { * * @returns the onboarding patch for the BYOK first-time flow before entering the API Key configuration page. */ -export function buildByokOnboardingSetupPatch(): OnboardingStateDto { +export function buildByokOnboardingSetupPatch( + installationState?: Pick +): OnboardingStateDto { return { completed: false, currentStep: "byok_setup_required", hasAcceptedTerms: true, acceptedTermsVersion: null, - scanPermission: "unset", + scanPermission: installationState?.scanPermission ?? "unset", + ...(installationState?.firstEncounterReportStatus + ? { firstEncounterReportStatus: installationState.firstEncounterReportStatus } + : {}), improvementProgram: "not_applicable", completedAt: null }; @@ -710,13 +731,19 @@ export function buildByokOnboardingSetupPatch(): OnboardingStateDto { * * @returns the patch for entering `/onboarding` after BYOK model configuration completes. */ -export function buildByokOnboardingGuidePatch(): OnboardingStateDto { +export function buildByokOnboardingGuidePatch( + installationState?: Pick +): OnboardingStateDto { return { - ...buildByokOnboardingSetupPatch(), + ...buildByokOnboardingSetupPatch(installationState), currentStep: "scan_permission_required" }; } +export function shouldShowFirstEncounterReport(onboarding: OnboardingStateDto): boolean { + return (onboarding.firstEncounterReportStatus ?? "pending") === "pending"; +} + /** * Resolves the target route for the given launch-form preference. * diff --git a/App/frontend/desktop/src/app/tests/routes.test.ts b/App/frontend/desktop/src/app/tests/routes.test.ts index 8baab83c8..6556f70d5 100644 --- a/App/frontend/desktop/src/app/tests/routes.test.ts +++ b/App/frontend/desktop/src/app/tests/routes.test.ts @@ -28,6 +28,7 @@ import { resolvePreferredLaunchMode, resolveReloadedInitialView, shouldExitPetLaunchForRoute, + shouldShowFirstEncounterReport, shouldShowTokenExhaustedModal, routeTable, writeCurrentRoute, @@ -163,6 +164,7 @@ describe("desktop route table", () => { ...baseBootstrap.onboarding, completed: true, currentStep: "completed" as const, + firstEncounterReportStatus: "shown" as const, completedAt: "2026-06-04T00:00:00.000Z" } }; @@ -191,6 +193,41 @@ describe("desktop route table", () => { })).toBe("/main"); }); + it("keeps a pending BYOK first report after completion carryover once a model is configured", () => { + const carriedBootstrap = { + ...baseBootstrap, + app: { ...baseBootstrap.app, userMode: "byok" as const }, + onboarding: { + ...baseBootstrap.onboarding, + completed: true, + currentStep: "completed" as const, + firstEncounterReportStatus: "pending" as const, + completedAt: "2026-06-04T00:00:00.000Z" + } + }; + const reconciled = reconcileInitialOnboarding({ bootstrap: carriedBootstrap }); + + expect(reconciled.onboarding).toMatchObject({ + completed: false, + currentStep: "scan_permission_required", + firstEncounterReportStatus: "pending" + }); + expect(resolveInitialView({ + bootstrap: reconciled, + preferredMode: "full", + modelConfig: { + catalog: { modelAssignments: { byok: { agent: { candidates: [] } } } } + } + })).toBe("/api-key"); + expect(resolveInitialView({ + bootstrap: reconciled, + preferredMode: "full", + modelConfig: { + catalog: { modelAssignments: { byok: { agent: { candidates: ["local-agent"] } } } } + } + })).toBe("/onboarding"); + }); + it("respects the preferred full or pet mode after onboarding is complete", () => { const completedBootstrap = { ...baseBootstrap, @@ -199,6 +236,7 @@ describe("desktop route table", () => { ...baseBootstrap.onboarding, completed: true, currentStep: "completed" as const, + firstEncounterReportStatus: "shown" as const, completedAt: "2026-06-01T00:00:00.000Z" } }; @@ -231,6 +269,7 @@ describe("desktop route table", () => { ...baseBootstrap.onboarding, completed: false, currentStep: "scan_permission_required" as const, + firstEncounterReportStatus: "shown" as const, completedAt: null } }; @@ -358,7 +397,8 @@ describe("desktop route table", () => { resolveInitialView({ bootstrap: { ...baseBootstrap, - app: { ...baseBootstrap.app, userMode: "account" } + app: { ...baseBootstrap.app, userMode: "account" }, + onboarding: { ...baseBootstrap.onboarding, firstEncounterReportStatus: "shown" } }, preferredMode: "full", accountSession: { @@ -380,6 +420,35 @@ describe("desktop route table", () => { ).toBe("/main"); }); + it("shows the local first encounter flow for an old cloud account on a new installation", () => { + expect( + resolveInitialView({ + bootstrap: { + ...baseBootstrap, + app: { ...baseBootstrap.app, userMode: "account" }, + onboarding: { ...baseBootstrap.onboarding, firstEncounterReportStatus: "pending" } + }, + preferredMode: "full", + guidanceCompleted: true, + accountSession: { + authenticated: true, + isNewUser: false, + profile: { + userId: "old-user", + email: "old@example.com", + phoneNumber: null, + nickname: "Old User", + avatarUrl: null, + planType: null, + hasFinishedGuide: true, + region: null, + registeredAt: "2025-01-01T00:00:00.000Z" + } + } + }) + ).toBe("/onboarding"); + }); + it("continues onboarding for authenticated account users whose guide is unfinished", () => { expect( resolveInitialView({ @@ -416,6 +485,7 @@ describe("desktop route table", () => { completed: true, currentStep: "completed" as const, scanPermission: "scan_only" as const, + firstEncounterReportStatus: "shown" as const, improvementProgram: "accepted" as const, completedAt: "2026-06-04T00:00:00.000Z" } @@ -440,7 +510,7 @@ describe("desktop route table", () => { accountSession: unfinishedAccountSession }); - expect(reconciled.onboarding).toMatchObject(buildAccountOnboardingStartPatch()); + expect(reconciled.onboarding).toMatchObject(buildAccountOnboardingStartPatch(staleCompletedBootstrap.onboarding)); expect(resolveInitialView({ bootstrap: reconciled, preferredMode: "full", accountSession: unfinishedAccountSession })).toBe("/onboarding"); }); @@ -575,6 +645,18 @@ describe("desktop route table", () => { improvementProgram: "not_applicable", completedAt: null }); + expect(buildAccountOnboardingStartPatch({ + scanPermission: "scan_only", + firstEncounterReportStatus: "shown" + })).toMatchObject({ + scanPermission: "scan_only", + firstEncounterReportStatus: "shown" + }); + expect(shouldShowFirstEncounterReport(buildAccountOnboardingStartPatch())).toBe(true); + expect(shouldShowFirstEncounterReport(buildAccountOnboardingStartPatch({ + scanPermission: "none", + firstEncounterReportStatus: "skipped" + }))).toBe(false); }); it("resolves the first route from the saved launch mode preference", () => { diff --git a/App/frontend/desktop/src/app/tests/update-coordinator.test.tsx b/App/frontend/desktop/src/app/tests/update-coordinator.test.tsx index 3e471a665..f4e990c7e 100644 --- a/App/frontend/desktop/src/app/tests/update-coordinator.test.tsx +++ b/App/frontend/desktop/src/app/tests/update-coordinator.test.tsx @@ -50,7 +50,9 @@ describe("UpdateCoordinatorProvider", () => { name: "Memmy", version: "2.1.0", platform: "darwin", - arch: "arm64" + arch: "arm64", + isPackaged: true, + isWindowsStore: false })), checkForUpdates, downloadUpdate @@ -124,7 +126,9 @@ describe("UpdateCoordinatorProvider", () => { name: "Memmy", version: "2.1.0", platform: "darwin", - arch: "arm64" + arch: "arm64", + isPackaged: true, + isWindowsStore: false })), checkForUpdates, downloadUpdate @@ -185,7 +189,9 @@ describe("UpdateCoordinatorProvider", () => { name: "Memmy", version: "2.1.0", platform: "darwin", - arch: "arm64" + arch: "arm64", + isPackaged: true, + isWindowsStore: false })), checkForUpdates, openUpdateInstaller @@ -242,7 +248,9 @@ describe("UpdateCoordinatorProvider", () => { name: "Memmy", version: "2.1.0", platform: "darwin", - arch: "arm64" + arch: "arm64", + isPackaged: true, + isWindowsStore: false })), checkForUpdates: vi.fn(async () => ({ status: "available" as const, diff --git a/App/frontend/desktop/src/global.d.ts b/App/frontend/desktop/src/global.d.ts index 7286ba7c7..7272742ae 100644 --- a/App/frontend/desktop/src/global.d.ts +++ b/App/frontend/desktop/src/global.d.ts @@ -47,6 +47,8 @@ declare global { exportDiagnosticsReport(): Promise; getLogLevel(): Promise<"error" | "warn" | "info" | "debug">; setLogLevel(level: "error" | "warn" | "info" | "debug"): Promise; + getLaunchAtLogin(): Promise; + setLaunchAtLogin(enabled: boolean): Promise; getMicrophoneAccessStatus(): Promise; requestMicrophoneAccess(): Promise; selectProjectDirectory(): Promise; diff --git a/App/frontend/desktop/src/i18n/messages.ts b/App/frontend/desktop/src/i18n/messages.ts index d64593e9a..390ee02b0 100644 --- a/App/frontend/desktop/src/i18n/messages.ts +++ b/App/frontend/desktop/src/i18n/messages.ts @@ -316,6 +316,8 @@ export const zhCNMessages = { "apiKey.modelPage.skillSubtitle": "持续打磨你的 Agent 技能与偏好", "apiKey.modelPage.reusePrevious": "沿用上一步的 Agent 任务模型", "apiKey.modelPage.reuseAgent": "沿用 Agent 任务模型", + "apiKey.modelPage.reuseEvolution": "继承技能进化模型", + "apiKey.modelPage.reuseAgentChat": "继承 Agent Chat 模型", "onboarding.permission.title": "Memmy 需要你的授权", "onboarding.permission.subtitle": "首次导入历史，之后让每个 AI 自动接上上下文", "onboarding.permission.scanTitle": "扫描已有 Agent 对话", @@ -825,6 +827,10 @@ export const zhCNMessages = { "memory.preferences": "自动同步", "memory.autoScan": "自动同步会话", "memory.autoScanDescription": "自动从已接入的 Agent 采集新对话，无需手动点「同步新增」", + "memory.startupScan": "启动时主动扫描", + "memory.startupScanDescription": "Memmy 启动后自动扫描已接入 Agent 的新增会话", + "memory.scheduledScan": "定时扫描", + "memory.scheduledScanDescription": "Memmy 运行期间每小时扫描一次已接入 Agent 的新增会话", "memory.autoInject": "发现新 Agent 时自动接入", "memory.autoInjectDescription": "自动安装接入组件；关闭后只出现在下方列表，由你手动接入", "memory.scan": "同步新增", @@ -1209,6 +1215,7 @@ export const zhCNMessages = { "memory.localData": "数据管理", "memory.localDataPath": "本地数据存储位置", "memory.cliPath": "~/.local/bin/memmy-memory", + "memory.cliPathWindows": "memmy-memory.cmd", "memory.cliInstallDone": "已安装到 {path}", "memory.cliInstallDonePathUpdated": "已安装到 {path}。已写入 {profiles}，重开终端后生效。", "memory.cliInstallUnavailable": "当前运行环境不支持安装 CLI", @@ -1415,9 +1422,9 @@ export const zhCNMessages = { "settings.modelWorkspace.platformName": "Memmy Platform", "settings.modelWorkspace.defaultModel": "默认", "settings.modelWorkspace.setDefaultModel": "设为默认", - "settings.modelWorkspace.platformEmbedding": "Memmy Platform · Embedding", - "settings.modelWorkspace.localEmbedding": "本地 · Xenova/all-MiniLM-L6-v2", - "settings.modelWorkspace.localEmbeddingShort": "本地 Embedding", + "settings.modelWorkspace.platformEmbedding": "Memmy Platform 云端 Embedding", + "settings.modelWorkspace.localEmbedding": "Memmy Platform 本地 Embedding", + "settings.modelWorkspace.localEmbeddingShort": "Memmy Platform 本地 Embedding", "settings.modelWorkspace.specialBuiltins": "内置能力", "settings.modelWorkspace.saveFailed": "模型配置保存失败，请重试；如仍失败，请重启应用后再试。", "settings.modelWorkspace.saveBusy": "模型配置正在被其他操作占用，请稍后重试。", @@ -1531,6 +1538,8 @@ export const zhCNMessages = { "settings.window.menuBarIcon": "显示菜单栏图标", "settings.window.menuBarIconDesc": "在 macOS 状态栏常驻 Memmy 图标，便于随时呼出", "settings.window.menuBarIconDescWindows": "在 Windows 状态栏常驻 Memmy 图标，便于随时呼出", + "settings.window.stopMemoryOnExit": "退出后停止记忆服务", + "settings.window.stopMemoryOnExitDesc": "默认关闭；开启后退出 Memmy Desktop 时同时停止独立记忆服务", "settings.notifications": "通知", "settings.notifications.update": "软件更新通知", "settings.notifications.updateDesc": "有新版本时发送系统通知", @@ -1943,6 +1952,8 @@ export const enUSMessages: Record = { "apiKey.modelPage.skillSubtitle": "Continuously refine your Agent skills and preferences", "apiKey.modelPage.reusePrevious": "Reuse the Agent task model from the previous step", "apiKey.modelPage.reuseAgent": "Reuse Agent task model", + "apiKey.modelPage.reuseEvolution": "Inherit skill evolution model", + "apiKey.modelPage.reuseAgentChat": "Inherit Agent Chat model", "onboarding.permission.title": "Memmy needs your authorization", "onboarding.permission.subtitle": "Import history once, then let every AI pick up the context", "onboarding.permission.scanTitle": "Scan existing Agent conversations", @@ -2452,6 +2463,10 @@ export const enUSMessages: Record = { "memory.preferences": "Auto sync", "memory.autoScan": "Auto-sync conversations", "memory.autoScanDescription": "Automatically collect new conversations from connected Agents—no need to click Sync new", + "memory.startupScan": "Scan on startup", + "memory.startupScanDescription": "Scan connected Agents for new conversations after Memmy starts", + "memory.scheduledScan": "Scheduled scan", + "memory.scheduledScanDescription": "Scan connected Agents for new conversations every hour while Memmy is running", "memory.autoInject": "Auto-connect newly found Agents", "memory.autoInjectDescription": "Install the integration automatically; when off, new Agents only appear in the list below for you to connect manually", "memory.scan": "Sync new", @@ -2835,6 +2850,7 @@ export const enUSMessages: Record = { "memory.localData": "Data management", "memory.localDataPath": "Local data storage location", "memory.cliPath": "~/.local/bin/memmy-memory", + "memory.cliPathWindows": "memmy-memory.cmd", "memory.cliInstallDone": "Installed to {path}", "memory.cliInstallDonePathUpdated": "Installed to {path}. Added {profiles}; open a new terminal for it to take effect.", "memory.cliInstallUnavailable": "CLI install is unavailable in this runtime", @@ -3042,9 +3058,9 @@ export const enUSMessages: Record = { "settings.modelWorkspace.platformName": "Memmy Platform", "settings.modelWorkspace.defaultModel": "Default", "settings.modelWorkspace.setDefaultModel": "Set default", - "settings.modelWorkspace.platformEmbedding": "Memmy Platform · Embedding", - "settings.modelWorkspace.localEmbedding": "Local · Xenova/all-MiniLM-L6-v2", - "settings.modelWorkspace.localEmbeddingShort": "Local Embedding", + "settings.modelWorkspace.platformEmbedding": "Memmy Platform Cloud Embedding", + "settings.modelWorkspace.localEmbedding": "Memmy Platform Local Embedding", + "settings.modelWorkspace.localEmbeddingShort": "Memmy Platform Local Embedding", "settings.modelWorkspace.specialBuiltins": "Built-in capabilities", "settings.modelWorkspace.saveFailed": "Could not save the model configuration. Try again, or restart the app if the problem continues.", "settings.modelWorkspace.saveBusy": "The model configuration is busy with another operation. Try again shortly.", @@ -3158,6 +3174,8 @@ export const enUSMessages: Record = { "settings.window.menuBarIcon": "Show menu bar icon", "settings.window.menuBarIconDesc": "Keep a Memmy icon in the macOS status bar for quick access", "settings.window.menuBarIconDescWindows": "Keep a Memmy icon in the Windows system tray for quick access", + "settings.window.stopMemoryOnExit": "Stop Memory when quitting", + "settings.window.stopMemoryOnExitDesc": "Off by default; when enabled, quitting Memmy Desktop also stops the standalone Memory service", "settings.notifications": "Notifications", "settings.notifications.update": "Software update notifications", "settings.notifications.updateDesc": "Send a system notification when a new version is available", diff --git a/App/frontend/desktop/src/pages/app-frame.tsx b/App/frontend/desktop/src/pages/app-frame.tsx index e1f4a6fa3..73b78608b 100644 --- a/App/frontend/desktop/src/pages/app-frame.tsx +++ b/App/frontend/desktop/src/pages/app-frame.tsx @@ -63,7 +63,7 @@ import { Wand2 } from "./memory/memory-prototype-icons.js"; import { SETTINGS_NAV_ITEMS, type SettingsTabId } from "./settings-nav.js"; -import { Check, CheckCheck, ChevronDown, ChevronRight, ChevronsDownUp, ChevronsUpDown, Download, Folder, FolderOpen, FolderPlus, ListFilter, MoreHorizontal, Plus, RotateCcw } from "lucide-react"; +import { ArrowDown, Check, CheckCheck, ChevronDown, ChevronRight, ChevronsDownUp, ChevronsUpDown, Folder, FolderOpen, FolderPlus, ListFilter, MoreHorizontal, Plus, RotateCcw } from "lucide-react"; export interface SettingsSidebarNav { activeTab: SettingsTabId; @@ -188,6 +188,7 @@ interface SidebarUpdateActionView { ariaLabel: string; title: string; disabled: boolean; + progress: number | null; } const navItems: NavItem[] = [ @@ -1489,7 +1490,7 @@ export function AppFrame(props: AppFrameProps) { void update?.requestInlineAction(); }} > - {renderSidebarUpdateActionIcon(sidebarUpdateAction.kind)} + {renderSidebarUpdateActionIcon(sidebarUpdateAction)} {sidebarUpdateAction.label} ); } diff --git a/App/frontend/desktop/src/pages/memory/tests/memory-refresh-button.interaction.test.tsx b/App/frontend/desktop/src/pages/memory/tests/memory-refresh-button.interaction.test.tsx new file mode 100644 index 000000000..e505360db --- /dev/null +++ b/App/frontend/desktop/src/pages/memory/tests/memory-refresh-button.interaction.test.tsx @@ -0,0 +1,56 @@ +// @vitest-environment happy-dom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "../../../i18n/i18n-provider.js"; +import { MemoryRefreshButton } from "../memory-refresh-button.js"; + +describe("MemoryRefreshButton interaction", () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(() => { + vi.useFakeTimers(); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.useRealTimers(); + }); + + it("shows pending, success, then returns to the refresh icon state", async () => { + let resolveRefresh!: () => void; + const onClick = vi.fn(() => new Promise((resolve) => { + resolveRefresh = resolve; + })); + + act(() => { + root.render( + + + + ); + }); + + const button = container.querySelector("button")!; + act(() => button.click()); + expect(button.classList.contains("memory-refresh-button--pending")).toBe(true); + expect(button.getAttribute("aria-label")).toBe("刷新中"); + + await act(async () => { + resolveRefresh(); + await Promise.resolve(); + }); + expect(button.classList.contains("memory-refresh-button--success")).toBe(true); + expect(button.getAttribute("aria-label")).toBe("已刷新"); + expect(button.querySelector("[data-icon='check']")).not.toBeNull(); + + act(() => vi.advanceTimersByTime(1_400)); + expect(button.classList.contains("memory-refresh-button--idle")).toBe(true); + expect(button.getAttribute("aria-label")).toBe("刷新本页"); + }); +}); diff --git a/App/frontend/desktop/src/pages/memory/tests/sources-sub-page-path.interaction.test.tsx b/App/frontend/desktop/src/pages/memory/tests/sources-sub-page-path.interaction.test.tsx index 9d30ab840..64ae10175 100644 --- a/App/frontend/desktop/src/pages/memory/tests/sources-sub-page-path.interaction.test.tsx +++ b/App/frontend/desktop/src/pages/memory/tests/sources-sub-page-path.interaction.test.tsx @@ -49,6 +49,15 @@ describe("SourcesSubPage local data path", () => { agentSources: { listSources }, + config: { + async getScanPreferences() { + return { + autoScanKnownAgents: true, + watchFileChanges: true, + autoInjectSkill: false + }; + } + }, memoryRuntime: { async health() { return { ok: true, storage: { ready: true } }; diff --git a/App/frontend/desktop/src/pages/memory/tests/sources-sub-page.test.tsx b/App/frontend/desktop/src/pages/memory/tests/sources-sub-page.test.tsx index f28b638ec..09fe3e8bd 100644 --- a/App/frontend/desktop/src/pages/memory/tests/sources-sub-page.test.tsx +++ b/App/frontend/desktop/src/pages/memory/tests/sources-sub-page.test.tsx @@ -20,6 +20,7 @@ import { resolveAgentSourceScanButtonState, resolveAgentSourceConnectionAction, resolveManagedAgentSourceSyncButtonState, + resolveMemoryCliPathMessageKey, resolveMemoryDocsUrl, resolveAgentSourceStatusLabelKey, resolveScanContinueSourceId, @@ -29,6 +30,16 @@ import { import { SourcesSubPage } from "../sources-sub-page.js"; describe("SourcesSubPage", () => { + it("为 Windows CLI 卡片提供 .cmd 命令入口而不是 macOS 安装路径", () => { + expect((zhCNMessages as Record)["memory.cliPathWindows"]).toBe("memmy-memory.cmd"); + expect((enUSMessages as Record)["memory.cliPathWindows"]).toBe("memmy-memory.cmd"); + expect(resolveMemoryCliPathMessageKey("win32", true, false)).toBe("memory.cliPathWindows"); + expect(resolveMemoryCliPathMessageKey("win32", false, false)).toBe("memory.cliPath"); + expect(resolveMemoryCliPathMessageKey("win32", true, true)).toBe("memory.cliPath"); + expect(resolveMemoryCliPathMessageKey("darwin", true, false)).toBe("memory.cliPath"); + expect(resolveMemoryCliPathMessageKey(undefined, undefined, undefined)).toBe("memory.cliPath"); + }); + it("只用 Hook 描述 Cursor、Claude Code 和 Codex 的接入操作", () => { expect(zhCNMessages["memory.hookInstalled"]).toBe("已安装 Hook"); expect(zhCNMessages["memory.hookNotInstalled"]).toBe("未安装 Hook"); diff --git a/App/frontend/desktop/src/pages/memory/user-memories-sub-page.tsx b/App/frontend/desktop/src/pages/memory/user-memories-sub-page.tsx index 7106dde6c..0e2744c31 100644 --- a/App/frontend/desktop/src/pages/memory/user-memories-sub-page.tsx +++ b/App/frontend/desktop/src/pages/memory/user-memories-sub-page.tsx @@ -79,7 +79,9 @@ export function UserMemoriesSubPage(props: UserMemoriesSubPageProps) {

{t("memory.userMemories.description")}

- void refresh()} /> + { + await refresh(); + }} />
diff --git a/App/frontend/desktop/src/pages/model-config.ts b/App/frontend/desktop/src/pages/model-config.ts index 51f8a4e65..8ff6dd404 100644 --- a/App/frontend/desktop/src/pages/model-config.ts +++ b/App/frontend/desktop/src/pages/model-config.ts @@ -251,6 +251,18 @@ export function createModelFormValues(config: ModelConfig, primary: PrimaryModel }; } +/** Converts resolved form values into the inheritance source for a weaker model role. */ +export function modelFormValuesAsPrimary(values: ModelConfigFormValues): PrimaryModelValues { + return { + protocol: toProtocol(values.provider), + modelId: values.model, + endpoint: values.endpoint, + apiKey: values.apiKey, + apiKeyMasked: values.apiKeyMasked, + configured: Boolean(values.apiKey.trim() || values.hasExistingApiKey) + }; +} + export function hydrateModelConfigForm( saved: ModelProviderConfig, defaultEmbeddingMode: ModelConfigEmbeddingMode @@ -310,6 +322,11 @@ export function hydrateModelConfigForm( imageGenApiKey, imageGenApiKeyMasked ); + const skillModel = hydrateRoleModelConfig(saved.memmyMemory?.evolution, primary); + const memoryModel = hydrateRoleModelConfig( + saved.memmyMemory?.summary, + modelFormValuesAsPrimary(createModelFormValues(skillModel, primary)) + ); return { protocol, @@ -335,8 +352,8 @@ export function hydrateModelConfigForm( imageGenApiKey, imageGenApiKeyMasked, imageGenValidation: hasImageGenApiKey(imageGenValues) ? createSavedValidation(imageGenValues) : createIdleValidation(), - memoryModel: hydrateRoleModelConfig(saved.memmyMemory?.summary, primary), - skillModel: hydrateRoleModelConfig(saved.memmyMemory?.evolution, primary) + memoryModel, + skillModel }; } @@ -352,13 +369,18 @@ export function createMemmyMemoryProviderConfig( skillModel: ModelConfig, primary: PrimaryModelValues ): MemmyMemoryProviderConfig { + const evolutionValues = createModelFormValues(skillModel, primary); + const summaryValues = createModelFormValues( + memoryModel, + modelFormValuesAsPrimary(evolutionValues) + ); return { summary: { - ...toRoleModelProviderConfig(createModelFormValues(memoryModel, primary)), + ...toRoleModelProviderConfig(summaryValues), mode: memoryModel.reuse ? "follow" : "fixed" }, evolution: { - ...toRoleModelProviderConfig(createModelFormValues(skillModel, primary)), + ...toRoleModelProviderConfig(evolutionValues), mode: skillModel.reuse ? "follow" : "fixed" } }; diff --git a/App/frontend/desktop/src/pages/model-page.tsx b/App/frontend/desktop/src/pages/model-page.tsx index cf9213b77..aef50d6a9 100644 --- a/App/frontend/desktop/src/pages/model-page.tsx +++ b/App/frontend/desktop/src/pages/model-page.tsx @@ -23,6 +23,7 @@ import { PROTOCOL_OPTIONS, canUseModelConfig, createModelFormValues, + modelFormValuesAsPrimary, createModelProtocolPatch, createTestModelConnectionMessages, hydrateModelConfigForm, @@ -43,6 +44,7 @@ interface ModelCardProps { hint?: string; cfg: ModelConfig; primary: PrimaryModelValues; + reuseLabel: string; onPatch: (patch: Partial) => void; onTest: () => void; } @@ -94,8 +96,9 @@ export function ModelPage() { const [skill, setSkill] = useState(() => initialModelForm.skillModel); const [savePending, setSavePending] = useState(false); const [saveFeedback, setSaveFeedback] = useState<{ text: string; tone: "error" | "success" } | null>(null); - const memoryValues = createModelFormValues(mem, primaryModel); const skillValues = createModelFormValues(skill, primaryModel); + const evolutionModel = modelFormValuesAsPrimary(skillValues); + const memoryValues = createModelFormValues(mem, evolutionModel); const canContinue = canUseModelConfig(mem, memoryValues) && canUseModelConfig(skill, skillValues); /** Handles patch mem. */ @@ -109,8 +112,13 @@ export function ModelPage() { } /** Handles test model config connection. */ - function testModelConfigConnection(config: ModelConfig, patch: (patch: Partial) => void, secretTarget: "memory" | "skill") { - const values = createModelFormValues(config, primaryModel); + function testModelConfigConnection( + config: ModelConfig, + inheritedModel: PrimaryModelValues, + patch: (patch: Partial) => void, + secretTarget: "memory" | "skill" + ) { + const values = createModelFormValues(config, inheritedModel); testModelConnection({ configClient: clients?.config, values, @@ -132,25 +140,8 @@ export function ModelPage() { setSavePending(true); const latest = await clients.config.getModelConfig(); let workspace = createModelWorkspace(latest); - const memoryValues = createModelFormValues(mem, primaryModel); - const assignedMemoryEndpointId = mem.reuse - ? assignedCatalogEndpointId(workspace, "byok", "agent") - : assignedCatalogEndpointId(workspace, "byok", "memory_summary") - ?? (!memoryValues.apiKey.trim() && memoryValues.apiKeyMasked - ? assignedCatalogEndpointId(workspace, "byok", "agent") - : undefined); - const memory = upsertByokPreset(workspace, { - provider: memoryValues.provider, - ...(memoryValues.apiKeyMasked && assignedMemoryEndpointId ? { endpointId: assignedMemoryEndpointId } : {}), - endpoint: memoryValues.endpoint, - protocol: chatProtocol(memoryValues.provider), - ...(memoryValues.apiKey.trim() ? { apiKey: memoryValues.apiKey.trim() } : {}), - ...(memoryValues.apiKeyMasked ? { apiKeyMasked: memoryValues.apiKeyMasked } : {}), - model: memoryValues.model, - capabilities: ["memory_summary"] - }); - workspace = assignCatalogPreset(memory.workspace, "byok", "memory_summary", memory.presetId); const evolutionValues = createModelFormValues(skill, primaryModel); + const memoryValues = createModelFormValues(mem, modelFormValuesAsPrimary(evolutionValues)); const assignedEvolutionEndpointId = skill.reuse ? assignedCatalogEndpointId(workspace, "byok", "agent") : assignedCatalogEndpointId(workspace, "byok", "memory_evolution") @@ -168,6 +159,23 @@ export function ModelPage() { capabilities: ["memory_evolution"] }); workspace = assignCatalogPreset(evolution.workspace, "byok", "memory_evolution", evolution.presetId); + const assignedMemoryEndpointId = mem.reuse + ? evolution.endpointId + : assignedCatalogEndpointId(workspace, "byok", "memory_summary") + ?? (!memoryValues.apiKey.trim() && memoryValues.apiKeyMasked + ? evolution.endpointId + : undefined); + const memory = upsertByokPreset(workspace, { + provider: memoryValues.provider, + ...(memoryValues.apiKeyMasked && assignedMemoryEndpointId ? { endpointId: assignedMemoryEndpointId } : {}), + endpoint: memoryValues.endpoint, + protocol: chatProtocol(memoryValues.provider), + ...(memoryValues.apiKey.trim() ? { apiKey: memoryValues.apiKey.trim() } : {}), + ...(memoryValues.apiKeyMasked ? { apiKeyMasked: memoryValues.apiKeyMasked } : {}), + model: memoryValues.model, + capabilities: ["memory_summary"] + }); + workspace = assignCatalogPreset(memory.workspace, "byok", "memory_summary", memory.presetId); const savedConfig = await clients.config.saveModelCatalog(modelConfigInput(workspace)); dispatch(appActions.modelConfigUpdated(savedConfig)); dispatch(appActions.navigate("/api-key-optional")); @@ -208,9 +216,10 @@ export function ModelPage() { subtitle={t("apiKey.modelPage.memorySubtitle")} hint={t("apiKey.modelPage.memoryHint")} cfg={mem} - primary={primaryModel} + primary={evolutionModel} + reuseLabel={t("apiKey.modelPage.reuseEvolution")} onPatch={patchMem} - onTest={() => testModelConfigConnection(mem, patchMem, "memory")} + onTest={() => testModelConfigConnection(mem, evolutionModel, patchMem, "memory")} /> testModelConfigConnection(skill, patchSkill, "skill")} + onTest={() => testModelConfigConnection(skill, primaryModel, patchSkill, "skill")} /> {props.hint && ( diff --git a/App/frontend/desktop/src/pages/model-workspace-section.tsx b/App/frontend/desktop/src/pages/model-workspace-section.tsx index 32224b0bd..de69c60fc 100644 --- a/App/frontend/desktop/src/pages/model-workspace-section.tsx +++ b/App/frontend/desktop/src/pages/model-workspace-section.tsx @@ -1,5 +1,9 @@ import { AlertTriangle, Check, CheckCircle2, ChevronDown, ChevronUp, Database, Info, KeyRound, Loader2, Pencil, Plus, Trash2, Wrench, X, XCircle } from "lucide-react"; -import { MODEL_NAME_MAX_LENGTH, type ModelEndpointProtocol } from "@memmy/local-api-contracts"; +import { + BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, + MODEL_NAME_MAX_LENGTH, + type ModelEndpointProtocol +} from "@memmy/local-api-contracts"; import { useCallback, useEffect, useRef, useState } from "react"; import type { ConfigClient, ModelProviderConfig } from "../api/config-client.js"; import { Button } from "../components/button.js"; @@ -53,7 +57,6 @@ export type ModelKind = "text" | "embedding" | "asr" | "image"; const DEFAULT_TEXT_CAPABILITIES: ModelCapability[] = ["chat", "memorySummary", "memoryEvolution"]; const MODEL_KIND_OPTIONS = ["text", "embedding", "asr", "image"] as const; -const LOCAL_EMBEDDING_OPTION_VALUE = "builtin:local-embedding"; export function modelCapabilitiesForKind(kind: ModelKind): ModelCapability[] { if (kind === "text") return [...DEFAULT_TEXT_CAPABILITIES]; @@ -586,12 +589,7 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { } function updateAssignment(kind: ModelAssignmentKind, candidateId: string) { - const assignment = kind === "embedding" - && props.mode === "byok" - && candidateId === LOCAL_EMBEDDING_OPTION_VALUE - ? null - : candidateId; - commitWorkspace(setModelAssignment(workspace, props.mode, kind, assignment)); + commitWorkspace(setModelAssignment(workspace, props.mode, kind, candidateId)); } function toggleTaskCandidate(candidateId: string) { @@ -624,18 +622,27 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { candidate.source, candidate.provider )); - const embeddingModelOptions = embeddingCandidates.map((candidate) => candidateOption( - candidate.id, - candidate.source === "platform" - ? t("settings.modelWorkspace.platformName") - : connectionProtocolLabel(candidate.provider, t), - candidate.source === "platform" ? platformModelName(candidate.capability, t) : candidate.model, - candidate.source === "platform" - ? t("settings.modelWorkspace.platformModels") - : t("settings.modelWorkspace.byokConnections"), - candidate.source, - candidate.provider - )); + const customEmbeddingOptions = embeddingCandidates + .filter((candidate) => candidate.source === "byok") + .map((candidate) => candidateOption( + candidate.id, + connectionProtocolLabel(candidate.provider, t), + candidate.model, + t("settings.modelWorkspace.byokConnections"), + candidate.source, + candidate.provider + )); + const cloudEmbeddingOptions = props.mode === "account" + ? embeddingCandidates + .filter((candidate) => candidate.source === "platform") + .map((candidate): SelectOption => ({ + value: candidate.id, + label: t("settings.modelWorkspace.platformEmbedding"), + selectedLabel: t("settings.modelWorkspace.platformEmbedding"), + groupLabel: t("settings.modelWorkspace.platformModels"), + icon: + })) + : []; const asrOptions = asrCandidates.map((candidate) => candidateOption( candidate.id, candidate.source === "platform" @@ -660,19 +667,19 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { candidate.source, candidate.provider )); - const embeddingOptions: SelectOption[] = props.mode === "byok" - ? [ - { - value: LOCAL_EMBEDDING_OPTION_VALUE, - label: t("settings.modelWorkspace.localEmbedding"), - selectedLabel: t("settings.modelWorkspace.localEmbeddingShort"), - groupLabel: t("settings.modelWorkspace.specialBuiltins") - }, - ...embeddingModelOptions - ] - : embeddingModelOptions; + const embeddingOptions: SelectOption[] = [ + { + value: BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID, + label: t("settings.modelWorkspace.localEmbedding"), + selectedLabel: t("settings.modelWorkspace.localEmbeddingShort"), + groupLabel: t("settings.modelWorkspace.platformModels"), + icon: + }, + ...cloudEmbeddingOptions, + ...customEmbeddingOptions + ]; const embeddingAssignment = space.assignments.embedding - ?? (props.mode === "byok" ? LOCAL_EMBEDDING_OPTION_VALUE : undefined); + ?? (props.mode === "byok" ? BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID : undefined); const editorExistingConnection = editor?.connectionId ? space.connections.find((connection) => connection.id === editor.connectionId) : undefined; @@ -693,9 +700,7 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { && (editor.models.length > 0 || (editor.addingModel && editor.modelDraft.trim())) ); const selectedTaskModelsText = taskCandidates.length > 0 - ? taskCandidates.map((candidate) => candidate.source === "platform" - ? platformModelName(candidate.capability, t) - : candidate.model).join("、") + ? taskCandidates.map((candidate) => candidate.model).join("、") : t("settings.modelWorkspace.notConfigured"); return ( @@ -775,21 +780,6 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { {t("settings.modelWorkspace.platformProvided")}
- {modelsExpanded ? ( - ({ - id: model.id, - model: platformModelName(model.capability, t), - capabilities: [model.capability] - }))} - /> - ) : ( -

- {t("settings.modelWorkspace.platformManaged")} · {t("settings.modelWorkspace.modelCount", { - count: platformCandidates.length - })} -

- )} )} @@ -939,11 +929,11 @@ export function ModelWorkspaceSection(props: ModelWorkspaceSectionProps) { {selected &&