diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 120cfbe..de04943 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -192,7 +192,7 @@ Data is persisted in the directory configured by `storagePath` (default: `~/.ope └── .cache/ # HuggingFace model cache (Xenova/nomic-embed-text-v1) ``` -Each shard database contains a `memories` table with columns for content, vector blob, scoring fields (recency, frequency, importance, utility, novelty, confidence, interference, strength), lifecycle fields (store_type, decay_rate, is_deprecated, is_pinned), and metadata (tags, type, user info, project info). The `schema_version` table tracks applied migrations. +Each shard database contains a `memories` table with columns for content, vector blob, scoring fields (recency, frequency, importance, utility, novelty, confidence, interference, strength), lifecycle fields (store_type, decay_rate, is_deprecated, is_pinned), and metadata (tags, type, user info, project info). An FTS5 virtual table `memories_fts` (external-content over `memories`, kept in sync by INSERT/UPDATE/DELETE triggers) indexes `id`, `content`, and `tags` for keyword search. The `schema_version` table tracks applied migrations. The `ConnectionManager` maintains up to 20 LRU-cached connections with WAL journaling, 64MB cache, and batch write support. When a shard exceeds `maxVectorsPerShard` (default: 50,000 vectors), the `ShardManager` rotates to a new shard file. @@ -204,7 +204,7 @@ Search follows a multi-stage pipeline: 2. **Shard selection** — Resolve scope (project vs all-projects) and fetch matching shards. 3. **Per-shard search** — For each shard: - Vector backend returns top-K candidates by cosine similarity (over-fetch with 2× base multiplier, adaptive up to 8×). - - FTS5 text search adds keyword-matching candidates. + - FTS5 text search against `memories_fts` adds keyword-matching candidates (MATCH on content/tags; created on new shards and backfilled on existing ones). - Results are merged and deduplicated. 4. **Reranking** — Apply `RetrievalContext` scoring: - Context boost: memories matching project path, recent files, or query topics get up to 1.5× boost. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ec98595..fdd1aca 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Warmup timeout race no longer triggers an unhandled promise rejection.** - **Auto-capture and profile learning now wait for opencode provider state instead of racing it at startup.** - **The forget tool now reports actual deletion failures instead of always claiming success.** +### Fixed + +- **Re-embed migrations now update vectors in place, and migration operations report success only when every shard succeeded — failures propagate to the admin UI instead of being logged away.** +- **Exact-duplicate cleanup is now transactional (no partial purges on crash) and no longer writes memory content into host logs.** +- **Memory archival now commits its sqlite transaction before touching the vector index — no more async work inside BEGIN IMMEDIATE, and archived ids are reliably deleted from the index.** +- **Batched writes are no longer dropped when their connection was evicted before flush.** +- **SQL connection pool no longer evicts (and mid-checkpoint-closes) connections with an open transaction.** +- **SQLite write transactions are no longer held open across async vector-index updates — eliminating nested-transaction/SQLITE_BUSY risks under concurrent captures, decay, and admin operations.** +- **Keyword search and FTS boost now work: the memories_fts FTS5 virtual table (absent since shards were created without it) is created on new and existing shards — keyword search no longer silently degrades to a full-table LIKE scan.** +- **Decay cycle now rotates through all decayable memories (ordered by last_decay_at) instead of repeatedly processing only the first batch; rows past the batch cap now decay and archive.** ## [2.23.1] - 2026-09-07 diff --git a/src/services/deduplication-service.ts b/src/services/deduplication-service.ts index afe2189..6a37cd6 100644 --- a/src/services/deduplication-service.ts +++ b/src/services/deduplication-service.ts @@ -74,7 +74,7 @@ export class DeduplicationService { continue; } - exactDeleted += this._deleteExactDuplicates(memories, db, shard); + exactDeleted += await this._deleteExactDuplicates(memories, db, shard); const contentMap = this.buildContentMap(memories); // Every group has at least one entry (pushed in buildContentMap). @@ -115,34 +115,49 @@ export class DeduplicationService { return contentMap; } - private _deleteExactDuplicates( + private async _deleteExactDuplicates( memories: DedupMemoryRow[], db: Database, shard: ShardInfo - ): number { + ): Promise { const contentMap = this.buildContentMap(memories); - - let exactDeleted = 0; + const toDelete: DedupMemoryRow[] = []; for (const [, duplicates] of contentMap) { if (duplicates.length > 1) { duplicates.sort((left, right) => Number(right.created_at) - Number(left.created_at)); - const toDelete = duplicates.slice(1); - - for (const dup of toDelete) { - try { - vectorSearch.deleteVector(db, dup.id, shard); - shardManager.decrementVectorCount(shard.id); - exactDeleted++; - } catch (error) { - log("Deduplication: delete error", { - memoryId: dup.id, - error: String(error), - }); - } - } + toDelete.push(...duplicates.slice(1)); + } + } + if (toDelete.length === 0) return 0; + + db.run("BEGIN IMMEDIATE"); + try { + for (const dup of toDelete) { + db.run("DELETE FROM memories WHERE id = ?", dup.id); + } + db.run("COMMIT"); + } catch (error) { + try { + db.run("ROLLBACK"); + } catch (rollbackErr) { + log("Deduplication: rollback failed", { error: String(rollbackErr) }); + } + log("Deduplication: delete error", { error: String(error) }); + return 0; + } + + for (const dup of toDelete) { + try { + await vectorSearch.deleteVector(db, dup.id, shard); + shardManager.decrementVectorCount(shard.id); + } catch (error) { + log("Deduplication: delete error", { + memoryId: dup.id, + error: String(error), + }); } } - return exactDeleted; + return toDelete.length; } private _findNearDuplicates( @@ -338,7 +353,6 @@ export class DeduplicationService { existingId: match.candidate.id, containerTag, similarity: match.similarity, - content: content.slice(0, 80), }); return { isDuplicate: false, @@ -378,7 +392,6 @@ export class DeduplicationService { existingId: match.candidate.id, containerTag, similarity: match.similarity, - content: content.slice(0, 80), }); return { isDuplicate: true, existingId: match.candidate.id, merged: true }; diff --git a/src/services/memory-lifecycle.ts b/src/services/memory-lifecycle.ts index 08d0876..2fbe7fa 100644 --- a/src/services/memory-lifecycle.ts +++ b/src/services/memory-lifecycle.ts @@ -342,6 +342,7 @@ export async function applyDecay(): Promise<{ `SELECT id, strength, decay_rate, created_at, last_decay_at, store_type, access_count, type, is_pinned FROM memories WHERE (store_type = 'stm' OR (store_type = 'ltm' AND decay_rate > 0)) AND is_pinned = 0 + ORDER BY last_decay_at ASC, id ASC LIMIT ?` ) .all(decayBatchSize) as any[]; @@ -373,15 +374,25 @@ export async function applyDecay(): Promise<{ totalUpdated++; if (result.decayed) totalDecayed++; if (result.archived) { - archiveMemory(db, memory.id, shard); - toArchive.push({ id: memory.id, shard }); - totalArchived++; + if (archiveMemorySqlite(db, memory.id)) { + toArchive.push({ id: memory.id, shard }); + totalArchived++; + } } } } db.run("COMMIT"); inTxn = false; + + for (const item of toArchive) { + try { + await vectorSearch.deleteVector(db, item.id, item.shard); + shardManager.decrementVectorCount(item.shard.id); + } catch (err) { + log("archiveMemory vector delete error", { memoryId: item.id, error: String(err) }); + } + } } catch (error) { if (inTxn) { try { @@ -419,61 +430,42 @@ export async function applyDecay(): Promise<{ } } -/** - * Archive a memory by moving it to an archive table and deleting from memories. - */ -async function archiveMemory(db: any, memoryId: string, shard: any): Promise { - try { - // Create archive table if not exists - db.run(` - CREATE TABLE IF NOT EXISTS memories_archive ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - tags TEXT, - type TEXT, - created_at INTEGER NOT NULL, - archived_at INTEGER NOT NULL, - strength REAL, - access_count INTEGER, - container_tag TEXT, - metadata TEXT, - store_type TEXT, - decay_rate REAL - ) - `); - - // Copy to archive - const insertResult = db.run( - ` - INSERT INTO memories_archive - SELECT id, content, tags, type, created_at, ?, strength, access_count, - container_tag, metadata, store_type, decay_rate - FROM memories WHERE id = ? - `, - Date.now(), - memoryId - ); - - if (insertResult.changes === 0) { - log("archiveMemory: memory already removed, skipping delete", { memoryId }); - return; - } - - // Delete from memories - db.run("DELETE FROM memories WHERE id = ?", memoryId); - - // Ensure the vector is removed from the backend index - try { - await vectorSearch.deleteVector(db, memoryId, shard); - shardManager.decrementVectorCount(shard.id); - } catch (err) { - log("archiveMemory vector delete error", { memoryId, error: String(err) }); - } +function archiveMemorySqlite(db: any, memoryId: string): boolean { + db.run(` + CREATE TABLE IF NOT EXISTS memories_archive ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + archived_at INTEGER NOT NULL, + strength REAL, + access_count INTEGER, + container_tag TEXT, + metadata TEXT, + store_type TEXT, + decay_rate REAL + ) + `); + + const insertResult = db.run( + ` + INSERT INTO memories_archive + SELECT id, content, tags, type, created_at, ?, strength, access_count, + container_tag, metadata, store_type, decay_rate + FROM memories WHERE id = ? + `, + Date.now(), + memoryId + ); - log("Memory archived", { memoryId, shardId: shard.id }); - } catch (error) { - log("archiveMemory error", { memoryId, error: String(error) }); + if (insertResult.changes === 0) { + log("archiveMemory: memory already removed, skipping delete", { memoryId }); + return false; } + + db.run("DELETE FROM memories WHERE id = ?", memoryId); + return true; } /** diff --git a/src/services/migration-service.ts b/src/services/migration-service.ts index 4605a1e..dea3684 100644 --- a/src/services/migration-service.ts +++ b/src/services/migration-service.ts @@ -176,12 +176,15 @@ class MigrationService { total: mismatch.shardMismatches.length, }); + const expected = mismatch.shardMismatches.length; + const success = deletedShards === expected; return { - success: true, + success, strategy: "fresh-start", deletedShards, reEmbeddedMemories: 0, duration: Date.now() - startTime, + ...(success ? {} : { error: "Failed to delete one or more shards" }), }; } @@ -223,41 +226,12 @@ class MigrationService { memory: any, processedCount: number, totalMemories: number, - shardId: string + shardId: string, + db: ReturnType ): Promise<{ success: boolean; processedCount: number }> { try { const vector = await embeddingService.embedWithTimeout(memory.content); - const scope = memory.containerTag.includes("_user_") ? "user" : "project"; - const hash = memory.containerTag.split("_").slice(2).join("_"); - const newShard = shardManager.getWriteShard(scope, hash); - const newDb = connectionManager.getConnection(newShard.dbPath); - - await vectorSearch.insertVector( - newDb, - { - id: memory.id, - content: memory.content, - vector, - containerTag: memory.containerTag, - type: memory.type || undefined, - createdAt: memory.createdAt, - updatedAt: memory.updatedAt, - metadata: memory.metadata || undefined, - displayName: memory.displayName || undefined, - userName: memory.userName || undefined, - userEmail: memory.userEmail || undefined, - projectPath: memory.projectPath || undefined, - projectName: memory.projectName || undefined, - gitRepoUrl: memory.gitRepoUrl || undefined, - }, - newShard - ); - - if (memory.isPinned === 1) { - vectorSearch.pinMemory(newDb, memory.id); - } - - shardManager.incrementVectorCount(newShard.id); + await vectorSearch.updateVector(db, memory.id, vector); const nextCount = processedCount + 1; this.reportProgress({ @@ -291,7 +265,7 @@ class MigrationService { let reEmbeddedCount = 0; let processedCount = 0; - let deletedShards = 0; + let shardHadFailures = false; for (const shardInfo of mismatch.shardMismatches) { this.reportProgress({ @@ -307,32 +281,41 @@ class MigrationService { // Default 10000 is too low for shards with large memory counts. const memories = vectorSearch.getAllMemories(db, 1_000_000); const tempMemories = this._backupMemories(memories); - let shardHadFailures = false; + let thisShardFailed = false; for (const memory of tempMemories) { const result = await this._reEmbedSingleMemory( memory, processedCount, totalMemories, - String(shardInfo.shardId) + String(shardInfo.shardId), + db ); processedCount = result.processedCount; if (result.success) { reEmbeddedCount++; } else { + thisShardFailed = true; shardHadFailures = true; } } - if (!shardHadFailures) { - await shardManager.deleteShard(shardInfo.shardId); - deletedShards++; + if (!thisShardFailed) { + db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [ + "embedding_dimensions", + String(CONFIG.embeddingDimensions), + ]); + db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [ + "embedding_model", + CONFIG.embeddingModel, + ]); } else { log("Migration: keeping original shard due to re-embedding failures", { shardId: shardInfo.shardId, }); } } catch (error) { + shardHadFailures = true; log("Migration: error processing shard", { shardId: shardInfo.shardId, error: String(error), @@ -346,12 +329,14 @@ class MigrationService { total: totalMemories, }); + const success = !shardHadFailures && reEmbeddedCount === totalMemories; return { - success: true, + success, strategy: "re-embed", - deletedShards, + deletedShards: 0, reEmbeddedMemories: reEmbeddedCount, duration: Date.now() - startTime, + ...(success ? {} : { error: "One or more memories failed to re-embed" }), }; } diff --git a/src/services/sqlite/connection-manager.ts b/src/services/sqlite/connection-manager.ts index d3770cb..b0b32e7 100644 --- a/src/services/sqlite/connection-manager.ts +++ b/src/services/sqlite/connection-manager.ts @@ -2,7 +2,7 @@ import { getDatabase, type Database } from "./sqlite-bootstrap.js"; import { existsSync, mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { log } from "../logger.js"; -import { runMigrations } from "./schema.js"; +import { ensureMemoriesFts, runMigrations } from "./schema.js"; const DB = getDatabase(); @@ -16,6 +16,18 @@ class ConnectionManager { private isClosing = false; private readonly batches: Map> = new Map(); private readonly stmtCache = new WeakMap>(); + private readonly inTxn = new Set(); + + private trackTransactions(dbPath: string, db: Database): void { + const origRun = db.run.bind(db); + db.run = (sql: string, ...params: unknown[]) => { + const result = origRun(sql, ...params); + const kind = sql.trim().split(/\s+/, 1)[0]?.toUpperCase(); + if (kind === "BEGIN") this.inTxn.add(dbPath); + else if (kind === "COMMIT" || kind === "ROLLBACK") this.inTxn.delete(dbPath); + return result; + }; + } private touchAccessOrder(dbPath: string): void { this.accessOrder = this.accessOrder.filter((p) => p !== dbPath); @@ -52,11 +64,7 @@ class ConnectionManager { const batch = this.batches.get(dbPath); if (!batch || batch.length === 0) return; - const db = this.connections.get(dbPath); - if (!db) { - this.batches.delete(dbPath); - throw new Error(`No open connection for ${dbPath} — cannot flush batch`); - } + const db = this.connections.get(dbPath) ?? this.getConnection(dbPath); db.run("BEGIN IMMEDIATE"); try { @@ -105,6 +113,14 @@ class ConnectionManager { if (!hasTags && columns.length > 0) { db.run("ALTER TABLE memories ADD COLUMN tags TEXT"); } + if (columns.length > 0) { + try { + db.run("CREATE INDEX IF NOT EXISTS idx_last_decay_at ON memories(last_decay_at)"); + } catch { + // last_decay_at may be missing on pre-scoring shards; scoring migrate adds it later + } + ensureMemoriesFts(db); + } } catch (error) { log("Schema migration error", { error: String(error) }); } @@ -133,10 +149,12 @@ class ConnectionManager { try { if (this.connections.size >= MAX_CONNECTIONS) { - const oldestPath = this.accessOrder.shift(); - if (oldestPath) { - this.closeConnection(oldestPath); - log("ConnectionManager: evicted oldest connection", { path: oldestPath }); + const idlePath = this.accessOrder.find((p) => !this.inTxn.has(p)); + if (idlePath) { + this.closeConnection(idlePath); + log("ConnectionManager: evicted oldest idle connection", { path: idlePath }); + } else { + log("ConnectionManager: skipped eviction, all connections in transaction"); } } @@ -152,6 +170,7 @@ class ConnectionManager { } const db = new DB(dbPath); + this.trackTransactions(dbPath, db); this.connections.set(dbPath, db); this.accessOrder.push(dbPath); this.initDatabase(db); @@ -175,6 +194,7 @@ class ConnectionManager { this.connections.delete(dbPath); } this.accessOrder = this.accessOrder.filter((p) => p !== dbPath); + this.inTxn.delete(dbPath); } closeAll(): void { @@ -193,6 +213,7 @@ class ConnectionManager { } this.connections.clear(); this.accessOrder = []; + this.inTxn.clear(); } finally { this.isClosing = false; } diff --git a/src/services/sqlite/schema.ts b/src/services/sqlite/schema.ts index 706eb9a..a133943 100644 --- a/src/services/sqlite/schema.ts +++ b/src/services/sqlite/schema.ts @@ -35,6 +35,60 @@ function shouldSkipMigration( return false; } +export function ensureMemoriesFts(db: Database): void { + const memories = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memories'") + .get(); + if (!memories) return; + + const existing = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memories_fts'") + .get(); + + db.run(` + CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( + id, + content, + tags, + content='memories', + content_rowid='rowid' + ) + `); + + db.run(` + CREATE TRIGGER IF NOT EXISTS memories_fts_insert + AFTER INSERT ON memories BEGIN + INSERT INTO memories_fts(rowid, id, content, tags) + VALUES (new.rowid, new.id, new.content, new.tags); + END + `); + + db.run(` + CREATE TRIGGER IF NOT EXISTS memories_fts_delete + AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, id, content, tags) + VALUES ('delete', old.rowid, old.id, old.content, old.tags); + END + `); + + db.run(` + CREATE TRIGGER IF NOT EXISTS memories_fts_update + AFTER UPDATE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, id, content, tags) + VALUES ('delete', old.rowid, old.id, old.content, old.tags); + INSERT INTO memories_fts(rowid, id, content, tags) + VALUES (new.rowid, new.id, new.content, new.tags); + END + `); + + if (!existing) { + db.run(` + INSERT INTO memories_fts(rowid, id, content, tags) + SELECT rowid, id, content, tags FROM memories + `); + } +} + export function runMigrations( db: Database, targetVersion: number = CURRENT_SCHEMA_VERSION, diff --git a/src/services/sqlite/shard-manager.ts b/src/services/sqlite/shard-manager.ts index db11302..5183335 100644 --- a/src/services/sqlite/shard-manager.ts +++ b/src/services/sqlite/shard-manager.ts @@ -6,7 +6,7 @@ import { connectionManager } from "./connection-manager.js"; import { log } from "../logger.js"; import { vectorSearch } from "./vector-search.js"; import type { ShardInfo } from "./types.js"; -import { runMigrations } from "./schema.js"; +import { ensureMemoriesFts, runMigrations } from "./schema.js"; const METADATA_DB_NAME = "metadata.db"; @@ -196,6 +196,7 @@ class ShardManager { "idx_store_type ON memories(store_type)", "idx_decay_strength ON memories(strength, created_at)", "idx_is_deprecated ON memories(is_deprecated)", + "idx_last_decay_at ON memories(last_decay_at)", ]; for (const idx of INDEXES) { db.run(`CREATE INDEX IF NOT EXISTS ${idx}`); @@ -226,6 +227,7 @@ class ShardManager { this.migrateScoringColumns(db); this.migrateConflictColumns(db); + ensureMemoriesFts(db); } private migrateScoringColumns(db: Database): void { diff --git a/src/services/sqlite/vector-search.ts b/src/services/sqlite/vector-search.ts index 0c6b457..5ceae60 100644 --- a/src/services/sqlite/vector-search.ts +++ b/src/services/sqlite/vector-search.ts @@ -154,25 +154,12 @@ export class VectorSearch { async insertVector(db: Database, record: MemoryRecord, shard?: ShardInfo): Promise { const insertMemory = this.getStmt(db, MEMORIES_INSERT_SQL); + const backend = shard ? await this.getBackend() : undefined; db.run("BEGIN IMMEDIATE"); try { insertMemory.run(...recordToInsertParams(record)); - - if (shard) { - const backend = await this.getBackend(); - await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); - if (record.tagsVector) { - await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); - } - } - db.run("COMMIT"); - - if (shard) { - this.rebuildDirty.set(`${shard.id}:content`, true); - this.rebuildDirty.set(`${shard.id}:tags`, true); - } } catch (error) { try { db.run("ROLLBACK"); @@ -181,6 +168,18 @@ export class VectorSearch { } throw error; } + + if (shard && backend) { + try { + await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); + if (record.tagsVector) { + await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); + } + } finally { + this.rebuildDirty.set(`${shard.id}:content`, true); + this.rebuildDirty.set(`${shard.id}:tags`, true); + } + } } async batchInsertVectors( @@ -657,6 +656,8 @@ export class VectorSearch { shard?: ShardInfo, tagsVector?: Float32Array ): Promise { + const backend = shard ? await this.getBackend() : undefined; + db.run("BEGIN IMMEDIATE"); try { this.getStmt(db, "UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?").run( @@ -664,17 +665,6 @@ export class VectorSearch { toBlob(tagsVector), memoryId ); - - if (shard) { - const backend = await this.getBackend(); - await backend.insert({ id: memoryId, vector, shard, kind: "content" }); - if (tagsVector) { - await backend.insert({ id: memoryId, vector: tagsVector, shard, kind: "tags" }); - } else { - await backend.delete({ id: memoryId, shard, kind: "tags" }); - } - } - db.run("COMMIT"); } catch (error) { try { @@ -684,6 +674,21 @@ export class VectorSearch { } throw error; } + + if (shard && backend) { + try { + await backend.insert({ id: memoryId, vector, shard, kind: "content" }); + if (tagsVector) { + await backend.insert({ id: memoryId, vector: tagsVector, shard, kind: "tags" }); + } else { + await backend.delete({ id: memoryId, shard, kind: "tags" }); + } + } catch (error) { + this.rebuildDirty.set(`${shard.id}:content`, true); + this.rebuildDirty.set(`${shard.id}:tags`, true); + throw error; + } + } } async replaceVector( @@ -693,22 +698,12 @@ export class VectorSearch { shard?: ShardInfo ): Promise { const insertMemory = this.getStmt(db, MEMORIES_INSERT_SQL); + const backend = shard ? await this.getBackend() : undefined; db.run("BEGIN IMMEDIATE"); try { this.getStmt(db, "DELETE FROM memories WHERE id = ?").run(memoryId); insertMemory.run(...recordToInsertParams(record)); - - if (shard) { - const backend = await this.getBackend(); - await backend.delete({ id: memoryId, shard, kind: "content" }); - await backend.delete({ id: memoryId, shard, kind: "tags" }); - await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); - if (record.tagsVector) { - await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); - } - } - db.run("COMMIT"); } catch (error) { try { @@ -718,6 +713,21 @@ export class VectorSearch { } throw error; } + + if (shard && backend) { + try { + await backend.delete({ id: memoryId, shard, kind: "content" }); + await backend.delete({ id: memoryId, shard, kind: "tags" }); + await backend.insert({ id: record.id, vector: record.vector, shard, kind: "content" }); + if (record.tagsVector) { + await backend.insert({ id: record.id, vector: record.tagsVector, shard, kind: "tags" }); + } + } catch (error) { + this.rebuildDirty.set(`${shard.id}:content`, true); + this.rebuildDirty.set(`${shard.id}:tags`, true); + throw error; + } + } } listMemories(db: Database, containerTag: string, limit: number): any[] { diff --git a/tests/archive-ordering.test.ts b/tests/archive-ordering.test.ts new file mode 100644 index 0000000..a34d04a --- /dev/null +++ b/tests/archive-ordering.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/services/sqlite/shard-manager.js", () => ({ + getAllShards: vi.fn(), + shardManager: { decrementVectorCount: vi.fn() }, + extractScopeFromContainerTag: vi.fn(), +})); + +import { CONFIG } from "../src/config.js"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { getAllShards } from "../src/services/sqlite/shard-manager.js"; +import { vectorSearch } from "../src/services/sqlite/vector-search.js"; +import { applyDecay } from "../src/services/memory-lifecycle.js"; + +const THIRTY_ONE_DAYS = 31 * 24 * 60 * 60 * 1000; + +describe("archive ordering", () => { + let testDir: string; + let dbPath: string; + + beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "archive-order-")); + dbPath = join(testDir, "shard.db"); + vi.mocked(getAllShards).mockReturnValue([ + { + id: 1, + scope: "user", + scopeHash: "h", + shardIndex: 0, + dbPath, + vectorCount: 1, + isActive: true, + createdAt: Date.now(), + }, + ]); + + const db = connectionManager.getConnection(dbPath); + db.run(` + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL DEFAULT '', + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + last_decay_at INTEGER, + strength REAL DEFAULT 0.1, + decay_rate REAL DEFAULT 0.05, + store_type TEXT DEFAULT 'stm', + access_count INTEGER DEFAULT 0, + is_pinned INTEGER DEFAULT 0, + recency_score REAL DEFAULT 0.5, + container_tag TEXT, + metadata TEXT + ) + `); + db.run( + `INSERT INTO memories (id, content, strength, created_at, last_decay_at, store_type, is_pinned) + VALUES ('arch-1', 'x', 0.1, ?, NULL, 'stm', 0)`, + Date.now() - THIRTY_ONE_DAYS + ); + }); + + afterEach(() => { + connectionManager.closeAll(); + rmSync(testDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it("archives in sqlite then deletes the vector after COMMIT", async () => { + const previous = CONFIG.memoryLifecycle?.archiveThreshold; + if (CONFIG.memoryLifecycle) CONFIG.memoryLifecycle.archiveThreshold = 0.2; + + const order: string[] = []; + const db = connectionManager.getConnection(dbPath); + const origRun = db.run.bind(db); + db.run = (sql: string, ...params: unknown[]) => { + if (sql.trim().toUpperCase().startsWith("COMMIT")) order.push("COMMIT"); + return origRun(sql, ...params); + }; + vi.spyOn(vectorSearch, "deleteVector").mockImplementation(async () => { + order.push("deleteVector"); + }); + + try { + await applyDecay(); + } finally { + if (CONFIG.memoryLifecycle) CONFIG.memoryLifecycle.archiveThreshold = previous; + } + + const archived = db.prepare("SELECT id FROM memories_archive WHERE id = ?").get("arch-1") as + | { id: string } + | undefined; + const remaining = db.prepare("SELECT id FROM memories WHERE id = ?").get("arch-1"); + expect(archived?.id).toBe("arch-1"); + expect(remaining).toBeUndefined(); + expect(order).toEqual(["COMMIT", "deleteVector"]); + }); +}); diff --git a/tests/connection-eviction-txn.test.ts b/tests/connection-eviction-txn.test.ts new file mode 100644 index 0000000..640d81b --- /dev/null +++ b/tests/connection-eviction-txn.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; + +const POOL_CAP = 20; + +describe("connection pool eviction", () => { + let dir: string; + + afterEach(() => { + connectionManager.closeAll(); + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it("eviction skips connection with open txn", () => { + dir = mkdtempSync(join(tmpdir(), "conn-evict-txn-")); + connectionManager.closeAll(); + + const paths = Array.from({ length: POOL_CAP + 1 }, (_, i) => join(dir, `s${i}.db`)); + const dbs = paths.slice(0, POOL_CAP).map((p) => connectionManager.getConnection(p)); + const busy = dbs[0]; + const idleVictim = dbs[1]; + busy.run("BEGIN IMMEDIATE"); + + connectionManager.getConnection(paths[POOL_CAP]); + + expect(() => busy.run("COMMIT")).not.toThrow(); + expect(() => idleVictim.run("SELECT 1")).toThrow(); + }); +}); diff --git a/tests/decay-batch-starvation.test.ts b/tests/decay-batch-starvation.test.ts new file mode 100644 index 0000000..3184ba6 --- /dev/null +++ b/tests/decay-batch-starvation.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/services/sqlite/shard-manager.js", () => ({ + getAllShards: vi.fn(), + shardManager: { decrementVectorCount: vi.fn() }, + extractScopeFromContainerTag: vi.fn(), +})); + +import { CONFIG } from "../src/config.js"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { getAllShards } from "../src/services/sqlite/shard-manager.js"; +import { applyDecay } from "../src/services/memory-lifecycle.js"; + +const TWO_DAYS_MS = 2 * 24 * 60 * 60 * 1000; + +describe("decay batch starvation", () => { + let testDir: string; + let dbPath: string; + let previousBatchSize: number | undefined; + + beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "decay-starvation-")); + dbPath = join(testDir, "shard.db"); + previousBatchSize = CONFIG.memoryLifecycle?.decayBatchSize; + if (CONFIG.memoryLifecycle) CONFIG.memoryLifecycle.decayBatchSize = 2; + + vi.mocked(getAllShards).mockReturnValue([ + { + id: 1, + scope: "user", + scopeHash: "h", + shardIndex: 0, + dbPath, + vectorCount: 3, + isActive: true, + createdAt: Date.now(), + }, + ]); + + const db = connectionManager.getConnection(dbPath); + db.run(` + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL DEFAULT '', + strength REAL DEFAULT 0.9, + decay_rate REAL DEFAULT 0.05, + created_at INTEGER NOT NULL, + last_decay_at INTEGER, + store_type TEXT DEFAULT 'stm', + access_count INTEGER DEFAULT 0, + type TEXT, + is_pinned INTEGER DEFAULT 0, + recency_score REAL DEFAULT 0.5 + ) + `); + const createdAt = Date.now() - TWO_DAYS_MS; + for (const id of ["early-1", "early-2", "late-3"]) { + db.run( + `INSERT INTO memories (id, content, strength, created_at, last_decay_at, store_type, is_pinned) + VALUES (?, 'x', 0.9, ?, NULL, 'stm', 0)`, + id, + createdAt + ); + } + }); + + afterEach(() => { + if (CONFIG.memoryLifecycle) CONFIG.memoryLifecycle.decayBatchSize = previousBatchSize; + connectionManager.closeAll(); + rmSync(testDir, { recursive: true, force: true }); + }); + + it("second cycle decays rows past the batch cap instead of reprocessing the first batch", async () => { + await applyDecay(); + + const db = connectionManager.getConnection(dbPath); + const afterFirst = db + .prepare("SELECT id, last_decay_at, strength FROM memories ORDER BY id") + .all() as Array<{ id: string; last_decay_at: number | null; strength: number }>; + const firstTouched = afterFirst.filter((r) => r.last_decay_at != null).map((r) => r.id); + expect(firstTouched).toHaveLength(2); + expect(afterFirst.find((r) => r.id === "late-3")?.last_decay_at).toBeNull(); + + await applyDecay(); + + const afterSecond = db + .prepare("SELECT id, last_decay_at, strength FROM memories WHERE id = ?") + .get("late-3") as { id: string; last_decay_at: number | null; strength: number }; + expect(afterSecond.last_decay_at).not.toBeNull(); + expect(afterSecond.strength).toBeLessThan(0.9); + }); +}); diff --git a/tests/deduplication-service.test.ts b/tests/deduplication-service.test.ts index 2581bab..838c0f2 100644 --- a/tests/deduplication-service.test.ts +++ b/tests/deduplication-service.test.ts @@ -463,9 +463,11 @@ describe("deduplication-service", () => { const mockDb1 = { prepare: vi.fn().mockReturnValue({ run: vi.fn(), all: vi.fn() }), + run: vi.fn(() => ({ changes: 1, lastInsertRowid: 0 })), }; const mockDb2 = { prepare: vi.fn().mockReturnValue({ run: vi.fn(), all: vi.fn() }), + run: vi.fn(() => ({ changes: 1, lastInsertRowid: 0 })), }; vi.mocked(connectionManager.getConnection).mockImplementation((path: string) => { if (path === "path-1") return mockDb1 as any; @@ -562,7 +564,10 @@ describe("deduplication-service", () => { if (scope === "user") return [shard] as any; return []; }); - const mockDb = { prepare: vi.fn() }; + const mockDb = { + prepare: vi.fn(), + run: vi.fn(() => ({ changes: 1, lastInsertRowid: 0 })), + }; vi.mocked(connectionManager.getConnection).mockReturnValue(mockDb as any); const vec1 = new Float32Array([1, 0, 0]); @@ -588,7 +593,7 @@ describe("deduplication-service", () => { }); const result = await deduplicationService.detectAndRemoveDuplicates(); - expect(result.exactDuplicatesDeleted).toBe(0); // Failed to delete + expect(result.exactDuplicatesDeleted).toBe(1); }); it("skips malformed vectors in _findNearDuplicates", async () => { diff --git a/tests/exact-dedup-txn.test.ts b/tests/exact-dedup-txn.test.ts new file mode 100644 index 0000000..f0ad741 --- /dev/null +++ b/tests/exact-dedup-txn.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/services/sqlite/shard-manager.js", () => ({ + shardManager: { + getAllShards: vi.fn(), + decrementVectorCount: vi.fn(), + }, +})); + +import { CONFIG } from "../src/config.js"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { shardManager } from "../src/services/sqlite/shard-manager.js"; +import { DeduplicationService } from "../src/services/deduplication-service.js"; + +describe("exact-dedup transactions", () => { + let dir: string; + let dbPath: string; + let previousEnabled: boolean | undefined; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "exact-dedup-txn-")); + dbPath = join(dir, "s.db"); + previousEnabled = CONFIG.deduplicationEnabled; + (CONFIG as { deduplicationEnabled: boolean }).deduplicationEnabled = true; + + vi.mocked(shardManager.getAllShards).mockImplementation((scope: string) => + scope === "user" + ? [ + { + id: 1, + scope: "user", + scopeHash: "h", + shardIndex: 0, + dbPath, + vectorCount: 3, + isActive: true, + createdAt: Date.now(), + }, + ] + : [] + ); + + const db = connectionManager.getConnection(dbPath); + db.run(` + CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + container_tag TEXT NOT NULL, + created_at INTEGER NOT NULL, + vector BLOB, + is_deprecated INTEGER DEFAULT 0, + metadata TEXT + ) + `); + const now = Date.now(); + for (const [id, created] of [ + ["d1", now], + ["d2", now - 1], + ["d3", now - 2], + ] as const) { + db.run( + "INSERT INTO memories (id, content, container_tag, created_at) VALUES (?, ?, ?, ?)", + id, + "same", + "tag", + created + ); + } + }); + + afterEach(() => { + (CONFIG as { deduplicationEnabled: boolean }).deduplicationEnabled = previousEnabled as boolean; + connectionManager.closeAll(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("rolls back all exact-dup deletes if a later delete throws", async () => { + const db = connectionManager.getConnection(dbPath); + const origRun = db.run.bind(db); + let deletes = 0; + db.run = (sql: string, ...params: unknown[]) => { + if (/DELETE FROM memories/i.test(sql)) { + deletes++; + if (deletes === 2) throw new Error("boom"); + } + return origRun(sql, ...params); + }; + + const result = await new DeduplicationService().detectAndRemoveDuplicates(); + expect(result.exactDuplicatesDeleted).toBe(0); + + const rows = db.prepare("SELECT id FROM memories ORDER BY id").all() as Array<{ id: string }>; + expect(rows.map((r) => r.id)).toEqual(["d1", "d2", "d3"]); + }); +}); diff --git a/tests/memories-fts.test.ts b/tests/memories-fts.test.ts new file mode 100644 index 0000000..8861a18 --- /dev/null +++ b/tests/memories-fts.test.ts @@ -0,0 +1,157 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getDatabase } from "../src/services/sqlite/sqlite-bootstrap.js"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { CONFIG } from "../src/config.js"; +import { ExactScanBackend } from "../src/services/vector-backends/exact-scan-backend.js"; +import { VectorSearch } from "../src/services/sqlite/vector-search.js"; +import { shardManager } from "../src/services/sqlite/shard-manager.js"; + +const Database = getDatabase(); + +const MEMORIES_DDL = ` + CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + tags_vector BLOB, + container_tag TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + metadata TEXT, + display_name TEXT, + user_name TEXT, + user_email TEXT, + project_path TEXT, + project_name TEXT, + git_repo_url TEXT, + is_pinned INTEGER DEFAULT 0, + is_deprecated INTEGER DEFAULT 0, + recency_score REAL DEFAULT 0.5, + frequency_score REAL DEFAULT 0.5, + importance_score REAL DEFAULT 0.5, + utility_score REAL DEFAULT 0.5, + novelty_score REAL DEFAULT 0.5, + confidence_score REAL DEFAULT 0.5, + interference_penalty REAL DEFAULT 0, + strength REAL DEFAULT 0.5, + access_count INTEGER DEFAULT 0, + last_accessed INTEGER, + store_type TEXT, + decay_rate REAL, + last_decay_at INTEGER + ) +`; + +function ftsName(db: { prepare: (sql: string) => { get: (...args: unknown[]) => unknown } }) { + return db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memories_fts'") + .get() as { name: string } | undefined; +} + +describe("memories_fts", () => { + const dirs: string[] = []; + + afterEach(() => { + connectionManager.closeAll(); + while (dirs.length > 0) { + const dir = dirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } + }); + + it("creates memories_fts on a fresh shard and keyword search uses MATCH not LIKE", async () => { + const dir = mkdtempSync(join(tmpdir(), "memories-fts-fresh-")); + dirs.push(dir); + const previous = CONFIG.storagePath; + CONFIG.storagePath = dir; + const shard = shardManager.createShard("user", `fts${Date.now()}`, 0); + CONFIG.storagePath = previous; + + const db = connectionManager.getConnection(shard.dbPath); + expect(ftsName(db)?.name).toBe("memories_fts"); + + const now = Date.now(); + db.run( + `INSERT INTO memories (id, content, vector, container_tag, tags, created_at, updated_at, is_deprecated) + VALUES (?, ?, ?, ?, ?, ?, ?, 0)`, + "m1", + "uniquekeyword zebra memory", + new Uint8Array(16), + "mem_user_ftstest", + "alpha", + now, + now + ); + + const ftsCount = db.prepare("SELECT count(*) AS n FROM memories_fts").get() as { n: number }; + expect(Number(ftsCount.n)).toBe(1); + + const prepared: string[] = []; + const orig = db.prepare.bind(db); + vi.spyOn(db, "prepare").mockImplementation((sql: string) => { + prepared.push(sql); + return orig(sql); + }); + + const vectorSearch = new VectorSearch(new ExactScanBackend()); + const results = await vectorSearch.searchInShard( + shard, + null, + "mem_user_ftstest", + 10, + "uniquekeyword" + ); + + expect(results.some((r) => r.id === "m1")).toBe(true); + expect(prepared.some((s) => s.includes("memories_fts MATCH"))).toBe(true); + expect(prepared.some((s) => /content LIKE/i.test(s))).toBe(false); + }); + + it("migrates an existing shard: creates memories_fts and backfills rows", () => { + const dir = mkdtempSync(join(tmpdir(), "memories-fts-migrate-")); + dirs.push(dir); + const dbPath = join(dir, "old.db"); + + const raw = new Database(dbPath); + raw.run(MEMORIES_DDL); + const now = Date.now(); + raw.run( + `INSERT INTO memories (id, content, vector, container_tag, tags, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + "old-1", + "alpha content", + new Uint8Array(8), + "tag", + "t", + now, + now + ); + raw.run( + `INSERT INTO memories (id, content, vector, container_tag, tags, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + "old-2", + "beta content", + new Uint8Array(8), + "tag", + "t", + now, + now + ); + expect(ftsName(raw)).toBeUndefined(); + raw.close(); + + const db = connectionManager.getConnection(dbPath); + expect(ftsName(db)?.name).toBe("memories_fts"); + const ftsCount = db.prepare("SELECT count(*) AS n FROM memories_fts").get() as { n: number }; + expect(Number(ftsCount.n)).toBe(2); + const hit = db + .prepare("SELECT id FROM memories_fts WHERE memories_fts MATCH ?") + .get("alpha") as { id: string } | undefined; + expect(hit?.id).toBe("old-1"); + }); +}); diff --git a/tests/reembed-honesty.test.ts b/tests/reembed-honesty.test.ts new file mode 100644 index 0000000..92e4ed1 --- /dev/null +++ b/tests/reembed-honesty.test.ts @@ -0,0 +1,178 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +vi.mock("../src/services/sqlite/shard-manager.js", () => ({ + shardManager: { + getAllShards: vi.fn(), + getWriteShard: vi.fn(), + deleteShard: vi.fn(), + incrementVectorCount: vi.fn(), + }, +})); + +const embedMock = vi.hoisted(() => ({ + warmup: vi.fn(async () => undefined), + clearCache: vi.fn(), + embedWithTimeout: vi.fn(), +})); + +vi.mock("../src/services/embedding.js", () => ({ + embeddingService: embedMock, +})); + +import { CONFIG } from "../src/config.js"; +import { connectionManager } from "../src/services/sqlite/connection-manager.js"; +import { shardManager } from "../src/services/sqlite/shard-manager.js"; +import { migrationService } from "../src/services/migration-service.js"; + +const NEW_DIMS = 8; +const OLD_DIMS = 4; + +function vectorBlob(dims: number): Uint8Array { + return new Uint8Array(new Float32Array(dims).fill(1).buffer); +} + +describe("re-embed honesty", () => { + let dir: string; + let dbPath: string; + let prevDims: number; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "reembed-")); + dbPath = join(dir, "s.db"); + prevDims = CONFIG.embeddingDimensions; + CONFIG.embeddingDimensions = NEW_DIMS; + embedMock.embedWithTimeout.mockReset(); + embedMock.warmup.mockClear(); + embedMock.clearCache.mockClear(); + + vi.mocked(shardManager.getAllShards).mockImplementation((scope: string) => + scope === "user" + ? [ + { + id: 1, + scope: "user", + scopeHash: "h", + shardIndex: 0, + dbPath, + vectorCount: 2, + isActive: true, + createdAt: Date.now(), + }, + ] + : [] + ); + vi.mocked(shardManager.deleteShard).mockResolvedValue(undefined); + vi.mocked(shardManager.getWriteShard).mockReturnValue({ + id: 1, + scope: "user", + scopeHash: "h", + shardIndex: 0, + dbPath, + vectorCount: 2, + isActive: true, + createdAt: Date.now(), + }); + + const db = connectionManager.getConnection(dbPath); + db.run(` + CREATE TABLE IF NOT EXISTS shard_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `); + db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [ + "embedding_dimensions", + String(OLD_DIMS), + ]); + db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [ + "embedding_model", + "old-model", + ]); + db.run(` + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + tags_vector BLOB, + container_tag TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + metadata TEXT, + display_name TEXT, + user_name TEXT, + user_email TEXT, + project_path TEXT, + project_name TEXT, + git_repo_url TEXT, + is_pinned INTEGER DEFAULT 0, + is_deprecated INTEGER DEFAULT 0 + ) + `); + const now = Date.now(); + db.run( + `INSERT INTO memories (id, content, vector, container_tag, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`, + "m1", + "one", + vectorBlob(OLD_DIMS), + "mem_user_h", + now - 2, + now + ); + db.run( + `INSERT INTO memories (id, content, vector, container_tag, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`, + "m2", + "two", + vectorBlob(OLD_DIMS), + "mem_user_h", + now - 1, + now + ); + }); + + afterEach(() => { + CONFIG.embeddingDimensions = prevDims; + connectionManager.closeAll(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("updates vectors in place and reports success", async () => { + embedMock.embedWithTimeout.mockResolvedValue(new Float32Array(NEW_DIMS).fill(0.5)); + + const result = await migrationService.migrateToNewModel("re-embed"); + expect(result.success).toBe(true); + expect(result.reEmbeddedMemories).toBe(2); + expect(shardManager.deleteShard).not.toHaveBeenCalled(); + expect(shardManager.getWriteShard).not.toHaveBeenCalled(); + + const db = connectionManager.getConnection(dbPath); + const rows = db.prepare("SELECT id, vector FROM memories ORDER BY id").all() as Array<{ + id: string; + vector: Uint8Array; + }>; + expect(rows).toHaveLength(2); + for (const row of rows) { + expect(new Uint8Array(row.vector).byteLength).toBe(NEW_DIMS * 4); + } + const dims = db + .prepare("SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'") + .get() as { value: string }; + expect(dims.value).toBe(String(NEW_DIMS)); + }); + + it("reports success false when a row fails to embed", async () => { + embedMock.embedWithTimeout + .mockRejectedValueOnce(new Error("embed failed")) + .mockResolvedValue(new Float32Array(NEW_DIMS).fill(0.5)); + + const result = await migrationService.migrateToNewModel("re-embed"); + expect(result.success).toBe(false); + expect(result.error).toBeTruthy(); + }); +}); diff --git a/tests/vector-insert-txn.test.ts b/tests/vector-insert-txn.test.ts new file mode 100644 index 0000000..a457029 --- /dev/null +++ b/tests/vector-insert-txn.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getDatabase } from "../src/services/sqlite/sqlite-bootstrap.js"; +import { VectorSearch } from "../src/services/sqlite/vector-search.js"; +import type { VectorBackend } from "../src/services/vector-backends/types.js"; +import type { MemoryRecord, ShardInfo } from "../src/services/sqlite/types.js"; + +const Database = getDatabase(); + +function record(id: string): MemoryRecord { + return { + id, + content: id, + vector: new Float32Array([1, 0, 0, 0]), + containerTag: "mem_user_t", + createdAt: Date.now(), + updatedAt: Date.now(), + }; +} + +describe("insertVector transaction does not span await", () => { + const dirs: string[] = []; + + afterEach(() => { + while (dirs.length > 0) { + const dir = dirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } + }); + + it("second insertVector during deferred backend insert does not nested-BEGIN and both rows land", async () => { + const dir = mkdtempSync(join(tmpdir(), "insert-txn-")); + dirs.push(dir); + const dbPath = join(dir, "t.db"); + const db = new Database(dbPath); + db.run(` + CREATE TABLE memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + vector BLOB NOT NULL, + tags_vector BLOB, + container_tag TEXT NOT NULL, + tags TEXT, + type TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + metadata TEXT, + display_name TEXT, + user_name TEXT, + user_email TEXT, + project_path TEXT, + project_name TEXT, + git_repo_url TEXT, + recency_score REAL DEFAULT 0.5, + frequency_score REAL DEFAULT 0, + importance_score REAL DEFAULT 0.5, + utility_score REAL DEFAULT 0.3, + novelty_score REAL DEFAULT 0.5, + confidence_score REAL DEFAULT 0.7, + interference_penalty REAL DEFAULT 0, + strength REAL DEFAULT 0.5, + access_count INTEGER DEFAULT 0, + last_accessed INTEGER, + store_type TEXT DEFAULT 'stm', + decay_rate REAL DEFAULT 0.05 + ) + `); + + const unlocks: Array<() => void> = []; + const backend: VectorBackend = { + getBackendName: () => "deferred", + insert: () => new Promise((resolve) => unlocks.push(resolve)), + insertBatch: () => {}, + delete: () => {}, + search: () => [], + rebuildFromShard: () => {}, + deleteShardIndexes: () => {}, + }; + + const vectorSearch = new VectorSearch(backend); + const shard: ShardInfo = { + id: 1, + scope: "user", + scopeHash: "t", + shardIndex: 0, + dbPath, + vectorCount: 0, + isActive: true, + createdAt: Date.now(), + }; + + const first = vectorSearch.insertVector(db, record("a"), shard); + await new Promise((r) => setImmediate(r)); + expect(unlocks.length).toBeGreaterThan(0); + + let secondErr: unknown; + const second = vectorSearch.insertVector(db, record("b"), shard).catch((e: unknown) => { + secondErr = e; + }); + await new Promise((r) => setImmediate(r)); + expect(secondErr).toBeUndefined(); + + unlocks.forEach((u) => u()); + await first; + await second; + expect(secondErr).toBeUndefined(); + + const rows = db.prepare("SELECT id FROM memories ORDER BY id").all() as Array<{ id: string }>; + expect(rows.map((r) => r.id)).toEqual(["a", "b"]); + db.close(); + }); +}); diff --git a/tests/wal-batch.test.ts b/tests/wal-batch.test.ts index 6a4b97c..54c8863 100644 --- a/tests/wal-batch.test.ts +++ b/tests/wal-batch.test.ts @@ -152,4 +152,12 @@ describe("WAL batch write API", () => { .all() as any[]; expect(rows2).toHaveLength(1); }); + + it("flushBatch after eviction does not drop queued writes", () => { + connectionManager.closeConnection(dbPath); + connectionManager.batchWrite(dbPath, "INSERT INTO test_table (value) VALUES (?)", ["evicted"]); + connectionManager.flushBatch(dbPath); + const rows = getDb().prepare("SELECT * FROM test_table WHERE value = 'evicted'").all() as any[]; + expect(rows).toHaveLength(1); + }); });