fix(sqlite): transactional integrity, FTS5 for memories, decay rotation

docs/ARCHITECTURE.md

@@ -192,7 +192,7 @@ Data is persisted in the directory configured by `storagePath` (default: `~/.ope
 └── .cache/                  # HuggingFace model cache (Xenova/nomic-embed-text-v1)
 ```
 
-Each shard database contains a `memories` table with columns for content, vector blob, scoring fields (recency, frequency, importance, utility, novelty, confidence, interference, strength), lifecycle fields (store_type, decay_rate, is_deprecated, is_pinned), and metadata (tags, type, user info, project info). The `schema_version` table tracks applied migrations.
+Each shard database contains a `memories` table with columns for content, vector blob, scoring fields (recency, frequency, importance, utility, novelty, confidence, interference, strength), lifecycle fields (store_type, decay_rate, is_deprecated, is_pinned), and metadata (tags, type, user info, project info). An FTS5 virtual table `memories_fts` (external-content over `memories`, kept in sync by INSERT/UPDATE/DELETE triggers) indexes `id`, `content`, and `tags` for keyword search. The `schema_version` table tracks applied migrations.
 
 The `ConnectionManager` maintains up to 20 LRU-cached connections with WAL journaling, 64MB cache, and batch write support. When a shard exceeds `maxVectorsPerShard` (default: 50,000 vectors), the `ShardManager` rotates to a new shard file.
 
@@ -204,7 +204,7 @@ Search follows a multi-stage pipeline:
 2. **Shard selection** — Resolve scope (project vs all-projects) and fetch matching shards.
 3. **Per-shard search** — For each shard:
    - Vector backend returns top-K candidates by cosine similarity (over-fetch with 2× base multiplier, adaptive up to 8×).
-   - FTS5 text search adds keyword-matching candidates.
+   - FTS5 text search against `memories_fts` adds keyword-matching candidates (MATCH on content/tags; created on new shards and backfilled on existing ones).
    - Results are merged and deduplicated.
 4. **Reranking** — Apply `RetrievalContext` scoring:
    - Context boost: memories matching project path, recent files, or query topics get up to 1.5× boost.

docs/CHANGELOG.md

@@ -18,6 +18,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Warmup timeout race no longer triggers an unhandled promise rejection.**
 - **Auto-capture and profile learning now wait for opencode provider state instead of racing it at startup.**
 - **The forget tool now reports actual deletion failures instead of always claiming success.**
+### Fixed
+
+- **Re-embed migrations now update vectors in place, and migration operations report success only when every shard succeeded — failures propagate to the admin UI instead of being logged away.**
+- **Exact-duplicate cleanup is now transactional (no partial purges on crash) and no longer writes memory content into host logs.**
+- **Memory archival now commits its sqlite transaction before touching the vector index — no more async work inside BEGIN IMMEDIATE, and archived ids are reliably deleted from the index.**
+- **Batched writes are no longer dropped when their connection was evicted before flush.**
+- **SQL connection pool no longer evicts (and mid-checkpoint-closes) connections with an open transaction.**
+- **SQLite write transactions are no longer held open across async vector-index updates — eliminating nested-transaction/SQLITE_BUSY risks under concurrent captures, decay, and admin operations.**
+- **Keyword search and FTS boost now work: the memories_fts FTS5 virtual table (absent since shards were created without it) is created on new and existing shards — keyword search no longer silently degrades to a full-table LIKE scan.**
+- **Decay cycle now rotates through all decayable memories (ordered by last_decay_at) instead of repeatedly processing only the first batch; rows past the batch cap now decay and archive.**
 
 ## [2.23.1] - 2026-09-07
 

src/services/deduplication-service.ts

@@ -74,7 +74,7 @@ export class DeduplicationService {
           continue;
         }
 
-        exactDeleted += this._deleteExactDuplicates(memories, db, shard);
+        exactDeleted += await this._deleteExactDuplicates(memories, db, shard);
 
         const contentMap = this.buildContentMap(memories);
         // Every group has at least one entry (pushed in buildContentMap).
@@ -115,34 +115,49 @@ export class DeduplicationService {
     return contentMap;
   }
 
-  private _deleteExactDuplicates(
+  private async _deleteExactDuplicates(
     memories: DedupMemoryRow[],
     db: Database,
     shard: ShardInfo
-  ): number {
+  ): Promise<number> {
     const contentMap = this.buildContentMap(memories);
-
-    let exactDeleted = 0;
+    const toDelete: DedupMemoryRow[] = [];
     for (const [, duplicates] of contentMap) {
       if (duplicates.length > 1) {
         duplicates.sort((left, right) => Number(right.created_at) - Number(left.created_at));
-        const toDelete = duplicates.slice(1);
-
-        for (const dup of toDelete) {
-          try {
-            vectorSearch.deleteVector(db, dup.id, shard);
-            shardManager.decrementVectorCount(shard.id);
-            exactDeleted++;
-          } catch (error) {
-            log("Deduplication: delete error", {
-              memoryId: dup.id,
-              error: String(error),
-            });
-          }
-        }
+        toDelete.push(...duplicates.slice(1));
+      }
+    }
+    if (toDelete.length === 0) return 0;
+
+    db.run("BEGIN IMMEDIATE");
+    try {
+      for (const dup of toDelete) {
+        db.run("DELETE FROM memories WHERE id = ?", dup.id);
+      }
+      db.run("COMMIT");
+    } catch (error) {
+      try {
+        db.run("ROLLBACK");
+      } catch (rollbackErr) {
+        log("Deduplication: rollback failed", { error: String(rollbackErr) });
+      }
+      log("Deduplication: delete error", { error: String(error) });
+      return 0;
+    }
+
+    for (const dup of toDelete) {
+      try {
+        await vectorSearch.deleteVector(db, dup.id, shard);
+        shardManager.decrementVectorCount(shard.id);
+      } catch (error) {
+        log("Deduplication: delete error", {
+          memoryId: dup.id,
+          error: String(error),
+        });
       }
     }
-    return exactDeleted;
+    return toDelete.length;
   }
 
   private _findNearDuplicates(
@@ -338,7 +353,6 @@ export class DeduplicationService {
           existingId: match.candidate.id,
           containerTag,
           similarity: match.similarity,
-          content: content.slice(0, 80),
         });
         return {
           isDuplicate: false,
@@ -378,7 +392,6 @@ export class DeduplicationService {
       existingId: match.candidate.id,
       containerTag,
       similarity: match.similarity,
-      content: content.slice(0, 80),
     });
 
     return { isDuplicate: true, existingId: match.candidate.id, merged: true };

src/services/memory-lifecycle.ts

@@ -342,6 +342,7 @@ export async function applyDecay(): Promise<{
             `SELECT id, strength, decay_rate, created_at, last_decay_at, store_type, access_count, type, is_pinned
              FROM memories
              WHERE (store_type = 'stm' OR (store_type = 'ltm' AND decay_rate > 0)) AND is_pinned = 0
+             ORDER BY last_decay_at ASC, id ASC
              LIMIT ?`
           )
           .all(decayBatchSize) as any[];
@@ -373,15 +374,25 @@ export async function applyDecay(): Promise<{
             totalUpdated++;
             if (result.decayed) totalDecayed++;
             if (result.archived) {
-              archiveMemory(db, memory.id, shard);
-              toArchive.push({ id: memory.id, shard });
-              totalArchived++;
+              if (archiveMemorySqlite(db, memory.id)) {
+                toArchive.push({ id: memory.id, shard });
+                totalArchived++;
+              }
             }
           }
         }
 
         db.run("COMMIT");
         inTxn = false;
+
+        for (const item of toArchive) {
+          try {
+            await vectorSearch.deleteVector(db, item.id, item.shard);
+            shardManager.decrementVectorCount(item.shard.id);
+          } catch (err) {
+            log("archiveMemory vector delete error", { memoryId: item.id, error: String(err) });
+          }
+        }
       } catch (error) {
         if (inTxn) {
           try {
@@ -419,61 +430,42 @@ export async function applyDecay(): Promise<{
   }
 }
 
-/**
- * Archive a memory by moving it to an archive table and deleting from memories.
- */
-async function archiveMemory(db: any, memoryId: string, shard: any): Promise<void> {
-  try {
-    // Create archive table if not exists
-    db.run(`
-      CREATE TABLE IF NOT EXISTS memories_archive (
-        id TEXT PRIMARY KEY,
-        content TEXT NOT NULL,
-        tags TEXT,
-        type TEXT,
-        created_at INTEGER NOT NULL,
-        archived_at INTEGER NOT NULL,
-        strength REAL,
-        access_count INTEGER,
-        container_tag TEXT,
-        metadata TEXT,
-        store_type TEXT,
-        decay_rate REAL
-      )
-    `);
-
-    // Copy to archive
-    const insertResult = db.run(
-      `
-      INSERT INTO memories_archive
-      SELECT id, content, tags, type, created_at, ?, strength, access_count,
-             container_tag, metadata, store_type, decay_rate
-      FROM memories WHERE id = ?
-    `,
-      Date.now(),
-      memoryId
-    );
-
-    if (insertResult.changes === 0) {
-      log("archiveMemory: memory already removed, skipping delete", { memoryId });
-      return;
-    }
-
-    // Delete from memories
-    db.run("DELETE FROM memories WHERE id = ?", memoryId);
-
-    // Ensure the vector is removed from the backend index
-    try {
-      await vectorSearch.deleteVector(db, memoryId, shard);
-      shardManager.decrementVectorCount(shard.id);
-    } catch (err) {
-      log("archiveMemory vector delete error", { memoryId, error: String(err) });
-    }
+function archiveMemorySqlite(db: any, memoryId: string): boolean {
+  db.run(`
+    CREATE TABLE IF NOT EXISTS memories_archive (
+      id TEXT PRIMARY KEY,
+      content TEXT NOT NULL,
+      tags TEXT,
+      type TEXT,
+      created_at INTEGER NOT NULL,
+      archived_at INTEGER NOT NULL,
+      strength REAL,
+      access_count INTEGER,
+      container_tag TEXT,
+      metadata TEXT,
+      store_type TEXT,
+      decay_rate REAL
+    )
+  `);
+
+  const insertResult = db.run(
+    `
+    INSERT INTO memories_archive
+    SELECT id, content, tags, type, created_at, ?, strength, access_count,
+           container_tag, metadata, store_type, decay_rate
+    FROM memories WHERE id = ?
+  `,
+    Date.now(),
+    memoryId
+  );
 
-    log("Memory archived", { memoryId, shardId: shard.id });
-  } catch (error) {
-    log("archiveMemory error", { memoryId, error: String(error) });
+  if (insertResult.changes === 0) {
+    log("archiveMemory: memory already removed, skipping delete", { memoryId });
+    return false;
   }
+
+  db.run("DELETE FROM memories WHERE id = ?", memoryId);
+  return true;
 }
 
 /**

src/services/migration-service.ts

@@ -176,12 +176,15 @@ class MigrationService {
       total: mismatch.shardMismatches.length,
     });
 
+    const expected = mismatch.shardMismatches.length;
+    const success = deletedShards === expected;
     return {
-      success: true,
+      success,
       strategy: "fresh-start",
       deletedShards,
       reEmbeddedMemories: 0,
       duration: Date.now() - startTime,
+      ...(success ? {} : { error: "Failed to delete one or more shards" }),
     };
   }
 
@@ -223,41 +226,12 @@ class MigrationService {
     memory: any,
     processedCount: number,
     totalMemories: number,
-    shardId: string
+    shardId: string,
+    db: ReturnType<typeof connectionManager.getConnection>
   ): Promise<{ success: boolean; processedCount: number }> {
     try {
       const vector = await embeddingService.embedWithTimeout(memory.content);
-      const scope = memory.containerTag.includes("_user_") ? "user" : "project";
-      const hash = memory.containerTag.split("_").slice(2).join("_");
-      const newShard = shardManager.getWriteShard(scope, hash);
-      const newDb = connectionManager.getConnection(newShard.dbPath);
-
-      await vectorSearch.insertVector(
-        newDb,
-        {
-          id: memory.id,
-          content: memory.content,
-          vector,
-          containerTag: memory.containerTag,
-          type: memory.type || undefined,
-          createdAt: memory.createdAt,
-          updatedAt: memory.updatedAt,
-          metadata: memory.metadata || undefined,
-          displayName: memory.displayName || undefined,
-          userName: memory.userName || undefined,
-          userEmail: memory.userEmail || undefined,
-          projectPath: memory.projectPath || undefined,
-          projectName: memory.projectName || undefined,
-          gitRepoUrl: memory.gitRepoUrl || undefined,
-        },
-        newShard
-      );
-
-      if (memory.isPinned === 1) {
-        vectorSearch.pinMemory(newDb, memory.id);
-      }
-
-      shardManager.incrementVectorCount(newShard.id);
+      await vectorSearch.updateVector(db, memory.id, vector);
       const nextCount = processedCount + 1;
 
       this.reportProgress({
@@ -291,7 +265,7 @@ class MigrationService {
 
     let reEmbeddedCount = 0;
     let processedCount = 0;
-    let deletedShards = 0;
+    let shardHadFailures = false;
 
     for (const shardInfo of mismatch.shardMismatches) {
       this.reportProgress({
@@ -307,32 +281,41 @@ class MigrationService {
         // Default 10000 is too low for shards with large memory counts.
         const memories = vectorSearch.getAllMemories(db, 1_000_000);
         const tempMemories = this._backupMemories(memories);
-        let shardHadFailures = false;
+        let thisShardFailed = false;
 
         for (const memory of tempMemories) {
           const result = await this._reEmbedSingleMemory(
             memory,
             processedCount,
             totalMemories,
-            String(shardInfo.shardId)
+            String(shardInfo.shardId),
+            db
           );
           processedCount = result.processedCount;
           if (result.success) {
             reEmbeddedCount++;
           } else {
+            thisShardFailed = true;
             shardHadFailures = true;
           }
         }
 
-        if (!shardHadFailures) {
-          await shardManager.deleteShard(shardInfo.shardId);
-          deletedShards++;
+        if (!thisShardFailed) {
+          db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [
+            "embedding_dimensions",
+            String(CONFIG.embeddingDimensions),
+          ]);
+          db.run("INSERT OR REPLACE INTO shard_metadata (key, value) VALUES (?, ?)", [
+            "embedding_model",
+            CONFIG.embeddingModel,
+          ]);
         } else {
           log("Migration: keeping original shard due to re-embedding failures", {
             shardId: shardInfo.shardId,
           });
         }
       } catch (error) {
+        shardHadFailures = true;
         log("Migration: error processing shard", {
           shardId: shardInfo.shardId,
           error: String(error),
@@ -346,12 +329,14 @@ class MigrationService {
       total: totalMemories,
     });
 
+    const success = !shardHadFailures && reEmbeddedCount === totalMemories;
     return {
-      success: true,
+      success,
       strategy: "re-embed",
-      deletedShards,
+      deletedShards: 0,
       reEmbeddedMemories: reEmbeddedCount,
       duration: Date.now() - startTime,
+      ...(success ? {} : { error: "One or more memories failed to re-embed" }),
     };
   }