diff --git a/docs/implementation/v2.1-sqlite-vec.md b/docs/implementation/v2.1-sqlite-vec.md new file mode 100644 index 0000000..845c43e --- /dev/null +++ b/docs/implementation/v2.1-sqlite-vec.md @@ -0,0 +1,262 @@ +# Implementierungsplan: sqlite-vec Vektorindex (Issue #2) + +> **Branch:** `feat/v2.1-sqlite-vec` +> **Milestone:** V2.1 Retrieval +> **Issue:** [#2](https://github.com/KniggeMS/IFlemma/issues/2) +> **Abhängigkeit:** Keine — erster Schritt in V2.1 +> **Folge-Issue:** #3 (Hybrid Ranking) baut direkt darauf auf + +--- + +## Ziel + +`sqlite-vec` als native SQLite-Extension einbinden, `memory_embeddings` Tabelle anlegen und lokale Embedding-Generation via `@xenova/transformers` (Modell: `all-MiniLM-L6-v2`, 384 Dimensionen) implementieren. Fallback auf FTS5-only wenn Extension nicht verfügbar. + +--- + +## Schritt-für-Schritt + +### 1. Dependency installieren + +```bash +npm install sqlite-vec @xenova/transformers +``` + +In `package.json` prüfen ob `better-sqlite3` >= 12.10 vorhanden (bereits in v0.16.0 ✅). + +--- + +### 2. Migration: `memory_embeddings` Tabelle + +**Datei:** `src/db/migrations/` (neue Migrationsdatei, z.B. `020_add_embeddings.ts`) + +```sql +CREATE TABLE IF NOT EXISTS memory_embeddings ( + id TEXT PRIMARY KEY, + fragment_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + model TEXT NOT NULL DEFAULT 'all-MiniLM-L6-v2', + vector BLOB NOT NULL, -- Float32Array, 384 Dimensionen + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_embeddings_fragment + ON memory_embeddings(fragment_id); +``` + +Migration in den bestehenden Migrations-Runner eintragen (Muster der bestehenden Migrations in `src/db/migrations/` prüfen). + +--- + +### 3. sqlite-vec Extension laden + +**Datei:** `src/db/connection.ts` (oder wo die DB-Verbindung initialisiert wird — prüfen!) + +```typescript +import * as sqliteVec from 'sqlite-vec'; + +// Nach db.open() / beim Initialisieren: +try { + sqliteVec.load(db); + console.log('[lemma] sqlite-vec geladen:', db.prepare('SELECT vec_version()').pluck().get()); +} catch (err) { + console.warn('[lemma] sqlite-vec nicht verfügbar — Fallback auf FTS5-only:', err); + // Flag setzen: vectorSearchAvailable = false +} +``` + +**Wichtig:** Fehler abfangen und als `vectorSearchAvailable = false` Flag weiterreichen — kein Hard-Crash. + +--- + +### 4. Embedding-Service + +**Neue Datei:** `src/retrieval/embeddings.ts` + +```typescript +import { pipeline } from '@xenova/transformers'; + +const MODEL = 'Xenova/all-MiniLM-L6-v2'; +let embedder: Awaited> | null = null; + +export async function getEmbedder() { + if (!embedder) { + embedder = await pipeline('feature-extraction', MODEL, { + quantized: true, // kleinere Modellgröße + }); + } + return embedder; +} + +export async function generateEmbedding(text: string): Promise { + const pipe = await getEmbedder(); + const output = await pipe(text, { pooling: 'mean', normalize: true }); + return output.data as Float32Array; +} + +export function embeddingToBlob(vec: Float32Array): Buffer { + return Buffer.from(vec.buffer); +} + +export function blobToEmbedding(blob: Buffer): Float32Array { + return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4); +} +``` + +--- + +### 5. Embeddings beim Speichern generieren + +In `lemma_memory_add` / `lemma_memory_merge` (bestehende Tools) nach dem INSERT: + +```typescript +// Nach erfolgreichem INSERT in memories: +if (vectorSearchAvailable) { + try { + const vec = await generateEmbedding(content); + db.prepare(` + INSERT OR REPLACE INTO memory_embeddings (id, fragment_id, model, vector, created_at) + VALUES (?, ?, ?, ?, datetime('now')) + `).run(nanoid(), fragmentId, 'all-MiniLM-L6-v2', embeddingToBlob(vec)); + } catch (err) { + // Embedding-Fehler darf das Memory-Speichern nicht blockieren + console.warn('[lemma] Embedding-Generierung fehlgeschlagen:', err); + } +} +``` + +--- + +### 6. Startup: fehlende Embeddings nachgenerieren + +**Neue Datei:** `src/retrieval/backfill.ts` + +```typescript +export async function backfillEmbeddings(db: Database, vectorSearchAvailable: boolean) { + if (!vectorSearchAvailable) return; + + const missing = db.prepare(` + SELECT m.id, m.content FROM memories m + LEFT JOIN memory_embeddings e ON e.fragment_id = m.id + WHERE e.id IS NULL + LIMIT 100 + `).all() as { id: string; content: string }[]; + + if (missing.length === 0) return; + + console.log(`[lemma] Backfill: ${missing.length} fehlende Embeddings werden generiert...`); + + for (const { id, content } of missing) { + try { + const vec = await generateEmbedding(content); + db.prepare(` + INSERT OR IGNORE INTO memory_embeddings (id, fragment_id, model, vector, created_at) + VALUES (?, ?, ?, ?, datetime('now')) + `).run(nanoid(), id, 'all-MiniLM-L6-v2', embeddingToBlob(vec)); + } catch { + // Einzelne Fehler überspringen + } + } + + console.log('[lemma] Backfill abgeschlossen.'); +} +``` + +Aufruf beim Server-Start (nach DB-Init, non-blocking): +```typescript +backfillEmbeddings(db, vectorSearchAvailable).catch(console.warn); +``` + +--- + +### 7. Vektor-Suche (Vorbereitung für Issue #3) + +**In `src/retrieval/` neue Datei `vectorSearch.ts`:** + +```typescript +export function searchByVector( + db: Database, + queryVec: Float32Array, + limit: number = 20 +): Array<{ fragment_id: string; distance: number }> { + // sqlite-vec vec_distance_cosine für Ranking + return db.prepare(` + SELECT e.fragment_id, + vec_distance_cosine(e.vector, ?) AS distance + FROM memory_embeddings e + ORDER BY distance ASC + LIMIT ? + `).all(embeddingToBlob(queryVec), limit) as any; +} +``` + +Diese Funktion wird von Issue #3 (RRF Fusion) aufgerufen — hier nur implementieren, noch nicht in Retrieval-Pipeline einbinden. + +--- + +### 8. Fallback sicherstellen + +In der bestehenden `searchAndSortFragments()` oder equivalent: + +```typescript +if (!vectorSearchAvailable) { + // bisheriges FTS5-only Verhalten — keine Änderung + return fts5Search(query, limit); +} +// später (Issue #3): Hybrid Ranking +``` + +--- + +## Tests + +**Neue Testdatei:** `tests/retrieval/embeddings.test.ts` + +- [ ] `generateEmbedding()` gibt Float32Array mit 384 Dimensionen zurück +- [ ] `embeddingToBlob()` / `blobToEmbedding()` Round-Trip verlustfrei +- [ ] Migration legt `memory_embeddings` Tabelle korrekt an +- [ ] INSERT in `memories` → Embedding wird in `memory_embeddings` gespeichert +- [ ] Fallback: wenn `sqlite-vec` nicht verfügbar → kein Crash, FTS5-only läuft weiter +- [ ] `backfillEmbeddings()`: fragmentierte Memories werden nachgefüllt +- [ ] `searchByVector()`: gibt sortierte Ergebnisse nach Cosinus-Distanz zurück +- [ ] Latenz-Test: 1000 Fragments → Vektor-Suche < 50ms + +--- + +## Akzeptanzkriterien + +- [ ] `npm test` grün (alle 701 bestehenden + neue Tests) +- [ ] `memory_embeddings` Tabelle wird bei Migration angelegt +- [ ] Embedding wird bei `lemma_memory_add` automatisch generiert +- [ ] Backfill läuft beim Start ohne Fehler durch +- [ ] `vectorSearchAvailable = false` führt zu sauberem FTS5-Fallback, kein Crash +- [ ] Latenz Vektor-Suche: < 50ms bei 1000 Fragmenten +- [ ] Keine Breaking Changes an bestehenden 26 Tools + +--- + +## Dateien die erstellt/geändert werden + +| Datei | Aktion | +|---|---| +| `src/db/migrations/020_add_embeddings.ts` | NEU — Migration | +| `src/db/connection.ts` | ÄNDERN — sqlite-vec laden | +| `src/retrieval/embeddings.ts` | NEU — Embedding Service | +| `src/retrieval/backfill.ts` | NEU — Startup Backfill | +| `src/retrieval/vectorSearch.ts` | NEU — Vektor-Suche (für Issue #3 vorbereiten) | +| `src/tools/memory_add.ts` | ÄNDERN — Embedding nach INSERT | +| `src/tools/memory_merge.ts` | ÄNDERN — Embedding nach Merge | +| `tests/retrieval/embeddings.test.ts` | NEU — Tests | +| `package.json` | ÄNDERN — sqlite-vec + @xenova/transformers | + +--- + +## Commit-Konvention für diesen Branch + +``` +feat(db): add memory_embeddings migration +feat(retrieval): add embedding service (all-MiniLM-L6-v2) +feat(retrieval): add startup backfill for missing embeddings +feat(retrieval): add vectorSearch with cosine distance +feat(tools): generate embedding on memory_add and memory_merge +test(retrieval): add embedding service tests +``` diff --git a/src/db/schema.ts b/src/db/schema.ts index f2fe997..1332ce2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -257,4 +257,24 @@ CREATE TABLE IF NOT EXISTS improvement_suggestions ( CREATE INDEX IF NOT EXISTS idx_suggestions_status ON improvement_suggestions(status); `; -export const MIGRATIONS: [number, string][] = [[1, SCHEMA_V1], [2, SCHEMA_V2]]; +export const SCHEMA_V3 = ` +-- sqlite-vec virtual table: 384-dim embeddings (all-MiniLM-L6-v2) +-- Each row links a memory_id to its float32 embedding vector. +-- vec0 requires the PRIMARY KEY column to be the first column. +CREATE VIRTUAL TABLE IF NOT EXISTS memory_embeddings USING vec0( + memory_id INTEGER PRIMARY KEY, + embedding FLOAT[384] +); + +-- Track which embedding model version generated the stored vector. +-- NULL = no embedding yet (pending backfill). +ALTER TABLE memories ADD COLUMN embedding_version TEXT DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_memories_embedding_version ON memories(embedding_version); +`; + +export const MIGRATIONS: [number, string][] = [ + [1, SCHEMA_V1], + [2, SCHEMA_V2], + [3, SCHEMA_V3], +]; diff --git a/src/retrieval/embedder.ts b/src/retrieval/embedder.ts new file mode 100644 index 0000000..ea59b65 --- /dev/null +++ b/src/retrieval/embedder.ts @@ -0,0 +1,119 @@ +/** + * EmbedderService — wraps @xenova/transformers to produce + * 384-dimensional sentence embeddings (all-MiniLM-L6-v2). + * + * Usage: + * const svc = EmbedderService.getInstance(); + * const vec = await svc.embed("some text"); + */ + +import { logger } from "../logger.js"; + +const MODEL_NAME = "Xenova/all-MiniLM-L6-v2"; +const EMBEDDING_DIM = 384; + +export class EmbedderError extends Error { + constructor(message: string, public readonly cause?: unknown) { + super(message); + this.name = "EmbedderError"; + } +} + +type PipelineFn = (texts: string | string[], opts: Record) => Promise<{ data: Float32Array }[]>; + +export class EmbedderService { + private static instance: EmbedderService | null = null; + private pipeline: PipelineFn | null = null; + private initPromise: Promise | null = null; + + private constructor() {} + + static getInstance(): EmbedderService { + if (!EmbedderService.instance) { + EmbedderService.instance = new EmbedderService(); + } + return EmbedderService.instance; + } + + /** Reset singleton — only for tests. */ + static _reset(): void { + EmbedderService.instance = null; + } + + isReady(): boolean { + return this.pipeline !== null; + } + + getModelName(): string { + return MODEL_NAME; + } + + getDim(): number { + return EMBEDDING_DIM; + } + + private async init(): Promise { + if (this.pipeline) return; + if (this.initPromise) return this.initPromise; + + this.initPromise = (async () => { + logger.info("EmbedderService: loading model", { model: MODEL_NAME }); + try { + // Dynamic import keeps startup fast when embeddings aren't needed. + const { pipeline } = await import("@xenova/transformers"); + this.pipeline = (await pipeline("feature-extraction", MODEL_NAME)) as PipelineFn; + logger.info("EmbedderService: model ready", { model: MODEL_NAME }); + } catch (err) { + this.initPromise = null; + throw new EmbedderError(`Failed to load embedding model ${MODEL_NAME}`, err); + } + })(); + + return this.initPromise; + } + + /** + * Embed a single text string. + * Returns a normalised Float32Array of length EMBEDDING_DIM. + */ + async embed(text: string): Promise { + if (!text || !text.trim()) { + throw new EmbedderError("Cannot embed empty text"); + } + await this.init(); + + try { + const output = await this.pipeline!(text, { + pooling: "mean", + normalize: true, + }); + // @xenova/transformers returns an array of Tensors; first element is our embedding. + const raw = output[0]?.data; + if (!raw || raw.length !== EMBEDDING_DIM) { + throw new EmbedderError( + `Unexpected embedding dimension: got ${raw?.length ?? 0}, expected ${EMBEDDING_DIM}` + ); + } + return raw instanceof Float32Array ? raw : new Float32Array(raw); + } catch (err) { + if (err instanceof EmbedderError) throw err; + throw new EmbedderError("Embedding failed", err); + } + } + + /** + * Embed multiple texts sequentially. + * onProgress is called after each item: onProgress(done, total). + */ + async embedBatch( + texts: string[], + onProgress?: (done: number, total: number) => void + ): Promise { + const results: Float32Array[] = []; + for (let i = 0; i < texts.length; i++) { + results.push(await this.embed(texts[i])); + onProgress?.(i + 1, texts.length); + } + return results; + } +}