diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..47b1f9f --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,80 @@ +# Architecture notes + +Long-lived decisions that outlive a single feature. Feature-specific design lives +in the issue tracker and in `PRODUCT.md` / `DESIGN.md`; this file records the +seams that other work is allowed to depend on. + +## Source provenance + +A document is an identity. Everything derived from it can be rebuilt: + +```text +PDF / DOCX / URL / Note + ↓ +documents ← source identity (id, content, structure, localFilePath) + ↓ +document_blocks ← derived: page / heading level / bbox / char span + ↓ +chunks + chunk_blocks ← derived: char offsets anchored to documents.content + ↓ +embeddings + vectors ← derived +``` + +Two invariants hold across the whole chain: + +1. `chunk.content` and `block.text` are always `documents.content.slice(start, end)`. + The chunker and the block builder never pre-process text, so offsets and content + cannot drift apart. +2. Re-indexing rebuilds the derived layer in place. It never creates a second + `documents` row, so a citation that stored a `documentId` stays valid. The parse + structure is persisted on the source row so a rebuild does not need to re-parse a + file that may have changed on disk. + +## Retrieval seam + +`src/main/services/retrieval/` is the boundary between "how do we search" and +"what do we do with the results". Anything — Chat, the search panel, the MCP +server (#80), the eval harness (#75) — goes through it instead of embedding its own +RAG path. + +```ts +interface Retriever { + search(notebookId: string, query: string, options?: RetrieveOptions): Promise +} +``` + +```ts +interface RetrievedEvidence { + chunkId: string + documentId: string + content: string + score: number + chunkIndex: number + source: { title: string; type: string } + locator: { + pageStart: number | null + pageEnd: number | null + blocks: EvidenceBlock[] + } + metadata?: Record +} +``` + +Rules: + +- **Provenance travels with the result.** A retriever must return a `locator`, not a + bare chunk, so the citation layer (#69) never has to query the database again. +- **The embedding-space guard is a precondition, not a strategy.** It stays in + `KnowledgeService.search()`, where the notebook-level index identity is checked, + before any strategy runs. +- **Strategies are swappable and measurable.** `DenseRetriever` is the only + implementation today. BM25, hybrid fusion and rerankers (#77) implement the same + interface; the eval harness (#75) measures them without touching callers. +- **Hydration is batch.** `hydrateEvidence()` resolves chunk, document and + provenance in a constant number of queries regardless of `topK`. Per-chunk + `resolveChunkProvenance()` calls are not allowed on the hot path. + +`KnowledgeService.search()` keeps its historical `SearchResult[]` shape for IPC +compatibility and delegates to the default retriever. `SearchResult` gained a +`locator` field, which is populated from `RetrievedEvidence`; ranking, scores and +the existing fields are unchanged. diff --git a/src/main/db/migrations/0016_known_norrin_radd.sql b/src/main/db/migrations/0016_known_norrin_radd.sql new file mode 100644 index 0000000..49e7a2a --- /dev/null +++ b/src/main/db/migrations/0016_known_norrin_radd.sql @@ -0,0 +1 @@ +ALTER TABLE `documents` ADD `structure` text; \ No newline at end of file diff --git a/src/main/db/migrations/meta/0016_snapshot.json b/src/main/db/migrations/meta/0016_snapshot.json new file mode 100644 index 0000000..440eff1 --- /dev/null +++ b/src/main/db/migrations/meta/0016_snapshot.json @@ -0,0 +1,1605 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "2ae791e5-410f-4941-8484-64436eba26fb", + "prevId": "03fce19e-c08b-4bf0-9635-f54f7b00a883", + "tables": { + "anki_cards": { + "name": "anki_cards", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "cards_data": { + "name": "cards_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_ankicards_notebook": { + "name": "idx_ankicards_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_ankicards_version": { + "name": "idx_ankicards_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "anki_cards_notebook_id_notebooks_id_fk": { + "name": "anki_cards_notebook_id_notebooks_id_fk", + "tableFrom": "anki_cards", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_content": { + "name": "reasoning_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_session": { + "name": "idx_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_messages_session_id_chat_sessions_id_fk": { + "name": "chat_messages_session_id_chat_sessions_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_sessions": { + "name": "chat_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "parent_session_id": { + "name": "parent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_sessions_notebook": { + "name": "idx_sessions_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_sessions_notebook_id_notebooks_id_fk": { + "name": "chat_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_sessions_parent_session_id_chat_sessions_id_fk": { + "name": "chat_sessions_parent_session_id_chat_sessions_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "chat_sessions", + "columnsFrom": [ + "parent_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunk_blocks": { + "name": "chunk_blocks", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_in_block": { + "name": "start_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_in_block": { + "name": "end_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunk_blocks_block": { + "name": "idx_chunk_blocks_block", + "columns": [ + "block_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunk_blocks_chunk_id_chunks_id_fk": { + "name": "chunk_blocks_chunk_id_chunks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunk_blocks_block_id_document_blocks_id_fk": { + "name": "chunk_blocks_block_id_document_blocks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "document_blocks", + "columnsFrom": [ + "block_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "chunk_blocks_chunk_id_block_id_pk": { + "columns": [ + "chunk_id", + "block_id" + ], + "name": "chunk_blocks_chunk_id_block_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunks": { + "name": "chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_start": { + "name": "page_start", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_end": { + "name": "page_end", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunks_document": { + "name": "idx_chunks_document", + "columns": [ + "document_id" + ], + "isUnique": false + }, + "idx_chunks_notebook": { + "name": "idx_chunks_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunks_document_id_documents_id_fk": { + "name": "chunks_document_id_documents_id_fk", + "tableFrom": "chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunks_notebook_id_notebooks_id_fk": { + "name": "chunks_notebook_id_notebooks_id_fk", + "tableFrom": "chunks", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_blocks": { + "name": "document_blocks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bbox": { + "name": "bbox", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_blocks_document_order": { + "name": "idx_blocks_document_order", + "columns": [ + "document_id", + "order" + ], + "isUnique": false + }, + "idx_blocks_document_page": { + "name": "idx_blocks_document_page", + "columns": [ + "document_id", + "page" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_blocks_document_id_documents_id_fk": { + "name": "document_blocks_document_id_documents_id_fk", + "tableFrom": "document_blocks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "local_file_path": { + "name": "local_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_note_id": { + "name": "source_note_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "structure": { + "name": "structure", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_documents_notebook": { + "name": "idx_documents_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_documents_status": { + "name": "idx_documents_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_notebook_id_notebooks_id_fk": { + "name": "documents_notebook_id_notebooks_id_fk", + "tableFrom": "documents", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_source_note_id_notes_id_fk": { + "name": "documents_source_note_id_notes_id_fk", + "tableFrom": "documents", + "tableTo": "notes", + "columnsFrom": [ + "source_note_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embeddings": { + "name": "embeddings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeddings_chunk": { + "name": "idx_embeddings_chunk", + "columns": [ + "chunk_id" + ], + "isUnique": false + }, + "idx_embeddings_notebook": { + "name": "idx_embeddings_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + }, + "idx_embeddings_model": { + "name": "idx_embeddings_model", + "columns": [ + "model" + ], + "isUnique": false + } + }, + "foreignKeys": { + "embeddings_chunk_id_chunks_id_fk": { + "name": "embeddings_chunk_id_chunks_id_fk", + "tableFrom": "embeddings", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embeddings_notebook_id_notebooks_id_fk": { + "name": "embeddings_notebook_id_notebooks_id_fk", + "tableFrom": "embeddings", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "items": { + "name": "items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_items_notebook_order": { + "name": "idx_items_notebook_order", + "columns": [ + "notebook_id", + "order" + ], + "isUnique": false + }, + "idx_items_type": { + "name": "idx_items_type", + "columns": [ + "type" + ], + "isUnique": false + }, + "idx_items_resource": { + "name": "idx_items_resource", + "columns": [ + "type", + "resource_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "items_notebook_id_notebooks_id_fk": { + "name": "items_notebook_id_notebooks_id_fk", + "tableFrom": "items", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mind_maps": { + "name": "mind_maps", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "tree_data": { + "name": "tree_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_mindmaps_notebook": { + "name": "idx_mindmaps_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_mindmaps_version": { + "name": "idx_mindmaps_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mind_maps_notebook_id_notebooks_id_fk": { + "name": "mind_maps_notebook_id_notebooks_id_fk", + "tableFrom": "mind_maps", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebook_embedding_spaces": { + "name": "notebook_embedding_spaces", + "columns": { + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "backend": { + "name": "backend", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "notebook_embedding_spaces_notebook_id_notebooks_id_fk": { + "name": "notebook_embedding_spaces_notebook_id_notebooks_id_fk", + "tableFrom": "notebook_embedding_spaces", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebooks": { + "name": "notebooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notebooks_updated": { + "name": "idx_notebooks_updated", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notes": { + "name": "notes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notes_notebook": { + "name": "idx_notes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "notes_notebook_id_notebooks_id_fk": { + "name": "notes_notebook_id_notebooks_id_fk", + "tableFrom": "notes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quiz_sessions": { + "name": "quiz_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answers": { + "name": "answers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_questions": { + "name": "total_questions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "correct_count": { + "name": "correct_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quiz_sessions_quiz": { + "name": "idx_quiz_sessions_quiz", + "columns": [ + "quiz_id", + "created_at" + ], + "isUnique": false + }, + "idx_quiz_sessions_notebook": { + "name": "idx_quiz_sessions_notebook", + "columns": [ + "notebook_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quiz_sessions_quiz_id_quizzes_id_fk": { + "name": "quiz_sessions_quiz_id_quizzes_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "quizzes", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_sessions_notebook_id_notebooks_id_fk": { + "name": "quiz_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quizzes": { + "name": "quizzes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "questions_data": { + "name": "questions_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quizzes_notebook": { + "name": "idx_quizzes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_quizzes_version": { + "name": "idx_quizzes_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quizzes_notebook_id_notebooks_id_fk": { + "name": "quizzes_notebook_id_notebooks_id_fk", + "tableFrom": "quizzes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/main/db/migrations/meta/_journal.json b/src/main/db/migrations/meta/_journal.json index c83a95e..b9b8156 100644 --- a/src/main/db/migrations/meta/_journal.json +++ b/src/main/db/migrations/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1790316032660, "tag": "0015_colossal_solo", "breakpoints": true + }, + { + "idx": 16, + "version": "6", + "when": 1790320058521, + "tag": "0016_known_norrin_radd", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/main/db/schema.ts b/src/main/db/schema.ts index 3fb4f58..8d191ea 100644 --- a/src/main/db/schema.ts +++ b/src/main/db/schema.ts @@ -1,5 +1,6 @@ import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlite-core' import type { QuizQuestion } from '../../shared/types/quiz' +import type { DocumentStructure } from '../services/loaders/types' /** * 笔记本表 @@ -129,6 +130,10 @@ export const documents = sqliteTable( localFilePath: text('local_file_path'), // 本地拷贝文件路径 sourceNoteId: text('source_note_id').references(() => notes.id, { onDelete: 'set null' }), content: text('content'), // 原始内容（可选存储） + // 解析器给出的结构（页/章节）。这是 source record 的一部分,不是派生索引: + // 重新索引必须能重建出与首次导入一致的 document_blocks,所以不能靠重新解析 + // 文件来恢复（文件可能已变化,内容又是规范文本）。 + structure: text('structure', { mode: 'json' }).$type(), contentHash: text('content_hash'), // 内容哈希，用于检测变更 mimeType: text('mime_type'), // 文件类型 fileSize: integer('file_size'), // 文件大小（字节） diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 0e65803..f6a60bf 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -36,12 +36,15 @@ import { EmbeddingService } from './EmbeddingService' import type { EmbeddingSpace } from '../../shared/types' import { ChunkingService, type ChunkOptions, type ChunkResult } from './ChunkingService' import { FileParserService } from './FileParserService' +import type { DocumentStructure } from './loaders/types' import { buildDocumentBlocks, assignBlockIds, type IdentifiedBlockDraft } from './blocks/documentBlocks' import { resolveChunkProvenance, type ChunkProvenance } from './chunkProvenance' +import { DenseRetriever } from './retrieval' +import type { EvidenceLocator, Retriever } from './retrieval' import { WebFetchService } from './WebFetchService' import { vectorStoreManager } from '../vectorstore' import Logger from '../../shared/utils/logger' @@ -72,6 +75,9 @@ export interface SearchOptions { /** * 搜索结果 + * + * `locator` 是检索 → 引用的 seam：页码区间与块区间随结果一起交付，#69 直接用它 + * 组装 citation，不必再回头查库。 */ export interface SearchResult { chunkId: string @@ -81,6 +87,7 @@ export interface SearchResult { content: string score: number chunkIndex: number + locator: EvidenceLocator metadata?: Record } @@ -96,6 +103,7 @@ export type IndexProgressCallback = (stage: string, progress: number) => void export class KnowledgeService { private embeddingService: EmbeddingService private chunkingService: ChunkingService + private retriever: Retriever private fileParserService: FileParserService private webFetchService: WebFetchService private knowledgeFilesDir: string @@ -103,6 +111,7 @@ export class KnowledgeService { constructor(embeddingService: EmbeddingService) { this.embeddingService = embeddingService this.chunkingService = new ChunkingService() + this.retriever = new DenseRetriever(embeddingService) this.fileParserService = new FileParserService() this.webFetchService = new WebFetchService() // 知识库文件存储目录 @@ -178,145 +187,41 @@ export class KnowledgeService { // 计算内容哈希 const contentHash = createHash('md5').update(options.content).digest('hex') - try { - // 1. 创建文档记录 - onProgress?.('creating_document', 0) - - const newDoc: NewDocument = { - id: documentId, - notebookId, - title: options.title, - type: options.type, - sourceUri: options.sourceUri, - sourceNoteId: options.sourceNoteId, - content: options.content, - contentHash, - mimeType: options.mimeType, - fileSize: options.fileSize, - metadata: options.metadata, - status: 'processing', - chunkCount: 0, - createdAt: now, - updatedAt: now - } - - db.insert(documents).values(newDoc).run() - - // 1b. 持久化文档块（文本/URL/笔记没有结构，按段落平铺），并把同一批块交给分块器 - const blocks = assignBlockIds(documentId, buildDocumentBlocks({ content: options.content })) - this.persistDocumentBlocks(blocks) - - // 2. 分块（偏移锚定 options.content，即 documents.content） - onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunkBlocks( - options.content, - blocks, - options.chunkOptions - ) - - if (chunkResults.length === 0) { - throw new Error('No chunks generated from document') - } - - Logger.info('KnowledgeService', `Document ${documentId}: ${chunkResults.length} chunks`) - - // 3. 保存分块与 chunk↔block 映射（同一事务，不会出现没有映射的 chunk） - onProgress?.('saving_chunks', 20) - const { chunkIds, chunkContents } = this.saveChunks(documentId, notebookId, chunkResults, now) - - // 4. 生成嵌入向量 - // 不指定维度:维度由 embedding 模型决定,写死维度会让用别的模型的笔记本直接 - // 索引失败(vec0 表的宽度在创建时固定,见 #33)。测出真实维度之后再用它建表。 - const embeddingResults = await this.embedDocumentChunks(notebookId, chunkContents, onProgress) - - if (embeddingResults.length === 0) { - throw new Error('Embedding 服务没有返回任何向量') - } - - // space 由模型决定;与既有 space 不一致时重建向量表并把文档标回待索引 - const detectedDimensions = embeddingResults[0].dimensions - await this.reconcileEmbeddingSpace( - notebookId, - await this.embeddingService.getSpace(), - detectedDimensions - ) - Logger.info('KnowledgeService', `Embedding dimensions: ${detectedDimensions}`) - - // 5. 保存嵌入元数据并添加到向量存储 - onProgress?.('saving_embeddings', 85) - const vectorStore = await vectorStoreManager.getStore( - notebookId, - undefined, - detectedDimensions - ) - const vectorItems: Array<{ - id: string - chunkId: string - vector: Float32Array - metadata?: Record - }> = [] - - for (let i = 0; i < chunkIds.length; i++) { - const embeddingId = `emb_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` - const embResult = embeddingResults[i] - - // 保存嵌入元数据到数据库 - const newEmbedding: NewEmbedding = { - id: embeddingId, - chunkId: chunkIds[i], - notebookId, - model: embResult.model, - dimensions: embResult.dimensions, - createdAt: now - } - - db.insert(embeddings).values(newEmbedding).run() + // 1. 创建 source 记录。Document 是 source identity：之后的每一次重新索引都复用 + // 这一行，不会再生成新的 documentId。 + onProgress?.('creating_document', 0) - // 准备向量数据 - vectorItems.push({ - id: embeddingId, - chunkId: chunkIds[i], - vector: embResult.embedding, - metadata: { model: embResult.model, documentId } - }) - } - - // 批量添加到向量存储 - await vectorStore.upsert(vectorItems) - - // 6. 更新文档状态 - onProgress?.('finalizing', 95) - db.update(documents) - .set({ - status: 'indexed', - chunkCount: chunkResults.length, - updatedAt: new Date() - }) - .where(eq(documents.id, documentId)) - .run() + const newDoc: NewDocument = { + id: documentId, + notebookId, + title: options.title, + type: options.type, + sourceUri: options.sourceUri, + sourceNoteId: options.sourceNoteId, + content: options.content, + contentHash, + mimeType: options.mimeType, + fileSize: options.fileSize, + metadata: options.metadata, + status: 'processing', + chunkCount: 0, + createdAt: now, + updatedAt: now + } - executeCheckpoint('PASSIVE') - onProgress?.('completed', 100) + db.insert(documents).values(newDoc).run() - Logger.info( - 'KnowledgeService', - `Document indexed: ${documentId}, ${chunkResults.length} chunks` - ) - return documentId - } catch (error) { - // 更新文档状态为失败 - db.update(documents) - .set({ - status: 'failed', - errorMessage: (error as Error).message, - updatedAt: new Date() - }) - .where(eq(documents.id, documentId)) - .run() + // 2. 建立派生索引（blocks → chunks → embeddings → 向量表）。失败时由 + // `indexDocument()` 把这一行标成 failed，source 行保留以便重试或重新索引。 + await this.indexDocument( + documentId, + options.content, + undefined, + { chunkOptions: options.chunkOptions }, + onProgress + ) - Logger.error('KnowledgeService', 'Failed to add document:', error) - throw error - } + return documentId } /** @@ -330,6 +235,8 @@ export class KnowledgeService { const db = getDatabase() const documentId = `doc_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` let localFilePath: string | undefined + let parsedContent = '' + let parsedStructure: DocumentStructure | undefined try { onProgress?.('parsing_file', 0) @@ -338,12 +245,14 @@ export class KnowledgeService { localFilePath = await this.copyFileToKnowledgeDir(filePath, documentId) const parseResult = await this.fileParserService.parseFile(filePath) + parsedContent = parseResult.content + parsedStructure = parseResult.structure ?? undefined // 计算内容哈希 const contentHash = createHash('md5').update(parseResult.content).digest('hex') const now = new Date() - // 1. 创建文档记录（包含 localFilePath） + // 1. 创建 source 记录（包含 localFilePath 与解析结构） onProgress?.('creating_document', 0) // 根据 MIME 类型决定文档类型 @@ -361,6 +270,8 @@ export class KnowledgeService { sourceUri: filePath, localFilePath: localFilePath, content: parseResult.content, + // 结构随 source 一起持久化，重新索引才能不加解析地重建同一批块 + structure: parsedStructure, contentHash, mimeType: parseResult.mimeType, fileSize: parseResult.metadata?.fileSize as number | undefined, @@ -372,17 +283,65 @@ export class KnowledgeService { } db.insert(documents).values(newDoc).run() + } catch (error) { + // 拷贝/解析/建行失败时删除已拷贝的文件，并把（可能已创建的）行标成失败 + if (localFilePath) { + await this.deleteLocalFile(localFilePath) + } - // 1b. 持久化文档块，保留解析器给出的页/标题结构，并把同一批块交给分块器 - const blocks = assignBlockIds( - documentId, - buildDocumentBlocks({ content: parseResult.content, structure: parseResult.structure }) - ) + db.update(documents) + .set({ + status: 'failed', + errorMessage: (error as Error).message, + updatedAt: new Date() + }) + .where(eq(documents.id, documentId)) + .run() + + Logger.error('KnowledgeService', 'Failed to add document from file:', error) + throw error + } + + // 2. 建立派生索引。索引失败由 `indexDocument()` 标记；这里保留本地文件， + // 用户可以重新索引，而不是重新导入。 + await this.indexDocument(documentId, parsedContent, parsedStructure, {}, onProgress) + + return documentId + } + + /** + * 为一份已经存在的 source（`documents` 行）建立派生索引： + * blocks → chunks → embeddings → 向量表。 + * + * Document 是 source identity，blocks/chunks/embeddings 是派生索引。这个方法只写 + * 派生索引，从不插入或删除 `documents` 行，所以对同一个 documentId 反复调用不会改 + * 变来源身份（历史 citation 仍然指向同一个来源）。 + * + * 调用前该文档不应带有旧的派生索引；重新索引要先 `clearDerivedIndex()`。 + * 失败时把该行标成 `failed` 并抛出，由调用方决定是否清理 source。 + */ + private async indexDocument( + documentId: string, + content: string, + structure: DocumentStructure | undefined, + options: { chunkOptions?: ChunkOptions } = {}, + onProgress?: IndexProgressCallback + ): Promise { + const db = getDatabase() + const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() + if (!doc) throw new Error(`Document ${documentId} not found`) + + const notebookId = doc.notebookId + const now = new Date() + + try { + // 1. 文档块。偏移锚定 content，即 documents.content。 + const blocks = assignBlockIds(documentId, buildDocumentBlocks({ content, structure })) this.persistDocumentBlocks(blocks) - // 2. 分块（偏移锚定 parseResult.content，即 documents.content） + // 2. 分块（偏移同样锚定 content） onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunkBlocks(parseResult.content, blocks) + const chunkResults = this.chunkingService.chunkBlocks(content, blocks, options.chunkOptions) if (chunkResults.length === 0) { throw new Error('No chunks generated from document') @@ -460,6 +419,7 @@ export class KnowledgeService { .set({ status: 'indexed', chunkCount: chunkResults.length, + errorMessage: null, updatedAt: new Date() }) .where(eq(documents.id, documentId)) @@ -472,14 +432,9 @@ export class KnowledgeService { 'KnowledgeService', `Document indexed: ${documentId}, ${chunkResults.length} chunks` ) - return documentId + return chunkResults.length } catch (error) { - // 失败时删除已拷贝的文件 - if (localFilePath) { - await this.deleteLocalFile(localFilePath) - } - - // 更新文档状态为失败 + // 派生索引可以重建，所以失败只是状态问题，不需要删掉 source 行 db.update(documents) .set({ status: 'failed', @@ -489,11 +444,46 @@ export class KnowledgeService { .where(eq(documents.id, documentId)) .run() - Logger.error('KnowledgeService', 'Failed to add document from file:', error) + Logger.error('KnowledgeService', 'Failed to index document:', error) throw error } } + /** + * 删除一份文档的全部派生索引（向量、chunk↔block 映射、chunks、embeddings、 + * document_blocks），并把它标回 `processing`。`documents` 行本身保留。 + */ + private async clearDerivedIndex(documentId: string): Promise { + const db = getDatabase() + const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() + if (!doc) return + + const oldChunks = db + .select({ id: chunks.id }) + .from(chunks) + .where(eq(chunks.documentId, documentId)) + .all() + + if (oldChunks.length > 0) { + const chunkIds = oldChunks.map((chunk) => chunk.id) + + const vectorStore = await vectorStoreManager.getStore(doc.notebookId) + await vectorStore.deleteByChunkIds(chunkIds) + + // 映射与 embeddings 不依赖外键级联（连接上 foreign_keys 默认是关的），显式删除 + db.delete(chunkBlocks).where(inArray(chunkBlocks.chunkId, chunkIds)).run() + db.delete(embeddings).where(inArray(embeddings.chunkId, chunkIds)).run() + db.delete(chunks).where(eq(chunks.documentId, documentId)).run() + } + + db.delete(documentBlocks).where(eq(documentBlocks.documentId, documentId)).run() + + db.update(documents) + .set({ status: 'processing', errorMessage: null, chunkCount: 0, updatedAt: new Date() }) + .where(eq(documents.id, documentId)) + .run() + } + /** * 持久化文档块。id 已由 `assignBlockIds()` 生成，与分块器看到的是同一批块。 */ @@ -751,14 +741,17 @@ export class KnowledgeService { } /** - * 语义搜索 + * 语义搜索。 + * + * 保持原有签名与返回形状，内部委托给默认的 `Retriever`（当前是 `DenseRetriever`）。 + * embedding space 校验留在这里：它是前置条件，不是检索策略的一部分。 */ async search( notebookId: string, query: string, options: SearchOptions = {} ): Promise { - const { topK = 5, threshold = 0.5, includeContent = true } = options + const { includeContent = true } = options const db = getDatabase() // 0. 索引身份校验：当前模型与建索引时不一致，向量不可比，明确要求重新索引 @@ -774,64 +767,31 @@ export class KnowledgeService { ) } - // 1. 生成查询向量（E5 要求 query 前缀，与索引时的 document 前缀区分） - await this.embeddingService.ensureReady() - const queryEmbedding = await this.embeddingService.embed(query, 'query') - - // 2. 向量检索 - const vectorStore = await vectorStoreManager.getStore(notebookId) - const vectorResults = await vectorStore.query(queryEmbedding.embedding, { - topK, - threshold + // 1. 检索（向量 → 批量补齐来源/定位信息） + const evidence = await this.retriever.search(notebookId, query, { + topK: options.topK, + threshold: options.threshold }) - if (vectorResults.length === 0) { - return [] - } - - // 3. 获取 chunk 详情 - const chunkIds = vectorResults.map((r) => r.chunkId) - - const chunkDetails = db.select().from(chunks).where(inArray(chunks.id, chunkIds)).all() - - // 获取文档信息 - const documentIds = [...new Set(chunkDetails.map((c) => c.documentId))] - const documentDetails = db - .select() - .from(documents) - .where(inArray(documents.id, documentIds)) - .all() - - const documentMap = new Map(documentDetails.map((d) => [d.id, d])) - const chunkMap = new Map(chunkDetails.map((c) => [c.id, c])) - - // 4. 组装结果 - const results: SearchResult[] = [] - - for (const vr of vectorResults) { - const chunk = chunkMap.get(vr.chunkId) - if (!chunk) continue - - const doc = documentMap.get(chunk.documentId) - + // 2. 映射回兼容的 SearchResult 形状 + return evidence.map((item) => { const result: SearchResult = { - chunkId: vr.chunkId, - documentId: chunk.documentId, - documentTitle: doc?.title || 'Unknown', - documentType: doc?.type || 'unknown', - content: includeContent ? chunk.content : '', - score: vr.score, - chunkIndex: chunk.chunkIndex + chunkId: item.chunkId, + documentId: item.documentId, + documentTitle: item.source.title, + documentType: item.source.type, + content: includeContent ? item.content : '', + score: item.score, + chunkIndex: item.chunkIndex, + locator: item.locator } - if (chunk.metadata) { - result.metadata = chunk.metadata + if (item.metadata) { + result.metadata = item.metadata } - results.push(result) - } - - return results + return result + }) } /** @@ -873,38 +833,14 @@ export class KnowledgeService { const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() if (!doc) return - // 获取所有 chunk IDs - const docChunks = db - .select({ id: chunks.id }) - .from(chunks) - .where(eq(chunks.documentId, documentId)) - .all() - - // 从向量存储删除 - if (docChunks.length > 0) { - const vectorStore = await vectorStoreManager.getStore(doc.notebookId) - await vectorStore.deleteByChunkIds(docChunks.map((c) => c.id)) - } + // 删除全部派生索引（向量、映射、chunks、embeddings、blocks） + await this.clearDerivedIndex(documentId) // 删除本地拷贝的文件（如果存在） if (doc.localFilePath) { await this.deleteLocalFile(doc.localFilePath) } - // 块与映射不依赖外键级联（连接上 foreign_keys 默认是关的），显式删除 - if (docChunks.length > 0) { - db.delete(chunkBlocks) - .where( - inArray( - chunkBlocks.chunkId, - docChunks.map((c) => c.id) - ) - ) - .run() - } - db.delete(documentBlocks).where(eq(documentBlocks.documentId, documentId)).run() - - // 级联删除会自动清理 chunks 和 embeddings db.delete(documents).where(eq(documents.id, documentId)).run() executeCheckpoint('PASSIVE') @@ -912,7 +848,13 @@ export class KnowledgeService { } /** - * 重建文档索引 + * 重建文档索引。 + * + * 只重建派生索引：documentId、localFilePath、sourceUri、metadata 与解析结构都 + * 保持不变。重新索引不会让历史 citation 指向另一个来源。 + * + * 升级前导入的文档 `structure` 为 NULL（migration 只加列，不回填）。这时先从本地 + * 副本恢复结构再删除旧索引：顺序很重要，否则恢复失败会把已有的 page/bbox 也删掉。 */ async reindexDocument(documentId: string, onProgress?: IndexProgressCallback): Promise { const db = getDatabase() @@ -922,57 +864,42 @@ export class KnowledgeService { throw new Error(`Document ${documentId} not found or has no content`) } - // 删除旧的 chunks 和 embeddings - const oldChunks = db - .select({ id: chunks.id }) - .from(chunks) - .where(eq(chunks.documentId, documentId)) - .all() - - if (oldChunks.length > 0) { - const vectorStore = await vectorStoreManager.getStore(doc.notebookId) - await vectorStore.deleteByChunkIds(oldChunks.map((c) => c.id)) - db.delete(chunkBlocks) - .where( - inArray( - chunkBlocks.chunkId, - oldChunks.map((c) => c.id) - ) - ) - .run() + let structure = doc.structure ?? undefined + if (!structure) { + structure = await this.recoverStructure(doc) + if (structure) { + db.update(documents).set({ structure }).where(eq(documents.id, documentId)).run() + Logger.info('KnowledgeService', `Recovered structure for legacy document ${documentId}`) + } } - db.delete(chunks).where(eq(chunks.documentId, documentId)).run() - db.delete(embeddings) - .where( - inArray( - embeddings.chunkId, - oldChunks.map((c) => c.id) - ) - ) - .run() + await this.clearDerivedIndex(documentId) + await this.indexDocument(documentId, doc.content, structure, {}, onProgress) + } - // 更新状态 - db.update(documents) - .set({ status: 'processing', updatedAt: new Date() }) - .where(eq(documents.id, documentId)) - .run() + /** + * 从本地副本重新解析出结构，用于升级前没有持久化 `structure` 的文档。 + * + * 只在 `localFilePath` 存在且重新解析出的内容与规范内容完全一致时采用 —— 文件被改 + * 过的话 offsets 就指不到同一份文本，宁可退化成平铺块也不能错误对齐。 + */ + private async recoverStructure(doc: Document): Promise { + if (!doc.localFilePath || !doc.content) return undefined - // 重新索引（复用 addDocument 逻辑的核心部分） - // 为简化实现，这里直接调用内部处理 - await this.addDocument( - doc.notebookId, - { - title: doc.title, - type: doc.type as 'file' | 'note' | 'url' | 'text', - content: doc.content, - sourceUri: doc.sourceUri || undefined, - sourceNoteId: doc.sourceNoteId || undefined, - mimeType: doc.mimeType || undefined, - metadata: doc.metadata || undefined - }, - onProgress - ) + try { + const parsed = await this.fileParserService.parseFile(doc.localFilePath) + if (parsed.content !== doc.content) { + Logger.warn( + 'KnowledgeService', + `Cannot recover structure for ${doc.id}: content changed since import` + ) + return undefined + } + return parsed.structure ?? undefined + } catch (error) { + Logger.warn('KnowledgeService', `Cannot recover structure for ${doc.id}:`, error) + return undefined + } } /** diff --git a/src/main/services/blocks/documentBlocks.ts b/src/main/services/blocks/documentBlocks.ts index 743266f..483dfd6 100644 --- a/src/main/services/blocks/documentBlocks.ts +++ b/src/main/services/blocks/documentBlocks.ts @@ -36,8 +36,8 @@ export interface IdentifiedBlockDraft extends DocumentBlockDraft { } /** - * 给块分配 id。id 由 documentId 与顺序确定，天然唯一且可读；重新索引会换新 - * documentId，不会与旧块冲突。 + * 给块分配 id。id 由 documentId 与顺序确定，天然唯一且可读。重新索引复用同一个 + * documentId，但会先删除该文档的旧块，所以顺序 id 不会与旧块冲突。 */ export function assignBlockIds( documentId: string, @@ -87,12 +87,34 @@ export function buildDocumentBlocks(source: BlockSource): DocumentBlockDraft[] { } /** - * 分页文档：页偏移已经由 loader 保证与规范文本一致，块文本直接切出来。 + * 分页文档：优先用 loader 已经分好的页内段落（PDF），每个段落一个块；loader 没有 + * 提供页内块时（例如旧的 / 非 PDF 分页格式）回退到整页一块。页偏移已由 loader + * 保证与规范文本一致，块文本直接切出来。 */ function buildPageBlocks(content: string, pages: PageInfo[]): Omit[] { const blocks: Omit[] = [] for (const page of pages) { + if (page.blocks && page.blocks.length > 0) { + for (const pageBlock of page.blocks) { + const text = content.slice(pageBlock.startOffset, pageBlock.endOffset) + if (text.trim().length === 0) continue + + blocks.push({ + kind: 'paragraph', + page: page.pageNumber, + level: null, + text, + // 用实际切出的文本长度回写 endOffset，避免 loader 的结尾空白把块撑出内容范围 + startOffset: pageBlock.startOffset, + endOffset: pageBlock.startOffset + text.length, + bbox: pageBlock.bbox ?? null, + metadata: null + }) + } + continue + } + const text = content.slice(page.startOffset, page.endOffset) if (text.trim().length === 0) continue @@ -101,7 +123,6 @@ function buildPageBlocks(content: string, pages: PageInfo[]): Omit @@ -21,6 +22,8 @@ export interface ChunkProvenanceBlock { kind: BlockKind level: number | null page: number | null + /** 归一化页面坐标（仅分页格式）；引用要高亮到段落就需要它。 */ + bbox: NormalizedBox | null text: string startOffset: number endOffset: number @@ -60,6 +63,7 @@ export function projectChunkProvenance( kind: row.kind, level: row.level, page: row.page, + bbox: row.bbox, text: row.text, startOffset: row.startOffset, endOffset: row.endOffset, @@ -84,19 +88,25 @@ export function insertChunkBlocks(db: Db, chunkId: string, spans: ChunkBlockSpan } /** - * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 - * 一次 join chunks → chunk_blocks → document_blocks。chunk 不存在时返回 undefined。 + * 批量解析多个 chunk 的来源。检索一次返回 topK 个 chunk，逐个调用 + * `resolveChunkProvenance()` 会自然形成 N+1；这里用一次 join 取回全部映射，再在 JS 里 + * 按 chunkId 分组。返回顺序与 `chunkIds` 无关，由 `Map` 承载。 */ -export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance | undefined { - const chunk = db.select().from(chunks).where(eq(chunks.id, chunkId)).get() - if (!chunk) return undefined +export function resolveChunksProvenance(db: Db, chunkIds: string[]): Map { + const result = new Map() + if (chunkIds.length === 0) return result const rows = db .select({ + chunkId: chunks.id, + documentId: chunks.documentId, + pageStart: chunks.pageStart, + pageEnd: chunks.pageEnd, blockId: documentBlocks.id, kind: documentBlocks.kind, level: documentBlocks.level, page: documentBlocks.page, + bbox: documentBlocks.bbox, text: documentBlocks.text, startOffset: documentBlocks.startOffset, endOffset: documentBlocks.endOffset, @@ -104,10 +114,66 @@ export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance startInBlock: chunkBlocks.startInBlock, endInBlock: chunkBlocks.endInBlock }) - .from(chunkBlocks) - .innerJoin(documentBlocks, eq(chunkBlocks.blockId, documentBlocks.id)) - .where(eq(chunkBlocks.chunkId, chunkId)) + .from(chunks) + .leftJoin(chunkBlocks, eq(chunkBlocks.chunkId, chunks.id)) + .leftJoin(documentBlocks, eq(chunkBlocks.blockId, documentBlocks.id)) + .where(inArray(chunks.id, chunkIds)) .all() - return projectChunkProvenance(chunkId, chunk.documentId, chunk.pageStart, chunk.pageEnd, rows) + const grouped = new Map< + string, + { + documentId: string + pageStart: number | null + pageEnd: number | null + rows: ChunkProvenanceRow[] + } + >() + + for (const row of rows) { + let entry = grouped.get(row.chunkId) + if (!entry) { + entry = { + documentId: row.documentId, + pageStart: row.pageStart, + pageEnd: row.pageEnd, + rows: [] + } + grouped.set(row.chunkId, entry) + } + + // 没有映射的 chunk 仍然解析（blocks 为空），左连接因此会给出一个全 NULL 块 + if (row.blockId === null || row.order === null) continue + + entry.rows.push({ + blockId: row.blockId, + kind: row.kind as BlockKind, + level: row.level, + page: row.page, + bbox: row.bbox, + text: row.text ?? '', + startOffset: row.startOffset ?? 0, + endOffset: row.endOffset ?? 0, + order: row.order, + startInBlock: row.startInBlock ?? 0, + endInBlock: row.endInBlock ?? 0 + }) + } + + for (const [chunkId, entry] of grouped) { + result.set( + chunkId, + projectChunkProvenance(chunkId, entry.documentId, entry.pageStart, entry.pageEnd, entry.rows) + ) + } + + return result +} + +/** + * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 + * 一次 join chunks → chunk_blocks → document_blocks。chunk 不存在时返回 undefined。 + */ +export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance | undefined { + return resolveChunksProvenance(db, [chunkId]).get(chunkId) } diff --git a/src/main/services/loaders/PdfLoader.ts b/src/main/services/loaders/PdfLoader.ts index fe55d27..36aceb7 100644 --- a/src/main/services/loaders/PdfLoader.ts +++ b/src/main/services/loaders/PdfLoader.ts @@ -9,10 +9,12 @@ import { extname } from 'path' import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs' import type { PDFDocumentProxy, PDFPageProxy, TextItem } from 'pdfjs-dist/types/src/display/api' import Logger from '../../../shared/utils/logger' +import { layoutPageText, type PositionedTextItem } from './pdfTextLayout' import type { IDocumentLoader, DocumentLoadResult, LoadOptions, + PageBlockInfo, PageInfo, NormalizedBox } from './types' @@ -86,22 +88,38 @@ export class PdfLoader implements IDocumentLoader { const textContent = await page.getTextContent() const viewport = page.getViewport({ scale: 1.0 }) - // 提取页面文本 + // 把文本项按几何位置还原成页内段落，引用才能落到"第 12 页右下角那一段" const textItems = textContent.items.filter((item): item is TextItem => 'str' in item) - const pageText = textItems.map((item) => item.str).join(' ') - - // 清理文本 - const cleanedText = this.cleanPDFText(pageText) - const pageTextWithNewline = cleanedText + '\n\n' + const paragraphs = layoutPageText( + this.toPositionedItems(textItems, viewport), + viewport.width, + viewport.height + ) + + // 页内容由段落拼出，段落偏移随后锚定到这份规范字符串上 + const pageContent = paragraphs.map((paragraph) => paragraph.text).join('\n\n') + const pageBlocks: PageBlockInfo[] = [] + let pageCursor = 0 + for (const paragraph of paragraphs) { + const startOffset = currentOffset + pageCursor + pageBlocks.push({ + text: paragraph.text, + startOffset, + endOffset: startOffset + paragraph.text.length, + bbox: paragraph.bbox + }) + pageCursor += paragraph.text.length + 2 // '\n\n' + } // 记录页面信息 if (opts.preserveStructure) { pages.push({ pageNumber: pageNum, - content: cleanedText, + content: pageContent, startOffset: currentOffset, - endOffset: currentOffset + cleanedText.length, - bbox: this.computeNormalizedBBox(textItems, viewport), + endOffset: currentOffset + pageContent.length, + bbox: this.unionBBox(paragraphs.map((paragraph) => paragraph.bbox)), + blocks: pageBlocks, metadata: { width: viewport.width, height: viewport.height @@ -109,6 +127,7 @@ export class PdfLoader implements IDocumentLoader { }) } + const pageTextWithNewline = pageContent + '\n\n' fullText += pageTextWithNewline currentOffset += pageTextWithNewline.length @@ -147,62 +166,41 @@ export class PdfLoader implements IDocumentLoader { } /** - * 把页面上所有文本项映射到页面空间,取并集后归一化到 0..1。 - * - * 用 viewport.transform 把 PDF 用户空间（原点在左下）转成视口空间（原点在左上）， - * 这样渲染层不需要知道缩放比例。没有任何文本项时返回 undefined。 + * 把 pdf.js 的文本项转换到视口坐标（原点左上，y 向下）。渲染层因此不需要知道缩放。 */ - private computeNormalizedBBox( + private toPositionedItems( items: TextItem[], viewport: ReturnType - ): NormalizedBox | undefined { - let minX = Infinity - let minY = Infinity - let maxX = -Infinity - let maxY = -Infinity + ): PositionedTextItem[] { + const positioned: PositionedTextItem[] = [] for (const item of items) { + if (!item.str) continue + const textTransform = pdfjsLib.Util.transform(viewport.transform, item.transform) const fontHeight = Math.hypot(textTransform[2], textTransform[3]) - const left = textTransform[4] - const right = left + Math.abs(item.width) - const top = textTransform[5] - fontHeight - const bottom = textTransform[5] - - minX = Math.min(minX, left, right) - maxX = Math.max(maxX, left, right) - minY = Math.min(minY, top, bottom) - maxY = Math.max(maxY, top, bottom) + + positioned.push({ + str: item.str, + left: textTransform[4], + baseline: textTransform[5], + width: Math.abs(item.width), + height: fontHeight > 0 ? fontHeight : 1 + }) } - if (!Number.isFinite(minX) || !Number.isFinite(minY)) return undefined + return positioned + } - const clamp = (value: number): number => Math.min(1, Math.max(0, value)) - const width = viewport.width || 1 - const height = viewport.height || 1 + /** 多个归一化包围盒的并集（页面 bbox）。没有有效盒时返回 undefined。 */ + private unionBBox(boxes: NormalizedBox[]): NormalizedBox | undefined { + if (boxes.length === 0) return undefined - return { - x: clamp(minX / width), - y: clamp(minY / height), - w: clamp((maxX - minX) / width), - h: clamp((maxY - minY) / height) - } - } + const left = Math.min(...boxes.map((box) => box.x)) + const top = Math.min(...boxes.map((box) => box.y)) + const right = Math.max(...boxes.map((box) => box.x + box.w)) + const bottom = Math.max(...boxes.map((box) => box.y + box.h)) - /** - * 清理 PDF 提取的文本 - */ - private cleanPDFText(text: string): string { - return ( - text - // 移除多余的空格 - .replace(/[ \t]+/g, ' ') - // 修复断行（连续的行可能是同一段落） - .replace(/([^\n])\n([^\n])/g, '$1 $2') - // 统一多个换行为两个 - .replace(/\n{3,}/g, '\n\n') - // 移除首尾空白 - .trim() - ) + return { x: left, y: top, w: right - left, h: bottom - top } } } diff --git a/src/main/services/loaders/pdfTextLayout.ts b/src/main/services/loaders/pdfTextLayout.ts new file mode 100644 index 0000000..a9c49b4 --- /dev/null +++ b/src/main/services/loaders/pdfTextLayout.ts @@ -0,0 +1,254 @@ +/** + * pdfTextLayout + * 把 pdf.js 的文本项按几何位置还原成「行 → 段落」。 + * + * `PdfLoader` 之前把一页所有文本项用空格拼成一个字符串，块粒度因此停在一页一句。 + * 引用要落到「第 12 页右下角那一段」，就需要在页内再分一层。这里只做几何分组， + * 不碰文本内容 —— 规范文本与偏移由 `PdfLoader` 在拼出内容时统一计算，从而保证 + * `content.slice(startOffset, endOffset) === text` 继续成立。 + * + * 纯函数：输入是已经转换到视口坐标的文本项，便于用合成数据单测，不需要 PDF fixture。 + */ + +import type { NormalizedBox } from './types' + +/** 已转换到视口坐标（原点左上，y 向下）的文本项。 */ +export interface PositionedTextItem { + str: string + /** 文本左边缘 x */ + left: number + /** 文本基线 y（视口坐标） */ + baseline: number + /** 文本宽度 */ + width: number + /** 字号（视口坐标下的行高） */ + height: number +} + +/** 一行文本。 */ +export interface TextLine { + text: string + left: number + right: number + top: number + bottom: number + height: number +} + +/** 一个段落：文本 + 归一化到页面的包围盒。 */ +export interface TextParagraph { + text: string + bbox: NormalizedBox +} + +/** 同一行的基线容差（相对最小字号）。 */ +const LINE_TOLERANCE = 0.5 +/** 行间垂直空隙超过多少倍行高视为新段落。 */ +const PARAGRAPH_GAP = 0.6 +/** 行首相对段首左边缘缩进超过多少倍字号视为新段落。 */ +const INDENT_RATIO = 1 +/** 同一行内文本项间距超过多少倍字号时补一个空格。 */ +const SPACE_GAP = 0.25 + +/** + * 把一页的文本项还原成段落列表。顺序为阅读顺序：先按基线自上而下，行内自左向右。 + * 文本项全部无效时返回空数组。 + */ +export function layoutPageText( + items: PositionedTextItem[], + pageWidth: number, + pageHeight: number +): TextParagraph[] { + const lines = groupLines(items) + const paragraphs = groupParagraphs(lines) + + return paragraphs.map((paragraph) => ({ + text: paragraph.text, + bbox: normalizeBox(paragraph.box, pageWidth, pageHeight) + })) +} + +interface LineDraft { + items: PositionedTextItem[] + baseline: number + height: number + left: number + right: number + top: number + bottom: number +} + +/** 按基线把文本项聚成行，行内按 x 排序后拼出文本。 */ +function groupLines(items: PositionedTextItem[]): TextLine[] { + const usable = items.filter( + (item) => + item.str.trim().length > 0 && + Number.isFinite(item.left) && + Number.isFinite(item.baseline) && + Number.isFinite(item.width) && + Number.isFinite(item.height) + ) + + const sorted = [...usable].sort((a, b) => a.baseline - b.baseline || a.left - b.left || 0) + + const drafts: LineDraft[] = [] + + for (const item of sorted) { + const last = drafts[drafts.length - 1] + const tolerance = Math.max( + 1, + Math.min(last?.height ?? item.height, item.height) * LINE_TOLERANCE + ) + const sameLine = last !== undefined && Math.abs(item.baseline - last.baseline) <= tolerance + + if (sameLine && last) { + last.items.push(item) + const count = last.items.length + last.baseline = (last.baseline * (count - 1) + item.baseline) / count + last.height = Math.max(last.height, item.height) + last.left = Math.min(last.left, item.left) + last.right = Math.max(last.right, item.left + item.width) + last.top = Math.min(last.top, item.baseline - item.height) + last.bottom = Math.max(last.bottom, item.baseline) + } else { + drafts.push({ + items: [item], + baseline: item.baseline, + height: item.height, + left: item.left, + right: item.left + item.width, + top: item.baseline - item.height, + bottom: item.baseline + }) + } + } + + return drafts + .filter((draft) => draft.items.some((item) => item.str.trim().length > 0)) + .map((draft) => ({ + text: lineText(draft.items), + left: draft.left, + right: draft.right, + top: draft.top, + bottom: draft.bottom, + height: draft.height + })) + .filter((line) => line.text.length > 0) +} + +/** 行内按 x 排序，间距足够大时补空格（同一词被拆成多个文本项时不补）。 */ +function lineText(items: PositionedTextItem[]): string { + const ordered = [...items].sort((a, b) => a.left - b.left) + + let text = '' + let previousRight: number | null = null + let previousHeight = 0 + + for (const item of ordered) { + if (previousRight !== null) { + const gap = item.left - previousRight + if (gap > SPACE_GAP * Math.max(previousHeight, item.height)) text += ' ' + } + text += item.str + previousRight = item.left + item.width + previousHeight = item.height + } + + return collapseWhitespace(text) +} + +interface ParagraphDraft { + lines: TextLine[] + left: number + top: number + bottom: number +} + +/** 按垂直间距与首行缩进把行聚成段落。 */ +function groupParagraphs(lines: TextLine[]): Array<{ text: string; box: Box }> { + const paragraphs: ParagraphDraft[] = [] + let current: ParagraphDraft | null = null + + const flush = (): void => { + if (current && current.lines.length > 0) paragraphs.push(current) + current = null + } + + for (const line of lines) { + if (!current) { + current = { + lines: [line], + left: line.left, + top: line.top, + bottom: line.bottom + } + continue + } + + const previous = current.lines[current.lines.length - 1] + const gap = line.top - previous.bottom + const lineHeight = Math.max(previous.height, line.height) + const newParagraph = + gap > PARAGRAPH_GAP * lineHeight || + (gap > 0 && line.left - current.left > INDENT_RATIO * line.height) + + if (newParagraph) { + flush() + current = { + lines: [line], + left: line.left, + top: line.top, + bottom: line.bottom + } + continue + } + + current.lines.push(line) + current.top = Math.min(current.top, line.top) + current.bottom = Math.max(current.bottom, line.bottom) + } + + flush() + + return paragraphs + .map((paragraph) => ({ + text: collapseWhitespace(paragraph.lines.map((line) => line.text).join(' ')), + box: { + left: Math.min(...paragraph.lines.map((line) => line.left)), + top: paragraph.top, + right: Math.max(...paragraph.lines.map((line) => line.right)), + bottom: paragraph.bottom + } + })) + .filter((paragraph) => paragraph.text.length > 0) +} + +interface Box { + left: number + top: number + right: number + bottom: number +} + +/** 视口坐标 → 0..1 归一化页面坐标。 */ +function normalizeBox(box: Box, pageWidth: number, pageHeight: number): NormalizedBox { + const clamp = (value: number): number => Math.min(1, Math.max(0, value)) + const width = pageWidth || 1 + const height = pageHeight || 1 + + const left = Math.min(box.left, box.right) + const right = Math.max(box.left, box.right) + const top = Math.min(box.top, box.bottom) + const bottom = Math.max(box.top, box.bottom) + + return { + x: clamp(left / width), + y: clamp(top / height), + w: clamp((right - left) / width), + h: clamp((bottom - top) / height) + } +} + +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim() +} diff --git a/src/main/services/loaders/types.ts b/src/main/services/loaders/types.ts index b40e91c..5bab4e1 100644 --- a/src/main/services/loaders/types.ts +++ b/src/main/services/loaders/types.ts @@ -13,6 +13,17 @@ export interface NormalizedBox { h: number } +/** + * 页内块（分页文档的段落）。文本与偏移锚定整份规范内容（`documents.content`）， + * bbox 归一化到页面。`PdfLoader` 逐段产出，找不到几何分组时回退到整页一块。 + */ +export interface PageBlockInfo { + text: string + startOffset: number + endOffset: number + bbox?: NormalizedBox +} + /** * 页面信息（用于 PDF、PPT 等分页文档） */ @@ -22,6 +33,7 @@ export interface PageInfo { startOffset: number // 在完整文本中的起始位置 endOffset: number // 在完整文本中的结束位置 bbox?: NormalizedBox // 页面文本的归一化包围盒（可选） + blocks?: PageBlockInfo[] // 页内段落块（可选；供 document_blocks 落到段落级） metadata?: { width?: number height?: number diff --git a/src/main/services/retrieval/DenseRetriever.ts b/src/main/services/retrieval/DenseRetriever.ts new file mode 100644 index 0000000..bc94d15 --- /dev/null +++ b/src/main/services/retrieval/DenseRetriever.ts @@ -0,0 +1,35 @@ +/** + * DenseRetriever + * 当前的默认检索策略：查询向量 → 向量库 KNN → 批量补齐证据。 + * + * 只负责"怎么检索"。embedding space 校验留在 `KnowledgeService`（那是前置条件， + * 不是策略），因此这个类可以单独被 eval harness 与后续的 BM25 / hybrid 策略替换。 + */ + +import { getDatabase } from '../../db' +import { vectorStoreManager } from '../../vectorstore' +import type { EmbeddingService } from '../EmbeddingService' +import { hydrateEvidence } from './evidence' +import type { RetrieveOptions, RetrievedEvidence, Retriever } from './types' + +export class DenseRetriever implements Retriever { + constructor(private readonly embeddingService: EmbeddingService) {} + + async search( + notebookId: string, + query: string, + options: RetrieveOptions = {} + ): Promise { + const { topK = 5, threshold = 0.5 } = options + + // E5 要求 query 前缀，与索引时的 document 前缀区分 + await this.embeddingService.ensureReady() + const queryEmbedding = await this.embeddingService.embed(query, 'query') + + const vectorStore = await vectorStoreManager.getStore(notebookId) + const hits = await vectorStore.query(queryEmbedding.embedding, { topK, threshold }) + if (hits.length === 0) return [] + + return hydrateEvidence(getDatabase(), hits) + } +} diff --git a/src/main/services/retrieval/evidence.ts b/src/main/services/retrieval/evidence.ts new file mode 100644 index 0000000..38243d7 --- /dev/null +++ b/src/main/services/retrieval/evidence.ts @@ -0,0 +1,106 @@ +/** + * evidence + * 把一次检索的命中（chunkId + score）补齐成 `RetrievedEvidence[]`。 + * + * 上层拿到的不再是裸 chunk：来源标题/类型、规范化页码区间和块区间都在这里一次补 + * 齐。补齐是批量的 —— 无论 topK 多大，SQL 查询次数恒定，不会形成 N+1。 + */ + +import { inArray } from 'drizzle-orm' +import { chunks, documents } from '../../db/schema' +import type { getDatabase } from '../../db' +import { resolveChunksProvenance, type ChunkProvenance } from '../chunkProvenance' +import type { RetrievedEvidence } from './types' + +type Db = ReturnType + +/** 一次向量检索的命中。 */ +export interface EvidenceHit { + chunkId: string + score: number +} + +/** 组装证据所需的最小 chunk / document 字段。 */ +export interface EvidenceChunk { + documentId: string + content: string + chunkIndex: number + metadata?: Record | null +} + +export interface EvidenceDocument { + title: string + type: string +} + +/** + * 纯映射：命中 + chunk + 来源 + 来源结构 → 证据。便于单测，不碰数据库。 + */ +export function assembleEvidence( + hit: EvidenceHit, + chunk: EvidenceChunk, + document: EvidenceDocument, + provenance: ChunkProvenance | undefined +): RetrievedEvidence { + return { + chunkId: hit.chunkId, + documentId: chunk.documentId, + content: chunk.content, + score: hit.score, + chunkIndex: chunk.chunkIndex, + source: { title: document.title, type: document.type }, + locator: { + pageStart: provenance?.pageStart ?? null, + pageEnd: provenance?.pageEnd ?? null, + blocks: provenance?.blocks ?? [] + }, + metadata: chunk.metadata ?? undefined + } +} + +/** + * 批量补齐证据。保持 `hits` 的顺序（即向量库的相关性排序）；找不到 chunk 的命中被 + * 丢弃，而不是用占位符污染结果。 + */ +export function hydrateEvidence(db: Db, hits: EvidenceHit[]): RetrievedEvidence[] { + if (hits.length === 0) return [] + + const chunkIds = hits.map((hit) => hit.chunkId) + + const chunkRows = db + .select({ + id: chunks.id, + documentId: chunks.documentId, + content: chunks.content, + chunkIndex: chunks.chunkIndex, + metadata: chunks.metadata + }) + .from(chunks) + .where(inArray(chunks.id, chunkIds)) + .all() + + const documentIds = [...new Set(chunkRows.map((row) => row.documentId))] + const documentRows = + documentIds.length > 0 + ? db + .select({ id: documents.id, title: documents.title, type: documents.type }) + .from(documents) + .where(inArray(documents.id, documentIds)) + .all() + : [] + + const chunkMap = new Map(chunkRows.map((row) => [row.id, row])) + const documentMap = new Map(documentRows.map((row) => [row.id, row])) + const provenance = resolveChunksProvenance(db, chunkIds) + + const evidence: RetrievedEvidence[] = [] + for (const hit of hits) { + const chunk = chunkMap.get(hit.chunkId) + if (!chunk) continue + + const document = documentMap.get(chunk.documentId) ?? { title: 'Unknown', type: 'unknown' } + evidence.push(assembleEvidence(hit, chunk, document, provenance.get(hit.chunkId))) + } + + return evidence +} diff --git a/src/main/services/retrieval/index.ts b/src/main/services/retrieval/index.ts new file mode 100644 index 0000000..5f950d2 --- /dev/null +++ b/src/main/services/retrieval/index.ts @@ -0,0 +1,7 @@ +/** + * Retrieval 模块入口 + */ + +export * from './types' +export { DenseRetriever } from './DenseRetriever' +export { hydrateEvidence, assembleEvidence, type EvidenceHit } from './evidence' diff --git a/src/main/services/retrieval/types.ts b/src/main/services/retrieval/types.ts new file mode 100644 index 0000000..44b8290 --- /dev/null +++ b/src/main/services/retrieval/types.ts @@ -0,0 +1,57 @@ +/** + * Retrieval contract + * + * 检索是一等能力，不是 Chat 的附属逻辑。Chat、MCP、Search 面板都通过同一个 + * `Retriever` 拿到同一份 `RetrievedEvidence`，因此不必各自重写一遍 RAG，也让 + * #75 的 eval harness 有一个稳定的入口。 + * + * `RetrievedEvidence` 与 `SearchResult` 的关键区别是 provenance 是一等字段：引用 + * 要落到"哪个来源、哪一页、哪一段"，检索结果在交付给上层时就必须带着它。 + */ + +import type { ChunkProvenanceBlock } from '../chunkProvenance' + +/** 检索结果覆盖的单个块（`chunk_blocks` 映射 + `document_blocks`）。 */ +export type EvidenceBlock = ChunkProvenanceBlock + +/** 证据在来源里的位置。`blocks` 按文档顺序排列。 */ +export interface EvidenceLocator { + pageStart: number | null + pageEnd: number | null + blocks: EvidenceBlock[] +} + +/** 证据来自哪份来源。 */ +export interface EvidenceSource { + title: string + type: string +} + +/** 一条可供引用/展示的检索证据。 */ +export interface RetrievedEvidence { + chunkId: string + documentId: string + content: string + score: number + chunkIndex: number + source: EvidenceSource + locator: EvidenceLocator + metadata?: Record +} + +/** + * 检索选项。`includeContent` 不在这里：截断内容由 legacy `SearchResult` 映射决定， + * 不是检索策略的事（策略应当总是返回真实内容）。 + */ +export interface RetrieveOptions { + topK?: number + threshold?: number +} + +/** + * 检索策略的稳定契约。当前只有 `DenseRetriever`；#77 的 BM25 / hybrid / reranker + * 只要实现这个接口就能被 eval harness 直接度量与替换。 + */ +export interface Retriever { + search(notebookId: string, query: string, options?: RetrieveOptions): Promise +} diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 0884795..8434f40 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -46,6 +46,9 @@ import { SQLiteVectorStore } from './vectorstore/SQLiteVectorStore' import { assignBlockIds, buildDocumentBlocks } from './services/blocks/documentBlocks' import { ChunkingService } from './services/ChunkingService' import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProvenance' +import { KnowledgeService } from './services/KnowledgeService' +import { DenseRetriever } from './services/retrieval' +import type { EmbeddingService } from './services/EmbeddingService' export const SMOKE_TEST_FLAG = '--smoke-test' @@ -95,6 +98,38 @@ function assertThrows(run: () => unknown, message: string): void { throw new Error(message) } +/** + * A deterministic stand-in for `EmbeddingService` so the smoke test can drive + * the whole index lifecycle (blocks → chunks → embeddings → vectors) without + * downloading model weights. Only the methods `KnowledgeService` calls are + * implemented. + */ +function fakeEmbeddingService(dimensions = 16): EmbeddingService { + const result = (): { embedding: Float32Array; model: string; dimensions: number } => { + const embedding = new Float32Array(dimensions) + embedding[0] = 1 + return { embedding, model: 'smoke-embed', dimensions } + } + + // SAFETY: only `ensureReady`, `embedBatch` and `getSpace` are reached by the + // index path; the cast hides the rest of the real service's surface from the + // compiler on purpose, so a new call site here fails loudly at runtime. + return { + ensureReady: async () => undefined, + embed: async () => result(), + embedBatch: async (texts: string[]) => texts.map(() => result()), + getSpace: async () => ({ + id: 'smoke-embed-space', + backend: 'local', + model: 'smoke-embed', + revision: '', + dimensions, + pooling: 'mean', + normalize: true + }) + } as unknown as EmbeddingService +} + /** * Rewinds the database to the old layout: one global vec_embeddings table whose * width is fixed at 1024, plus a notebook that owns a vector in it. @@ -424,6 +459,106 @@ async function runChecks(): Promise { db.delete(documentBlocks).where(eq(documentBlocks.documentId, provenanceDocId)).run() db.delete(documents).where(eq(documents.id, provenanceDocId)).run() + // --- re-index keeps the source identity ----------------------------------- + // The bug this guards: `reindexDocument()` used to call `addDocument()`, which + // minted a new `documentId` and dropped `localFilePath`. A citation persisted + // against the old id would then point at a source that no longer exists. This + // drives the real service end to end with a fake embedding backend. + const reindexNotebook = 'smoke-notebook-reindex' + const reindexNow = Date.now() + database + .prepare('INSERT INTO notebooks (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)') + .run(reindexNotebook, 'Smoke re-index notebook', reindexNow, reindexNow) + + const knowledge = new KnowledgeService(fakeEmbeddingService()) + const reindexDocId = await knowledge.addDocumentFromFile( + reindexNotebook, + join(fixtures, 'sample.pdf') + ) + + const before = knowledge.getDocument(reindexDocId) + assert(before?.status === 'indexed', `imported document status was ${before?.status}`) + assert(Boolean(before?.localFilePath), 'imported document lost its localFilePath') + const blocksBefore = knowledge.getDocumentBlocks(reindexDocId) + assert(blocksBefore.length > 0, 'imported document produced no blocks') + + await knowledge.reindexDocument(reindexDocId) + + const after = knowledge.getDocument(reindexDocId) + assert(Boolean(after), 'the document disappeared after a re-index') + assert(after!.status === 'indexed', `re-index left status ${after!.status}`) + assert(after!.localFilePath === before!.localFilePath, 're-index did not preserve localFilePath') + assert( + !knowledge + .getDocuments(reindexNotebook) + .some((doc) => doc.id !== reindexDocId && doc.title === before!.title), + 're-index created a second document instead of rebuilding in place' + ) + assert( + JSON.stringify(knowledge.getDocumentBlocks(reindexDocId)) === JSON.stringify(blocksBefore), + 're-index rebuilt a different block sequence from the persisted structure' + ) + const chunksAfter = knowledge.getDocumentChunks(reindexDocId) + assert( + chunksAfter.length > 0 && after!.chunkCount === chunksAfter.length, + 're-index left the document without chunks' + ) + + // Retrieval delivers provenance, not a bare chunk (#74). The re-index above + // just rebuilt chunks, embeddings, vectors and the chunk↔block mapping, so + // this drives the real DenseRetriever against them. + const evidence = await new DenseRetriever(fakeEmbeddingService()).search( + reindexNotebook, + 'sample query' + ) + assert(evidence.length > 0, 'DenseRetriever returned no evidence for an indexed notebook') + const first = evidence[0] + assert(first.documentId === reindexDocId, `evidence came from ${first.documentId}`) + assert(first.source.title.length > 0, 'evidence lost its source title') + assert(first.content.length > 0, 'evidence lost its content') + assert(first.locator.pageStart === 1, `evidence page was ${first.locator.pageStart}`) + assert(first.locator.blocks.length > 0, 'evidence carried no block locator') + assert( + first.locator.blocks.every((block) => block.page === 1), + 'an evidence block lost its page' + ) + assert( + first.locator.blocks.some((block) => block.bbox && block.bbox.w > 0 && block.bbox.h > 0), + 'evidence lost the paragraph bbox needed to highlight it (#72)' + ) + pass('DenseRetriever returns retrieved evidence with a page/block locator') + + // Documents imported before `documents.structure` existed have NULL there. + // Re-indexing them must recover the structure from the local copy *before* + // clearing the old index, or one re-index would downgrade a page/paragraph + // citation to a flat block. + database.prepare('UPDATE documents SET structure = NULL WHERE id = ?').run(reindexDocId) + assert( + knowledge.getDocument(reindexDocId)?.structure == null, + 'could not simulate a legacy document without structure' + ) + + await knowledge.reindexDocument(reindexDocId) + + assert( + Boolean(knowledge.getDocument(reindexDocId)?.structure), + 'legacy re-index did not backfill the structure' + ) + const legacyBlocks = knowledge.getDocumentBlocks(reindexDocId) + assert(legacyBlocks.length > 0, 'legacy re-index produced no blocks') + assert( + legacyBlocks.every((block) => block.page === 1), + 'legacy re-index dropped page provenance instead of recovering it' + ) + pass('re-index recovers structure for documents imported before the column existed') + + await knowledge.deleteDocument(reindexDocId) + assert( + knowledge.getDocument(reindexDocId) === undefined, + 'deleting the re-indexed document did not remove it' + ) + pass('re-index rebuilds the derived index in place and keeps the source identity') + const { ApkgExporter } = await import('./services/exporters/ApkgExporter') const { buffer, summary } = await new ApkgExporter().export( [{ id: 'smoke-card', type: 'basic', front: 'front', back: 'back', tags: ['smoke'] }], diff --git a/test/chunkBlocks.test.ts b/test/chunkBlocks.test.ts index 4818750..34edf0e 100644 --- a/test/chunkBlocks.test.ts +++ b/test/chunkBlocks.test.ts @@ -271,6 +271,7 @@ test('a chunk resolves to its ordered block spans', () => { kind: block.kind, level: block.level, page: block.page, + bbox: null, text: block.text, startOffset: block.startOffset, endOffset: block.endOffset, diff --git a/test/pdfTextLayout.test.ts b/test/pdfTextLayout.test.ts new file mode 100644 index 0000000..513c7af --- /dev/null +++ b/test/pdfTextLayout.test.ts @@ -0,0 +1,194 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { + layoutPageText, + type PositionedTextItem +} from '../src/main/services/loaders/pdfTextLayout.ts' +import { + buildDocumentBlocks, + type DocumentBlockDraft +} from '../src/main/services/blocks/documentBlocks.ts' +import type { DocumentStructure } from '../src/main/services/loaders/types.ts' + +/** + * The PDF loader used to join every `TextItem` on a page into one string, so a + * citation could only ever say "page 12", never "the paragraph at the bottom of + * page 12". `layoutPageText()` groups items geometrically into lines and then + * paragraphs; these tests pin that geometry with synthetic items, so the rule + * is testable without a PDF fixture. + * + * The second half checks that `buildDocumentBlocks()` turns those paragraphs + * into blocks whose offsets still slice back out of `documents.content` — the + * one invariant the whole provenance chain rests on. + */ + +/** A viewport-space text item; font height defaults to 10. */ +function item(str: string, left: number, baseline: number, width: number, height = 10) { + return { str, left, baseline, width, height } +} + +/** Two lines in one paragraph, then a second paragraph after a blank line. */ +function twoParagraphPage(): PositionedTextItem[] { + return [ + item('First', 50, 100, 40), + item('line', 96, 100, 20), + item('continues', 50, 114, 60), + item('Second paragraph.', 50, 140, 110) + ] +} + +test('items on the same baseline become one line and words are spaced apart', () => { + const [first] = layoutPageText(twoParagraphPage(), 800, 1000).slice(0, 1) + assert.ok(first) + // "First" ends at x=90, "line" starts at x=96: a gap, so a space is inserted. + assert.equal(first.text, 'First line continues') +}) + +test('a word split across items is not given an inner space', () => { + const words = [item('Know', 50, 100, 32), item('Note', 82, 100, 34)] + const [paragraph] = layoutPageText(words, 800, 1000) + assert.ok(paragraph) + assert.equal(paragraph.text, 'KnowNote') +}) + +test('a vertical gap starts a new paragraph, preserving reading order', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['First line continues', 'Second paragraph.'] + ) +}) + +test('a first-line indent starts a new paragraph even without a vertical gap', () => { + const page = [ + item('one line', 50, 100, 60), + item('and the next', 70, 112, 70) // left indented by 20 > height(10), gap 2 + ] + const paragraphs = layoutPageText(page, 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['one line', 'and the next'] + ) +}) + +test('paragraph bboxes are normalized to the page and cover only their own items', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + const [first, second] = paragraphs + + // First paragraph spans x 50..116, y top 90..bottom 114. + assert.deepEqual(first.bbox, { x: 50 / 800, y: 90 / 1000, w: 66 / 800, h: 24 / 1000 }) + assert.deepEqual(second.bbox, { x: 50 / 800, y: 130 / 1000, w: 110 / 800, h: 10 / 1000 }) +}) + +test('items arrive in any order but read top-to-bottom, left-to-right', () => { + const shuffled = [ + item('second', 50, 114, 40), + item('world', 100, 100, 40), + item('hello', 50, 100, 40) + ] + const paragraphs = layoutPageText(shuffled, 800, 1000) + // Both lines are only 4 apart (top 104 vs 104... 114-10=104) so they merge + // into one paragraph; order inside the line still follows x. + assert.equal(paragraphs[0].text, 'hello world second') +}) + +test('blank and non-finite items are ignored', () => { + const page = [ + item(' ', 50, 100, 40), + item('real', 50, 100, 40), + { str: 'bad', left: NaN, baseline: 100, width: 10, height: 10 } + ] + const paragraphs = layoutPageText(page, 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['real'] + ) +}) + +test('an empty page produces no paragraphs', () => { + assert.deepEqual(layoutPageText([], 800, 1000), []) +}) + +// --- block consumption ------------------------------------------------------- + +/** + * Collapse the two-line page above into a canonical content string the same way + * `PdfLoader` does: paragraphs joined by a blank line. + */ +function pageStructure( + paragraphs: { + text: string + bbox: import('../src/main/services/loaders/types.ts').NormalizedBox + }[] +): { + content: string + structure: DocumentStructure +} { + const content = paragraphs.map((paragraph) => paragraph.text).join('\n\n') + + let cursor = 0 + const blocks = paragraphs.map((paragraph) => { + const startOffset = cursor + cursor += paragraph.text.length + 2 + return { + text: paragraph.text, + startOffset, + endOffset: startOffset + paragraph.text.length, + bbox: paragraph.bbox + } + }) + + // Offsets in `blocks` must address the *trimmed* canonical content. + const trimmed = content.trim() + const shift = content.length - content.trimStart().length + for (const block of blocks) { + block.startOffset -= shift + block.endOffset -= shift + } + + return { + content: trimmed, + structure: { + type: 'pages', + pages: [ + { pageNumber: 1, content: trimmed, startOffset: 0, endOffset: trimmed.length, blocks } + ] + } + } +} + +function assertBlocks(content: string, blocks: DocumentBlockDraft[]): void { + for (const block of blocks) { + assert.equal( + content.slice(block.startOffset, block.endOffset), + block.text, + `block ${block.order} does not slice back to its text` + ) + } +} + +test('a multi-paragraph page becomes multiple blocks in reading order', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + const { content, structure } = pageStructure(paragraphs) + const blocks = buildDocumentBlocks({ content, structure }) + + assert.deepEqual( + blocks.map((block) => block.text), + ['First line continues', 'Second paragraph.'] + ) + assert.ok(blocks.every((block) => block.page === 1)) + assertBlocks(content, blocks) +}) + +test('a page without paragraph blocks still falls back to one page-level block', () => { + const content = 'legacy page text' + const structure: DocumentStructure = { + type: 'pages', + pages: [{ pageNumber: 1, content, startOffset: 0, endOffset: content.length }] + } + const blocks = buildDocumentBlocks({ content, structure }) + + assert.equal(blocks.length, 1) + assert.equal(blocks[0].text, content) + assertBlocks(content, blocks) +}) diff --git a/test/retrievalEvidence.test.ts b/test/retrievalEvidence.test.ts new file mode 100644 index 0000000..40d1653 --- /dev/null +++ b/test/retrievalEvidence.test.ts @@ -0,0 +1,101 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { + assembleEvidence, + type EvidenceChunk, + type EvidenceDocument +} from '../src/main/services/retrieval/evidence.ts' +import type { ChunkProvenance } from '../src/main/services/chunkProvenance.ts' + +/** + * `RetrievedEvidence` is the seam between retrieval and citations (#74): the + * retriever has to hand over provenance, not just the chunk text, or #69 cannot + * build a citation without querying the database again. + * + * `assembleEvidence()` is the pure step that attaches it. `hydrateEvidence()` + * only adds the batch queries around it; the shape asserted here is what every + * retriever strategy must produce. + */ + +const hit = { chunkId: 'chunk_1', score: 0.87 } +const chunk: EvidenceChunk = { + documentId: 'doc_1', + content: 'the extracted passage', + chunkIndex: 4 +} +const document: EvidenceDocument = { title: 'Attention Is All You Need', type: 'file' } + +const provenance: ChunkProvenance = { + chunkId: 'chunk_1', + documentId: 'doc_1', + pageStart: 12, + pageEnd: 12, + blocks: [ + { + blockId: 'blk_doc_1_7', + kind: 'paragraph', + level: null, + page: 12, + bbox: { x: 0.5, y: 0.25, w: 0.4, h: 0.05 }, + text: 'the extracted passage', + startOffset: 100, + endOffset: 121, + startInBlock: 0, + endInBlock: 21 + } + ] +} + +test('evidence carries score, source and locator together', () => { + const evidence = assembleEvidence(hit, chunk, document, provenance) + + assert.deepEqual(evidence, { + chunkId: 'chunk_1', + documentId: 'doc_1', + content: 'the extracted passage', + score: 0.87, + chunkIndex: 4, + source: { title: 'Attention Is All You Need', type: 'file' }, + locator: { + pageStart: 12, + pageEnd: 12, + blocks: provenance.blocks + }, + metadata: undefined + }) +}) + +test('a chunk without a provenance row still produces a valid locator', () => { + const evidence = assembleEvidence(hit, chunk, document, undefined) + + assert.deepEqual(evidence.locator, { pageStart: null, pageEnd: null, blocks: [] }) +}) + +test('block geometry survives so a citation can highlight the paragraph (#72)', () => { + const evidence = assembleEvidence(hit, chunk, document, provenance) + const [block] = evidence.locator.blocks + + assert.equal(block.page, 12) + assert.deepEqual(block.bbox, { x: 0.5, y: 0.25, w: 0.4, h: 0.05 }) +}) + +test('chunk metadata survives the mapping', () => { + const evidence = assembleEvidence( + hit, + { ...chunk, metadata: { source: 'legacy' } }, + document, + provenance + ) + assert.deepEqual(evidence.metadata, { source: 'legacy' }) +}) + +test('the locator keeps block spans in the order provenance resolved them', () => { + const evidence = assembleEvidence(hit, chunk, document, { + ...provenance, + blocks: [provenance.blocks[0], { ...provenance.blocks[0], blockId: 'b2' }] + }) + assert.deepEqual( + evidence.locator.blocks.map((block) => block.blockId), + ['blk_doc_1_7', 'b2'] + ) +})