From 7594983f1e5f11433c7283854048e4bdd64d7e49 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 15:09:29 +0800 Subject: [PATCH 1/5] fix(indexing): reindex in place and preserve source identity reindexDocument() used to delete the old derived index and then call addDocument(), which minted a new documentId, dropped localFilePath and left the old row stuck in 'processing'. Once citations persist a documentId, that silently repoints every historical answer at a different source. Split the indexing path so a document is a stable source identity: addDocument / addDocumentFromFile -> create source -> indexDocument(id) reindexDocument -> clearDerivedIndex(id) -> indexDocument(id) indexDocument() owns only the derived chain (blocks -> chunks -> embeddings -> vectors -> status) and never inserts or deletes documents. clearDerivedIndex() is the symmetric teardown, shared with deleteDocument. Persist the parse structure on the source row (documents.structure) so a re-index rebuilds the same page/paragraph blocks without re-parsing a file that may have changed on disk. Fixes #109 --- .../db/migrations/0016_known_norrin_radd.sql | 1 + .../db/migrations/meta/0016_snapshot.json | 1605 +++++++++++++++++ src/main/db/migrations/meta/_journal.json | 7 + src/main/db/schema.ts | 5 + src/main/services/KnowledgeService.ts | 370 ++-- src/main/smokeTest.ts | 85 + 6 files changed, 1844 insertions(+), 229 deletions(-) create mode 100644 src/main/db/migrations/0016_known_norrin_radd.sql create mode 100644 src/main/db/migrations/meta/0016_snapshot.json diff --git a/src/main/db/migrations/0016_known_norrin_radd.sql b/src/main/db/migrations/0016_known_norrin_radd.sql new file mode 100644 index 0000000..49e7a2a --- /dev/null +++ b/src/main/db/migrations/0016_known_norrin_radd.sql @@ -0,0 +1 @@ +ALTER TABLE `documents` ADD `structure` text; \ No newline at end of file diff --git a/src/main/db/migrations/meta/0016_snapshot.json b/src/main/db/migrations/meta/0016_snapshot.json new file mode 100644 index 0000000..440eff1 --- /dev/null +++ b/src/main/db/migrations/meta/0016_snapshot.json @@ -0,0 +1,1605 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "2ae791e5-410f-4941-8484-64436eba26fb", + "prevId": "03fce19e-c08b-4bf0-9635-f54f7b00a883", + "tables": { + "anki_cards": { + "name": "anki_cards", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "cards_data": { + "name": "cards_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_ankicards_notebook": { + "name": "idx_ankicards_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_ankicards_version": { + "name": "idx_ankicards_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "anki_cards_notebook_id_notebooks_id_fk": { + "name": "anki_cards_notebook_id_notebooks_id_fk", + "tableFrom": "anki_cards", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_content": { + "name": "reasoning_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_session": { + "name": "idx_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_messages_session_id_chat_sessions_id_fk": { + "name": "chat_messages_session_id_chat_sessions_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_sessions": { + "name": "chat_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "parent_session_id": { + "name": "parent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_sessions_notebook": { + "name": "idx_sessions_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_sessions_notebook_id_notebooks_id_fk": { + "name": "chat_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_sessions_parent_session_id_chat_sessions_id_fk": { + "name": "chat_sessions_parent_session_id_chat_sessions_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "chat_sessions", + "columnsFrom": [ + "parent_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunk_blocks": { + "name": "chunk_blocks", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_in_block": { + "name": "start_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_in_block": { + "name": "end_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunk_blocks_block": { + "name": "idx_chunk_blocks_block", + "columns": [ + "block_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunk_blocks_chunk_id_chunks_id_fk": { + "name": "chunk_blocks_chunk_id_chunks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunk_blocks_block_id_document_blocks_id_fk": { + "name": "chunk_blocks_block_id_document_blocks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "document_blocks", + "columnsFrom": [ + "block_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "chunk_blocks_chunk_id_block_id_pk": { + "columns": [ + "chunk_id", + "block_id" + ], + "name": "chunk_blocks_chunk_id_block_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunks": { + "name": "chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_start": { + "name": "page_start", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_end": { + "name": "page_end", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunks_document": { + "name": "idx_chunks_document", + "columns": [ + "document_id" + ], + "isUnique": false + }, + "idx_chunks_notebook": { + "name": "idx_chunks_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunks_document_id_documents_id_fk": { + "name": "chunks_document_id_documents_id_fk", + "tableFrom": "chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunks_notebook_id_notebooks_id_fk": { + "name": "chunks_notebook_id_notebooks_id_fk", + "tableFrom": "chunks", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_blocks": { + "name": "document_blocks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bbox": { + "name": "bbox", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_blocks_document_order": { + "name": "idx_blocks_document_order", + "columns": [ + "document_id", + "order" + ], + "isUnique": false + }, + "idx_blocks_document_page": { + "name": "idx_blocks_document_page", + "columns": [ + "document_id", + "page" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_blocks_document_id_documents_id_fk": { + "name": "document_blocks_document_id_documents_id_fk", + "tableFrom": "document_blocks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "local_file_path": { + "name": "local_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_note_id": { + "name": "source_note_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "structure": { + "name": "structure", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_documents_notebook": { + "name": "idx_documents_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_documents_status": { + "name": "idx_documents_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_notebook_id_notebooks_id_fk": { + "name": "documents_notebook_id_notebooks_id_fk", + "tableFrom": "documents", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_source_note_id_notes_id_fk": { + "name": "documents_source_note_id_notes_id_fk", + "tableFrom": "documents", + "tableTo": "notes", + "columnsFrom": [ + "source_note_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embeddings": { + "name": "embeddings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeddings_chunk": { + "name": "idx_embeddings_chunk", + "columns": [ + "chunk_id" + ], + "isUnique": false + }, + "idx_embeddings_notebook": { + "name": "idx_embeddings_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + }, + "idx_embeddings_model": { + "name": "idx_embeddings_model", + "columns": [ + "model" + ], + "isUnique": false + } + }, + "foreignKeys": { + "embeddings_chunk_id_chunks_id_fk": { + "name": "embeddings_chunk_id_chunks_id_fk", + "tableFrom": "embeddings", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embeddings_notebook_id_notebooks_id_fk": { + "name": "embeddings_notebook_id_notebooks_id_fk", + "tableFrom": "embeddings", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "items": { + "name": "items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_items_notebook_order": { + "name": "idx_items_notebook_order", + "columns": [ + "notebook_id", + "order" + ], + "isUnique": false + }, + "idx_items_type": { + "name": "idx_items_type", + "columns": [ + "type" + ], + "isUnique": false + }, + "idx_items_resource": { + "name": "idx_items_resource", + "columns": [ + "type", + "resource_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "items_notebook_id_notebooks_id_fk": { + "name": "items_notebook_id_notebooks_id_fk", + "tableFrom": "items", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mind_maps": { + "name": "mind_maps", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "tree_data": { + "name": "tree_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_mindmaps_notebook": { + "name": "idx_mindmaps_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_mindmaps_version": { + "name": "idx_mindmaps_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mind_maps_notebook_id_notebooks_id_fk": { + "name": "mind_maps_notebook_id_notebooks_id_fk", + "tableFrom": "mind_maps", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebook_embedding_spaces": { + "name": "notebook_embedding_spaces", + "columns": { + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "backend": { + "name": "backend", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "notebook_embedding_spaces_notebook_id_notebooks_id_fk": { + "name": "notebook_embedding_spaces_notebook_id_notebooks_id_fk", + "tableFrom": "notebook_embedding_spaces", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebooks": { + "name": "notebooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notebooks_updated": { + "name": "idx_notebooks_updated", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notes": { + "name": "notes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notes_notebook": { + "name": "idx_notes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "notes_notebook_id_notebooks_id_fk": { + "name": "notes_notebook_id_notebooks_id_fk", + "tableFrom": "notes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quiz_sessions": { + "name": "quiz_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answers": { + "name": "answers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_questions": { + "name": "total_questions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "correct_count": { + "name": "correct_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quiz_sessions_quiz": { + "name": "idx_quiz_sessions_quiz", + "columns": [ + "quiz_id", + "created_at" + ], + "isUnique": false + }, + "idx_quiz_sessions_notebook": { + "name": "idx_quiz_sessions_notebook", + "columns": [ + "notebook_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quiz_sessions_quiz_id_quizzes_id_fk": { + "name": "quiz_sessions_quiz_id_quizzes_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "quizzes", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_sessions_notebook_id_notebooks_id_fk": { + "name": "quiz_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quizzes": { + "name": "quizzes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "questions_data": { + "name": "questions_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quizzes_notebook": { + "name": "idx_quizzes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_quizzes_version": { + "name": "idx_quizzes_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quizzes_notebook_id_notebooks_id_fk": { + "name": "quizzes_notebook_id_notebooks_id_fk", + "tableFrom": "quizzes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/main/db/migrations/meta/_journal.json b/src/main/db/migrations/meta/_journal.json index c83a95e..b9b8156 100644 --- a/src/main/db/migrations/meta/_journal.json +++ b/src/main/db/migrations/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1790316032660, "tag": "0015_colossal_solo", "breakpoints": true + }, + { + "idx": 16, + "version": "6", + "when": 1790320058521, + "tag": "0016_known_norrin_radd", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/main/db/schema.ts b/src/main/db/schema.ts index 3fb4f58..8d191ea 100644 --- a/src/main/db/schema.ts +++ b/src/main/db/schema.ts @@ -1,5 +1,6 @@ import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlite-core' import type { QuizQuestion } from '../../shared/types/quiz' +import type { DocumentStructure } from '../services/loaders/types' /** * 笔记本表 @@ -129,6 +130,10 @@ export const documents = sqliteTable( localFilePath: text('local_file_path'), // 本地拷贝文件路径 sourceNoteId: text('source_note_id').references(() => notes.id, { onDelete: 'set null' }), content: text('content'), // 原始内容（可选存储） + // 解析器给出的结构（页/章节）。这是 source record 的一部分,不是派生索引: + // 重新索引必须能重建出与首次导入一致的 document_blocks,所以不能靠重新解析 + // 文件来恢复（文件可能已变化,内容又是规范文本）。 + structure: text('structure', { mode: 'json' }).$type(), contentHash: text('content_hash'), // 内容哈希，用于检测变更 mimeType: text('mime_type'), // 文件类型 fileSize: integer('file_size'), // 文件大小（字节） diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 0e65803..6723795 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -36,6 +36,7 @@ import { EmbeddingService } from './EmbeddingService' import type { EmbeddingSpace } from '../../shared/types' import { ChunkingService, type ChunkOptions, type ChunkResult } from './ChunkingService' import { FileParserService } from './FileParserService' +import type { DocumentStructure } from './loaders/types' import { buildDocumentBlocks, assignBlockIds, @@ -178,145 +179,41 @@ export class KnowledgeService { // 计算内容哈希 const contentHash = createHash('md5').update(options.content).digest('hex') - try { - // 1. 创建文档记录 - onProgress?.('creating_document', 0) - - const newDoc: NewDocument = { - id: documentId, - notebookId, - title: options.title, - type: options.type, - sourceUri: options.sourceUri, - sourceNoteId: options.sourceNoteId, - content: options.content, - contentHash, - mimeType: options.mimeType, - fileSize: options.fileSize, - metadata: options.metadata, - status: 'processing', - chunkCount: 0, - createdAt: now, - updatedAt: now - } - - db.insert(documents).values(newDoc).run() - - // 1b. 持久化文档块（文本/URL/笔记没有结构，按段落平铺），并把同一批块交给分块器 - const blocks = assignBlockIds(documentId, buildDocumentBlocks({ content: options.content })) - this.persistDocumentBlocks(blocks) - - // 2. 分块（偏移锚定 options.content，即 documents.content） - onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunkBlocks( - options.content, - blocks, - options.chunkOptions - ) - - if (chunkResults.length === 0) { - throw new Error('No chunks generated from document') - } - - Logger.info('KnowledgeService', `Document ${documentId}: ${chunkResults.length} chunks`) - - // 3. 保存分块与 chunk↔block 映射（同一事务，不会出现没有映射的 chunk） - onProgress?.('saving_chunks', 20) - const { chunkIds, chunkContents } = this.saveChunks(documentId, notebookId, chunkResults, now) - - // 4. 生成嵌入向量 - // 不指定维度:维度由 embedding 模型决定,写死维度会让用别的模型的笔记本直接 - // 索引失败(vec0 表的宽度在创建时固定,见 #33)。测出真实维度之后再用它建表。 - const embeddingResults = await this.embedDocumentChunks(notebookId, chunkContents, onProgress) - - if (embeddingResults.length === 0) { - throw new Error('Embedding 服务没有返回任何向量') - } - - // space 由模型决定;与既有 space 不一致时重建向量表并把文档标回待索引 - const detectedDimensions = embeddingResults[0].dimensions - await this.reconcileEmbeddingSpace( - notebookId, - await this.embeddingService.getSpace(), - detectedDimensions - ) - Logger.info('KnowledgeService', `Embedding dimensions: ${detectedDimensions}`) - - // 5. 保存嵌入元数据并添加到向量存储 - onProgress?.('saving_embeddings', 85) - const vectorStore = await vectorStoreManager.getStore( - notebookId, - undefined, - detectedDimensions - ) - const vectorItems: Array<{ - id: string - chunkId: string - vector: Float32Array - metadata?: Record - }> = [] - - for (let i = 0; i < chunkIds.length; i++) { - const embeddingId = `emb_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` - const embResult = embeddingResults[i] - - // 保存嵌入元数据到数据库 - const newEmbedding: NewEmbedding = { - id: embeddingId, - chunkId: chunkIds[i], - notebookId, - model: embResult.model, - dimensions: embResult.dimensions, - createdAt: now - } - - db.insert(embeddings).values(newEmbedding).run() + // 1. 创建 source 记录。Document 是 source identity：之后的每一次重新索引都复用 + // 这一行，不会再生成新的 documentId。 + onProgress?.('creating_document', 0) - // 准备向量数据 - vectorItems.push({ - id: embeddingId, - chunkId: chunkIds[i], - vector: embResult.embedding, - metadata: { model: embResult.model, documentId } - }) - } - - // 批量添加到向量存储 - await vectorStore.upsert(vectorItems) - - // 6. 更新文档状态 - onProgress?.('finalizing', 95) - db.update(documents) - .set({ - status: 'indexed', - chunkCount: chunkResults.length, - updatedAt: new Date() - }) - .where(eq(documents.id, documentId)) - .run() + const newDoc: NewDocument = { + id: documentId, + notebookId, + title: options.title, + type: options.type, + sourceUri: options.sourceUri, + sourceNoteId: options.sourceNoteId, + content: options.content, + contentHash, + mimeType: options.mimeType, + fileSize: options.fileSize, + metadata: options.metadata, + status: 'processing', + chunkCount: 0, + createdAt: now, + updatedAt: now + } - executeCheckpoint('PASSIVE') - onProgress?.('completed', 100) + db.insert(documents).values(newDoc).run() - Logger.info( - 'KnowledgeService', - `Document indexed: ${documentId}, ${chunkResults.length} chunks` - ) - return documentId - } catch (error) { - // 更新文档状态为失败 - db.update(documents) - .set({ - status: 'failed', - errorMessage: (error as Error).message, - updatedAt: new Date() - }) - .where(eq(documents.id, documentId)) - .run() + // 2. 建立派生索引（blocks → chunks → embeddings → 向量表）。失败时由 + // `indexDocument()` 把这一行标成 failed，source 行保留以便重试或重新索引。 + await this.indexDocument( + documentId, + options.content, + undefined, + { chunkOptions: options.chunkOptions }, + onProgress + ) - Logger.error('KnowledgeService', 'Failed to add document:', error) - throw error - } + return documentId } /** @@ -330,6 +227,8 @@ export class KnowledgeService { const db = getDatabase() const documentId = `doc_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` let localFilePath: string | undefined + let parsedContent = '' + let parsedStructure: DocumentStructure | undefined try { onProgress?.('parsing_file', 0) @@ -338,12 +237,14 @@ export class KnowledgeService { localFilePath = await this.copyFileToKnowledgeDir(filePath, documentId) const parseResult = await this.fileParserService.parseFile(filePath) + parsedContent = parseResult.content + parsedStructure = parseResult.structure ?? undefined // 计算内容哈希 const contentHash = createHash('md5').update(parseResult.content).digest('hex') const now = new Date() - // 1. 创建文档记录（包含 localFilePath） + // 1. 创建 source 记录（包含 localFilePath 与解析结构） onProgress?.('creating_document', 0) // 根据 MIME 类型决定文档类型 @@ -361,6 +262,8 @@ export class KnowledgeService { sourceUri: filePath, localFilePath: localFilePath, content: parseResult.content, + // 结构随 source 一起持久化，重新索引才能不加解析地重建同一批块 + structure: parsedStructure, contentHash, mimeType: parseResult.mimeType, fileSize: parseResult.metadata?.fileSize as number | undefined, @@ -372,17 +275,65 @@ export class KnowledgeService { } db.insert(documents).values(newDoc).run() + } catch (error) { + // 拷贝/解析/建行失败时删除已拷贝的文件，并把（可能已创建的）行标成失败 + if (localFilePath) { + await this.deleteLocalFile(localFilePath) + } - // 1b. 持久化文档块，保留解析器给出的页/标题结构，并把同一批块交给分块器 - const blocks = assignBlockIds( - documentId, - buildDocumentBlocks({ content: parseResult.content, structure: parseResult.structure }) - ) + db.update(documents) + .set({ + status: 'failed', + errorMessage: (error as Error).message, + updatedAt: new Date() + }) + .where(eq(documents.id, documentId)) + .run() + + Logger.error('KnowledgeService', 'Failed to add document from file:', error) + throw error + } + + // 2. 建立派生索引。索引失败由 `indexDocument()` 标记；这里保留本地文件， + // 用户可以重新索引，而不是重新导入。 + await this.indexDocument(documentId, parsedContent, parsedStructure, {}, onProgress) + + return documentId + } + + /** + * 为一份已经存在的 source（`documents` 行）建立派生索引： + * blocks → chunks → embeddings → 向量表。 + * + * Document 是 source identity，blocks/chunks/embeddings 是派生索引。这个方法只写 + * 派生索引，从不插入或删除 `documents` 行，所以对同一个 documentId 反复调用不会改 + * 变来源身份（历史 citation 仍然指向同一个来源）。 + * + * 调用前该文档不应带有旧的派生索引；重新索引要先 `clearDerivedIndex()`。 + * 失败时把该行标成 `failed` 并抛出，由调用方决定是否清理 source。 + */ + private async indexDocument( + documentId: string, + content: string, + structure: DocumentStructure | undefined, + options: { chunkOptions?: ChunkOptions } = {}, + onProgress?: IndexProgressCallback + ): Promise { + const db = getDatabase() + const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() + if (!doc) throw new Error(`Document ${documentId} not found`) + + const notebookId = doc.notebookId + const now = new Date() + + try { + // 1. 文档块。偏移锚定 content，即 documents.content。 + const blocks = assignBlockIds(documentId, buildDocumentBlocks({ content, structure })) this.persistDocumentBlocks(blocks) - // 2. 分块（偏移锚定 parseResult.content，即 documents.content） + // 2. 分块（偏移同样锚定 content） onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunkBlocks(parseResult.content, blocks) + const chunkResults = this.chunkingService.chunkBlocks(content, blocks, options.chunkOptions) if (chunkResults.length === 0) { throw new Error('No chunks generated from document') @@ -460,6 +411,7 @@ export class KnowledgeService { .set({ status: 'indexed', chunkCount: chunkResults.length, + errorMessage: null, updatedAt: new Date() }) .where(eq(documents.id, documentId)) @@ -472,14 +424,9 @@ export class KnowledgeService { 'KnowledgeService', `Document indexed: ${documentId}, ${chunkResults.length} chunks` ) - return documentId + return chunkResults.length } catch (error) { - // 失败时删除已拷贝的文件 - if (localFilePath) { - await this.deleteLocalFile(localFilePath) - } - - // 更新文档状态为失败 + // 派生索引可以重建，所以失败只是状态问题，不需要删掉 source 行 db.update(documents) .set({ status: 'failed', @@ -489,11 +436,46 @@ export class KnowledgeService { .where(eq(documents.id, documentId)) .run() - Logger.error('KnowledgeService', 'Failed to add document from file:', error) + Logger.error('KnowledgeService', 'Failed to index document:', error) throw error } } + /** + * 删除一份文档的全部派生索引（向量、chunk↔block 映射、chunks、embeddings、 + * document_blocks），并把它标回 `processing`。`documents` 行本身保留。 + */ + private async clearDerivedIndex(documentId: string): Promise { + const db = getDatabase() + const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() + if (!doc) return + + const oldChunks = db + .select({ id: chunks.id }) + .from(chunks) + .where(eq(chunks.documentId, documentId)) + .all() + + if (oldChunks.length > 0) { + const chunkIds = oldChunks.map((chunk) => chunk.id) + + const vectorStore = await vectorStoreManager.getStore(doc.notebookId) + await vectorStore.deleteByChunkIds(chunkIds) + + // 映射与 embeddings 不依赖外键级联（连接上 foreign_keys 默认是关的），显式删除 + db.delete(chunkBlocks).where(inArray(chunkBlocks.chunkId, chunkIds)).run() + db.delete(embeddings).where(inArray(embeddings.chunkId, chunkIds)).run() + db.delete(chunks).where(eq(chunks.documentId, documentId)).run() + } + + db.delete(documentBlocks).where(eq(documentBlocks.documentId, documentId)).run() + + db.update(documents) + .set({ status: 'processing', errorMessage: null, chunkCount: 0, updatedAt: new Date() }) + .where(eq(documents.id, documentId)) + .run() + } + /** * 持久化文档块。id 已由 `assignBlockIds()` 生成，与分块器看到的是同一批块。 */ @@ -873,38 +855,14 @@ export class KnowledgeService { const doc = db.select().from(documents).where(eq(documents.id, documentId)).get() if (!doc) return - // 获取所有 chunk IDs - const docChunks = db - .select({ id: chunks.id }) - .from(chunks) - .where(eq(chunks.documentId, documentId)) - .all() - - // 从向量存储删除 - if (docChunks.length > 0) { - const vectorStore = await vectorStoreManager.getStore(doc.notebookId) - await vectorStore.deleteByChunkIds(docChunks.map((c) => c.id)) - } + // 删除全部派生索引（向量、映射、chunks、embeddings、blocks） + await this.clearDerivedIndex(documentId) // 删除本地拷贝的文件（如果存在） if (doc.localFilePath) { await this.deleteLocalFile(doc.localFilePath) } - // 块与映射不依赖外键级联（连接上 foreign_keys 默认是关的），显式删除 - if (docChunks.length > 0) { - db.delete(chunkBlocks) - .where( - inArray( - chunkBlocks.chunkId, - docChunks.map((c) => c.id) - ) - ) - .run() - } - db.delete(documentBlocks).where(eq(documentBlocks.documentId, documentId)).run() - - // 级联删除会自动清理 chunks 和 embeddings db.delete(documents).where(eq(documents.id, documentId)).run() executeCheckpoint('PASSIVE') @@ -912,7 +870,10 @@ export class KnowledgeService { } /** - * 重建文档索引 + * 重建文档索引。 + * + * 只重建派生索引：documentId、localFilePath、sourceUri、metadata 与解析结构都 + * 保持不变。重新索引不会让历史 citation 指向另一个来源。 */ async reindexDocument(documentId: string, onProgress?: IndexProgressCallback): Promise { const db = getDatabase() @@ -922,57 +883,8 @@ export class KnowledgeService { throw new Error(`Document ${documentId} not found or has no content`) } - // 删除旧的 chunks 和 embeddings - const oldChunks = db - .select({ id: chunks.id }) - .from(chunks) - .where(eq(chunks.documentId, documentId)) - .all() - - if (oldChunks.length > 0) { - const vectorStore = await vectorStoreManager.getStore(doc.notebookId) - await vectorStore.deleteByChunkIds(oldChunks.map((c) => c.id)) - db.delete(chunkBlocks) - .where( - inArray( - chunkBlocks.chunkId, - oldChunks.map((c) => c.id) - ) - ) - .run() - } - - db.delete(chunks).where(eq(chunks.documentId, documentId)).run() - db.delete(embeddings) - .where( - inArray( - embeddings.chunkId, - oldChunks.map((c) => c.id) - ) - ) - .run() - - // 更新状态 - db.update(documents) - .set({ status: 'processing', updatedAt: new Date() }) - .where(eq(documents.id, documentId)) - .run() - - // 重新索引（复用 addDocument 逻辑的核心部分） - // 为简化实现，这里直接调用内部处理 - await this.addDocument( - doc.notebookId, - { - title: doc.title, - type: doc.type as 'file' | 'note' | 'url' | 'text', - content: doc.content, - sourceUri: doc.sourceUri || undefined, - sourceNoteId: doc.sourceNoteId || undefined, - mimeType: doc.mimeType || undefined, - metadata: doc.metadata || undefined - }, - onProgress - ) + await this.clearDerivedIndex(documentId) + await this.indexDocument(documentId, doc.content, doc.structure ?? undefined, {}, onProgress) } /** diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 0884795..ec2cef6 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -46,6 +46,8 @@ import { SQLiteVectorStore } from './vectorstore/SQLiteVectorStore' import { assignBlockIds, buildDocumentBlocks } from './services/blocks/documentBlocks' import { ChunkingService } from './services/ChunkingService' import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProvenance' +import { KnowledgeService } from './services/KnowledgeService' +import type { EmbeddingService } from './services/EmbeddingService' export const SMOKE_TEST_FLAG = '--smoke-test' @@ -95,6 +97,37 @@ function assertThrows(run: () => unknown, message: string): void { throw new Error(message) } +/** + * A deterministic stand-in for `EmbeddingService` so the smoke test can drive + * the whole index lifecycle (blocks → chunks → embeddings → vectors) without + * downloading model weights. Only the methods `KnowledgeService` calls are + * implemented. + */ +function fakeEmbeddingService(dimensions = 16): EmbeddingService { + const result = (): { embedding: Float32Array; model: string; dimensions: number } => { + const embedding = new Float32Array(dimensions) + embedding[0] = 1 + return { embedding, model: 'smoke-embed', dimensions } + } + + // SAFETY: only `ensureReady`, `embedBatch` and `getSpace` are reached by the + // index path; the cast hides the rest of the real service's surface from the + // compiler on purpose, so a new call site here fails loudly at runtime. + return { + ensureReady: async () => undefined, + embedBatch: async (texts: string[]) => texts.map(() => result()), + getSpace: async () => ({ + id: 'smoke-embed-space', + backend: 'local', + model: 'smoke-embed', + revision: '', + dimensions, + pooling: 'mean', + normalize: true + }) + } as unknown as EmbeddingService +} + /** * Rewinds the database to the old layout: one global vec_embeddings table whose * width is fixed at 1024, plus a notebook that owns a vector in it. @@ -424,6 +457,58 @@ async function runChecks(): Promise { db.delete(documentBlocks).where(eq(documentBlocks.documentId, provenanceDocId)).run() db.delete(documents).where(eq(documents.id, provenanceDocId)).run() + // --- re-index keeps the source identity ----------------------------------- + // The bug this guards: `reindexDocument()` used to call `addDocument()`, which + // minted a new `documentId` and dropped `localFilePath`. A citation persisted + // against the old id would then point at a source that no longer exists. This + // drives the real service end to end with a fake embedding backend. + const reindexNotebook = 'smoke-notebook-reindex' + const reindexNow = Date.now() + database + .prepare('INSERT INTO notebooks (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)') + .run(reindexNotebook, 'Smoke re-index notebook', reindexNow, reindexNow) + + const knowledge = new KnowledgeService(fakeEmbeddingService()) + const reindexDocId = await knowledge.addDocumentFromFile( + reindexNotebook, + join(fixtures, 'sample.pdf') + ) + + const before = knowledge.getDocument(reindexDocId) + assert(before?.status === 'indexed', `imported document status was ${before?.status}`) + assert(Boolean(before?.localFilePath), 'imported document lost its localFilePath') + const blocksBefore = knowledge.getDocumentBlocks(reindexDocId) + assert(blocksBefore.length > 0, 'imported document produced no blocks') + + await knowledge.reindexDocument(reindexDocId) + + const after = knowledge.getDocument(reindexDocId) + assert(Boolean(after), 'the document disappeared after a re-index') + assert(after!.status === 'indexed', `re-index left status ${after!.status}`) + assert(after!.localFilePath === before!.localFilePath, 're-index did not preserve localFilePath') + assert( + !knowledge + .getDocuments(reindexNotebook) + .some((doc) => doc.id !== reindexDocId && doc.title === before!.title), + 're-index created a second document instead of rebuilding in place' + ) + assert( + JSON.stringify(knowledge.getDocumentBlocks(reindexDocId)) === JSON.stringify(blocksBefore), + 're-index rebuilt a different block sequence from the persisted structure' + ) + const chunksAfter = knowledge.getDocumentChunks(reindexDocId) + assert( + chunksAfter.length > 0 && after!.chunkCount === chunksAfter.length, + 're-index left the document without chunks' + ) + + await knowledge.deleteDocument(reindexDocId) + assert( + knowledge.getDocument(reindexDocId) === undefined, + 'deleting the re-indexed document did not remove it' + ) + pass('re-index rebuilds the derived index in place and keeps the source identity') + const { ApkgExporter } = await import('./services/exporters/ApkgExporter') const { buffer, summary } = await new ApkgExporter().export( [{ id: 'smoke-card', type: 'basic', front: 'front', back: 'back', tags: ['smoke'] }], From 8ddd75a76e0c42f4637c9bca61d694864408ce72 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 15:11:16 +0800 Subject: [PATCH 2/5] feat(parsers): paragraph-level PDF blocks for precise citation anchors PdfLoader joined every TextItem on a page with a space and emitted one page-level block, so a citation could say "page 12" but never "the paragraph at the bottom of page 12". Add a pure pdfTextLayout module that groups items into lines by baseline and lines into paragraphs by vertical gap and first-line indent, each with its own normalized bbox. PdfLoader builds the canonical content from those paragraphs, so content.slice(startOffset, endOffset) === block.text keeps holding; buildPageBlocks() consumes page.blocks when present and falls back to the old one-block-per-page shape otherwise. Geometry is unit-tested with synthetic items; the existing single-run fixtures still produce one block per page. Fixes #110 --- src/main/services/blocks/documentBlocks.ts | 25 +- src/main/services/loaders/PdfLoader.ts | 106 +++++---- src/main/services/loaders/pdfTextLayout.ts | 254 +++++++++++++++++++++ src/main/services/loaders/types.ts | 12 + test/pdfTextLayout.test.ts | 194 ++++++++++++++++ 5 files changed, 535 insertions(+), 56 deletions(-) create mode 100644 src/main/services/loaders/pdfTextLayout.ts create mode 100644 test/pdfTextLayout.test.ts diff --git a/src/main/services/blocks/documentBlocks.ts b/src/main/services/blocks/documentBlocks.ts index 743266f..ea0f934 100644 --- a/src/main/services/blocks/documentBlocks.ts +++ b/src/main/services/blocks/documentBlocks.ts @@ -87,12 +87,34 @@ export function buildDocumentBlocks(source: BlockSource): DocumentBlockDraft[] { } /** - * 分页文档：页偏移已经由 loader 保证与规范文本一致，块文本直接切出来。 + * 分页文档：优先用 loader 已经分好的页内段落（PDF），每个段落一个块；loader 没有 + * 提供页内块时（例如旧的 / 非 PDF 分页格式）回退到整页一块。页偏移已由 loader + * 保证与规范文本一致，块文本直接切出来。 */ function buildPageBlocks(content: string, pages: PageInfo[]): Omit[] { const blocks: Omit[] = [] for (const page of pages) { + if (page.blocks && page.blocks.length > 0) { + for (const pageBlock of page.blocks) { + const text = content.slice(pageBlock.startOffset, pageBlock.endOffset) + if (text.trim().length === 0) continue + + blocks.push({ + kind: 'paragraph', + page: page.pageNumber, + level: null, + text, + // 用实际切出的文本长度回写 endOffset，避免 loader 的结尾空白把块撑出内容范围 + startOffset: pageBlock.startOffset, + endOffset: pageBlock.startOffset + text.length, + bbox: pageBlock.bbox ?? null, + metadata: null + }) + } + continue + } + const text = content.slice(page.startOffset, page.endOffset) if (text.trim().length === 0) continue @@ -101,7 +123,6 @@ function buildPageBlocks(content: string, pages: PageInfo[]): Omit 'str' in item) - const pageText = textItems.map((item) => item.str).join(' ') - - // 清理文本 - const cleanedText = this.cleanPDFText(pageText) - const pageTextWithNewline = cleanedText + '\n\n' + const paragraphs = layoutPageText( + this.toPositionedItems(textItems, viewport), + viewport.width, + viewport.height + ) + + // 页内容由段落拼出，段落偏移随后锚定到这份规范字符串上 + const pageContent = paragraphs.map((paragraph) => paragraph.text).join('\n\n') + const pageBlocks: PageBlockInfo[] = [] + let pageCursor = 0 + for (const paragraph of paragraphs) { + const startOffset = currentOffset + pageCursor + pageBlocks.push({ + text: paragraph.text, + startOffset, + endOffset: startOffset + paragraph.text.length, + bbox: paragraph.bbox + }) + pageCursor += paragraph.text.length + 2 // '\n\n' + } // 记录页面信息 if (opts.preserveStructure) { pages.push({ pageNumber: pageNum, - content: cleanedText, + content: pageContent, startOffset: currentOffset, - endOffset: currentOffset + cleanedText.length, - bbox: this.computeNormalizedBBox(textItems, viewport), + endOffset: currentOffset + pageContent.length, + bbox: this.unionBBox(paragraphs.map((paragraph) => paragraph.bbox)), + blocks: pageBlocks, metadata: { width: viewport.width, height: viewport.height @@ -109,6 +127,7 @@ export class PdfLoader implements IDocumentLoader { }) } + const pageTextWithNewline = pageContent + '\n\n' fullText += pageTextWithNewline currentOffset += pageTextWithNewline.length @@ -147,62 +166,41 @@ export class PdfLoader implements IDocumentLoader { } /** - * 把页面上所有文本项映射到页面空间,取并集后归一化到 0..1。 - * - * 用 viewport.transform 把 PDF 用户空间（原点在左下）转成视口空间（原点在左上）， - * 这样渲染层不需要知道缩放比例。没有任何文本项时返回 undefined。 + * 把 pdf.js 的文本项转换到视口坐标（原点左上，y 向下）。渲染层因此不需要知道缩放。 */ - private computeNormalizedBBox( + private toPositionedItems( items: TextItem[], viewport: ReturnType - ): NormalizedBox | undefined { - let minX = Infinity - let minY = Infinity - let maxX = -Infinity - let maxY = -Infinity + ): PositionedTextItem[] { + const positioned: PositionedTextItem[] = [] for (const item of items) { + if (!item.str) continue + const textTransform = pdfjsLib.Util.transform(viewport.transform, item.transform) const fontHeight = Math.hypot(textTransform[2], textTransform[3]) - const left = textTransform[4] - const right = left + Math.abs(item.width) - const top = textTransform[5] - fontHeight - const bottom = textTransform[5] - - minX = Math.min(minX, left, right) - maxX = Math.max(maxX, left, right) - minY = Math.min(minY, top, bottom) - maxY = Math.max(maxY, top, bottom) + + positioned.push({ + str: item.str, + left: textTransform[4], + baseline: textTransform[5], + width: Math.abs(item.width), + height: fontHeight > 0 ? fontHeight : 1 + }) } - if (!Number.isFinite(minX) || !Number.isFinite(minY)) return undefined + return positioned + } - const clamp = (value: number): number => Math.min(1, Math.max(0, value)) - const width = viewport.width || 1 - const height = viewport.height || 1 + /** 多个归一化包围盒的并集（页面 bbox）。没有有效盒时返回 undefined。 */ + private unionBBox(boxes: NormalizedBox[]): NormalizedBox | undefined { + if (boxes.length === 0) return undefined - return { - x: clamp(minX / width), - y: clamp(minY / height), - w: clamp((maxX - minX) / width), - h: clamp((maxY - minY) / height) - } - } + const left = Math.min(...boxes.map((box) => box.x)) + const top = Math.min(...boxes.map((box) => box.y)) + const right = Math.max(...boxes.map((box) => box.x + box.w)) + const bottom = Math.max(...boxes.map((box) => box.y + box.h)) - /** - * 清理 PDF 提取的文本 - */ - private cleanPDFText(text: string): string { - return ( - text - // 移除多余的空格 - .replace(/[ \t]+/g, ' ') - // 修复断行（连续的行可能是同一段落） - .replace(/([^\n])\n([^\n])/g, '$1 $2') - // 统一多个换行为两个 - .replace(/\n{3,}/g, '\n\n') - // 移除首尾空白 - .trim() - ) + return { x: left, y: top, w: right - left, h: bottom - top } } } diff --git a/src/main/services/loaders/pdfTextLayout.ts b/src/main/services/loaders/pdfTextLayout.ts new file mode 100644 index 0000000..a9c49b4 --- /dev/null +++ b/src/main/services/loaders/pdfTextLayout.ts @@ -0,0 +1,254 @@ +/** + * pdfTextLayout + * 把 pdf.js 的文本项按几何位置还原成「行 → 段落」。 + * + * `PdfLoader` 之前把一页所有文本项用空格拼成一个字符串，块粒度因此停在一页一句。 + * 引用要落到「第 12 页右下角那一段」，就需要在页内再分一层。这里只做几何分组， + * 不碰文本内容 —— 规范文本与偏移由 `PdfLoader` 在拼出内容时统一计算，从而保证 + * `content.slice(startOffset, endOffset) === text` 继续成立。 + * + * 纯函数：输入是已经转换到视口坐标的文本项，便于用合成数据单测，不需要 PDF fixture。 + */ + +import type { NormalizedBox } from './types' + +/** 已转换到视口坐标（原点左上，y 向下）的文本项。 */ +export interface PositionedTextItem { + str: string + /** 文本左边缘 x */ + left: number + /** 文本基线 y（视口坐标） */ + baseline: number + /** 文本宽度 */ + width: number + /** 字号（视口坐标下的行高） */ + height: number +} + +/** 一行文本。 */ +export interface TextLine { + text: string + left: number + right: number + top: number + bottom: number + height: number +} + +/** 一个段落：文本 + 归一化到页面的包围盒。 */ +export interface TextParagraph { + text: string + bbox: NormalizedBox +} + +/** 同一行的基线容差（相对最小字号）。 */ +const LINE_TOLERANCE = 0.5 +/** 行间垂直空隙超过多少倍行高视为新段落。 */ +const PARAGRAPH_GAP = 0.6 +/** 行首相对段首左边缘缩进超过多少倍字号视为新段落。 */ +const INDENT_RATIO = 1 +/** 同一行内文本项间距超过多少倍字号时补一个空格。 */ +const SPACE_GAP = 0.25 + +/** + * 把一页的文本项还原成段落列表。顺序为阅读顺序：先按基线自上而下，行内自左向右。 + * 文本项全部无效时返回空数组。 + */ +export function layoutPageText( + items: PositionedTextItem[], + pageWidth: number, + pageHeight: number +): TextParagraph[] { + const lines = groupLines(items) + const paragraphs = groupParagraphs(lines) + + return paragraphs.map((paragraph) => ({ + text: paragraph.text, + bbox: normalizeBox(paragraph.box, pageWidth, pageHeight) + })) +} + +interface LineDraft { + items: PositionedTextItem[] + baseline: number + height: number + left: number + right: number + top: number + bottom: number +} + +/** 按基线把文本项聚成行，行内按 x 排序后拼出文本。 */ +function groupLines(items: PositionedTextItem[]): TextLine[] { + const usable = items.filter( + (item) => + item.str.trim().length > 0 && + Number.isFinite(item.left) && + Number.isFinite(item.baseline) && + Number.isFinite(item.width) && + Number.isFinite(item.height) + ) + + const sorted = [...usable].sort((a, b) => a.baseline - b.baseline || a.left - b.left || 0) + + const drafts: LineDraft[] = [] + + for (const item of sorted) { + const last = drafts[drafts.length - 1] + const tolerance = Math.max( + 1, + Math.min(last?.height ?? item.height, item.height) * LINE_TOLERANCE + ) + const sameLine = last !== undefined && Math.abs(item.baseline - last.baseline) <= tolerance + + if (sameLine && last) { + last.items.push(item) + const count = last.items.length + last.baseline = (last.baseline * (count - 1) + item.baseline) / count + last.height = Math.max(last.height, item.height) + last.left = Math.min(last.left, item.left) + last.right = Math.max(last.right, item.left + item.width) + last.top = Math.min(last.top, item.baseline - item.height) + last.bottom = Math.max(last.bottom, item.baseline) + } else { + drafts.push({ + items: [item], + baseline: item.baseline, + height: item.height, + left: item.left, + right: item.left + item.width, + top: item.baseline - item.height, + bottom: item.baseline + }) + } + } + + return drafts + .filter((draft) => draft.items.some((item) => item.str.trim().length > 0)) + .map((draft) => ({ + text: lineText(draft.items), + left: draft.left, + right: draft.right, + top: draft.top, + bottom: draft.bottom, + height: draft.height + })) + .filter((line) => line.text.length > 0) +} + +/** 行内按 x 排序，间距足够大时补空格（同一词被拆成多个文本项时不补）。 */ +function lineText(items: PositionedTextItem[]): string { + const ordered = [...items].sort((a, b) => a.left - b.left) + + let text = '' + let previousRight: number | null = null + let previousHeight = 0 + + for (const item of ordered) { + if (previousRight !== null) { + const gap = item.left - previousRight + if (gap > SPACE_GAP * Math.max(previousHeight, item.height)) text += ' ' + } + text += item.str + previousRight = item.left + item.width + previousHeight = item.height + } + + return collapseWhitespace(text) +} + +interface ParagraphDraft { + lines: TextLine[] + left: number + top: number + bottom: number +} + +/** 按垂直间距与首行缩进把行聚成段落。 */ +function groupParagraphs(lines: TextLine[]): Array<{ text: string; box: Box }> { + const paragraphs: ParagraphDraft[] = [] + let current: ParagraphDraft | null = null + + const flush = (): void => { + if (current && current.lines.length > 0) paragraphs.push(current) + current = null + } + + for (const line of lines) { + if (!current) { + current = { + lines: [line], + left: line.left, + top: line.top, + bottom: line.bottom + } + continue + } + + const previous = current.lines[current.lines.length - 1] + const gap = line.top - previous.bottom + const lineHeight = Math.max(previous.height, line.height) + const newParagraph = + gap > PARAGRAPH_GAP * lineHeight || + (gap > 0 && line.left - current.left > INDENT_RATIO * line.height) + + if (newParagraph) { + flush() + current = { + lines: [line], + left: line.left, + top: line.top, + bottom: line.bottom + } + continue + } + + current.lines.push(line) + current.top = Math.min(current.top, line.top) + current.bottom = Math.max(current.bottom, line.bottom) + } + + flush() + + return paragraphs + .map((paragraph) => ({ + text: collapseWhitespace(paragraph.lines.map((line) => line.text).join(' ')), + box: { + left: Math.min(...paragraph.lines.map((line) => line.left)), + top: paragraph.top, + right: Math.max(...paragraph.lines.map((line) => line.right)), + bottom: paragraph.bottom + } + })) + .filter((paragraph) => paragraph.text.length > 0) +} + +interface Box { + left: number + top: number + right: number + bottom: number +} + +/** 视口坐标 → 0..1 归一化页面坐标。 */ +function normalizeBox(box: Box, pageWidth: number, pageHeight: number): NormalizedBox { + const clamp = (value: number): number => Math.min(1, Math.max(0, value)) + const width = pageWidth || 1 + const height = pageHeight || 1 + + const left = Math.min(box.left, box.right) + const right = Math.max(box.left, box.right) + const top = Math.min(box.top, box.bottom) + const bottom = Math.max(box.top, box.bottom) + + return { + x: clamp(left / width), + y: clamp(top / height), + w: clamp((right - left) / width), + h: clamp((bottom - top) / height) + } +} + +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim() +} diff --git a/src/main/services/loaders/types.ts b/src/main/services/loaders/types.ts index b40e91c..5bab4e1 100644 --- a/src/main/services/loaders/types.ts +++ b/src/main/services/loaders/types.ts @@ -13,6 +13,17 @@ export interface NormalizedBox { h: number } +/** + * 页内块（分页文档的段落）。文本与偏移锚定整份规范内容（`documents.content`）， + * bbox 归一化到页面。`PdfLoader` 逐段产出，找不到几何分组时回退到整页一块。 + */ +export interface PageBlockInfo { + text: string + startOffset: number + endOffset: number + bbox?: NormalizedBox +} + /** * 页面信息（用于 PDF、PPT 等分页文档） */ @@ -22,6 +33,7 @@ export interface PageInfo { startOffset: number // 在完整文本中的起始位置 endOffset: number // 在完整文本中的结束位置 bbox?: NormalizedBox // 页面文本的归一化包围盒（可选） + blocks?: PageBlockInfo[] // 页内段落块（可选；供 document_blocks 落到段落级） metadata?: { width?: number height?: number diff --git a/test/pdfTextLayout.test.ts b/test/pdfTextLayout.test.ts new file mode 100644 index 0000000..513c7af --- /dev/null +++ b/test/pdfTextLayout.test.ts @@ -0,0 +1,194 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { + layoutPageText, + type PositionedTextItem +} from '../src/main/services/loaders/pdfTextLayout.ts' +import { + buildDocumentBlocks, + type DocumentBlockDraft +} from '../src/main/services/blocks/documentBlocks.ts' +import type { DocumentStructure } from '../src/main/services/loaders/types.ts' + +/** + * The PDF loader used to join every `TextItem` on a page into one string, so a + * citation could only ever say "page 12", never "the paragraph at the bottom of + * page 12". `layoutPageText()` groups items geometrically into lines and then + * paragraphs; these tests pin that geometry with synthetic items, so the rule + * is testable without a PDF fixture. + * + * The second half checks that `buildDocumentBlocks()` turns those paragraphs + * into blocks whose offsets still slice back out of `documents.content` — the + * one invariant the whole provenance chain rests on. + */ + +/** A viewport-space text item; font height defaults to 10. */ +function item(str: string, left: number, baseline: number, width: number, height = 10) { + return { str, left, baseline, width, height } +} + +/** Two lines in one paragraph, then a second paragraph after a blank line. */ +function twoParagraphPage(): PositionedTextItem[] { + return [ + item('First', 50, 100, 40), + item('line', 96, 100, 20), + item('continues', 50, 114, 60), + item('Second paragraph.', 50, 140, 110) + ] +} + +test('items on the same baseline become one line and words are spaced apart', () => { + const [first] = layoutPageText(twoParagraphPage(), 800, 1000).slice(0, 1) + assert.ok(first) + // "First" ends at x=90, "line" starts at x=96: a gap, so a space is inserted. + assert.equal(first.text, 'First line continues') +}) + +test('a word split across items is not given an inner space', () => { + const words = [item('Know', 50, 100, 32), item('Note', 82, 100, 34)] + const [paragraph] = layoutPageText(words, 800, 1000) + assert.ok(paragraph) + assert.equal(paragraph.text, 'KnowNote') +}) + +test('a vertical gap starts a new paragraph, preserving reading order', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['First line continues', 'Second paragraph.'] + ) +}) + +test('a first-line indent starts a new paragraph even without a vertical gap', () => { + const page = [ + item('one line', 50, 100, 60), + item('and the next', 70, 112, 70) // left indented by 20 > height(10), gap 2 + ] + const paragraphs = layoutPageText(page, 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['one line', 'and the next'] + ) +}) + +test('paragraph bboxes are normalized to the page and cover only their own items', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + const [first, second] = paragraphs + + // First paragraph spans x 50..116, y top 90..bottom 114. + assert.deepEqual(first.bbox, { x: 50 / 800, y: 90 / 1000, w: 66 / 800, h: 24 / 1000 }) + assert.deepEqual(second.bbox, { x: 50 / 800, y: 130 / 1000, w: 110 / 800, h: 10 / 1000 }) +}) + +test('items arrive in any order but read top-to-bottom, left-to-right', () => { + const shuffled = [ + item('second', 50, 114, 40), + item('world', 100, 100, 40), + item('hello', 50, 100, 40) + ] + const paragraphs = layoutPageText(shuffled, 800, 1000) + // Both lines are only 4 apart (top 104 vs 104... 114-10=104) so they merge + // into one paragraph; order inside the line still follows x. + assert.equal(paragraphs[0].text, 'hello world second') +}) + +test('blank and non-finite items are ignored', () => { + const page = [ + item(' ', 50, 100, 40), + item('real', 50, 100, 40), + { str: 'bad', left: NaN, baseline: 100, width: 10, height: 10 } + ] + const paragraphs = layoutPageText(page, 800, 1000) + assert.deepEqual( + paragraphs.map((paragraph) => paragraph.text), + ['real'] + ) +}) + +test('an empty page produces no paragraphs', () => { + assert.deepEqual(layoutPageText([], 800, 1000), []) +}) + +// --- block consumption ------------------------------------------------------- + +/** + * Collapse the two-line page above into a canonical content string the same way + * `PdfLoader` does: paragraphs joined by a blank line. + */ +function pageStructure( + paragraphs: { + text: string + bbox: import('../src/main/services/loaders/types.ts').NormalizedBox + }[] +): { + content: string + structure: DocumentStructure +} { + const content = paragraphs.map((paragraph) => paragraph.text).join('\n\n') + + let cursor = 0 + const blocks = paragraphs.map((paragraph) => { + const startOffset = cursor + cursor += paragraph.text.length + 2 + return { + text: paragraph.text, + startOffset, + endOffset: startOffset + paragraph.text.length, + bbox: paragraph.bbox + } + }) + + // Offsets in `blocks` must address the *trimmed* canonical content. + const trimmed = content.trim() + const shift = content.length - content.trimStart().length + for (const block of blocks) { + block.startOffset -= shift + block.endOffset -= shift + } + + return { + content: trimmed, + structure: { + type: 'pages', + pages: [ + { pageNumber: 1, content: trimmed, startOffset: 0, endOffset: trimmed.length, blocks } + ] + } + } +} + +function assertBlocks(content: string, blocks: DocumentBlockDraft[]): void { + for (const block of blocks) { + assert.equal( + content.slice(block.startOffset, block.endOffset), + block.text, + `block ${block.order} does not slice back to its text` + ) + } +} + +test('a multi-paragraph page becomes multiple blocks in reading order', () => { + const paragraphs = layoutPageText(twoParagraphPage(), 800, 1000) + const { content, structure } = pageStructure(paragraphs) + const blocks = buildDocumentBlocks({ content, structure }) + + assert.deepEqual( + blocks.map((block) => block.text), + ['First line continues', 'Second paragraph.'] + ) + assert.ok(blocks.every((block) => block.page === 1)) + assertBlocks(content, blocks) +}) + +test('a page without paragraph blocks still falls back to one page-level block', () => { + const content = 'legacy page text' + const structure: DocumentStructure = { + type: 'pages', + pages: [{ pageNumber: 1, content, startOffset: 0, endOffset: content.length }] + } + const blocks = buildDocumentBlocks({ content, structure }) + + assert.equal(blocks.length, 1) + assert.equal(blocks[0].text, content) + assertBlocks(content, blocks) +}) From 1eca956905f6f0c429fdbe362aafe845a377f547 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 15:13:10 +0800 Subject: [PATCH 3/5] refactor(retrieval): extract a Retriever seam that returns evidence Retrieval was inline in KnowledgeService.search(): verify the embedding space, embed the query, query the vector store, then join chunks and documents. A new strategy could not be added without editing the service, and the eval harness (#75) had no stable entry point. Introduce src/main/services/retrieval/: interface Retriever { search(notebookId, query, opts): Promise } RetrievedEvidence carries provenance as a first-class field (source title/type, page range, ordered block spans), so the citation layer (#69) never queries the database again. DenseRetriever is the current strategy; the embedding-space guard stays in KnowledgeService because it is a precondition, not a strategy concern. Provenance hydration is now batch: resolveChunksProvenance() resolves any number of chunks with one join instead of one query per chunk, avoiding the N+1 that topK=5 would otherwise create. hydrateEvidence() wires the batch queries into RetrievedEvidence[]. KnowledgeService.search() keeps its signature and delegates; SearchResult gains a locator field and is otherwise byte-for-byte the same. Fixes #74 --- docs/architecture.md | 84 ++++++++++++++ src/main/services/KnowledgeService.ts | 83 +++++--------- src/main/services/chunkProvenance.ts | 80 +++++++++++-- src/main/services/retrieval/DenseRetriever.ts | 35 ++++++ src/main/services/retrieval/evidence.ts | 106 ++++++++++++++++++ src/main/services/retrieval/index.ts | 7 ++ src/main/services/retrieval/types.ts | 55 +++++++++ src/main/smokeTest.ts | 22 ++++ test/retrievalEvidence.test.ts | 92 +++++++++++++++ 9 files changed, 500 insertions(+), 64 deletions(-) create mode 100644 docs/architecture.md create mode 100644 src/main/services/retrieval/DenseRetriever.ts create mode 100644 src/main/services/retrieval/evidence.ts create mode 100644 src/main/services/retrieval/index.ts create mode 100644 src/main/services/retrieval/types.ts create mode 100644 test/retrievalEvidence.test.ts diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..7e3ec21 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,84 @@ +# Architecture notes + +Long-lived decisions that outlive a single feature. Feature-specific design lives +in the issue tracker and in `PRODUCT.md` / `DESIGN.md`; this file records the +seams that other work is allowed to depend on. + +## Source provenance + +A document is an identity. Everything derived from it can be rebuilt: + +```text +PDF / DOCX / URL / Note + ↓ +documents ← source identity (id, content, structure, localFilePath) + ↓ +document_blocks ← derived: page / heading level / bbox / char span + ↓ +chunks + chunk_blocks ← derived: char offsets anchored to documents.content + ↓ +embeddings + vectors ← derived +``` + +Two invariants hold across the whole chain: + +1. `chunk.content` and `block.text` are always `documents.content.slice(start, end)`. + The chunker and the block builder never pre-process text, so offsets and content + cannot drift apart. +2. Re-indexing rebuilds the derived layer in place. It never creates a second + `documents` row, so a citation that stored a `documentId` stays valid. The parse + structure is persisted on the source row so a rebuild does not need to re-parse a + file that may have changed on disk. + +## Retrieval seam + +`src/main/services/retrieval/` is the boundary between "how do we search" and +"what do we do with the results". Anything — Chat, the search panel, the MCP +server (#80), the eval harness (#75) — goes through it instead of embedding its own +RAG path. + +```ts +interface Retriever { + search( + notebookId: string, + query: string, + options?: RetrieveOptions + ): Promise +} +``` + +```ts +interface RetrievedEvidence { + chunkId: string + documentId: string + content: string + score: number + chunkIndex: number + source: { title: string; type: string } + locator: { + pageStart: number | null + pageEnd: number | null + blocks: EvidenceBlock[] + } + metadata?: Record +} +``` + +Rules: + +- **Provenance travels with the result.** A retriever must return a `locator`, not a + bare chunk, so the citation layer (#69) never has to query the database again. +- **The embedding-space guard is a precondition, not a strategy.** It stays in + `KnowledgeService.search()`, where the notebook-level index identity is checked, + before any strategy runs. +- **Strategies are swappable and measurable.** `DenseRetriever` is the only + implementation today. BM25, hybrid fusion and rerankers (#77) implement the same + interface; the eval harness (#75) measures them without touching callers. +- **Hydration is batch.** `hydrateEvidence()` resolves chunk, document and + provenance in a constant number of queries regardless of `topK`. Per-chunk + `resolveChunkProvenance()` calls are not allowed on the hot path. + +`KnowledgeService.search()` keeps its historical `SearchResult[]` shape for IPC +compatibility and delegates to the default retriever. `SearchResult` gained a +`locator` field, which is populated from `RetrievedEvidence`; ranking, scores and +the existing fields are unchanged. diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 6723795..1f9c3fe 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -43,6 +43,8 @@ import { type IdentifiedBlockDraft } from './blocks/documentBlocks' import { resolveChunkProvenance, type ChunkProvenance } from './chunkProvenance' +import { DenseRetriever } from './retrieval' +import type { EvidenceLocator, Retriever } from './retrieval' import { WebFetchService } from './WebFetchService' import { vectorStoreManager } from '../vectorstore' import Logger from '../../shared/utils/logger' @@ -73,6 +75,9 @@ export interface SearchOptions { /** * 搜索结果 + * + * `locator` 是检索 → 引用的 seam：页码区间与块区间随结果一起交付，#69 直接用它 + * 组装 citation，不必再回头查库。 */ export interface SearchResult { chunkId: string @@ -82,6 +87,7 @@ export interface SearchResult { content: string score: number chunkIndex: number + locator: EvidenceLocator metadata?: Record } @@ -97,6 +103,7 @@ export type IndexProgressCallback = (stage: string, progress: number) => void export class KnowledgeService { private embeddingService: EmbeddingService private chunkingService: ChunkingService + private retriever: Retriever private fileParserService: FileParserService private webFetchService: WebFetchService private knowledgeFilesDir: string @@ -104,6 +111,7 @@ export class KnowledgeService { constructor(embeddingService: EmbeddingService) { this.embeddingService = embeddingService this.chunkingService = new ChunkingService() + this.retriever = new DenseRetriever(embeddingService) this.fileParserService = new FileParserService() this.webFetchService = new WebFetchService() // 知识库文件存储目录 @@ -733,14 +741,17 @@ export class KnowledgeService { } /** - * 语义搜索 + * 语义搜索。 + * + * 保持原有签名与返回形状，内部委托给默认的 `Retriever`（当前是 `DenseRetriever`）。 + * embedding space 校验留在这里：它是前置条件，不是检索策略的一部分。 */ async search( notebookId: string, query: string, options: SearchOptions = {} ): Promise { - const { topK = 5, threshold = 0.5, includeContent = true } = options + const { includeContent = true } = options const db = getDatabase() // 0. 索引身份校验：当前模型与建索引时不一致，向量不可比，明确要求重新索引 @@ -756,64 +767,28 @@ export class KnowledgeService { ) } - // 1. 生成查询向量（E5 要求 query 前缀，与索引时的 document 前缀区分） - await this.embeddingService.ensureReady() - const queryEmbedding = await this.embeddingService.embed(query, 'query') - - // 2. 向量检索 - const vectorStore = await vectorStoreManager.getStore(notebookId) - const vectorResults = await vectorStore.query(queryEmbedding.embedding, { - topK, - threshold - }) - - if (vectorResults.length === 0) { - return [] - } - - // 3. 获取 chunk 详情 - const chunkIds = vectorResults.map((r) => r.chunkId) - - const chunkDetails = db.select().from(chunks).where(inArray(chunks.id, chunkIds)).all() - - // 获取文档信息 - const documentIds = [...new Set(chunkDetails.map((c) => c.documentId))] - const documentDetails = db - .select() - .from(documents) - .where(inArray(documents.id, documentIds)) - .all() - - const documentMap = new Map(documentDetails.map((d) => [d.id, d])) - const chunkMap = new Map(chunkDetails.map((c) => [c.id, c])) - - // 4. 组装结果 - const results: SearchResult[] = [] - - for (const vr of vectorResults) { - const chunk = chunkMap.get(vr.chunkId) - if (!chunk) continue - - const doc = documentMap.get(chunk.documentId) + // 1. 检索（向量 → 批量补齐来源/定位信息） + const evidence = await this.retriever.search(notebookId, query, options) + // 2. 映射回兼容的 SearchResult 形状 + return evidence.map((item) => { const result: SearchResult = { - chunkId: vr.chunkId, - documentId: chunk.documentId, - documentTitle: doc?.title || 'Unknown', - documentType: doc?.type || 'unknown', - content: includeContent ? chunk.content : '', - score: vr.score, - chunkIndex: chunk.chunkIndex + chunkId: item.chunkId, + documentId: item.documentId, + documentTitle: item.source.title, + documentType: item.source.type, + content: includeContent ? item.content : '', + score: item.score, + chunkIndex: item.chunkIndex, + locator: item.locator } - if (chunk.metadata) { - result.metadata = chunk.metadata + if (item.metadata) { + result.metadata = item.metadata } - results.push(result) - } - - return results + return result + }) } /** diff --git a/src/main/services/chunkProvenance.ts b/src/main/services/chunkProvenance.ts index 3996735..bcdba90 100644 --- a/src/main/services/chunkProvenance.ts +++ b/src/main/services/chunkProvenance.ts @@ -6,7 +6,7 @@ * `resolveChunkProvenance()` 一次 join 同时拿到有序块区间与反规范化的页码区间。 */ -import { eq } from 'drizzle-orm' +import { eq, inArray } from 'drizzle-orm' import { chunks, documentBlocks, chunkBlocks } from '../db/schema' import type { getDatabase } from '../db' import type { ChunkBlockSpan } from './ChunkingService' @@ -84,15 +84,20 @@ export function insertChunkBlocks(db: Db, chunkId: string, spans: ChunkBlockSpan } /** - * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 - * 一次 join chunks → chunk_blocks → document_blocks。chunk 不存在时返回 undefined。 + * 批量解析多个 chunk 的来源。检索一次返回 topK 个 chunk，逐个调用 + * `resolveChunkProvenance()` 会自然形成 N+1；这里用一次 join 取回全部映射，再在 JS 里 + * 按 chunkId 分组。返回顺序与 `chunkIds` 无关，由 `Map` 承载。 */ -export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance | undefined { - const chunk = db.select().from(chunks).where(eq(chunks.id, chunkId)).get() - if (!chunk) return undefined +export function resolveChunksProvenance(db: Db, chunkIds: string[]): Map { + const result = new Map() + if (chunkIds.length === 0) return result const rows = db .select({ + chunkId: chunks.id, + documentId: chunks.documentId, + pageStart: chunks.pageStart, + pageEnd: chunks.pageEnd, blockId: documentBlocks.id, kind: documentBlocks.kind, level: documentBlocks.level, @@ -104,10 +109,65 @@ export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance startInBlock: chunkBlocks.startInBlock, endInBlock: chunkBlocks.endInBlock }) - .from(chunkBlocks) - .innerJoin(documentBlocks, eq(chunkBlocks.blockId, documentBlocks.id)) - .where(eq(chunkBlocks.chunkId, chunkId)) + .from(chunks) + .leftJoin(chunkBlocks, eq(chunkBlocks.chunkId, chunks.id)) + .leftJoin(documentBlocks, eq(chunkBlocks.blockId, documentBlocks.id)) + .where(inArray(chunks.id, chunkIds)) .all() - return projectChunkProvenance(chunkId, chunk.documentId, chunk.pageStart, chunk.pageEnd, rows) + const grouped = new Map< + string, + { + documentId: string + pageStart: number | null + pageEnd: number | null + rows: ChunkProvenanceRow[] + } + >() + + for (const row of rows) { + let entry = grouped.get(row.chunkId) + if (!entry) { + entry = { + documentId: row.documentId, + pageStart: row.pageStart, + pageEnd: row.pageEnd, + rows: [] + } + grouped.set(row.chunkId, entry) + } + + // 没有映射的 chunk 仍然解析（blocks 为空），左连接因此会给出一个全 NULL 块 + if (row.blockId === null || row.order === null) continue + + entry.rows.push({ + blockId: row.blockId, + kind: row.kind as BlockKind, + level: row.level, + page: row.page, + text: row.text ?? '', + startOffset: row.startOffset ?? 0, + endOffset: row.endOffset ?? 0, + order: row.order, + startInBlock: row.startInBlock ?? 0, + endInBlock: row.endInBlock ?? 0 + }) + } + + for (const [chunkId, entry] of grouped) { + result.set( + chunkId, + projectChunkProvenance(chunkId, entry.documentId, entry.pageStart, entry.pageEnd, entry.rows) + ) + } + + return result +} + +/** + * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 + * 一次 join chunks → chunk_blocks → document_blocks。chunk 不存在时返回 undefined。 + */ +export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance | undefined { + return resolveChunksProvenance(db, [chunkId]).get(chunkId) } diff --git a/src/main/services/retrieval/DenseRetriever.ts b/src/main/services/retrieval/DenseRetriever.ts new file mode 100644 index 0000000..bc94d15 --- /dev/null +++ b/src/main/services/retrieval/DenseRetriever.ts @@ -0,0 +1,35 @@ +/** + * DenseRetriever + * 当前的默认检索策略：查询向量 → 向量库 KNN → 批量补齐证据。 + * + * 只负责"怎么检索"。embedding space 校验留在 `KnowledgeService`（那是前置条件， + * 不是策略），因此这个类可以单独被 eval harness 与后续的 BM25 / hybrid 策略替换。 + */ + +import { getDatabase } from '../../db' +import { vectorStoreManager } from '../../vectorstore' +import type { EmbeddingService } from '../EmbeddingService' +import { hydrateEvidence } from './evidence' +import type { RetrieveOptions, RetrievedEvidence, Retriever } from './types' + +export class DenseRetriever implements Retriever { + constructor(private readonly embeddingService: EmbeddingService) {} + + async search( + notebookId: string, + query: string, + options: RetrieveOptions = {} + ): Promise { + const { topK = 5, threshold = 0.5 } = options + + // E5 要求 query 前缀，与索引时的 document 前缀区分 + await this.embeddingService.ensureReady() + const queryEmbedding = await this.embeddingService.embed(query, 'query') + + const vectorStore = await vectorStoreManager.getStore(notebookId) + const hits = await vectorStore.query(queryEmbedding.embedding, { topK, threshold }) + if (hits.length === 0) return [] + + return hydrateEvidence(getDatabase(), hits) + } +} diff --git a/src/main/services/retrieval/evidence.ts b/src/main/services/retrieval/evidence.ts new file mode 100644 index 0000000..38243d7 --- /dev/null +++ b/src/main/services/retrieval/evidence.ts @@ -0,0 +1,106 @@ +/** + * evidence + * 把一次检索的命中（chunkId + score）补齐成 `RetrievedEvidence[]`。 + * + * 上层拿到的不再是裸 chunk：来源标题/类型、规范化页码区间和块区间都在这里一次补 + * 齐。补齐是批量的 —— 无论 topK 多大，SQL 查询次数恒定，不会形成 N+1。 + */ + +import { inArray } from 'drizzle-orm' +import { chunks, documents } from '../../db/schema' +import type { getDatabase } from '../../db' +import { resolveChunksProvenance, type ChunkProvenance } from '../chunkProvenance' +import type { RetrievedEvidence } from './types' + +type Db = ReturnType + +/** 一次向量检索的命中。 */ +export interface EvidenceHit { + chunkId: string + score: number +} + +/** 组装证据所需的最小 chunk / document 字段。 */ +export interface EvidenceChunk { + documentId: string + content: string + chunkIndex: number + metadata?: Record | null +} + +export interface EvidenceDocument { + title: string + type: string +} + +/** + * 纯映射：命中 + chunk + 来源 + 来源结构 → 证据。便于单测，不碰数据库。 + */ +export function assembleEvidence( + hit: EvidenceHit, + chunk: EvidenceChunk, + document: EvidenceDocument, + provenance: ChunkProvenance | undefined +): RetrievedEvidence { + return { + chunkId: hit.chunkId, + documentId: chunk.documentId, + content: chunk.content, + score: hit.score, + chunkIndex: chunk.chunkIndex, + source: { title: document.title, type: document.type }, + locator: { + pageStart: provenance?.pageStart ?? null, + pageEnd: provenance?.pageEnd ?? null, + blocks: provenance?.blocks ?? [] + }, + metadata: chunk.metadata ?? undefined + } +} + +/** + * 批量补齐证据。保持 `hits` 的顺序（即向量库的相关性排序）；找不到 chunk 的命中被 + * 丢弃，而不是用占位符污染结果。 + */ +export function hydrateEvidence(db: Db, hits: EvidenceHit[]): RetrievedEvidence[] { + if (hits.length === 0) return [] + + const chunkIds = hits.map((hit) => hit.chunkId) + + const chunkRows = db + .select({ + id: chunks.id, + documentId: chunks.documentId, + content: chunks.content, + chunkIndex: chunks.chunkIndex, + metadata: chunks.metadata + }) + .from(chunks) + .where(inArray(chunks.id, chunkIds)) + .all() + + const documentIds = [...new Set(chunkRows.map((row) => row.documentId))] + const documentRows = + documentIds.length > 0 + ? db + .select({ id: documents.id, title: documents.title, type: documents.type }) + .from(documents) + .where(inArray(documents.id, documentIds)) + .all() + : [] + + const chunkMap = new Map(chunkRows.map((row) => [row.id, row])) + const documentMap = new Map(documentRows.map((row) => [row.id, row])) + const provenance = resolveChunksProvenance(db, chunkIds) + + const evidence: RetrievedEvidence[] = [] + for (const hit of hits) { + const chunk = chunkMap.get(hit.chunkId) + if (!chunk) continue + + const document = documentMap.get(chunk.documentId) ?? { title: 'Unknown', type: 'unknown' } + evidence.push(assembleEvidence(hit, chunk, document, provenance.get(hit.chunkId))) + } + + return evidence +} diff --git a/src/main/services/retrieval/index.ts b/src/main/services/retrieval/index.ts new file mode 100644 index 0000000..5f950d2 --- /dev/null +++ b/src/main/services/retrieval/index.ts @@ -0,0 +1,7 @@ +/** + * Retrieval 模块入口 + */ + +export * from './types' +export { DenseRetriever } from './DenseRetriever' +export { hydrateEvidence, assembleEvidence, type EvidenceHit } from './evidence' diff --git a/src/main/services/retrieval/types.ts b/src/main/services/retrieval/types.ts new file mode 100644 index 0000000..7367056 --- /dev/null +++ b/src/main/services/retrieval/types.ts @@ -0,0 +1,55 @@ +/** + * Retrieval contract + * + * 检索是一等能力，不是 Chat 的附属逻辑。Chat、MCP、Search 面板都通过同一个 + * `Retriever` 拿到同一份 `RetrievedEvidence`，因此不必各自重写一遍 RAG，也让 + * #75 的 eval harness 有一个稳定的入口。 + * + * `RetrievedEvidence` 与 `SearchResult` 的关键区别是 provenance 是一等字段：引用 + * 要落到"哪个来源、哪一页、哪一段"，检索结果在交付给上层时就必须带着它。 + */ + +import type { ChunkProvenanceBlock } from '../chunkProvenance' + +/** 检索结果覆盖的单个块（`chunk_blocks` 映射 + `document_blocks`）。 */ +export type EvidenceBlock = ChunkProvenanceBlock + +/** 证据在来源里的位置。`blocks` 按文档顺序排列。 */ +export interface EvidenceLocator { + pageStart: number | null + pageEnd: number | null + blocks: EvidenceBlock[] +} + +/** 证据来自哪份来源。 */ +export interface EvidenceSource { + title: string + type: string +} + +/** 一条可供引用/展示的检索证据。 */ +export interface RetrievedEvidence { + chunkId: string + documentId: string + content: string + score: number + chunkIndex: number + source: EvidenceSource + locator: EvidenceLocator + metadata?: Record +} + +/** 检索选项。 */ +export interface RetrieveOptions { + topK?: number + threshold?: number + includeContent?: boolean +} + +/** + * 检索策略的稳定契约。当前只有 `DenseRetriever`；#77 的 BM25 / hybrid / reranker + * 只要实现这个接口就能被 eval harness 直接度量与替换。 + */ +export interface Retriever { + search(notebookId: string, query: string, options?: RetrieveOptions): Promise +} diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index ec2cef6..2e0678e 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -47,6 +47,7 @@ import { assignBlockIds, buildDocumentBlocks } from './services/blocks/documentB import { ChunkingService } from './services/ChunkingService' import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProvenance' import { KnowledgeService } from './services/KnowledgeService' +import { DenseRetriever } from './services/retrieval' import type { EmbeddingService } from './services/EmbeddingService' export const SMOKE_TEST_FLAG = '--smoke-test' @@ -115,6 +116,7 @@ function fakeEmbeddingService(dimensions = 16): EmbeddingService { // compiler on purpose, so a new call site here fails loudly at runtime. return { ensureReady: async () => undefined, + embed: async () => result(), embedBatch: async (texts: string[]) => texts.map(() => result()), getSpace: async () => ({ id: 'smoke-embed-space', @@ -502,6 +504,26 @@ async function runChecks(): Promise { 're-index left the document without chunks' ) + // Retrieval delivers provenance, not a bare chunk (#74). The re-index above + // just rebuilt chunks, embeddings, vectors and the chunk↔block mapping, so + // this drives the real DenseRetriever against them. + const evidence = await new DenseRetriever(fakeEmbeddingService()).search( + reindexNotebook, + 'sample query' + ) + assert(evidence.length > 0, 'DenseRetriever returned no evidence for an indexed notebook') + const first = evidence[0] + assert(first.documentId === reindexDocId, `evidence came from ${first.documentId}`) + assert(first.source.title.length > 0, 'evidence lost its source title') + assert(first.content.length > 0, 'evidence lost its content') + assert(first.locator.pageStart === 1, `evidence page was ${first.locator.pageStart}`) + assert(first.locator.blocks.length > 0, 'evidence carried no block locator') + assert( + first.locator.blocks.every((block) => block.page === 1), + 'an evidence block lost its page' + ) + pass('DenseRetriever returns retrieved evidence with a page/block locator') + await knowledge.deleteDocument(reindexDocId) assert( knowledge.getDocument(reindexDocId) === undefined, diff --git a/test/retrievalEvidence.test.ts b/test/retrievalEvidence.test.ts new file mode 100644 index 0000000..25c7128 --- /dev/null +++ b/test/retrievalEvidence.test.ts @@ -0,0 +1,92 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { + assembleEvidence, + type EvidenceChunk, + type EvidenceDocument +} from '../src/main/services/retrieval/evidence.ts' +import type { ChunkProvenance } from '../src/main/services/chunkProvenance.ts' + +/** + * `RetrievedEvidence` is the seam between retrieval and citations (#74): the + * retriever has to hand over provenance, not just the chunk text, or #69 cannot + * build a citation without querying the database again. + * + * `assembleEvidence()` is the pure step that attaches it. `hydrateEvidence()` + * only adds the batch queries around it; the shape asserted here is what every + * retriever strategy must produce. + */ + +const hit = { chunkId: 'chunk_1', score: 0.87 } +const chunk: EvidenceChunk = { + documentId: 'doc_1', + content: 'the extracted passage', + chunkIndex: 4 +} +const document: EvidenceDocument = { title: 'Attention Is All You Need', type: 'file' } + +const provenance: ChunkProvenance = { + chunkId: 'chunk_1', + documentId: 'doc_1', + pageStart: 12, + pageEnd: 12, + blocks: [ + { + blockId: 'blk_doc_1_7', + kind: 'paragraph', + level: null, + page: 12, + text: 'the extracted passage', + startOffset: 100, + endOffset: 121, + startInBlock: 0, + endInBlock: 21 + } + ] +} + +test('evidence carries score, source and locator together', () => { + const evidence = assembleEvidence(hit, chunk, document, provenance) + + assert.deepEqual(evidence, { + chunkId: 'chunk_1', + documentId: 'doc_1', + content: 'the extracted passage', + score: 0.87, + chunkIndex: 4, + source: { title: 'Attention Is All You Need', type: 'file' }, + locator: { + pageStart: 12, + pageEnd: 12, + blocks: provenance.blocks + }, + metadata: undefined + }) +}) + +test('a chunk without a provenance row still produces a valid locator', () => { + const evidence = assembleEvidence(hit, chunk, document, undefined) + + assert.deepEqual(evidence.locator, { pageStart: null, pageEnd: null, blocks: [] }) +}) + +test('chunk metadata survives the mapping', () => { + const evidence = assembleEvidence( + hit, + { ...chunk, metadata: { source: 'legacy' } }, + document, + provenance + ) + assert.deepEqual(evidence.metadata, { source: 'legacy' }) +}) + +test('the locator keeps block spans in the order provenance resolved them', () => { + const evidence = assembleEvidence(hit, chunk, document, { + ...provenance, + blocks: [provenance.blocks[0], { ...provenance.blocks[0], blockId: 'b2' }] + }) + assert.deepEqual( + evidence.locator.blocks.map((block) => block.blockId), + ['blk_doc_1_7', 'b2'] + ) +}) From 85f148e43f99d4102caf995df08ee2535c4026f9 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 15:26:56 +0800 Subject: [PATCH 4/5] fix(provenance): carry block bbox through retrieved evidence PdfLoader produces per-paragraph bboxes and document_blocks stores them, but ChunkProvenanceBlock never read document_blocks.bbox, so the value was dropped before RetrievedEvidence. #69 could build a citation on the current shape and #72 would then only know the page and blockId, not where on the page the paragraph sits. Select document_blocks.bbox in the provenance join, project it into ChunkProvenanceBlock and assert it survives into RetrievedEvidence.locator.blocks in the unit test, the evidence test and the packaged smoke check. Also drop includeContent from RetrieveOptions: which fields the legacy SearchResult exposes is a mapping concern, not a retrieval strategy one, and DenseRetriever never read it. --- docs/architecture.md | 6 +----- src/main/services/KnowledgeService.ts | 5 ++++- src/main/services/blocks/documentBlocks.ts | 4 ++-- src/main/services/chunkProvenance.ts | 6 ++++++ src/main/services/retrieval/types.ts | 6 ++++-- src/main/smokeTest.ts | 4 ++++ test/chunkBlocks.test.ts | 1 + test/retrievalEvidence.test.ts | 9 +++++++++ 8 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7e3ec21..47b1f9f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,11 +39,7 @@ RAG path. ```ts interface Retriever { - search( - notebookId: string, - query: string, - options?: RetrieveOptions - ): Promise + search(notebookId: string, query: string, options?: RetrieveOptions): Promise } ``` diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 1f9c3fe..547ad66 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -768,7 +768,10 @@ export class KnowledgeService { } // 1. 检索（向量 → 批量补齐来源/定位信息） - const evidence = await this.retriever.search(notebookId, query, options) + const evidence = await this.retriever.search(notebookId, query, { + topK: options.topK, + threshold: options.threshold + }) // 2. 映射回兼容的 SearchResult 形状 return evidence.map((item) => { diff --git a/src/main/services/blocks/documentBlocks.ts b/src/main/services/blocks/documentBlocks.ts index ea0f934..483dfd6 100644 --- a/src/main/services/blocks/documentBlocks.ts +++ b/src/main/services/blocks/documentBlocks.ts @@ -36,8 +36,8 @@ export interface IdentifiedBlockDraft extends DocumentBlockDraft { } /** - * 给块分配 id。id 由 documentId 与顺序确定，天然唯一且可读；重新索引会换新 - * documentId，不会与旧块冲突。 + * 给块分配 id。id 由 documentId 与顺序确定，天然唯一且可读。重新索引复用同一个 + * documentId，但会先删除该文档的旧块，所以顺序 id 不会与旧块冲突。 */ export function assignBlockIds( documentId: string, diff --git a/src/main/services/chunkProvenance.ts b/src/main/services/chunkProvenance.ts index bcdba90..3b7d8af 100644 --- a/src/main/services/chunkProvenance.ts +++ b/src/main/services/chunkProvenance.ts @@ -11,6 +11,7 @@ import { chunks, documentBlocks, chunkBlocks } from '../db/schema' import type { getDatabase } from '../db' import type { ChunkBlockSpan } from './ChunkingService' import type { BlockKind } from './blocks/documentBlocks' +import type { NormalizedBox } from './loaders/types' /** drizzle 实例类型（与 `getDatabase()` 的返回一致）。 */ type Db = ReturnType @@ -21,6 +22,8 @@ export interface ChunkProvenanceBlock { kind: BlockKind level: number | null page: number | null + /** 归一化页面坐标（仅分页格式）；引用要高亮到段落就需要它。 */ + bbox: NormalizedBox | null text: string startOffset: number endOffset: number @@ -60,6 +63,7 @@ export function projectChunkProvenance( kind: row.kind, level: row.level, page: row.page, + bbox: row.bbox, text: row.text, startOffset: row.startOffset, endOffset: row.endOffset, @@ -102,6 +106,7 @@ export function resolveChunksProvenance(db: Db, chunkIds: string[]): Map } -/** 检索选项。 */ +/** + * 检索选项。`includeContent` 不在这里：截断内容由 legacy `SearchResult` 映射决定， + * 不是检索策略的事（策略应当总是返回真实内容）。 + */ export interface RetrieveOptions { topK?: number threshold?: number - includeContent?: boolean } /** diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 2e0678e..0cb2727 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -522,6 +522,10 @@ async function runChecks(): Promise { first.locator.blocks.every((block) => block.page === 1), 'an evidence block lost its page' ) + assert( + first.locator.blocks.some((block) => block.bbox && block.bbox.w > 0 && block.bbox.h > 0), + 'evidence lost the paragraph bbox needed to highlight it (#72)' + ) pass('DenseRetriever returns retrieved evidence with a page/block locator') await knowledge.deleteDocument(reindexDocId) diff --git a/test/chunkBlocks.test.ts b/test/chunkBlocks.test.ts index 4818750..34edf0e 100644 --- a/test/chunkBlocks.test.ts +++ b/test/chunkBlocks.test.ts @@ -271,6 +271,7 @@ test('a chunk resolves to its ordered block spans', () => { kind: block.kind, level: block.level, page: block.page, + bbox: null, text: block.text, startOffset: block.startOffset, endOffset: block.endOffset, diff --git a/test/retrievalEvidence.test.ts b/test/retrievalEvidence.test.ts index 25c7128..40d1653 100644 --- a/test/retrievalEvidence.test.ts +++ b/test/retrievalEvidence.test.ts @@ -36,6 +36,7 @@ const provenance: ChunkProvenance = { kind: 'paragraph', level: null, page: 12, + bbox: { x: 0.5, y: 0.25, w: 0.4, h: 0.05 }, text: 'the extracted passage', startOffset: 100, endOffset: 121, @@ -70,6 +71,14 @@ test('a chunk without a provenance row still produces a valid locator', () => { assert.deepEqual(evidence.locator, { pageStart: null, pageEnd: null, blocks: [] }) }) +test('block geometry survives so a citation can highlight the paragraph (#72)', () => { + const evidence = assembleEvidence(hit, chunk, document, provenance) + const [block] = evidence.locator.blocks + + assert.equal(block.page, 12) + assert.deepEqual(block.bbox, { x: 0.5, y: 0.25, w: 0.4, h: 0.05 }) +}) + test('chunk metadata survives the mapping', () => { const evidence = assembleEvidence( hit, From fcabe6d4e49cbbc2b2cf06cf80a165f53c205500 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 15:28:36 +0800 Subject: [PATCH 5/5] fix(indexing): lazily recover structure when reindexing legacy documents The migration only adds documents.structure; documents imported before it have NULL there. The new reindexDocument() clears the derived index first, so re-indexing a pre-upgrade PDF would rebuild flat paragraphs and silently drop the page/bbox provenance it used to have. Recover the structure from localFilePath before clearing anything: re-parse the local copy, keep the result only when the content is byte-identical to the canonical documents.content, and persist it. If recovery fails the old index is still cleared, but we never destroy existing provenance to attempt recovery. Verified in the packaged smoke test by nulling structure on an imported PDF, re-indexing, and asserting the structure is backfilled and the blocks keep their page number. --- src/main/services/KnowledgeService.ts | 39 ++++++++++++++++++++++++++- src/main/smokeTest.ts | 24 +++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 547ad66..f6a60bf 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -852,6 +852,9 @@ export class KnowledgeService { * * 只重建派生索引：documentId、localFilePath、sourceUri、metadata 与解析结构都 * 保持不变。重新索引不会让历史 citation 指向另一个来源。 + * + * 升级前导入的文档 `structure` 为 NULL（migration 只加列，不回填）。这时先从本地 + * 副本恢复结构再删除旧索引：顺序很重要，否则恢复失败会把已有的 page/bbox 也删掉。 */ async reindexDocument(documentId: string, onProgress?: IndexProgressCallback): Promise { const db = getDatabase() @@ -861,8 +864,42 @@ export class KnowledgeService { throw new Error(`Document ${documentId} not found or has no content`) } + let structure = doc.structure ?? undefined + if (!structure) { + structure = await this.recoverStructure(doc) + if (structure) { + db.update(documents).set({ structure }).where(eq(documents.id, documentId)).run() + Logger.info('KnowledgeService', `Recovered structure for legacy document ${documentId}`) + } + } + await this.clearDerivedIndex(documentId) - await this.indexDocument(documentId, doc.content, doc.structure ?? undefined, {}, onProgress) + await this.indexDocument(documentId, doc.content, structure, {}, onProgress) + } + + /** + * 从本地副本重新解析出结构，用于升级前没有持久化 `structure` 的文档。 + * + * 只在 `localFilePath` 存在且重新解析出的内容与规范内容完全一致时采用 —— 文件被改 + * 过的话 offsets 就指不到同一份文本，宁可退化成平铺块也不能错误对齐。 + */ + private async recoverStructure(doc: Document): Promise { + if (!doc.localFilePath || !doc.content) return undefined + + try { + const parsed = await this.fileParserService.parseFile(doc.localFilePath) + if (parsed.content !== doc.content) { + Logger.warn( + 'KnowledgeService', + `Cannot recover structure for ${doc.id}: content changed since import` + ) + return undefined + } + return parsed.structure ?? undefined + } catch (error) { + Logger.warn('KnowledgeService', `Cannot recover structure for ${doc.id}:`, error) + return undefined + } } /** diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 0cb2727..8434f40 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -528,6 +528,30 @@ async function runChecks(): Promise { ) pass('DenseRetriever returns retrieved evidence with a page/block locator') + // Documents imported before `documents.structure` existed have NULL there. + // Re-indexing them must recover the structure from the local copy *before* + // clearing the old index, or one re-index would downgrade a page/paragraph + // citation to a flat block. + database.prepare('UPDATE documents SET structure = NULL WHERE id = ?').run(reindexDocId) + assert( + knowledge.getDocument(reindexDocId)?.structure == null, + 'could not simulate a legacy document without structure' + ) + + await knowledge.reindexDocument(reindexDocId) + + assert( + Boolean(knowledge.getDocument(reindexDocId)?.structure), + 'legacy re-index did not backfill the structure' + ) + const legacyBlocks = knowledge.getDocumentBlocks(reindexDocId) + assert(legacyBlocks.length > 0, 'legacy re-index produced no blocks') + assert( + legacyBlocks.every((block) => block.page === 1), + 'legacy re-index dropped page provenance instead of recovering it' + ) + pass('re-index recovers structure for documents imported before the column existed') + await knowledge.deleteDocument(reindexDocId) assert( knowledge.getDocument(reindexDocId) === undefined,