From 1e096ac8f05cb6a02b8672949c2ff951280495c3 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Fri, 25 Sep 2026 14:03:53 +0800 Subject: [PATCH] =?UTF-8?q?feat(provenance):=20persist=20chunk=E2=86=94blo?= =?UTF-8?q?ck=20mapping=20and=20page=20range?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A retrieval result only carries a chunk id. Without a persisted mapping it cannot be resolved back to a page, a heading or a character range without re-parsing the document — which is what #68 (and the second half of #67) requires. - `chunk_blocks` (chunk_id, block_id, start_in_block, end_in_block, PK on chunk+block) plus denormalized `chunks.page_start`/`page_end`. Additive migration 0015; existing documents keep working at document-level. - `chunkProvenance.ts`: `insertChunkBlocks()` writes the mapping, and `resolveChunkProvenance()` resolves a chunk id with a single join through `chunk_blocks` into `document_blocks`, returning the ordered block spans and the page range. `projectChunkProvenance()` (pure) does the ordering and projection. - `KnowledgeService.saveChunks()` writes each chunk and its mapping in one transaction, so a partially indexed document never has chunks without their mapping. `getChunkProvenance()` exposes the resolver. `deleteDocument` and `reindexDocument` delete the mapping explicitly rather than relying on a foreign-key cascade the connection does not enable. - The packaged smoke test now round-trips a fixture through the real database: document + blocks + chunk + mapping, then resolves the chunk and checks the block spans and page range. Verification: `test/chunkBlocks.test.ts` gains two tests — the ordered-span projection and the page-range invariant (page_start/page_end equal the min/max page of the mapped blocks) — 110 tests pass. `npm run typecheck`, `npm run build` and `npx eslint` are clean. The packaged smoke check runs in CI (no xvfb locally). Fixes #68 Refs #67 --- src/main/db/migrations/0015_colossal_solo.sql | 13 + .../db/migrations/meta/0015_snapshot.json | 1598 +++++++++++++++++ src/main/db/migrations/meta/_journal.json | 7 + src/main/db/schema.ts | 36 +- src/main/services/KnowledgeService.ts | 135 +- src/main/services/chunkProvenance.ts | 113 ++ src/main/smokeTest.ts | 93 + test/chunkBlocks.test.ts | 67 + 8 files changed, 2010 insertions(+), 52 deletions(-) create mode 100644 src/main/db/migrations/0015_colossal_solo.sql create mode 100644 src/main/db/migrations/meta/0015_snapshot.json create mode 100644 src/main/services/chunkProvenance.ts diff --git a/src/main/db/migrations/0015_colossal_solo.sql b/src/main/db/migrations/0015_colossal_solo.sql new file mode 100644 index 0000000..acc17a7 --- /dev/null +++ b/src/main/db/migrations/0015_colossal_solo.sql @@ -0,0 +1,13 @@ +CREATE TABLE `chunk_blocks` ( + `chunk_id` text NOT NULL, + `block_id` text NOT NULL, + `start_in_block` integer NOT NULL, + `end_in_block` integer NOT NULL, + PRIMARY KEY(`chunk_id`, `block_id`), + FOREIGN KEY (`chunk_id`) REFERENCES `chunks`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`block_id`) REFERENCES `document_blocks`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `idx_chunk_blocks_block` ON `chunk_blocks` (`block_id`);--> statement-breakpoint +ALTER TABLE `chunks` ADD `page_start` integer;--> statement-breakpoint +ALTER TABLE `chunks` ADD `page_end` integer; \ No newline at end of file diff --git a/src/main/db/migrations/meta/0015_snapshot.json b/src/main/db/migrations/meta/0015_snapshot.json new file mode 100644 index 0000000..5771e1a --- /dev/null +++ b/src/main/db/migrations/meta/0015_snapshot.json @@ -0,0 +1,1598 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "03fce19e-c08b-4bf0-9635-f54f7b00a883", + "prevId": "750fe638-c615-4d04-8556-79ecc7b2c87b", + "tables": { + "anki_cards": { + "name": "anki_cards", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "cards_data": { + "name": "cards_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_ankicards_notebook": { + "name": "idx_ankicards_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_ankicards_version": { + "name": "idx_ankicards_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "anki_cards_notebook_id_notebooks_id_fk": { + "name": "anki_cards_notebook_id_notebooks_id_fk", + "tableFrom": "anki_cards", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_content": { + "name": "reasoning_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_session": { + "name": "idx_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_messages_session_id_chat_sessions_id_fk": { + "name": "chat_messages_session_id_chat_sessions_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_sessions": { + "name": "chat_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "parent_session_id": { + "name": "parent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_sessions_notebook": { + "name": "idx_sessions_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_sessions_notebook_id_notebooks_id_fk": { + "name": "chat_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_sessions_parent_session_id_chat_sessions_id_fk": { + "name": "chat_sessions_parent_session_id_chat_sessions_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "chat_sessions", + "columnsFrom": [ + "parent_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunk_blocks": { + "name": "chunk_blocks", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_in_block": { + "name": "start_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_in_block": { + "name": "end_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunk_blocks_block": { + "name": "idx_chunk_blocks_block", + "columns": [ + "block_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunk_blocks_chunk_id_chunks_id_fk": { + "name": "chunk_blocks_chunk_id_chunks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunk_blocks_block_id_document_blocks_id_fk": { + "name": "chunk_blocks_block_id_document_blocks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "document_blocks", + "columnsFrom": [ + "block_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "chunk_blocks_chunk_id_block_id_pk": { + "columns": [ + "chunk_id", + "block_id" + ], + "name": "chunk_blocks_chunk_id_block_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunks": { + "name": "chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_start": { + "name": "page_start", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_end": { + "name": "page_end", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunks_document": { + "name": "idx_chunks_document", + "columns": [ + "document_id" + ], + "isUnique": false + }, + "idx_chunks_notebook": { + "name": "idx_chunks_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunks_document_id_documents_id_fk": { + "name": "chunks_document_id_documents_id_fk", + "tableFrom": "chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunks_notebook_id_notebooks_id_fk": { + "name": "chunks_notebook_id_notebooks_id_fk", + "tableFrom": "chunks", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_blocks": { + "name": "document_blocks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bbox": { + "name": "bbox", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_blocks_document_order": { + "name": "idx_blocks_document_order", + "columns": [ + "document_id", + "order" + ], + "isUnique": false + }, + "idx_blocks_document_page": { + "name": "idx_blocks_document_page", + "columns": [ + "document_id", + "page" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_blocks_document_id_documents_id_fk": { + "name": "document_blocks_document_id_documents_id_fk", + "tableFrom": "document_blocks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "local_file_path": { + "name": "local_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_note_id": { + "name": "source_note_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_documents_notebook": { + "name": "idx_documents_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_documents_status": { + "name": "idx_documents_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_notebook_id_notebooks_id_fk": { + "name": "documents_notebook_id_notebooks_id_fk", + "tableFrom": "documents", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_source_note_id_notes_id_fk": { + "name": "documents_source_note_id_notes_id_fk", + "tableFrom": "documents", + "tableTo": "notes", + "columnsFrom": [ + "source_note_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embeddings": { + "name": "embeddings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeddings_chunk": { + "name": "idx_embeddings_chunk", + "columns": [ + "chunk_id" + ], + "isUnique": false + }, + "idx_embeddings_notebook": { + "name": "idx_embeddings_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + }, + "idx_embeddings_model": { + "name": "idx_embeddings_model", + "columns": [ + "model" + ], + "isUnique": false + } + }, + "foreignKeys": { + "embeddings_chunk_id_chunks_id_fk": { + "name": "embeddings_chunk_id_chunks_id_fk", + "tableFrom": "embeddings", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embeddings_notebook_id_notebooks_id_fk": { + "name": "embeddings_notebook_id_notebooks_id_fk", + "tableFrom": "embeddings", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "items": { + "name": "items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_items_notebook_order": { + "name": "idx_items_notebook_order", + "columns": [ + "notebook_id", + "order" + ], + "isUnique": false + }, + "idx_items_type": { + "name": "idx_items_type", + "columns": [ + "type" + ], + "isUnique": false + }, + "idx_items_resource": { + "name": "idx_items_resource", + "columns": [ + "type", + "resource_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "items_notebook_id_notebooks_id_fk": { + "name": "items_notebook_id_notebooks_id_fk", + "tableFrom": "items", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mind_maps": { + "name": "mind_maps", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "tree_data": { + "name": "tree_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_mindmaps_notebook": { + "name": "idx_mindmaps_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_mindmaps_version": { + "name": "idx_mindmaps_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mind_maps_notebook_id_notebooks_id_fk": { + "name": "mind_maps_notebook_id_notebooks_id_fk", + "tableFrom": "mind_maps", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebook_embedding_spaces": { + "name": "notebook_embedding_spaces", + "columns": { + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "backend": { + "name": "backend", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "notebook_embedding_spaces_notebook_id_notebooks_id_fk": { + "name": "notebook_embedding_spaces_notebook_id_notebooks_id_fk", + "tableFrom": "notebook_embedding_spaces", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebooks": { + "name": "notebooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notebooks_updated": { + "name": "idx_notebooks_updated", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notes": { + "name": "notes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notes_notebook": { + "name": "idx_notes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "notes_notebook_id_notebooks_id_fk": { + "name": "notes_notebook_id_notebooks_id_fk", + "tableFrom": "notes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quiz_sessions": { + "name": "quiz_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answers": { + "name": "answers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_questions": { + "name": "total_questions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "correct_count": { + "name": "correct_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quiz_sessions_quiz": { + "name": "idx_quiz_sessions_quiz", + "columns": [ + "quiz_id", + "created_at" + ], + "isUnique": false + }, + "idx_quiz_sessions_notebook": { + "name": "idx_quiz_sessions_notebook", + "columns": [ + "notebook_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quiz_sessions_quiz_id_quizzes_id_fk": { + "name": "quiz_sessions_quiz_id_quizzes_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "quizzes", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_sessions_notebook_id_notebooks_id_fk": { + "name": "quiz_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quizzes": { + "name": "quizzes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "questions_data": { + "name": "questions_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quizzes_notebook": { + "name": "idx_quizzes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_quizzes_version": { + "name": "idx_quizzes_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quizzes_notebook_id_notebooks_id_fk": { + "name": "quizzes_notebook_id_notebooks_id_fk", + "tableFrom": "quizzes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/main/db/migrations/meta/_journal.json b/src/main/db/migrations/meta/_journal.json index 82a877f..c83a95e 100644 --- a/src/main/db/migrations/meta/_journal.json +++ b/src/main/db/migrations/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1790315288657, "tag": "0014_naive_lifeguard", "breakpoints": true + }, + { + "idx": 15, + "version": "6", + "when": 1790316032660, + "tag": "0015_colossal_solo", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/main/db/schema.ts b/src/main/db/schema.ts index 10685f8..3fb4f58 100644 --- a/src/main/db/schema.ts +++ b/src/main/db/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, index } from 'drizzle-orm/sqlite-core' +import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlite-core' import type { QuizQuestion } from '../../shared/types/quiz' /** @@ -166,8 +166,10 @@ export const chunks = sqliteTable( .references(() => notebooks.id, { onDelete: 'cascade' }), content: text('content').notNull(), chunkIndex: integer('chunk_index').notNull(), // 在文档中的顺序 - startOffset: integer('start_offset'), // 原文起始位置 - endOffset: integer('end_offset'), // 原文结束位置 + startOffset: integer('start_offset'), // 原文起始位置（相对 documents.content） + endOffset: integer('end_offset'), // 原文结束位置（相对 documents.content） + pageStart: integer('page_start'), // 覆盖块的最小页码；非分页文档为 NULL + pageEnd: integer('page_end'), // 覆盖块的最大页码；非分页文档为 NULL metadata: text('metadata', { mode: 'json' }).$type>(), tokenCount: integer('token_count'), // token 估算值 createdAt: integer('created_at', { mode: 'timestamp' }).notNull() @@ -224,6 +226,34 @@ export const documentBlocks = sqliteTable( export type DocumentBlock = typeof documentBlocks.$inferSelect export type NewDocumentBlock = typeof documentBlocks.$inferInsert +/** + * chunk ↔ block 映射表 + * + * 检索结果只有 chunk id，必须能一路回到块、页码和字符区间。`start_in_block` / + * `end_in_block` 是块内的字符区间（相对 `document_blocks.text`），所以一个 chunk + * 可以只覆盖某块的一部分。 + */ +export const chunkBlocks = sqliteTable( + 'chunk_blocks', + { + chunkId: text('chunk_id') + .notNull() + .references(() => chunks.id, { onDelete: 'cascade' }), + blockId: text('block_id') + .notNull() + .references(() => documentBlocks.id, { onDelete: 'cascade' }), + startInBlock: integer('start_in_block').notNull(), + endInBlock: integer('end_in_block').notNull() + }, + (table) => ({ + pk: primaryKey({ columns: [table.chunkId, table.blockId] }), + blockIdx: index('idx_chunk_blocks_block').on(table.blockId) + }) +) + +export type ChunkBlock = typeof chunkBlocks.$inferSelect +export type NewChunkBlock = typeof chunkBlocks.$inferInsert + /** * 向量嵌入表（元数据） * 存储 chunk 的向量元信息，实际向量存储在 vec0 虚拟表中 diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 89d90d6..0e65803 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -19,7 +19,8 @@ import { embeddings, notes, notebookEmbeddingSpaces, - documentBlocks + documentBlocks, + chunkBlocks } from '../db/schema' import type { Document, @@ -33,13 +34,14 @@ import type { import { eq, desc, inArray, sql } from 'drizzle-orm' import { EmbeddingService } from './EmbeddingService' import type { EmbeddingSpace } from '../../shared/types' -import { ChunkingService, type ChunkOptions } from './ChunkingService' +import { ChunkingService, type ChunkOptions, type ChunkResult } from './ChunkingService' import { FileParserService } from './FileParserService' import { buildDocumentBlocks, assignBlockIds, type IdentifiedBlockDraft } from './blocks/documentBlocks' +import { resolveChunkProvenance, type ChunkProvenance } from './chunkProvenance' import { WebFetchService } from './WebFetchService' import { vectorStoreManager } from '../vectorstore' import Logger from '../../shared/utils/logger' @@ -218,30 +220,9 @@ export class KnowledgeService { Logger.info('KnowledgeService', `Document ${documentId}: ${chunkResults.length} chunks`) - // 3. 保存分块 + // 3. 保存分块与 chunk↔block 映射（同一事务，不会出现没有映射的 chunk） onProgress?.('saving_chunks', 20) - const chunkIds: string[] = [] - const chunkContents: string[] = [] - - for (const chunk of chunkResults) { - const chunkId = `chunk_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` - chunkIds.push(chunkId) - chunkContents.push(chunk.content) - - const newChunk: NewChunk = { - id: chunkId, - documentId, - notebookId, - content: chunk.content, - chunkIndex: chunk.index, - startOffset: chunk.startOffset, - endOffset: chunk.endOffset, - tokenCount: chunk.tokenCount, - createdAt: now - } - - db.insert(chunks).values(newChunk).run() - } + const { chunkIds, chunkContents } = this.saveChunks(documentId, notebookId, chunkResults, now) // 4. 生成嵌入向量 // 不指定维度:维度由 embedding 模型决定,写死维度会让用别的模型的笔记本直接 @@ -409,30 +390,9 @@ export class KnowledgeService { Logger.info('KnowledgeService', `Document ${documentId}: ${chunkResults.length} chunks`) - // 3. 保存分块 + // 3. 保存分块与 chunk↔block 映射（同一事务，不会出现没有映射的 chunk） onProgress?.('saving_chunks', 20) - const chunkIds: string[] = [] - const chunkContents: string[] = [] - - for (const chunk of chunkResults) { - const chunkId = `chunk_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` - chunkIds.push(chunkId) - chunkContents.push(chunk.content) - - const newChunk: NewChunk = { - id: chunkId, - documentId, - notebookId, - content: chunk.content, - chunkIndex: chunk.index, - startOffset: chunk.startOffset, - endOffset: chunk.endOffset, - tokenCount: chunk.tokenCount, - createdAt: now - } - - db.insert(chunks).values(newChunk).run() - } + const { chunkIds, chunkContents } = this.saveChunks(documentId, notebookId, chunkResults, now) // 4. 生成嵌入向量 // 不指定维度:维度由 embedding 模型决定,写死维度会让用别的模型的笔记本直接 @@ -558,6 +518,58 @@ export class KnowledgeService { Logger.debug('KnowledgeService', `Document ${blocks[0].documentId}: ${rows.length} blocks`) } + /** + * 写入分块与 chunk↔block 映射。两者放在同一个事务里，所以不会出现没有映射的 + * chunk；`page_start/page_end` 直接取分块器算好的覆盖块页码区间。 + */ + private saveChunks( + documentId: string, + notebookId: string, + chunkResults: ChunkResult[], + now: Date + ): { chunkIds: string[]; chunkContents: string[] } { + const db = getDatabase() + const chunkIds: string[] = [] + const chunkContents: string[] = [] + + db.transaction((tx) => { + for (const chunk of chunkResults) { + const chunkId = `chunk_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` + chunkIds.push(chunkId) + chunkContents.push(chunk.content) + + const newChunk: NewChunk = { + id: chunkId, + documentId, + notebookId, + content: chunk.content, + chunkIndex: chunk.index, + startOffset: chunk.startOffset, + endOffset: chunk.endOffset, + pageStart: chunk.pageStart, + pageEnd: chunk.pageEnd, + tokenCount: chunk.tokenCount, + createdAt: now + } + + tx.insert(chunks).values(newChunk).run() + + for (const span of chunk.blockSpans) { + tx.insert(chunkBlocks) + .values({ + chunkId, + blockId: span.blockId, + startInBlock: span.startInBlock, + endInBlock: span.endInBlock + }) + .run() + } + } + }) + + return { chunkIds, chunkContents } + } + /** * 获取文档的所有块，按阅读顺序返回。 */ @@ -570,6 +582,13 @@ export class KnowledgeService { .all() } + /** + * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 + */ + getChunkProvenance(chunkId: string): ChunkProvenance | undefined { + return resolveChunkProvenance(getDatabase(), chunkId) + } + /** * 从 URL 添加文档 */ @@ -872,7 +891,17 @@ export class KnowledgeService { await this.deleteLocalFile(doc.localFilePath) } - // 块没有依赖向量表，显式删除而不依赖外键级联（连接上 foreign_keys 默认是关的） + // 块与映射不依赖外键级联（连接上 foreign_keys 默认是关的），显式删除 + if (docChunks.length > 0) { + db.delete(chunkBlocks) + .where( + inArray( + chunkBlocks.chunkId, + docChunks.map((c) => c.id) + ) + ) + .run() + } db.delete(documentBlocks).where(eq(documentBlocks.documentId, documentId)).run() // 级联删除会自动清理 chunks 和 embeddings @@ -903,6 +932,14 @@ export class KnowledgeService { if (oldChunks.length > 0) { const vectorStore = await vectorStoreManager.getStore(doc.notebookId) await vectorStore.deleteByChunkIds(oldChunks.map((c) => c.id)) + db.delete(chunkBlocks) + .where( + inArray( + chunkBlocks.chunkId, + oldChunks.map((c) => c.id) + ) + ) + .run() } db.delete(chunks).where(eq(chunks.documentId, documentId)).run() diff --git a/src/main/services/chunkProvenance.ts b/src/main/services/chunkProvenance.ts new file mode 100644 index 0000000..3996735 --- /dev/null +++ b/src/main/services/chunkProvenance.ts @@ -0,0 +1,113 @@ +/** + * chunkProvenance + * chunk ↔ block 映射的写入与解析。 + * + * 检索结果只带 chunk id；引用要落到"哪一页、哪一段"，必须凭映射一路查回块。 + * `resolveChunkProvenance()` 一次 join 同时拿到有序块区间与反规范化的页码区间。 + */ + +import { eq } from 'drizzle-orm' +import { chunks, documentBlocks, chunkBlocks } from '../db/schema' +import type { getDatabase } from '../db' +import type { ChunkBlockSpan } from './ChunkingService' +import type { BlockKind } from './blocks/documentBlocks' + +/** drizzle 实例类型（与 `getDatabase()` 的返回一致）。 */ +type Db = ReturnType + +/** 解析结果里的单个块。 */ +export interface ChunkProvenanceBlock { + blockId: string + kind: BlockKind + level: number | null + page: number | null + text: string + startOffset: number + endOffset: number + startInBlock: number + endInBlock: number +} + +export interface ChunkProvenance { + chunkId: string + documentId: string + pageStart: number | null + pageEnd: number | null + blocks: ChunkProvenanceBlock[] +} + +/** join 出来的原始行，`order` 用于排序，不进入结果。 */ +export interface ChunkProvenanceRow extends ChunkProvenanceBlock { + order: number +} + +/** 把 join 行排序并投影成解析结果。纯函数，便于单测。 */ +export function projectChunkProvenance( + chunkId: string, + documentId: string, + pageStart: number | null, + pageEnd: number | null, + rows: ChunkProvenanceRow[] +): ChunkProvenance { + const ordered = [...rows].sort((a, b) => a.order - b.order) + return { + chunkId, + documentId, + pageStart, + pageEnd, + blocks: ordered.map((row) => ({ + blockId: row.blockId, + kind: row.kind, + level: row.level, + page: row.page, + text: row.text, + startOffset: row.startOffset, + endOffset: row.endOffset, + startInBlock: row.startInBlock, + endInBlock: row.endInBlock + })) + } +} + +/** 写入一个 chunk 覆盖的全部块区间。 */ +export function insertChunkBlocks(db: Db, chunkId: string, spans: ChunkBlockSpan[]): void { + for (const span of spans) { + db.insert(chunkBlocks) + .values({ + chunkId, + blockId: span.blockId, + startInBlock: span.startInBlock, + endInBlock: span.endInBlock + }) + .run() + } +} + +/** + * 解析一个 chunk 的来源：文档 id、页码区间，以及按文档顺序排列的块区间。 + * 一次 join chunks → chunk_blocks → document_blocks。chunk 不存在时返回 undefined。 + */ +export function resolveChunkProvenance(db: Db, chunkId: string): ChunkProvenance | undefined { + const chunk = db.select().from(chunks).where(eq(chunks.id, chunkId)).get() + if (!chunk) return undefined + + const rows = db + .select({ + blockId: documentBlocks.id, + kind: documentBlocks.kind, + level: documentBlocks.level, + page: documentBlocks.page, + text: documentBlocks.text, + startOffset: documentBlocks.startOffset, + endOffset: documentBlocks.endOffset, + order: documentBlocks.order, + startInBlock: chunkBlocks.startInBlock, + endInBlock: chunkBlocks.endInBlock + }) + .from(chunkBlocks) + .innerJoin(documentBlocks, eq(chunkBlocks.blockId, documentBlocks.id)) + .where(eq(chunkBlocks.chunkId, chunkId)) + .all() + + return projectChunkProvenance(chunkId, chunk.documentId, chunk.pageStart, chunk.pageEnd, rows) +} diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 38165ad..0884795 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -25,10 +25,12 @@ import Logger from '../shared/utils/logger' import { readFile } from 'fs/promises' import { join } from 'path' +import { eq } from 'drizzle-orm' import { closeDatabase, createNotebookVectorTable, dropNotebookVectorTable, + getDatabase, getNotebookVectorTable, getSqlite, initDatabase, @@ -36,10 +38,14 @@ import { rebuildNotebookVectorTable, runMigrations } from './db' +import { documents, documentBlocks, chunks, chunkBlocks } from './db/schema' import type Database from 'better-sqlite3' import { FileParserService } from './services/FileParserService' import { WebLoader } from './services/loaders/WebLoader' import { SQLiteVectorStore } from './vectorstore/SQLiteVectorStore' +import { assignBlockIds, buildDocumentBlocks } from './services/blocks/documentBlocks' +import { ChunkingService } from './services/ChunkingService' +import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProvenance' export const SMOKE_TEST_FLAG = '--smoke-test' @@ -331,6 +337,93 @@ async function runChecks(): Promise { ) pass('HTML import extracts body text (jsdom + Readability)') + // --- provenance: blocks -> chunk -> chunk_blocks -> resolved page range ---- + // test/blocks.test.ts and test/chunkBlocks.test.ts cover the pure builder and + // chunker, but only the packaged app has a working better-sqlite3, so the + // database round trip is asserted here: insert a document, its blocks, a chunk + // and the mapping, then resolve the chunk back through the real join. + const provenanceDocId = 'smoke-provenance-doc' + const provenanceChunkId = 'smoke-provenance-chunk' + const provenanceBlocks = assignBlockIds( + provenanceDocId, + buildDocumentBlocks({ content: pdf.content, structure: pdf.structure }) + ) + const provenanceChunks = new ChunkingService().chunkBlocks(pdf.content, provenanceBlocks) + assert(provenanceChunks.length > 0, 'chunking the PDF fixture produced no chunks') + + const db = getDatabase() + const provenanceNow = new Date() + db.insert(documents) + .values({ + id: provenanceDocId, + notebookId: LEGACY_NOTEBOOK, + title: 'Smoke provenance document', + type: 'file', + content: pdf.content, + status: 'indexed', + chunkCount: provenanceChunks.length, + createdAt: provenanceNow, + updatedAt: provenanceNow + }) + .run() + db.insert(documentBlocks) + .values( + provenanceBlocks.map((block) => ({ + id: block.id, + documentId: block.documentId, + kind: block.kind, + order: block.order, + page: block.page, + level: block.level, + text: block.text, + startOffset: block.startOffset, + endOffset: block.endOffset, + bbox: block.bbox, + metadata: block.metadata + })) + ) + .run() + + const provenanceChunk = provenanceChunks[0] + db.insert(chunks) + .values({ + id: provenanceChunkId, + documentId: provenanceDocId, + notebookId: LEGACY_NOTEBOOK, + content: provenanceChunk.content, + chunkIndex: provenanceChunk.index, + startOffset: provenanceChunk.startOffset, + endOffset: provenanceChunk.endOffset, + pageStart: provenanceChunk.pageStart, + pageEnd: provenanceChunk.pageEnd, + tokenCount: provenanceChunk.tokenCount, + createdAt: provenanceNow + }) + .run() + insertChunkBlocks(db, provenanceChunkId, provenanceChunk.blockSpans) + + const resolved = resolveChunkProvenance(db, provenanceChunkId) + assert(resolved, 'a chunk with a mapping did not resolve') + assert(resolved.documentId === provenanceDocId, `resolved document was ${resolved.documentId}`) + assert( + resolved.blocks.length === provenanceChunk.blockSpans.length, + `resolved ${resolved.blocks.length} blocks, expected ${provenanceChunk.blockSpans.length}` + ) + assert( + resolved.pageStart === 1 && resolved.pageEnd === 1, + `resolved page range was ${resolved.pageStart}-${resolved.pageEnd}` + ) + assert( + resolved.blocks.every((block) => block.page === 1), + 'a resolved block lost its page' + ) + pass('document blocks, chunk mapping and page range round trip through the database') + + db.delete(chunkBlocks).where(eq(chunkBlocks.chunkId, provenanceChunkId)).run() + db.delete(chunks).where(eq(chunks.documentId, provenanceDocId)).run() + db.delete(documentBlocks).where(eq(documentBlocks.documentId, provenanceDocId)).run() + db.delete(documents).where(eq(documents.id, provenanceDocId)).run() + const { ApkgExporter } = await import('./services/exporters/ApkgExporter') const { buffer, summary } = await new ApkgExporter().export( [{ id: 'smoke-card', type: 'basic', front: 'front', back: 'back', tags: ['smoke'] }], diff --git a/test/chunkBlocks.test.ts b/test/chunkBlocks.test.ts index b5b5cbb..4818750 100644 --- a/test/chunkBlocks.test.ts +++ b/test/chunkBlocks.test.ts @@ -13,6 +13,10 @@ import { type ChunkBlock, type ChunkResult } from '../src/main/services/ChunkingService.ts' +import { + projectChunkProvenance, + type ChunkProvenanceRow +} from '../src/main/services/chunkProvenance.ts' /** * The chunker is the safety net for the whole provenance epic: if @@ -251,6 +255,69 @@ test('the char-window fallback also produces canonical offsets and overlap', () } }) +test('a chunk resolves to its ordered block spans', () => { + const { content, blocks } = synthetic() + const chunks = new ChunkingService().chunkBlocks(content, blocks) + const chunk = chunks[0] + + assert.equal(chunk.blockSpans.length, blocks.length, 'the whole synthetic document is one chunk') + + // Build the rows in reverse to prove the projection sorts by document order. + const rows: ChunkProvenanceRow[] = [...chunk.blockSpans].reverse().map((span) => { + const order = blocks.findIndex((block) => block.id === span.blockId) + const block = blocks[order] + return { + blockId: block.id, + kind: block.kind, + level: block.level, + page: block.page, + text: block.text, + startOffset: block.startOffset, + endOffset: block.endOffset, + order, + startInBlock: span.startInBlock, + endInBlock: span.endInBlock + } + }) + + const resolved = projectChunkProvenance( + 'chunk_1', + 'doc_synthetic', + chunk.pageStart, + chunk.pageEnd, + rows + ) + + assert.equal(resolved.chunkId, 'chunk_1') + assert.equal(resolved.documentId, 'doc_synthetic') + assert.deepEqual( + resolved.blocks.map((block) => block.blockId), + blocks.map((block) => block.id) + ) + assert.equal(resolved.blocks[0].text, blocks[0].text) +}) + +test('page_start/page_end equal the min/max page of the mapped blocks', async () => { + const result = await new PdfLoader().loadFromPath(fixture('multipage.pdf')) + const blocks = assignBlockIds( + 'doc_pdf', + buildDocumentBlocks({ content: result.content, structure: result.structure }) + ) + const chunks = new ChunkingService().chunkBlocks(result.content, blocks) + + for (const chunk of chunks) { + const pages = chunk.blockSpans + .map((span) => blocks.find((block) => block.id === span.blockId)!.page) + .filter((page): page is number => page !== null) + + assert.deepEqual( + [chunk.pageStart, chunk.pageEnd], + [Math.min(...pages), Math.max(...pages)], + `chunk ${chunk.index} page range does not match its blocks` + ) + } +}) + test('no blocks and no text produces no chunks', () => { const service = new ChunkingService() assert.deepEqual(service.chunkBlocks('', []), [])