From 4e803381bc20e9d8350ec0ca334b9dfd8cb1fae7 Mon Sep 17 00:00:00 2001 From: mrsibe Date: Mon, 28 Sep 2026 19:24:27 +0800 Subject: [PATCH] feat(watch): a watched folder keeps itself current, and never deletes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #158. A folder import is a snapshot; this is what keeps it current. - documents.source_state + source_mtime_ms (migration 0021) separate the SOURCE file's state from the index state: a deleted file becomes `missing` while its index, notes, excerpts and citations all stay. - folder_watches persists the watch (one row per notebook + folder). - diffFolder() decides added / changed / restored / missing. It is pure, so the decision table is unit-tested without a filesystem or a database — and "changed" and "missing" are determined, not guessed. - FolderWatchService reconciles on launch (so a change made while the app was closed is picked up) and live via fs.watch with an 800ms debounce. New files go through the same batch import as #98; a changed file is re-parsed into the SAME documentId (a plain reindex would reuse the stored content — exactly the thing that changed). - The source list shows the `missing` state; the Add menu gains "Watch folder". Verified: npm run typecheck; npm test (409 pass, incl. 8 diffFolder tests); npm run check:design; npm run build; eval baseline unchanged; electron . --smoke-test PASS (29 checks), incl. a watched folder that imports a newly added file and marks a deleted one missing without dropping its chunks. --- src/main/db/migrations/0021_aromatic_pyro.sql | 12 + .../db/migrations/meta/0021_snapshot.json | 1827 +++++++++++++++++ src/main/db/migrations/meta/_journal.json | 7 + src/main/db/schema.ts | 36 +- src/main/index.ts | 7 + src/main/ipc/knowledgeHandlers.ts | 126 +- src/main/ipc/validation.ts | 14 +- src/main/services/KnowledgeService.ts | 100 +- src/main/services/ingestion/folderDiff.ts | 61 + src/main/services/ingestion/folderScan.ts | 7 +- src/main/services/ingestion/folderWatch.ts | 217 ++ src/main/smokeTest.ts | 35 + src/preload/index.d.ts | 13 +- src/preload/index.ts | 7 +- .../src/components/notebook/SourcePanel.tsx | 47 +- .../notebook/source/DocumentList.tsx | 4 + src/renderer/src/locales/en-US/ui.json | 2 + src/renderer/src/locales/zh-CN/ui.json | 2 + src/renderer/src/store/knowledgeStore.ts | 12 +- test/folderDiff.test.ts | 109 + 20 files changed, 2563 insertions(+), 82 deletions(-) create mode 100644 src/main/db/migrations/0021_aromatic_pyro.sql create mode 100644 src/main/db/migrations/meta/0021_snapshot.json create mode 100644 src/main/services/ingestion/folderDiff.ts create mode 100644 src/main/services/ingestion/folderWatch.ts create mode 100644 test/folderDiff.test.ts diff --git a/src/main/db/migrations/0021_aromatic_pyro.sql b/src/main/db/migrations/0021_aromatic_pyro.sql new file mode 100644 index 0000000..98a5838 --- /dev/null +++ b/src/main/db/migrations/0021_aromatic_pyro.sql @@ -0,0 +1,12 @@ +CREATE TABLE `folder_watches` ( + `id` text PRIMARY KEY NOT NULL, + `notebook_id` text NOT NULL, + `path` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL, + FOREIGN KEY (`notebook_id`) REFERENCES `notebooks`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_folder_watches_notebook_path` ON `folder_watches` (`notebook_id`,`path`);--> statement-breakpoint +ALTER TABLE `documents` ADD `source_state` text DEFAULT 'available' NOT NULL;--> statement-breakpoint +ALTER TABLE `documents` ADD `source_mtime_ms` integer; \ No newline at end of file diff --git a/src/main/db/migrations/meta/0021_snapshot.json b/src/main/db/migrations/meta/0021_snapshot.json new file mode 100644 index 0000000..11bb8ef --- /dev/null +++ b/src/main/db/migrations/meta/0021_snapshot.json @@ -0,0 +1,1827 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "baab174f-764a-417c-8f5c-c1a56c3320fe", + "prevId": "7afec921-a432-43f3-aafe-054c28f80f89", + "tables": { + "anki_cards": { + "name": "anki_cards", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "cards_data": { + "name": "cards_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_ankicards_notebook": { + "name": "idx_ankicards_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_ankicards_version": { + "name": "idx_ankicards_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "anki_cards_notebook_id_notebooks_id_fk": { + "name": "anki_cards_notebook_id_notebooks_id_fk", + "tableFrom": "anki_cards", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_content": { + "name": "reasoning_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "usage": { + "name": "usage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt_of": { + "name": "attempt_of", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_session": { + "name": "idx_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_messages_session_id_chat_sessions_id_fk": { + "name": "chat_messages_session_id_chat_sessions_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_sessions": { + "name": "chat_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "parent_session_id": { + "name": "parent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retrieval_scope": { + "name": "retrieval_scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_sessions_notebook": { + "name": "idx_sessions_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chat_sessions_notebook_id_notebooks_id_fk": { + "name": "chat_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_sessions_parent_session_id_chat_sessions_id_fk": { + "name": "chat_sessions_parent_session_id_chat_sessions_id_fk", + "tableFrom": "chat_sessions", + "tableTo": "chat_sessions", + "columnsFrom": [ + "parent_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunk_blocks": { + "name": "chunk_blocks", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_in_block": { + "name": "start_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_in_block": { + "name": "end_in_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunk_blocks_block": { + "name": "idx_chunk_blocks_block", + "columns": [ + "block_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunk_blocks_chunk_id_chunks_id_fk": { + "name": "chunk_blocks_chunk_id_chunks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunk_blocks_block_id_document_blocks_id_fk": { + "name": "chunk_blocks_block_id_document_blocks_id_fk", + "tableFrom": "chunk_blocks", + "tableTo": "document_blocks", + "columnsFrom": [ + "block_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "chunk_blocks_chunk_id_block_id_pk": { + "columns": [ + "chunk_id", + "block_id" + ], + "name": "chunk_blocks_chunk_id_block_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chunks": { + "name": "chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_start": { + "name": "page_start", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "page_end": { + "name": "page_end", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_chunks_document": { + "name": "idx_chunks_document", + "columns": [ + "document_id" + ], + "isUnique": false + }, + "idx_chunks_notebook": { + "name": "idx_chunks_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chunks_document_id_documents_id_fk": { + "name": "chunks_document_id_documents_id_fk", + "tableFrom": "chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chunks_notebook_id_notebooks_id_fk": { + "name": "chunks_notebook_id_notebooks_id_fk", + "tableFrom": "chunks", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "document_blocks": { + "name": "document_blocks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bbox": { + "name": "bbox", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_blocks_document_order": { + "name": "idx_blocks_document_order", + "columns": [ + "document_id", + "order" + ], + "isUnique": false + }, + "idx_blocks_document_page": { + "name": "idx_blocks_document_page", + "columns": [ + "document_id", + "page" + ], + "isUnique": false + } + }, + "foreignKeys": { + "document_blocks_document_id_documents_id_fk": { + "name": "document_blocks_document_id_documents_id_fk", + "tableFrom": "document_blocks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "local_file_path": { + "name": "local_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_note_id": { + "name": "source_note_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "structure": { + "name": "structure", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "source_state": { + "name": "source_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'available'" + }, + "source_mtime_ms": { + "name": "source_mtime_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_documents_notebook": { + "name": "idx_documents_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_documents_status": { + "name": "idx_documents_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_notebook_id_notebooks_id_fk": { + "name": "documents_notebook_id_notebooks_id_fk", + "tableFrom": "documents", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_source_note_id_notes_id_fk": { + "name": "documents_source_note_id_notes_id_fk", + "tableFrom": "documents", + "tableTo": "notes", + "columnsFrom": [ + "source_note_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embeddings": { + "name": "embeddings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "chunk_id": { + "name": "chunk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeddings_chunk": { + "name": "idx_embeddings_chunk", + "columns": [ + "chunk_id" + ], + "isUnique": false + }, + "idx_embeddings_notebook": { + "name": "idx_embeddings_notebook", + "columns": [ + "notebook_id" + ], + "isUnique": false + }, + "idx_embeddings_model": { + "name": "idx_embeddings_model", + "columns": [ + "model" + ], + "isUnique": false + } + }, + "foreignKeys": { + "embeddings_chunk_id_chunks_id_fk": { + "name": "embeddings_chunk_id_chunks_id_fk", + "tableFrom": "embeddings", + "tableTo": "chunks", + "columnsFrom": [ + "chunk_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embeddings_notebook_id_notebooks_id_fk": { + "name": "embeddings_notebook_id_notebooks_id_fk", + "tableFrom": "embeddings", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "folder_watches": { + "name": "folder_watches", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_folder_watches_notebook_path": { + "name": "idx_folder_watches_notebook_path", + "columns": [ + "notebook_id", + "path" + ], + "isUnique": true + } + }, + "foreignKeys": { + "folder_watches_notebook_id_notebooks_id_fk": { + "name": "folder_watches_notebook_id_notebooks_id_fk", + "tableFrom": "folder_watches", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ingestion_runs": { + "name": "ingestion_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_ingestion_runs_document": { + "name": "idx_ingestion_runs_document", + "columns": [ + "document_id", + "started_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "ingestion_runs_document_id_documents_id_fk": { + "name": "ingestion_runs_document_id_documents_id_fk", + "tableFrom": "ingestion_runs", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "items": { + "name": "items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_items_notebook_order": { + "name": "idx_items_notebook_order", + "columns": [ + "notebook_id", + "order" + ], + "isUnique": false + }, + "idx_items_type": { + "name": "idx_items_type", + "columns": [ + "type" + ], + "isUnique": false + }, + "idx_items_resource": { + "name": "idx_items_resource", + "columns": [ + "type", + "resource_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "items_notebook_id_notebooks_id_fk": { + "name": "items_notebook_id_notebooks_id_fk", + "tableFrom": "items", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mind_maps": { + "name": "mind_maps", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "tree_data": { + "name": "tree_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_mindmaps_notebook": { + "name": "idx_mindmaps_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_mindmaps_version": { + "name": "idx_mindmaps_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mind_maps_notebook_id_notebooks_id_fk": { + "name": "mind_maps_notebook_id_notebooks_id_fk", + "tableFrom": "mind_maps", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebook_embedding_spaces": { + "name": "notebook_embedding_spaces", + "columns": { + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "backend": { + "name": "backend", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "notebook_embedding_spaces_notebook_id_notebooks_id_fk": { + "name": "notebook_embedding_spaces_notebook_id_notebooks_id_fk", + "tableFrom": "notebook_embedding_spaces", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notebooks": { + "name": "notebooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notebooks_updated": { + "name": "idx_notebooks_updated", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notes": { + "name": "notes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_notes_notebook": { + "name": "idx_notes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "notes_notebook_id_notebooks_id_fk": { + "name": "notes_notebook_id_notebooks_id_fk", + "tableFrom": "notes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quiz_sessions": { + "name": "quiz_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "quiz_id": { + "name": "quiz_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answers": { + "name": "answers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_questions": { + "name": "total_questions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "correct_count": { + "name": "correct_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quiz_sessions_quiz": { + "name": "idx_quiz_sessions_quiz", + "columns": [ + "quiz_id", + "created_at" + ], + "isUnique": false + }, + "idx_quiz_sessions_notebook": { + "name": "idx_quiz_sessions_notebook", + "columns": [ + "notebook_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quiz_sessions_quiz_id_quizzes_id_fk": { + "name": "quiz_sessions_quiz_id_quizzes_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "quizzes", + "columnsFrom": [ + "quiz_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "quiz_sessions_notebook_id_notebooks_id_fk": { + "name": "quiz_sessions_notebook_id_notebooks_id_fk", + "tableFrom": "quiz_sessions", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "quizzes": { + "name": "quizzes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "notebook_id": { + "name": "notebook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "questions_data": { + "name": "questions_data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_mapping": { + "name": "chunk_mapping", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'generating'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_quizzes_notebook": { + "name": "idx_quizzes_notebook", + "columns": [ + "notebook_id", + "updated_at" + ], + "isUnique": false + }, + "idx_quizzes_version": { + "name": "idx_quizzes_version", + "columns": [ + "notebook_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "quizzes_notebook_id_notebooks_id_fk": { + "name": "quizzes_notebook_id_notebooks_id_fk", + "tableFrom": "quizzes", + "tableTo": "notebooks", + "columnsFrom": [ + "notebook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/main/db/migrations/meta/_journal.json b/src/main/db/migrations/meta/_journal.json index db04dc0..fdbf467 100644 --- a/src/main/db/migrations/meta/_journal.json +++ b/src/main/db/migrations/meta/_journal.json @@ -148,6 +148,13 @@ "when": 1790587593731, "tag": "0020_normal_nextwave", "breakpoints": true + }, + { + "idx": 21, + "version": "6", + "when": 1790594238292, + "tag": "0021_aromatic_pyro", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/main/db/schema.ts b/src/main/db/schema.ts index 9d8736d..baba4e4 100644 --- a/src/main/db/schema.ts +++ b/src/main/db/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, index, primaryKey } from 'drizzle-orm/sqlite-core' +import { sqliteTable, text, integer, index, primaryKey, uniqueIndex } from 'drizzle-orm/sqlite-core' import type { QuizQuestion } from '../../shared/types/quiz' import type { ChatExecutionError, @@ -165,6 +165,15 @@ export const documents = sqliteTable( status: text('status', { enum: ['pending', 'processing', 'indexed', 'failed'] }) .notNull() .default('pending'), + /** + * 来源文件本身的状态（#158），与 `status`（索引状态）分开： + * 文件被删是 `missing`，索引只是过期；笔记、摘录、citation 都还在。 + */ + sourceState: text('source_state', { enum: ['available', 'missing', 'changed'] }) + .notNull() + .default('available'), + /** 上次导入/刷新时来源文件的 mtime（ms）；watch 靠它判断文件是否被改过。 */ + sourceMtimeMs: integer('source_mtime_ms'), errorMessage: text('error_message'), chunkCount: integer('chunk_count').default(0), createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), @@ -179,6 +188,31 @@ export const documents = sqliteTable( export type Document = typeof documents.$inferSelect export type NewDocument = typeof documents.$inferInsert +/** + * 被监听的文件夹（#158）。 + * + * 文件夹不是 document（document 是一份来源文件），所以监听关系单独一张表：一个 + * notebook 可以监听多个文件夹，同一个（notebook, path）不会重复（唯一索引）。 + */ +export const folderWatches = sqliteTable( + 'folder_watches', + { + id: text('id').primaryKey(), + notebookId: text('notebook_id') + .notNull() + .references(() => notebooks.id, { onDelete: 'cascade' }), + path: text('path').notNull(), + createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), + updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull() + }, + (table) => ({ + uniquePath: uniqueIndex('idx_folder_watches_notebook_path').on(table.notebookId, table.path) + }) +) + +export type FolderWatch = typeof folderWatches.$inferSelect +export type NewFolderWatch = typeof folderWatches.$inferInsert + /** * 索引尝试记录（#95）。 * diff --git a/src/main/index.ts b/src/main/index.ts index a6e24cf..c3b1497 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -137,6 +137,12 @@ app.whenReady().then(async () => { knowledgeService = new KnowledgeService(embeddingService) Logger.info('Main', 'Knowledge Service initialized') + // Watched folders (#158): reconcile once so changes made while the app was closed + // are picked up, then keep them current live. + Logger.info('Main', 'Reconciling watched folders...') + await knowledgeService.reconcileWatchedFolders() + knowledgeService.startFolderWatching() + // Initialize Update Service Logger.info('Main', 'Initializing Update Service...') updateService = new UpdateService() @@ -245,6 +251,7 @@ app.on('before-quit', () => { } Logger.info('Main', 'Closing database connection...') + knowledgeService?.stopFolderWatching() closeDatabase() }) diff --git a/src/main/ipc/knowledgeHandlers.ts b/src/main/ipc/knowledgeHandlers.ts index f624cf9..8b32cc0 100644 --- a/src/main/ipc/knowledgeHandlers.ts +++ b/src/main/ipc/knowledgeHandlers.ts @@ -337,64 +337,92 @@ export function registerKnowledgeHandlers(knowledgeService: KnowledgeService) { }) // 文件夹导入：一次快照（#98） - ipcMain.handle( - 'knowledge:add-folder', - async (event: IpcMainInvokeEvent, args: unknown) => { - const validated = await validate(KnowledgeSchemas.addFolder, async (params) => { - Logger.debug('KnowledgeHandlers', 'add-folder:', params) + ipcMain.handle('knowledge:add-folder', async (event: IpcMainInvokeEvent, args: unknown) => { + const validated = await validate(KnowledgeSchemas.addFolder, async (params) => { + Logger.debug('KnowledgeHandlers', 'add-folder:', params) - try { - const result = await knowledgeService.addFolder( - params.notebookId, - params.folderPath, - (stage, progress) => { - event.sender.send('knowledge:index-progress', { - notebookId: params.notebookId, - stage, - progress - }) - } - ) - return { success: true, ...result } - } catch (error) { - Logger.error('KnowledgeHandlers', 'Error adding folder:', error) - return { success: false, added: [], skipped: [], failed: [], error: (error as Error).message } + try { + const result = await knowledgeService.addFolder( + params.notebookId, + params.folderPath, + (stage, progress) => { + event.sender.send('knowledge:index-progress', { + notebookId: params.notebookId, + stage, + progress + }) + } + ) + + // 监听（#158）在快照之后建立：持久化监听关系、立刻对齐一次、开始实时监听。 + if (params.watch) { + await knowledgeService.watchFolder(params.notebookId, params.folderPath) } - })(event, args) - return validated - } + return { success: true, ...result } + } catch (error) { + Logger.error('KnowledgeHandlers', 'Error adding folder:', error) + return { + success: false, + added: [], + skipped: [], + failed: [], + error: (error as Error).message + } + } + })(event, args) + + return validated + }) + + // 监听文件夹（#158） + ipcMain.handle( + 'knowledge:list-folder-watches', + validate(KnowledgeSchemas.listFolderWatches, async (params) => { + return knowledgeService.listFolderWatches(params.notebookId) + }) ) - // 批量导入一组文件（#98，拖放/多选共用） ipcMain.handle( - 'knowledge:add-files', - async (event: IpcMainInvokeEvent, args: unknown) => { - const validated = await validate(KnowledgeSchemas.addFiles, async (params) => { - Logger.debug('KnowledgeHandlers', 'add-files:', params.paths.length) + 'knowledge:unwatch-folder', + validate(KnowledgeSchemas.unwatchFolder, async (params) => { + knowledgeService.unwatchFolder(params.watchId) + return { success: true } + }) + ) - try { - const result = await knowledgeService.addDocumentsFromPaths( - params.notebookId, - params.paths, - (stage, progress) => { - event.sender.send('knowledge:index-progress', { - notebookId: params.notebookId, - stage, - progress - }) - } - ) - return { success: true, ...result } - } catch (error) { - Logger.error('KnowledgeHandlers', 'Error adding files:', error) - return { success: false, added: [], skipped: [], failed: [], error: (error as Error).message } + // 批量导入一组文件（#98，拖放/多选共用） + ipcMain.handle('knowledge:add-files', async (event: IpcMainInvokeEvent, args: unknown) => { + const validated = await validate(KnowledgeSchemas.addFiles, async (params) => { + Logger.debug('KnowledgeHandlers', 'add-files:', params.paths.length) + + try { + const result = await knowledgeService.addDocumentsFromPaths( + params.notebookId, + params.paths, + (stage, progress) => { + event.sender.send('knowledge:index-progress', { + notebookId: params.notebookId, + stage, + progress + }) + } + ) + return { success: true, ...result } + } catch (error) { + Logger.error('KnowledgeHandlers', 'Error adding files:', error) + return { + success: false, + added: [], + skipped: [], + failed: [], + error: (error as Error).message } - })(event, args) + } + })(event, args) - return validated - } - ) + return validated + }) // 打开文档源文件 ipcMain.handle( diff --git a/src/main/ipc/validation.ts b/src/main/ipc/validation.ts index fee8194..e409938 100644 --- a/src/main/ipc/validation.ts +++ b/src/main/ipc/validation.ts @@ -146,10 +146,20 @@ export const KnowledgeSchemas = { filePath: z.string().min(1, '文件路径不能为空') }), - // 批量/文件夹导入（#98） + // 批量/文件夹导入（#98），以及监听文件夹（#158） addFolder: z.object({ notebookId: z.string().min(1, '笔记本 ID 不能为空'), - folderPath: z.string().min(1, '文件夹路径不能为空') + folderPath: z.string().min(1, '文件夹路径不能为空'), + /** true 时把文件夹改为监听来源（#158）：持续对齐，而不仅仅是一次快照。 */ + watch: z.boolean().optional() + }), + + listFolderWatches: z.object({ + notebookId: z.string().min(1, '笔记本 ID 不能为空') + }), + + unwatchFolder: z.object({ + watchId: z.string().min(1, '监听 ID 不能为空') }), addFiles: z.object({ diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 9a38a45..521b050 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -5,7 +5,7 @@ import { createHash } from 'crypto' import { app } from 'electron' -import { join, basename } from 'path' +import { join, basename, sep } from 'path' import { mkdir, copyFile, unlink, stat } from 'fs/promises' import { getDatabase, @@ -60,6 +60,15 @@ import type { Retriever } from './retrieval' import { WebFetchService } from './WebFetchService' +import type { WatchedDocument } from './ingestion/folderWatch' +import { + FolderWatchService, + addFolderWatch, + listFolderWatchesForNotebook, + removeFolderWatch, + type FolderWatchRecord, + type ReconcileResult +} from './ingestion/folderWatch' import { scanFolder } from './ingestion/folderScan' import { deleteDocumentChunksFts, ensureChunksFts, indexChunksFts, searchChunksFts } from './fts' import { vectorStoreManager } from '../vectorstore' @@ -182,6 +191,7 @@ export class KnowledgeService { private fileParserService: FileParserService private webFetchService: WebFetchService private knowledgeFilesDir: string + private folderWatch: FolderWatchService constructor(embeddingService: EmbeddingService) { this.embeddingService = embeddingService @@ -192,6 +202,8 @@ export class KnowledgeService { // 知识库文件存储目录 this.knowledgeFilesDir = join(app.getPath('userData'), 'knowledge-files') this.ensureKnowledgeFilesDir() + // 监听服务只持有本服务的引用，构造本身没有副作用（不会开数据库、不会开始监听）。 + this.folderWatch = new FolderWatchService(this) } /** @@ -468,6 +480,9 @@ export class KnowledgeService { fileSize: parseResult.metadata?.fileSize as number | undefined, metadata: parseResult.metadata, errorMessage: null, + // 文件回到 available，并记下这次看到的 mtime：watch（#158）靠它判断是否被改过。 + sourceState: 'available', + sourceMtimeMs: (await stat(filePath).catch(() => null))?.mtimeMs, updatedAt: new Date() }) .where(eq(documents.id, documentId)) @@ -1175,6 +1190,89 @@ export class KnowledgeService { return runsFor(documentId) } + /** 当前所有已注册 loader 认识的扩展名（#98 / #158 的扫描用它）。 */ + supportedExtensions(): string[] { + return this.fileParserService.supportedExtensions() + } + + /** + * 一个被监听文件夹下的所有 source（#158）。 + * + * 按 `sourceUri` 前缀匹配：文件导入后 sourceUri 仍是原路径，所以「这个文件夹下的 + * 来源」可以直接算出来，不需要另存一份父子关系。 + */ + getWatchedDocuments(notebookId: string, folderPath: string): WatchedDocument[] { + const prefix = folderPath.endsWith(sep) ? folderPath : folderPath + sep + + return getDatabase() + .select({ + documentId: documents.id, + sourceUri: documents.sourceUri, + sourceMtimeMs: documents.sourceMtimeMs, + sourceState: documents.sourceState + }) + .from(documents) + .where(eq(documents.notebookId, notebookId)) + .all() + .filter( + (row): row is WatchedDocument => + typeof row.sourceUri === 'string' && + (row.sourceUri === folderPath || row.sourceUri.startsWith(prefix)) + ) + } + + /** 标记来源文件本身的状态（#158）：`missing` 不删行，也不碰笔记与 citation。 */ + markSourceState(documentId: string, state: 'available' | 'missing' | 'changed'): void { + getDatabase() + .update(documents) + .set({ sourceState: state, updatedAt: new Date() }) + .where(eq(documents.id, documentId)) + .run() + } + + /** + * 来源文件变了：重新拷贝、解析并索引**同一个** documentId（#158）。 + * + * 与 reindex 的区别：reindex 用的是已持久化的 content，而这里文件本身被改过，必须 + * 重新解析。来源身份不变，所以历史 citation 与摘录仍然指向同一个来源。 + */ + async refreshDocumentFromFile(documentId: string, filePath: string): Promise { + await this.ingestFile(documentId, filePath, { copyFrom: filePath }, 'reindex') + } + + /** + * 监听一个文件夹（#158）：持久化监听关系，立刻对齐一次，然后开始实时监听。 + * 立刻对齐是必要的：watch 建立之前的改动也要被覆盖，而不仅仅依赖之后的 fs 事件。 + */ + async watchFolder(notebookId: string, folderPath: string): Promise { + addFolderWatch(notebookId, folderPath) + const result = await this.folderWatch.reconcileFolder(notebookId, folderPath) + this.folderWatch.start() + return result + } + + unwatchFolder(watchId: string): void { + removeFolderWatch(watchId) + this.folderWatch.stopWatch(watchId) + } + + listFolderWatches(notebookId: string): FolderWatchRecord[] { + return listFolderWatchesForNotebook(notebookId) + } + + /** 启动时对齐一次（#158）：app 关着的时候发生的改动不会被漏掉。 */ + async reconcileWatchedFolders(): Promise { + return this.folderWatch.reconcileAll() + } + + startFolderWatching(): void { + this.folderWatch.start() + } + + stopFolderWatching(): void { + this.folderWatch.stop() + } + /** * 从本地副本重新解析出结构，用于升级前没有持久化 `structure` 的文档。 * diff --git a/src/main/services/ingestion/folderDiff.ts b/src/main/services/ingestion/folderDiff.ts new file mode 100644 index 0000000..6cfc1c8 --- /dev/null +++ b/src/main/services/ingestion/folderDiff.ts @@ -0,0 +1,61 @@ +import type { ScannedFile } from './folderScan' + +/** + * The pure part of folder watching (#158). + * + * Kept out of `folderWatch.ts` on purpose: that module opens the database (and so + * loads Electron), and the decision table below is the part worth unit-testing. + * "Changed" and "missing" have to be *determined*, not guessed, so they are a pure + * function of the scan and the rows already imported. + */ + +/** A document already imported from a file inside the watched folder. */ +export interface WatchedDocument { + documentId: string + sourceUri: string + sourceMtimeMs: number | null + sourceState: 'available' | 'missing' | 'changed' +} + +export interface FolderDiff { + /** Files on disk with no document yet. */ + added: ScannedFile[] + /** Files whose mtime differs from what was recorded. */ + changed: ScannedFile[] + /** Documents whose file is gone. */ + missing: WatchedDocument[] + /** Documents previously not `available` whose file is back, mtime unchanged. */ + restored: WatchedDocument[] +} + +export function diffFolder( + scanned: readonly ScannedFile[], + existing: readonly WatchedDocument[] +): FolderDiff { + const byPath = new Map(existing.map((document) => [document.sourceUri, document])) + const seen = new Set() + + const added: ScannedFile[] = [] + const changed: ScannedFile[] = [] + const restored: WatchedDocument[] = [] + + for (const file of scanned) { + seen.add(file.path) + const document = byPath.get(file.path) + + if (!document) { + added.push(file) + continue + } + + const mtimeChanged = document.sourceMtimeMs === null || document.sourceMtimeMs !== file.mtimeMs + if (mtimeChanged) { + changed.push(file) + } else if (document.sourceState !== 'available') { + restored.push(document) + } + } + + const missing = existing.filter((document) => !seen.has(document.sourceUri)) + return { added, changed, missing, restored } +} diff --git a/src/main/services/ingestion/folderScan.ts b/src/main/services/ingestion/folderScan.ts index ff29c3c..ae567ae 100644 --- a/src/main/services/ingestion/folderScan.ts +++ b/src/main/services/ingestion/folderScan.ts @@ -1,4 +1,4 @@ -import { readdir } from 'fs/promises' +import { readdir, stat } from 'fs/promises' import { extname, join, relative } from 'path' /** @@ -15,6 +15,8 @@ export interface ScannedFile { path: string /** Path relative to the scanned root, for stable ordering and messages. */ relativePath: string + /** File modification time in ms — what a watch compares to decide "changed" (#158). */ + mtimeMs: number } /** @@ -46,7 +48,8 @@ export async function scanFolder( const extension = extname(entry.name).toLowerCase().slice(1) if (allowed.has(extension)) { - files.push({ path: fullPath, relativePath: relative(dir, fullPath) }) + const info = await stat(fullPath) + files.push({ path: fullPath, relativePath: relative(dir, fullPath), mtimeMs: info.mtimeMs }) } } } diff --git a/src/main/services/ingestion/folderWatch.ts b/src/main/services/ingestion/folderWatch.ts new file mode 100644 index 0000000..3262c31 --- /dev/null +++ b/src/main/services/ingestion/folderWatch.ts @@ -0,0 +1,217 @@ +import { watch, type FSWatcher } from 'fs' +import { eq } from 'drizzle-orm' +import { getDatabase } from '../../db' +import { folderWatches } from '../../db/schema' +import type { KnowledgeService } from '../KnowledgeService' +import { scanFolder, type ScannedFile } from './folderScan' +import { diffFolder } from './folderDiff' + +// Re-exported so the rest of the app keeps importing the watcher from one place; +// the pure decision table lives in `folderDiff.ts` so it is testable without Electron. +export { diffFolder, type FolderDiff, type WatchedDocument } from './folderDiff' + +/** + * Watched folders (#158). + * + * A folder import (#98) is a snapshot; this is what keeps a folder current. The + * two are deliberately separate code paths: the snapshot never runs on a timer, + * and the watcher never re-implements parsing. + * + * The division of labour: + * + * filesystem event → (debounced) reconcile that one folder + * new file → enqueue an import run (#95) + * changed file → re-parse + re-index the SAME documentId + * deleted file → mark the source `missing`; never delete the row, + * its notes, its excerpts or its citations + * + * Reconciliation is a *diff*, not a re-import: an unchanged file costs one stat + * and nothing else. + */ + +export interface FolderWatchRecord { + id: string + notebookId: string + path: string +} + +export function listFolderWatches(): FolderWatchRecord[] { + return getDatabase() + .select({ + id: folderWatches.id, + notebookId: folderWatches.notebookId, + path: folderWatches.path + }) + .from(folderWatches) + .all() +} + +export function listFolderWatchesForNotebook(notebookId: string): FolderWatchRecord[] { + return getDatabase() + .select({ + id: folderWatches.id, + notebookId: folderWatches.notebookId, + path: folderWatches.path + }) + .from(folderWatches) + .where(eq(folderWatches.notebookId, notebookId)) + .all() +} + +/** Add a watch. Adding the same folder twice is a no-op, not a second row. */ +export function addFolderWatch(notebookId: string, path: string): FolderWatchRecord { + const existing = listFolderWatchesForNotebook(notebookId).find((entry) => entry.path === path) + if (existing) return existing + + const now = new Date() + const id = `watch_${Date.now()}_${Math.random().toString(36).slice(2, 9)}` + getDatabase() + .insert(folderWatches) + .values({ id, notebookId, path, createdAt: now, updatedAt: now }) + .run() + + return { id, notebookId, path } +} + +export function removeFolderWatch(watchId: string): void { + getDatabase().delete(folderWatches).where(eq(folderWatches.id, watchId)).run() +} + +export interface ReconcileResult { + notebookId: string + path: string + added: number + changed: number + missing: number + restored: number +} + +/** + * Keeps watched folders current. + * + * `reconcileAll()` runs on launch, so a change made while the app was closed is + * not missed; `start()` adds live watchers on top of that. Both funnel through + * `reconcileFolder()`, which is the only place that decides what changed. + */ +export class FolderWatchService { + private readonly watchers = new Map() + private readonly timers = new Map() + private stopped = false + + constructor(private readonly knowledgeService: KnowledgeService) {} + + async reconcileAll(): Promise { + const results: ReconcileResult[] = [] + for (const watch of listFolderWatches()) { + results.push(await this.reconcileFolder(watch.notebookId, watch.path)) + } + return results + } + + async reconcileFolder(notebookId: string, folderPath: string): Promise { + const extensions = this.knowledgeService.supportedExtensions() + let scanned: ScannedFile[] + try { + scanned = await scanFolder(folderPath, extensions) + } catch { + // A watched folder can disappear (unmounted drive, deleted folder). Every + // source under it becomes `missing`; nothing is deleted. + scanned = [] + } + + const existing = this.knowledgeService.getWatchedDocuments(notebookId, folderPath) + const diff = diffFolder(scanned, existing) + + for (const document of diff.missing) { + this.knowledgeService.markSourceState(document.documentId, 'missing') + } + for (const document of diff.restored) { + this.knowledgeService.markSourceState(document.documentId, 'available') + } + for (const file of diff.changed) { + const document = existing.find((entry) => entry.sourceUri === file.path) + if (!document) continue + try { + await this.knowledgeService.refreshDocumentFromFile(document.documentId, file.path) + } catch { + // A refresh failure is recorded on the ingestion run and leaves the old + // index in place (#95); reconciliation itself must not throw. + } + } + + if (diff.added.length > 0) { + await this.knowledgeService.addDocumentsFromPaths( + notebookId, + diff.added.map((file) => file.path) + ) + } + + return { + notebookId, + path: folderPath, + added: diff.added.length, + changed: diff.changed.length, + missing: diff.missing.length, + restored: diff.restored.length + } + } + + /** Live watchers over the persisted watches. Safe to call once at startup. */ + start(): void { + for (const record of listFolderWatches()) { + this.watchFolder(record) + } + } + + stop(): void { + this.stopped = true + for (const timer of this.timers.values()) clearTimeout(timer) + this.timers.clear() + for (const watcher of this.watchers.values()) watcher.close() + this.watchers.clear() + } + + /** 停止监听单个文件夹（取消监听时）；行还在时下次 `start()` 不会把它加回来。 */ + stopWatch(watchId: string): void { + const timer = this.timers.get(watchId) + if (timer) clearTimeout(timer) + this.timers.delete(watchId) + + const watcher = this.watchers.get(watchId) + if (watcher) watcher.close() + this.watchers.delete(watchId) + } + + private watchFolder(record: FolderWatchRecord): void { + if (this.watchers.has(record.id)) return + + try { + // `recursive` is supported on macOS/Windows and on Linux since Node 20. If a + // platform refuses it, the launch-time reconcile still keeps the folder + // current — only live updates are lost, and that is not silent: it is this catch. + const watcher = watch(record.path, { recursive: true }, () => this.debounce(record)) + watcher.on('error', () => { + watcher.close() + this.watchers.delete(record.id) + }) + this.watchers.set(record.id, watcher) + } catch { + // Folder gone or recursive unsupported: reconcile-on-launch remains the fallback. + } + } + + /** One folder's burst of events becomes one reconcile. */ + private debounce(watch: FolderWatchRecord): void { + if (this.stopped) return + const pending = this.timers.get(watch.id) + if (pending) clearTimeout(pending) + + this.timers.set( + watch.id, + setTimeout(() => { + this.timers.delete(watch.id) + void this.reconcileFolder(watch.notebookId, watch.path) + }, 800) + ) + } +} diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index e07c777..70555c4 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -757,6 +757,41 @@ async function runChecks(): Promise { ) pass('a folder import is a snapshot: unsupported files are skipped and re-import is a no-op') + // --- watching a folder keeps it current, and never deletes (#158) ----------- + const watchedFolder = mkdtempSync(join(tmpdir(), 'knownote-watch-')) + copyFileSync(join(fixtures, 'sample.md'), join(watchedFolder, 'watched.md')) + await knowledge.watchFolder(reindexNotebook, watchedFolder) + + const watchedDoc = knowledge + .getDocuments(reindexNotebook) + .find((doc) => doc.sourceUri === join(watchedFolder, 'watched.md')) + assert(watchedDoc, 'watching a folder did not import the file already in it') + + // A new file appears: reconcile imports it without the user doing anything. + copyFileSync(join(fixtures, 'sample.md'), join(watchedFolder, 'added.md')) + await knowledge.reconcileWatchedFolders() + assert( + knowledge + .getDocuments(reindexNotebook) + .some((doc) => doc.sourceUri === join(watchedFolder, 'added.md')), + 'reconcile did not import a file added to a watched folder' + ) + + // A file disappears: the source is marked missing; the row, its chunks and its + // citations survive, because a citation is a snapshot. + rmSync(join(watchedFolder, 'watched.md')) + await knowledge.reconcileWatchedFolders() + const missingDoc = knowledge.getDocument(watchedDoc.id) + assert( + missingDoc?.sourceState === 'missing', + `a deleted file left source_state=${missingDoc?.sourceState}` + ) + assert( + knowledge.getDocumentChunks(watchedDoc.id).length > 0, + 'marking a source missing destroyed its derived index' + ) + pass('a watched folder picks up new files and marks deleted ones missing without deleting them') + // --- a failed re-index leaves the previous index usable (#95) --------------- // The failure this guards: `reindexDocument()` used to clear the derived index // *before* embedding, so an embedding failure at 70% left the document with no diff --git a/src/preload/index.d.ts b/src/preload/index.d.ts index 18bdb80..cef081e 100644 --- a/src/preload/index.d.ts +++ b/src/preload/index.d.ts @@ -260,8 +260,17 @@ declare global { notebookId: string, filePath: string ) => Promise<{ success: boolean; documentId?: string; error?: string }> - /** 文件夹导入：一次快照（#98）。 */ - addFolder: (notebookId: string, folderPath: string) => Promise + /** 文件夹导入：一次快照；`watch` 为 true 时同时改为监听来源（#158）。 */ + addFolder: ( + notebookId: string, + folderPath: string, + watch?: boolean + ) => Promise + /** 监听列表（#158）。 */ + listFolderWatches: ( + notebookId: string + ) => Promise> + unwatchFolder: (watchId: string) => Promise<{ success: boolean }> /** 批量导入一组文件（#98）：拖放与多选共用。 */ addFiles: (notebookId: string, paths: string[]) => Promise /** 拖放的 `File` → 绝对路径（#98）。 */ diff --git a/src/preload/index.ts b/src/preload/index.ts index 191c69a..f24b510 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -168,8 +168,11 @@ const api = { ipcRenderer.invoke('knowledge:add-document', { notebookId, options }), addDocumentFromFile: (notebookId: string, filePath: string) => ipcRenderer.invoke('knowledge:add-document-from-file', { notebookId, filePath }), - addFolder: (notebookId: string, folderPath: string) => - ipcRenderer.invoke('knowledge:add-folder', { notebookId, folderPath }), + addFolder: (notebookId: string, folderPath: string, watch = false) => + ipcRenderer.invoke('knowledge:add-folder', { notebookId, folderPath, watch }), + listFolderWatches: (notebookId: string) => + ipcRenderer.invoke('knowledge:list-folder-watches', { notebookId }), + unwatchFolder: (watchId: string) => ipcRenderer.invoke('knowledge:unwatch-folder', { watchId }), addFiles: (notebookId: string, paths: string[]) => ipcRenderer.invoke('knowledge:add-files', { notebookId, paths }), /** diff --git a/src/renderer/src/components/notebook/SourcePanel.tsx b/src/renderer/src/components/notebook/SourcePanel.tsx index 7268052..366331b 100644 --- a/src/renderer/src/components/notebook/SourcePanel.tsx +++ b/src/renderer/src/components/notebook/SourcePanel.tsx @@ -11,7 +11,8 @@ import { ArrowLeft, Quote, ExternalLink, - FolderOpen + FolderOpen, + FolderSync } from 'lucide-react' import { toast } from 'sonner' import { useKnowledgeStore, setupKnowledgeListeners } from '../../store/knowledgeStore' @@ -27,10 +28,7 @@ import { Card } from '../ui/card' import { PanelHeader } from '../ui/panel-header' import DocumentList from './source/DocumentList' import SourceReader from './source/reader/SourceReader' -import type { - KnowledgeDocument, - BatchImportOutcome -} from '../../../../shared/types/knowledge' +import type { KnowledgeDocument, BatchImportOutcome } from '../../../../shared/types/knowledge' import type { ReaderAnchor, ReaderSelection } from '../../../../shared/types/source' import { selectionToSourceAnchor, @@ -511,20 +509,23 @@ export default function SourcePanel(): ReactElement { reportBatch(await addFiles(notebookId, files)) }, [notebookId, hasEmbeddingModel, selectFiles, addFiles, reportBatch, t]) - // 处理文件夹导入（#98）：一次快照，不是监听（监听是 #158） - const handleFolderUpload = useCallback(async () => { - if (!notebookId) return - if (!hasEmbeddingModel) { - alert(t('noEmbeddingModelConfigured')) - return - } + // 处理文件夹导入（#98 快照 / #158 监听） + const handleFolderUpload = useCallback( + async (watch: boolean) => { + if (!notebookId) return + if (!hasEmbeddingModel) { + alert(t('noEmbeddingModelConfigured')) + return + } - const folders = await selectFolder() - setShowAddMenu(false) - for (const folder of folders) { - reportBatch(await addFolder(notebookId, folder)) - } - }, [notebookId, hasEmbeddingModel, selectFolder, addFolder, reportBatch, t]) + const folders = await selectFolder() + setShowAddMenu(false) + for (const folder of folders) { + reportBatch(await addFolder(notebookId, folder, watch)) + } + }, + [notebookId, hasEmbeddingModel, selectFolder, addFolder, reportBatch, t] + ) // 拖放文件/文件夹（#98）。Electron 39 下路径必须由 preload 的 webUtils 给出。 const handleDrop = useCallback( @@ -764,13 +765,21 @@ export default function SourcePanel(): ReactElement { {t('uploadFile')} + diff --git a/src/renderer/src/locales/en-US/ui.json b/src/renderer/src/locales/en-US/ui.json index f5d6b82..607b48e 100644 --- a/src/renderer/src/locales/en-US/ui.json +++ b/src/renderer/src/locales/en-US/ui.json @@ -86,6 +86,8 @@ "retryDocument": "Try again", "reindexDocument": "Re-index", "uploadFolder": "Add folder", + "watchFolder": "Watch folder", + "sourceMissing": "The file is no longer at its path", "dropToImport": "Drop files or a folder to import", "importAdded": "Imported {{count}}", "importSkipped": "skipped {{count}} (already imported)", diff --git a/src/renderer/src/locales/zh-CN/ui.json b/src/renderer/src/locales/zh-CN/ui.json index dbbb6e1..89e74ff 100644 --- a/src/renderer/src/locales/zh-CN/ui.json +++ b/src/renderer/src/locales/zh-CN/ui.json @@ -86,6 +86,8 @@ "retryDocument": "重试", "reindexDocument": "重建索引", "uploadFolder": "添加文件夹", + "watchFolder": "监听文件夹", + "sourceMissing": "文件已不在原路径", "dropToImport": "拖入文件或文件夹以导入", "importAdded": "已导入 {{count}} 个", "importSkipped": "跳过 {{count}} 个（已存在）", diff --git a/src/renderer/src/store/knowledgeStore.ts b/src/renderer/src/store/knowledgeStore.ts index a49c1d8..31bb093 100644 --- a/src/renderer/src/store/knowledgeStore.ts +++ b/src/renderer/src/store/knowledgeStore.ts @@ -70,8 +70,12 @@ interface KnowledgeStore { notebookId: string, noteId: string ) => Promise<{ success: boolean; documentId?: string; error?: string }> - /** 文件夹导入：一次快照（#98）。 */ - addFolder: (notebookId: string, folderPath: string) => Promise + /** 文件夹导入：一次快照；`watch` 为 true 时同时改为监听来源（#158）。 */ + addFolder: ( + notebookId: string, + folderPath: string, + watch?: boolean + ) => Promise /** 批量导入一组文件（#98）：拖放与多选共用同一路径。 */ addFiles: (notebookId: string, paths: string[]) => Promise selectFolder: () => Promise @@ -207,9 +211,9 @@ export const useKnowledgeStore = create()((set, get) => ({ }, // 文件夹 / 批量导入（#98） - addFolder: async (notebookId, folderPath) => { + addFolder: async (notebookId, folderPath, watch = false) => { set({ isIndexing: true, error: null }) - const result = await window.api.knowledge.addFolder(notebookId, folderPath) + const result = await window.api.knowledge.addFolder(notebookId, folderPath, watch) await refreshAfterImport(notebookId, result, set, get) return result }, diff --git a/test/folderDiff.test.ts b/test/folderDiff.test.ts new file mode 100644 index 0000000..a25031d --- /dev/null +++ b/test/folderDiff.test.ts @@ -0,0 +1,109 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { diffFolder, type WatchedDocument } from '../src/main/services/ingestion/folderDiff.ts' +import type { ScannedFile } from '../src/main/services/ingestion/folderScan.ts' + +/** + * The decision table behind watching a folder (#158). + * + * This is the part that must not be guessed: a document is re-indexed only when + * its file's mtime actually changed, and a document is only marked `missing` when + * its file is really gone. Everything downstream (re-index, mark missing) is a + * consequence of this diff. + */ + +const file = (path: string, mtimeMs: number): ScannedFile => ({ path, relativePath: path, mtimeMs }) + +const doc = ( + path: string, + mtimeMs: number | null, + sourceState: WatchedDocument['sourceState'] = 'available' +): WatchedDocument => ({ + documentId: `doc_${path}`, + sourceUri: path, + sourceMtimeMs: mtimeMs, + sourceState +}) + +test('a file with no document is added', () => { + const diff = diffFolder([file('/p/a.pdf', 100)], []) + assert.deepEqual( + diff.added.map((f) => f.path), + ['/p/a.pdf'] + ) + assert.equal(diff.changed.length + diff.missing.length + diff.restored.length, 0) +}) + +test('an mtime change is changed, not added', () => { + const diff = diffFolder([file('/p/a.pdf', 200)], [doc('/p/a.pdf', 100)]) + assert.deepEqual( + diff.changed.map((f) => f.path), + ['/p/a.pdf'] + ) + assert.equal(diff.added.length, 0) +}) + +test('an unchanged file is neither added nor changed', () => { + const diff = diffFolder([file('/p/a.pdf', 100)], [doc('/p/a.pdf', 100)]) + assert.equal( + diff.added.length + diff.changed.length + diff.missing.length + diff.restored.length, + 0 + ) +}) + +test('a document with no mtime recorded counts as changed, not as current', () => { + // An older row has no mtime; the first reconcile after the upgrade must treat it + // as out of date rather than assume it is current. + const diff = diffFolder([file('/p/a.pdf', 100)], [doc('/p/a.pdf', null)]) + assert.deepEqual( + diff.changed.map((f) => f.path), + ['/p/a.pdf'] + ) +}) + +test('a document whose file is gone is missing', () => { + const diff = diffFolder([], [doc('/p/gone.pdf', 100)]) + assert.deepEqual( + diff.missing.map((d) => d.sourceUri), + ['/p/gone.pdf'] + ) + assert.equal(diff.added.length + diff.changed.length, 0) +}) + +test('a missing document whose file is back and unchanged is restored', () => { + const diff = diffFolder([file('/p/a.pdf', 100)], [doc('/p/a.pdf', 100, 'missing')]) + assert.deepEqual( + diff.restored.map((d) => d.sourceUri), + ['/p/a.pdf'] + ) + assert.equal(diff.changed.length, 0) +}) + +test('a missing document whose file is back but changed is re-indexed', () => { + const diff = diffFolder([file('/p/a.pdf', 300)], [doc('/p/a.pdf', 100, 'missing')]) + assert.deepEqual( + diff.changed.map((f) => f.path), + ['/p/a.pdf'] + ) + assert.equal(diff.restored.length, 0) +}) + +test('added, changed and missing are computed together', () => { + const diff = diffFolder( + [file('/p/new.pdf', 10), file('/p/edited.pdf', 20), file('/p/same.pdf', 30)], + [doc('/p/edited.pdf', 5), doc('/p/same.pdf', 30), doc('/p/deleted.pdf', 40)] + ) + + assert.deepEqual( + diff.added.map((f) => f.path), + ['/p/new.pdf'] + ) + assert.deepEqual( + diff.changed.map((f) => f.path), + ['/p/edited.pdf'] + ) + assert.deepEqual( + diff.missing.map((d) => d.sourceUri), + ['/p/deleted.pdf'] + ) +})