From f96c33fe0fb8266d4833e0e643c01ccc10633ce2 Mon Sep 17 00:00:00 2001 From: markgewhite Date: Sat, 11 Apr 2026 23:05:57 +0100 Subject: [PATCH 1/3] Added folder scanner with file discovery and MD5 hash computation Co-Authored-By: Claude Opus 4.6 (1M context) --- src/ingestion/scanner.py | 29 +++++++++++++++++ tests/test_scanner.py | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/ingestion/scanner.py create mode 100644 tests/test_scanner.py diff --git a/src/ingestion/scanner.py b/src/ingestion/scanner.py new file mode 100644 index 0000000..96caffa --- /dev/null +++ b/src/ingestion/scanner.py @@ -0,0 +1,29 @@ +"""Folder scanning and file-level change detection.""" + +import hashlib +from pathlib import Path + +SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt", ".md"} + + +def compute_file_hash(path: Path) -> str: + """Compute MD5 hash of a file's raw contents.""" + return hashlib.md5(path.read_bytes()).hexdigest() + + +def scan_folder(path: Path, *, recursive: bool = True) -> list[Path]: + """Discover all supported document files in a folder. + + Returns sorted list of paths to supported files. + """ + if recursive: + all_files = path.rglob("*") + else: + all_files = path.glob("*") + + files = [ + f for f in all_files + if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS + ] + files.sort() + return files diff --git a/tests/test_scanner.py b/tests/test_scanner.py new file mode 100644 index 0000000..02cfd41 --- /dev/null +++ b/tests/test_scanner.py @@ -0,0 +1,67 @@ +"""Tests for the folder scanner module.""" + +import hashlib +from pathlib import Path + +import pytest + +from src.ingestion.scanner import compute_file_hash, scan_folder + + +@pytest.fixture +def doc_folder(tmp_path): + """Create a folder with test documents.""" + (tmp_path / "report.pdf").write_bytes(b"%PDF-1.4 test content") + (tmp_path / "notes.txt").write_text("Some notes") + (tmp_path / "readme.md").write_text("# Readme") + (tmp_path / "letter.docx").write_bytes(b"PK docx content") + (tmp_path / "image.png").write_bytes(b"PNG data") # unsupported + sub = tmp_path / "sub" + sub.mkdir() + (sub / "nested.txt").write_text("Nested content") + return tmp_path + + +class TestComputeFileHash: + def test_hash_is_md5_of_contents(self, tmp_path): + f = tmp_path / "test.txt" + f.write_bytes(b"hello world") + expected = hashlib.md5(b"hello world").hexdigest() + assert compute_file_hash(f) == expected + + def test_different_content_different_hash(self, tmp_path): + f1 = tmp_path / "a.txt" + f2 = tmp_path / "b.txt" + f1.write_bytes(b"content A") + f2.write_bytes(b"content B") + assert compute_file_hash(f1) != compute_file_hash(f2) + + +class TestScanFolder: + def test_discovers_supported_files(self, doc_folder): + files = scan_folder(doc_folder, recursive=True) + extensions = {f.suffix for f in files} + assert extensions == {".pdf", ".txt", ".md", ".docx"} + + def test_excludes_unsupported_files(self, doc_folder): + files = scan_folder(doc_folder, recursive=True) + names = {f.name for f in files} + assert "image.png" not in names + + def test_recursive_finds_nested(self, doc_folder): + files = scan_folder(doc_folder, recursive=True) + names = {f.name for f in files} + assert "nested.txt" in names + + def test_non_recursive_excludes_nested(self, doc_folder): + files = scan_folder(doc_folder, recursive=False) + names = {f.name for f in files} + assert "nested.txt" not in names + + def test_returns_sorted_paths(self, doc_folder): + files = scan_folder(doc_folder, recursive=True) + assert files == sorted(files) + + def test_empty_folder(self, tmp_path): + files = scan_folder(tmp_path, recursive=True) + assert files == [] From a8d23dd58aa0661d7892dae6ca0d1605611864c4 Mon Sep 17 00:00:00 2001 From: markgewhite Date: Sat, 11 Apr 2026 23:07:04 +0100 Subject: [PATCH 2/3] Added incremental ingestion with file-hash change detection and tests Co-Authored-By: Claude Opus 4.6 (1M context) --- src/ingestion/ingest.py | 77 +++++++++++++++++++++++++++++++++++++++++ src/ingestion/loader.py | 19 ++++++++++ tests/test_ingest.py | 67 +++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 src/ingestion/ingest.py create mode 100644 tests/test_ingest.py diff --git a/src/ingestion/ingest.py b/src/ingestion/ingest.py new file mode 100644 index 0000000..db687bf --- /dev/null +++ b/src/ingestion/ingest.py @@ -0,0 +1,77 @@ +"""Incremental document ingestion with change detection.""" + +import logging +from pathlib import Path + +from src.ingestion.chunker import chunk_documents +from src.ingestion.loader import load_file +from src.ingestion.scanner import compute_file_hash, scan_folder +from src.retrieval.vector_store import VectorStore + +logger = logging.getLogger(__name__) + + +def ingest_folder( + folder: Path, + store: VectorStore, + *, + recursive: bool = True, + chunk_size: int = 512, + chunk_overlap: int = 100, +) -> tuple[int, int]: + """Scan a folder and ingest new or changed documents. + + Returns (ingested_count, skipped_count) — files ingested vs unchanged. + """ + files = scan_folder(folder, recursive=recursive) + + if not files: + return 0, 0 + + existing_hashes = _get_stored_file_hashes(store) + + ingested = 0 + skipped = 0 + + for file_path in files: + file_hash = compute_file_hash(file_path) + if file_hash in existing_hashes: + skipped += 1 + continue + + docs = load_file(file_path, folder) + if not docs: + continue + + # Attach file_hash to document metadata before chunking + for doc in docs: + doc.metadata["file_hash"] = file_hash + + chunks = chunk_documents( + docs, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + ) + + # Propagate file_hash to chunk metadata + for chunk in chunks: + chunk.metadata["file_hash"] = file_hash + + store.add_chunks(chunks) + ingested += 1 + logger.info("Ingested %s (%d chunks)", file_path.name, len(chunks)) + + return ingested, skipped + + +def _get_stored_file_hashes(store: VectorStore) -> set[str]: + """Extract unique file_hash values from all stored chunks.""" + if store.count() == 0: + return set() + + _, metadatas = store.get_all_texts_and_metadatas() + return { + m["file_hash"] + for m in metadatas + if "file_hash" in m + } diff --git a/src/ingestion/loader.py b/src/ingestion/loader.py index 2294e8e..c8d5651 100644 --- a/src/ingestion/loader.py +++ b/src/ingestion/loader.py @@ -19,6 +19,25 @@ } +def load_file(file_path: Path, root: Path) -> list[Document]: + """Load a single supported document file. + + Args: + file_path: Path to the file. + root: Root folder for computing relative paths. + + Returns: + List of Documents (one per page for PDFs, one for other formats). + """ + doc_type = EXTENSION_TO_DOC_TYPE.get(file_path.suffix.lower()) + if doc_type is None: + return [] + + if doc_type == "pdf": + return _load_pdf(file_path, root) + return _load_single_file(file_path, root, doc_type) + + def load_folder(path: Path, *, recursive: bool = True) -> list[Document]: """Load all supported documents from a folder.""" path = Path(path) diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..24e80db --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,67 @@ +"""Tests for incremental ingestion.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from src.ingestion.ingest import ingest_folder + + +@pytest.fixture +def doc_folder(tmp_path): + """Create a folder with a test text file.""" + (tmp_path / "notes.txt").write_text("Some notes about testing.") + return tmp_path + + +@pytest.fixture +def mock_store(): + store = MagicMock() + store.count.return_value = 0 + store.get_all_texts_and_metadatas.return_value = ([], []) + return store + + +class TestIngestFolder: + def test_ingests_new_file(self, doc_folder, mock_store): + ingested, skipped = ingest_folder( + doc_folder, mock_store, chunk_size=512, chunk_overlap=100 + ) + assert ingested == 1 + assert skipped == 0 + mock_store.add_chunks.assert_called_once() + + def test_skips_unchanged_file(self, doc_folder, mock_store): + # Simulate file already in store by returning its hash + import hashlib + file_hash = hashlib.md5( + (doc_folder / "notes.txt").read_bytes() + ).hexdigest() + mock_store.count.return_value = 5 + mock_store.get_all_texts_and_metadatas.return_value = ( + ["text"], + [{"file_hash": file_hash}], + ) + + ingested, skipped = ingest_folder(doc_folder, mock_store) + assert ingested == 0 + assert skipped == 1 + mock_store.add_chunks.assert_not_called() + + def test_chunks_have_file_hash(self, doc_folder, mock_store): + ingest_folder(doc_folder, mock_store) + chunks = mock_store.add_chunks.call_args[0][0] + for chunk in chunks: + assert "file_hash" in chunk.metadata + + def test_empty_folder(self, tmp_path, mock_store): + ingested, skipped = ingest_folder(tmp_path, mock_store) + assert ingested == 0 + assert skipped == 0 + + def test_relative_path_in_metadata(self, doc_folder, mock_store): + ingest_folder(doc_folder, mock_store) + chunks = mock_store.add_chunks.call_args[0][0] + for chunk in chunks: + rel_path = chunk.metadata.get("relative_path", "") + assert not rel_path.startswith("/") From 4694fdc34a2ccf4d74e58671d5acc4bea6c49ac6 Mon Sep 17 00:00:00 2001 From: markgewhite Date: Sat, 11 Apr 2026 23:07:50 +0100 Subject: [PATCH 3/3] Wired incremental ingestion and Chainlit settings panel into app startup Co-Authored-By: Claude Opus 4.6 (1M context) --- app.py | 128 ++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 94 insertions(+), 34 deletions(-) diff --git a/app.py b/app.py index 4cd050a..03eadaf 100644 --- a/app.py +++ b/app.py @@ -8,8 +8,7 @@ from src.generation.answerer import answer, build_source_elements from src.generation.condenser import Condenser from src.health_check import check_models, check_ollama -from src.ingestion.chunker import chunk_documents -from src.ingestion.loader import load_folder +from src.ingestion.ingest import ingest_folder from src.retrieval.bm25_index import BM25Index from src.retrieval.embeddings import make_ollama_embed_fn from src.retrieval.hybrid import HybridRetriever @@ -19,6 +18,43 @@ logger = logging.getLogger(__name__) +def _build_retriever(store: VectorStore, config) -> HybridRetriever: + """Build BM25 index and hybrid retriever from current store contents.""" + bm25_index = BM25Index() + doc_count = store.count() + if doc_count > 0: + start = time.time() + texts, metadatas = store.get_all_texts_and_metadatas() + bm25_index.build(texts, metadatas) + elapsed = time.time() - start + logger.info("BM25 index built from %d chunks in %.2fs", doc_count, elapsed) + + return HybridRetriever( + vector_store=store, + bm25_index=bm25_index, + config=config.retrieval, + ) + + +async def _run_ingestion(store: VectorStore, config) -> int: + """Ingest documents from configured folder. Returns chunk count.""" + folder = config.paths.documents + if str(folder) and folder.exists(): + ingested, skipped = ingest_folder( + folder, + store, + recursive=config.scanning.recursive, + chunk_size=config.chunking.chunk_size, + chunk_overlap=config.chunking.chunk_overlap, + ) + if ingested > 0: + logger.info("Ingested %d new files, skipped %d unchanged", ingested, skipped) + elif skipped > 0: + logger.info("All %d files unchanged, skipped", skipped) + + return store.count() + + @cl.on_chat_start async def on_chat_start(): try: @@ -52,37 +88,11 @@ async def on_chat_start(): store = VectorStore(embed_fn=embed_fn, client=chroma_client) cl.user_session.set("store", store) - # Ingest documents if a folder is configured and store is empty - doc_count = store.count() - if doc_count == 0 and str(config.paths.documents) and config.paths.documents.exists(): - docs = load_folder( - config.paths.documents, - recursive=config.scanning.recursive, - ) - if docs: - chunks = chunk_documents( - docs, - chunk_size=config.chunking.chunk_size, - chunk_overlap=config.chunking.chunk_overlap, - ) - store.add_chunks(chunks) - doc_count = store.count() - - # Build BM25 index from stored chunks - bm25_index = BM25Index() - if doc_count > 0: - start = time.time() - texts, metadatas = store.get_all_texts_and_metadatas() - bm25_index.build(texts, metadatas) - elapsed = time.time() - start - logger.info("BM25 index built from %d chunks in %.2fs", doc_count, elapsed) + # Auto-scan and ingest new/changed documents on startup + doc_count = await _run_ingestion(store, config) - # Create hybrid retriever - retriever = HybridRetriever( - vector_store=store, - bm25_index=bm25_index, - config=config.retrieval, - ) + # Build retriever + retriever = _build_retriever(store, config) cl.user_session.set("retriever", retriever) # Load cross-encoder reranker @@ -94,16 +104,66 @@ async def on_chat_start(): cl.user_session.set("condenser", condenser) cl.user_session.set("chat_history", []) + # Set up Chainlit settings panel + settings = await cl.ChatSettings( + [ + cl.input_widget.TextInput( + id="documents_folder", + label="Documents Folder", + initial=str(config.paths.documents), + ), + cl.input_widget.Switch( + id="recursive_scan", + label="Recursive Scanning", + initial=config.scanning.recursive, + ), + ] + ).send() + if doc_count > 0: await cl.Message( content=f"{doc_count} chunks indexed. Ask me anything!" ).send() else: await cl.Message( - content="No documents indexed. Configure your documents folder in `config.yaml` to get started." + content="No documents indexed. Configure your documents folder in the settings panel (gear icon) or `config.yaml`." ).send() +@cl.on_settings_update +async def on_settings_update(settings: dict): + """Re-scan and ingest when settings change.""" + from pathlib import Path + + config = cl.user_session.get("config") + store = cl.user_session.get("store") + + folder = Path(settings["documents_folder"]).expanduser() + recursive = settings["recursive_scan"] + + if not folder.exists(): + await cl.Message( + content=f"Folder not found: `{folder}`" + ).send() + return + + # Update config in session + config.paths.documents = folder + config.scanning.recursive = recursive + + await cl.Message(content=f"Scanning `{folder}`...").send() + + doc_count = await _run_ingestion(store, config) + + # Rebuild retriever with updated store + retriever = _build_retriever(store, config) + cl.user_session.set("retriever", retriever) + + await cl.Message( + content=f"Done. {doc_count} chunks indexed." + ).send() + + @cl.on_message async def on_message(message: cl.Message): config = cl.user_session.get("config") @@ -116,7 +176,7 @@ async def on_message(message: cl.Message): store = cl.user_session.get("store") if store is None or store.count() == 0: await cl.Message( - content="No documents indexed yet. Configure your documents folder in `config.yaml` first." + content="No documents indexed yet. Configure your documents folder in the settings panel (gear icon) or `config.yaml`." ).send() return