Folder scanning + settings panel + auto-ingest

app.py

@@ -8,8 +8,7 @@
 from src.generation.answerer import answer, build_source_elements
 from src.generation.condenser import Condenser
 from src.health_check import check_models, check_ollama
-from src.ingestion.chunker import chunk_documents
-from src.ingestion.loader import load_folder
+from src.ingestion.ingest import ingest_folder
 from src.retrieval.bm25_index import BM25Index
 from src.retrieval.embeddings import make_ollama_embed_fn
 from src.retrieval.hybrid import HybridRetriever
@@ -19,6 +18,43 @@
 logger = logging.getLogger(__name__)
 
 
+def _build_retriever(store: VectorStore, config) -> HybridRetriever:
+    """Build BM25 index and hybrid retriever from current store contents."""
+    bm25_index = BM25Index()
+    doc_count = store.count()
+    if doc_count > 0:
+        start = time.time()
+        texts, metadatas = store.get_all_texts_and_metadatas()
+        bm25_index.build(texts, metadatas)
+        elapsed = time.time() - start
+        logger.info("BM25 index built from %d chunks in %.2fs", doc_count, elapsed)
+
+    return HybridRetriever(
+        vector_store=store,
+        bm25_index=bm25_index,
+        config=config.retrieval,
+    )
+
+
+async def _run_ingestion(store: VectorStore, config) -> int:
+    """Ingest documents from configured folder. Returns chunk count."""
+    folder = config.paths.documents
+    if str(folder) and folder.exists():
+        ingested, skipped = ingest_folder(
+            folder,
+            store,
+            recursive=config.scanning.recursive,
+            chunk_size=config.chunking.chunk_size,
+            chunk_overlap=config.chunking.chunk_overlap,
+        )
+        if ingested > 0:
+            logger.info("Ingested %d new files, skipped %d unchanged", ingested, skipped)
+        elif skipped > 0:
+            logger.info("All %d files unchanged, skipped", skipped)
+
+    return store.count()
+
+
 @cl.on_chat_start
 async def on_chat_start():
     try:
@@ -52,37 +88,11 @@ async def on_chat_start():
     store = VectorStore(embed_fn=embed_fn, client=chroma_client)
     cl.user_session.set("store", store)
 
-    # Ingest documents if a folder is configured and store is empty
-    doc_count = store.count()
-    if doc_count == 0 and str(config.paths.documents) and config.paths.documents.exists():
-        docs = load_folder(
-            config.paths.documents,
-            recursive=config.scanning.recursive,
-        )
-        if docs:
-            chunks = chunk_documents(
-                docs,
-                chunk_size=config.chunking.chunk_size,
-                chunk_overlap=config.chunking.chunk_overlap,
-            )
-            store.add_chunks(chunks)
-            doc_count = store.count()
-
-    # Build BM25 index from stored chunks
-    bm25_index = BM25Index()
-    if doc_count > 0:
-        start = time.time()
-        texts, metadatas = store.get_all_texts_and_metadatas()
-        bm25_index.build(texts, metadatas)
-        elapsed = time.time() - start
-        logger.info("BM25 index built from %d chunks in %.2fs", doc_count, elapsed)
+    # Auto-scan and ingest new/changed documents on startup
+    doc_count = await _run_ingestion(store, config)
 
-    # Create hybrid retriever
-    retriever = HybridRetriever(
-        vector_store=store,
-        bm25_index=bm25_index,
-        config=config.retrieval,
-    )
+    # Build retriever
+    retriever = _build_retriever(store, config)
     cl.user_session.set("retriever", retriever)
 
     # Load cross-encoder reranker
@@ -94,16 +104,66 @@ async def on_chat_start():
     cl.user_session.set("condenser", condenser)
     cl.user_session.set("chat_history", [])
 
+    # Set up Chainlit settings panel
+    settings = await cl.ChatSettings(
+        [
+            cl.input_widget.TextInput(
+                id="documents_folder",
+                label="Documents Folder",
+                initial=str(config.paths.documents),
+            ),
+            cl.input_widget.Switch(
+                id="recursive_scan",
+                label="Recursive Scanning",
+                initial=config.scanning.recursive,
+            ),
+        ]
+    ).send()
+
     if doc_count > 0:
         await cl.Message(
             content=f"{doc_count} chunks indexed. Ask me anything!"
         ).send()
     else:
         await cl.Message(
-            content="No documents indexed. Configure your documents folder in `config.yaml` to get started."
+            content="No documents indexed. Configure your documents folder in the settings panel (gear icon) or `config.yaml`."
         ).send()
 
 
+@cl.on_settings_update
+async def on_settings_update(settings: dict):
+    """Re-scan and ingest when settings change."""
+    from pathlib import Path
+
+    config = cl.user_session.get("config")
+    store = cl.user_session.get("store")
+
+    folder = Path(settings["documents_folder"]).expanduser()
+    recursive = settings["recursive_scan"]
+
+    if not folder.exists():
+        await cl.Message(
+            content=f"Folder not found: `{folder}`"
+        ).send()
+        return
+
+    # Update config in session
+    config.paths.documents = folder
+    config.scanning.recursive = recursive
+
+    await cl.Message(content=f"Scanning `{folder}`...").send()
+
+    doc_count = await _run_ingestion(store, config)
+
+    # Rebuild retriever with updated store
+    retriever = _build_retriever(store, config)
+    cl.user_session.set("retriever", retriever)
+
+    await cl.Message(
+        content=f"Done. {doc_count} chunks indexed."
+    ).send()
+
+
 @cl.on_message
 async def on_message(message: cl.Message):
     config = cl.user_session.get("config")
@@ -116,7 +176,7 @@ async def on_message(message: cl.Message):
     store = cl.user_session.get("store")
     if store is None or store.count() == 0:
         await cl.Message(
-            content="No documents indexed yet. Configure your documents folder in `config.yaml` first."
+            content="No documents indexed yet. Configure your documents folder in the settings panel (gear icon) or `config.yaml`."
         ).send()
         return
 

src/ingestion/ingest.py

@@ -0,0 +1,77 @@
+"""Incremental document ingestion with change detection."""
+
+import logging
+from pathlib import Path
+
+from src.ingestion.chunker import chunk_documents
+from src.ingestion.loader import load_file
+from src.ingestion.scanner import compute_file_hash, scan_folder
+from src.retrieval.vector_store import VectorStore
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_folder(
+    folder: Path,
+    store: VectorStore,
+    *,
+    recursive: bool = True,
+    chunk_size: int = 512,
+    chunk_overlap: int = 100,
+) -> tuple[int, int]:
+    """Scan a folder and ingest new or changed documents.
+
+    Returns (ingested_count, skipped_count) — files ingested vs unchanged.
+    """
+    files = scan_folder(folder, recursive=recursive)
+
+    if not files:
+        return 0, 0
+
+    existing_hashes = _get_stored_file_hashes(store)
+
+    ingested = 0
+    skipped = 0
+
+    for file_path in files:
+        file_hash = compute_file_hash(file_path)
+        if file_hash in existing_hashes:
+            skipped += 1
+            continue
+
+        docs = load_file(file_path, folder)
+        if not docs:
+            continue
+
+        # Attach file_hash to document metadata before chunking
+        for doc in docs:
+            doc.metadata["file_hash"] = file_hash
+
+        chunks = chunk_documents(
+            docs,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+
+        # Propagate file_hash to chunk metadata
+        for chunk in chunks:
+            chunk.metadata["file_hash"] = file_hash
+
+        store.add_chunks(chunks)
+        ingested += 1
+        logger.info("Ingested %s (%d chunks)", file_path.name, len(chunks))
+
+    return ingested, skipped
+
+
+def _get_stored_file_hashes(store: VectorStore) -> set[str]:
+    """Extract unique file_hash values from all stored chunks."""
+    if store.count() == 0:
+        return set()
+
+    _, metadatas = store.get_all_texts_and_metadatas()
+    return {
+        m["file_hash"]
+        for m in metadatas
+        if "file_hash" in m
+    }

src/ingestion/loader.py

@@ -19,6 +19,25 @@
 }
 
 
+def load_file(file_path: Path, root: Path) -> list[Document]:
+    """Load a single supported document file.
+
+    Args:
+        file_path: Path to the file.
+        root: Root folder for computing relative paths.
+
+    Returns:
+        List of Documents (one per page for PDFs, one for other formats).
+    """
+    doc_type = EXTENSION_TO_DOC_TYPE.get(file_path.suffix.lower())
+    if doc_type is None:
+        return []
+
+    if doc_type == "pdf":
+        return _load_pdf(file_path, root)
+    return _load_single_file(file_path, root, doc_type)
+
+
 def load_folder(path: Path, *, recursive: bool = True) -> list[Document]:
     """Load all supported documents from a folder."""
     path = Path(path)

src/ingestion/scanner.py

@@ -0,0 +1,29 @@
+"""Folder scanning and file-level change detection."""
+
+import hashlib
+from pathlib import Path
+
+SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt", ".md"}
+
+
+def compute_file_hash(path: Path) -> str:
+    """Compute MD5 hash of a file's raw contents."""
+    return hashlib.md5(path.read_bytes()).hexdigest()
+
+
+def scan_folder(path: Path, *, recursive: bool = True) -> list[Path]:
+    """Discover all supported document files in a folder.
+
+    Returns sorted list of paths to supported files.
+    """
+    if recursive:
+        all_files = path.rglob("*")
+    else:
+        all_files = path.glob("*")
+
+    files = [
+        f for f in all_files
+        if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
+    ]
+    files.sort()
+    return files

tests/test_ingest.py

@@ -0,0 +1,67 @@
+"""Tests for incremental ingestion."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from src.ingestion.ingest import ingest_folder
+
+
+@pytest.fixture
+def doc_folder(tmp_path):
+    """Create a folder with a test text file."""
+    (tmp_path / "notes.txt").write_text("Some notes about testing.")
+    return tmp_path
+
+
+@pytest.fixture
+def mock_store():
+    store = MagicMock()
+    store.count.return_value = 0
+    store.get_all_texts_and_metadatas.return_value = ([], [])
+    return store
+
+
+class TestIngestFolder:
+    def test_ingests_new_file(self, doc_folder, mock_store):
+        ingested, skipped = ingest_folder(
+            doc_folder, mock_store, chunk_size=512, chunk_overlap=100
+        )
+        assert ingested == 1
+        assert skipped == 0
+        mock_store.add_chunks.assert_called_once()
+
+    def test_skips_unchanged_file(self, doc_folder, mock_store):
+        # Simulate file already in store by returning its hash
+        import hashlib
+        file_hash = hashlib.md5(
+            (doc_folder / "notes.txt").read_bytes()
+        ).hexdigest()
+        mock_store.count.return_value = 5
+        mock_store.get_all_texts_and_metadatas.return_value = (
+            ["text"],
+            [{"file_hash": file_hash}],
+        )
+
+        ingested, skipped = ingest_folder(doc_folder, mock_store)
+        assert ingested == 0
+        assert skipped == 1
+        mock_store.add_chunks.assert_not_called()
+
+    def test_chunks_have_file_hash(self, doc_folder, mock_store):
+        ingest_folder(doc_folder, mock_store)
+        chunks = mock_store.add_chunks.call_args[0][0]
+        for chunk in chunks:
+            assert "file_hash" in chunk.metadata
+
+    def test_empty_folder(self, tmp_path, mock_store):
+        ingested, skipped = ingest_folder(tmp_path, mock_store)
+        assert ingested == 0
+        assert skipped == 0
+
+    def test_relative_path_in_metadata(self, doc_folder, mock_store):
+        ingest_folder(doc_folder, mock_store)
+        chunks = mock_store.add_chunks.call_args[0][0]
+        for chunk in chunks:
+            rel_path = chunk.metadata.get("relative_path", "")
+            assert not rel_path.startswith("/")