fix: harden knowledge base phase-1 index safety

.github/workflows/ci.yml

@@ -84,3 +84,23 @@ jobs:
           tests/unit/test_platform.py
           tests/unit/test_mcp_market_service.py
           tests/unit/test_skill_market_service.py
+
+  frozen-rag-smoke:
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: pip install -e ".[dev,build]"
+
+      - name: Build frozen application
+        run: pyinstaller misaka.spec
+
+      - name: Run frozen RAG smoke test
+        run: .\dist\Misaka\Misaka.exe --rag-smoke

.github/workflows/release.yml

@@ -55,6 +55,9 @@ jobs:
       - name: Build with PyInstaller
         run: pyinstaller misaka.spec
 
+      - name: Run frozen RAG smoke test
+        run: .\dist\Misaka\Misaka.exe --rag-smoke
+
       - name: Install Inno Setup
         run: choco install innosetup -y
 

misaka.spec

@@ -19,6 +19,7 @@ from pathlib import Path
 import certifi
 import flet
 import flet_desktop
+from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs
 
 block_cipher = None
 
@@ -62,6 +63,12 @@ _certifi_datas = [
     (str(_certifi_pem), "certifi"),
 ]
 
+# sqlite-vec loads vec0 dynamically at runtime, so PyInstaller cannot infer
+# the native library from the import graph.  Collect its package data and
+# dynamic libraries explicitly; rank-bm25 also requires NumPy at runtime.
+_sqlite_vec_datas = collect_data_files("sqlite_vec")
+_sqlite_vec_binaries = collect_dynamic_libs("sqlite_vec")
+
 _datas = [
     (str(_i18n_dir / "en.json"), "misaka/i18n"),
     (str(_i18n_dir / "zh_CN.json"), "misaka/i18n"),
@@ -70,12 +77,13 @@ _datas = [
     *_flet_datas,
     *_flet_desktop_datas,
     *_certifi_datas,
+    *_sqlite_vec_datas,
 ]
 
 a = Analysis(
     [str(project_root / "misaka" / "main.py")],
     pathex=[str(project_root)],
-    binaries=[],
+    binaries=_sqlite_vec_binaries,
     datas=_datas,
     hiddenimports=[
         "misaka",
@@ -99,6 +107,22 @@ a = Analysis(
         "misaka.services.file",
         "misaka.services.file.file_service",
         "misaka.services.file.update_check_service",
+        "misaka.services.knowledge",
+        "misaka.services.knowledge.document_service",
+        "misaka.services.knowledge.frozen_smoke",
+        "misaka.services.knowledge.index_manager",
+        "misaka.services.knowledge.job_coordinator",
+        "misaka.services.knowledge.kb_service",
+        "misaka.services.knowledge.rag",
+        "misaka.services.knowledge.rag.abstractions",
+        "misaka.services.knowledge.rag.factory",
+        "misaka.services.knowledge.rag.langchain.chunker",
+        "misaka.services.knowledge.rag.langchain.embedding",
+        "misaka.services.knowledge.rag.langchain.parser",
+        "misaka.services.knowledge.rag.langchain.reranker",
+        "misaka.services.knowledge.rag.langchain.retriever",
+        "misaka.services.knowledge.rag.langchain.vector_store",
+        "misaka.services.knowledge.rag_orchestrator",
         "misaka.services.mcp",
         "misaka.services.mcp.mcp_service",
         "misaka.services.session",
@@ -154,6 +178,9 @@ a = Analysis(
         "watchdog",
         "watchdog.observers",
         "sqlite3",
+        "sqlite_vec",
+        "rank_bm25",
+        "numpy",
         "certifi",
     ],
     hookspath=[],
@@ -165,7 +192,6 @@ a = Analysis(
     excludes=[
         "tkinter",
         "matplotlib",
-        "numpy",
         "pandas",
         "scipy",
         "IPython",

misaka/db/database.py

@@ -15,6 +15,7 @@
 from misaka.db.models import (
     ChatSession,
     KBChunk,
+    KBCleanupJob,
     KBDocument,
     KnowledgeBase,
     Message,
@@ -362,6 +363,27 @@ def get_kb_chunks_by_document(self, doc_id: str) -> list[KBChunk]:
     def get_kb_chunks_by_kb(self, kb_id: str) -> list[KBChunk]:
         """Return all chunks for a knowledge base, ordered by chunk_index."""
 
+    @abstractmethod
+    def get_kb_chunks_by_index(
+        self, kb_id: str, index_version: str,
+    ) -> list[KBChunk]:
+        """Return chunks belonging to one immutable KB index version."""
+
+    @abstractmethod
+    def activate_kb_index(
+        self,
+        kb_id: str,
+        index_version: str,
+        chunks: list[KBChunk],
+        document_updates: dict[str, dict[str, Any]],
+        dimensions: int,
+    ) -> None:
+        """Atomically publish staged chunks and their corresponding document state."""
+
+    @abstractmethod
+    def delete_kb_chunks_by_index(self, kb_id: str, index_version: str) -> None:
+        """Remove persisted chunks belonging to a retired index version."""
+
     @abstractmethod
     def delete_kb_chunks_by_document(self, doc_id: str) -> None:
         """Delete all chunks belonging to a specific document."""
@@ -370,6 +392,32 @@ def delete_kb_chunks_by_document(self, doc_id: str) -> None:
     def update_kb_chunk_embedded(self, chunk_ids: list[str]) -> None:
         """Mark chunks as embedded (``is_embedded = 1``)."""
 
+    # ----- KB background jobs and durable cleanup -----
+
+    @abstractmethod
+    def create_kb_job(self, kb_id: str, document_id: str, operation: str) -> str:
+        """Create and return a durable KB operation record."""
+
+    @abstractmethod
+    def update_kb_job(self, job_id: str, status: str, error_message: str = "") -> None:
+        """Update a KB operation record."""
+
+    @abstractmethod
+    def create_kb_cleanup_job(
+        self, kb_id: str, index_version: str, operation: str, error_message: str,
+    ) -> str:
+        """Persist vector cleanup that must be retried."""
+
+    @abstractmethod
+    def get_pending_kb_cleanup_jobs(self) -> list[KBCleanupJob]:
+        """Return all cleanup jobs still awaiting successful vector deletion."""
+
+    @abstractmethod
+    def update_kb_cleanup_job(
+        self, job_id: str, status: str, error_message: str = "",
+    ) -> None:
+        """Record a cleanup attempt or completion."""
+
     # ----- Dashboard aggregation -----
 
     @abstractmethod

misaka/db/migrations.py

@@ -12,7 +12,7 @@
 logger = logging.getLogger(__name__)
 
 # Current schema version. Increment when adding new migrations.
-SCHEMA_VERSION = 6
+SCHEMA_VERSION = 7
 
 
 def run_migrations(conn: sqlite3.Connection) -> None:
@@ -43,6 +43,9 @@ def run_migrations(conn: sqlite3.Connection) -> None:
     if current < 6:
         _migrate_v6(conn)
 
+    if current < 7:
+        _migrate_v7(conn)
+
     _set_version(conn, SCHEMA_VERSION)
     conn.commit()
 
@@ -295,3 +298,71 @@ def _migrate_v6(conn: sqlite3.Connection) -> None:
             updated_at TEXT NOT NULL DEFAULT (datetime('now'))
         )
     """)
+
+
+def _migrate_v7(conn: sqlite3.Connection) -> None:
+    """Migration v7: versioned KB indexes and durable cleanup jobs.
+
+    Existing rows continue to address the legacy vector-table name through
+    the empty version string.  New writes always receive an opaque version
+    and are made visible only after a complete index has been built.
+    """
+    logger.info("Running migration v7")
+    existing_tables = {
+        row[0]
+        for row in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        ).fetchall()
+    }
+    if "knowledge_bases" in existing_tables:
+        kb_columns = _get_column_names(conn, "knowledge_bases")
+        if "active_index_version" not in kb_columns:
+            conn.execute(
+                "ALTER TABLE knowledge_bases "
+                "ADD COLUMN active_index_version TEXT NOT NULL DEFAULT ''"
+            )
+
+    if "kb_chunks" in existing_tables:
+        chunk_columns = _get_column_names(conn, "kb_chunks")
+        if "index_version" not in chunk_columns:
+            conn.execute(
+                "ALTER TABLE kb_chunks "
+                "ADD COLUMN index_version TEXT NOT NULL DEFAULT ''"
+            )
+        conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_kb_chunks_index_version
+                ON kb_chunks(knowledge_base_id, index_version)
+        """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS kb_cleanup_jobs (
+            id TEXT PRIMARY KEY,
+            knowledge_base_id TEXT NOT NULL,
+            index_version TEXT NOT NULL DEFAULT '',
+            operation TEXT NOT NULL,
+            status TEXT NOT NULL DEFAULT 'pending',
+            attempts INTEGER NOT NULL DEFAULT 0,
+            error_message TEXT NOT NULL DEFAULT '',
+            created_at TEXT NOT NULL DEFAULT (datetime('now')),
+            updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_kb_cleanup_jobs_pending
+            ON kb_cleanup_jobs(status, created_at)
+    """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS kb_jobs (
+            id TEXT PRIMARY KEY,
+            knowledge_base_id TEXT NOT NULL,
+            document_id TEXT NOT NULL DEFAULT '',
+            operation TEXT NOT NULL,
+            status TEXT NOT NULL DEFAULT 'queued',
+            error_message TEXT NOT NULL DEFAULT '',
+            created_at TEXT NOT NULL DEFAULT (datetime('now')),
+            updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_kb_jobs_active
+            ON kb_jobs(knowledge_base_id, status, updated_at)
+    """)

misaka/db/models.py

@@ -383,6 +383,11 @@ class KnowledgeBase:
 
     status: Literal["active", "building", "error"] = "active"
 
+    # The version suffix of the vector index currently served to chat.  An
+    # empty value refers to the pre-versioning table name used by v6 and
+    # earlier databases.
+    active_index_version: str = ""
+
     created_at: str = ""
     updated_at: str = ""
 
@@ -428,9 +433,28 @@ class KBChunk:
 
     is_embedded: int = 0
 
+    # Chunks are staged under a new index version and become visible only
+    # once that version has been atomically activated.
+    index_version: str = ""
+
     created_at: str = ""
 
 
+@dataclass
+class KBCleanupJob:
+    """A durable retry record for vector-index cleanup."""
+
+    id: str
+    knowledge_base_id: str
+    index_version: str
+    operation: str
+    status: str = "pending"
+    attempts: int = 0
+    error_message: str = ""
+    created_at: str = ""
+    updated_at: str = ""
+
+
 @dataclass
 class KBSearchResult:
     """A single RAG retrieval result (runtime model, not persisted)."""

misaka/db/row_mappers.py

@@ -108,6 +108,7 @@ def row_to_knowledge_base(row: sqlite3.Row) -> KnowledgeBase:
         document_count=row["document_count"],
         chunk_count=row["chunk_count"],
         status=row["status"],
+        active_index_version=row["active_index_version"],
         created_at=row["created_at"],
         updated_at=row["updated_at"],
     )
@@ -143,5 +144,6 @@ def row_to_kb_chunk(row: sqlite3.Row) -> KBChunk:
         end_char=row["end_char"],
         metadata_json=row["metadata_json"],
         is_embedded=row["is_embedded"],
+        index_version=row["index_version"],
         created_at=row["created_at"],
     )