fix(code-index): delete a reindex's stale rows in one pass per FTS table

src/lemoncrow/pro/capabilities/code_context/engine.py

@@ -975,6 +975,31 @@ def _safe_relpath(repo_root: Path, path: Path) -> str:
         return str(resolved)
 
 
+def _dir_cached_relpath(repo_root: Path) -> Callable[[Path], str]:
+    """``_safe_relpath`` that resolves each directory once instead of once per file.
+
+    ``Path.resolve`` lstat()s every component of the path, so one call per file of
+    a 17k-file tree is ~2 s of syscalls on every index run. A file that is not
+    itself a symlink resolves to its parent's resolution joined with its name, and
+    parents repeat, so only they need resolving -- and only once each.
+    """
+    parents: dict[Path, Path] = {}
+
+    def relpath(path: Path) -> str:
+        if path.is_symlink():
+            return _safe_relpath(repo_root, path)
+        parent = parents.get(path.parent)
+        if parent is None:
+            parent = parents[path.parent] = path.parent.resolve()
+        resolved = parent / path.name
+        try:
+            return str(resolved.relative_to(repo_root))
+        except ValueError:
+            return str(resolved)
+
+    return relpath
+
+
 # Binary/non-text extensions to skip when walking non-source files for the
 # Python-text-search fallback.  Kept small and conservative — the walk is
 # only reached when rg is unavailable and the FTS index is empty.
@@ -4287,17 +4312,19 @@ def _index_repo_unsafe(
 
                 to_extract: list[tuple[Path, bytes]] = []  # (path, source_bytes)
                 current_paths: set[str] = set()
+                stale: list[str] = []
+                relpath = _dir_cached_relpath(self.repo_root)
 
                 for path in all_files:
-                    rel = _safe_relpath(self.repo_root, path)
+                    rel = relpath(path)
                     current_paths.add(rel)
                     try:
                         stat = path.stat()
                     except OSError:
                         continue
                     if stat.st_size > _MAX_FILE_BYTES:
                         if rel in existing:
-                            self._delete_file_index(conn, rel)
+                            stale.append(rel)
                         continue
                     previous = existing.get(rel)
                     # Fast path: a file whose (size, mtime) matches the indexed row
@@ -4328,12 +4355,11 @@ def _index_repo_unsafe(
                             (int(stat.st_mtime_ns), self.repo_id, rel),
                         )
                         continue
-                    self._delete_file_index(conn, rel)
+                    stale.append(rel)
                     to_extract.append((path, source_bytes))
 
                 removed_paths = set(existing.keys()) - current_paths
-                for rel in sorted(removed_paths):
-                    self._delete_file_index(conn, rel)
+                self._delete_files_index(conn, [*stale, *sorted(removed_paths)])
 
                 if to_extract:
                     paths = [item[0] for item in to_extract]
@@ -4468,47 +4494,47 @@ def _stamp_indexer_semantics_version(self, conn: sqlite3.Connection) -> None:
             (str(_CODE_INDEXER_SEMANTICS_VERSION),),
         )
 
-    def _delete_file_index(self, conn: sqlite3.Connection, rel: str) -> None:
-        conn.execute("DELETE FROM file_line_fts WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel))
-        conn.execute("DELETE FROM file_path_trigram WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel))
-        conn.execute(
-            """
-            DELETE FROM symbol_trigram
-            WHERE symbol_id IN (
-                SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ?
-            )
-            """,
-            (self.repo_id, rel),
-        )
+    def _delete_files_index(self, conn: sqlite3.Connection, rels: list[str]) -> None:
+        """Remove every indexed row for the files *rels*, in one pass per table.
+
+        The FTS5 tables are keyed by rowid alone -- ``file_path`` and ``symbol_id``
+        are UNINDEXED columns -- so any delete filtered on them scans the whole
+        table. Issued once per file, that was ~2 s per file on a 12M-line index:
+        an incremental run after a pull (hundreds of changed files) took longer
+        than the autosync subprocess's 600 s timeout, was killed before it
+        committed, and the next run started over. Filtering every table on the
+        whole batch at once costs one scan per FTS table per run. The regular
+        tables are indexed on these columns, so the batch costs them nothing.
+        """
+        if not rels:
+            return
+        batch = json.dumps(rels)
+        paths = "SELECT value FROM json_each(?)"
+        symbols = f"SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path IN ({paths})"
+        conn.execute(f"DELETE FROM file_line_fts WHERE repo_id = ? AND file_path IN ({paths})", (self.repo_id, batch))
         conn.execute(
-            """
-            DELETE FROM symbol_fts
-            WHERE symbol_id IN (
-                SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ?
-            )
-            """,
-            (self.repo_id, rel),
+            f"DELETE FROM file_path_trigram WHERE repo_id = ? AND file_path IN ({paths})", (self.repo_id, batch)
         )
-        # Prune persisted embeddings for this file's symbols *before* the symbols
+        conn.execute(f"DELETE FROM symbol_trigram WHERE symbol_id IN ({symbols})", (self.repo_id, batch))
+        conn.execute(f"DELETE FROM symbol_fts WHERE symbol_id IN ({symbols})", (self.repo_id, batch))
+        # Prune persisted embeddings for these files' symbols *before* the symbols
         # themselves. symbol_id encodes the file content hash, so an edited or
         # removed file yields fresh ids -- without this the old vectors orphan
         # (never overwritten, never cleaned) and pollute semantic ranking. The
         # vector table is created lazily, so guard against its absence.
         with contextlib.suppress(sqlite3.OperationalError):
             conn.execute(
-                """
-                DELETE FROM symbol_vectors
-                WHERE repo_id = ? AND symbol_id IN (
-                    SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ?
-                )
-                """,
-                (self.repo_id, self.repo_id, rel),
+                f"DELETE FROM symbol_vectors WHERE repo_id = ? AND symbol_id IN ({symbols})",
+                (self.repo_id, self.repo_id, batch),
             )
-        conn.execute("DELETE FROM symbols WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel))
-        conn.execute("DELETE FROM imports WHERE repo_id = ? AND source_file = ?", (self.repo_id, rel))
-        conn.execute('DELETE FROM "references" WHERE repo_id = ? AND file_path = ?', (self.repo_id, rel))
-        conn.execute("DELETE FROM call_edges WHERE repo_id = ? AND caller_file_path = ?", (self.repo_id, rel))
-        conn.execute("DELETE FROM files WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel))
+        for table, column in (
+            ("symbols", "file_path"),
+            ("imports", "source_file"),
+            ('"references"', "file_path"),
+            ("call_edges", "caller_file_path"),
+            ("files", "file_path"),
+        ):
+            conn.execute(f"DELETE FROM {table} WHERE repo_id = ? AND {column} IN ({paths})", (self.repo_id, batch))
 
     def tool_index(
         self,
@@ -8746,7 +8772,7 @@ def _build_symbol_embeddings(self, conn: sqlite3.Connection, index_version: int)
         # Skip symbols whose vector is already current. symbol_id encodes the file
         # content hash, so an unchanged symbol keeps its id across reindexes and is
         # skipped here; an edited symbol gets a new id and its stale vector is pruned
-        # by _delete_file_index. index_version stays provenance-only -- gating on it
+        # by _delete_files_index. index_version stays provenance-only -- gating on it
         # would make every reindex re-embed the whole repo instead of just the delta.
         fresh = self._ann_symbol_index.existing_stamped_ids(conn, embedder_name=embedder.name, embedding_dim=dim)
         pending = [sym for sym in (_row_to_symbol(row) for row in rows) if sym.symbol_id not in fresh]
@@ -11533,6 +11559,8 @@ def _ensure_indexed(self) -> None:
         # autosync always on in practice; index will be built by the worker
 
     def _excluded(self, path: Path, patterns: list[str]) -> bool:
+        if not patterns:
+            return False
         rel = _safe_relpath(self.repo_root, path)
         return any(fnmatch.fnmatch(rel, pattern) for pattern in patterns)
 
@@ -13454,8 +13482,7 @@ def _reindex_locked() -> None:
                     return
                 with self._connect() as conn:
                     self._init_schema(conn)
-                    for rel in rels:
-                        self._delete_file_index(conn, rel)
+                    self._delete_files_index(conn, rels)
                     results = (
                         self._parallel_extract(existing_paths, total=len(existing_paths)) if existing_paths else []
                     )

tests/core/test_code_context.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import re
 import sqlite3
 import subprocess
 import time
@@ -2209,6 +2210,112 @@ def test_incremental_index_updates_changed_and_removed_files(tmp_path: Path) ->
     assert int(row["n"]) == 0
 
 
+_FTS_TABLES = ("file_line_fts", "file_path_trigram", "symbol_fts", "symbol_trigram")
+_FTS_DELETE = re.compile(r"\s*DELETE FROM (" + "|".join(_FTS_TABLES) + r")\s+WHERE")
+
+
+def test_incremental_index_leaves_no_rows_for_replaced_or_removed_files(tmp_path: Path) -> None:
+    """Every table drops a changed file's old rows and a removed file's rows; the rest stay."""
+    _write_fixture_repo(tmp_path)
+    engine = CodeContextEngine(tmp_path, db_path=tmp_path / "code.sqlite", autosync_enabled=False)
+    engine.index_repo()
+    untouched = "tests/test_checkout.py"
+    with engine._connect() as conn:
+        untouched_lines = conn.execute(
+            "SELECT COUNT(*) FROM file_line_fts WHERE file_path = ?", (untouched,)
+        ).fetchone()[0]
+    assert untouched_lines > 0
+
+    (tmp_path / "src" / "orders.py").write_text("class RenamedService:\n    pass\n", encoding="utf-8")
+    (tmp_path / "src" / "checkout.py").unlink()
+    engine.index_repo(force=False)
+
+    with engine._connect() as conn:
+
+        def count(sql: str, *args: object) -> int:
+            return int(conn.execute(sql, args).fetchone()[0])
+
+        indexed = "SELECT file_path FROM files"
+        assert count(f"SELECT COUNT(*) FROM file_line_fts WHERE file_path NOT IN ({indexed})") == 0
+        assert count(f"SELECT COUNT(*) FROM file_path_trigram WHERE file_path NOT IN ({indexed})") == 0
+        assert count(f'SELECT COUNT(*) FROM "references" WHERE file_path NOT IN ({indexed})') == 0
+        assert count(f"SELECT COUNT(*) FROM call_edges WHERE caller_file_path NOT IN ({indexed})") == 0
+        assert count(f"SELECT COUNT(*) FROM imports WHERE source_file NOT IN ({indexed})") == 0
+        live_symbols = "SELECT symbol_id FROM symbols"
+        assert count(f"SELECT COUNT(*) FROM symbol_fts WHERE symbol_id NOT IN ({live_symbols})") == 0
+        assert count(f"SELECT COUNT(*) FROM symbol_trigram WHERE symbol_id NOT IN ({live_symbols})") == 0
+
+        assert count("SELECT COUNT(*) FROM files WHERE file_path = 'src/checkout.py'") == 0
+        assert (
+            count("SELECT COUNT(*) FROM file_line_fts WHERE file_path = 'src/orders.py' AND text LIKE '%OrderService%'")
+            == 0
+        )
+        assert (
+            count("SELECT COUNT(*) FROM symbols WHERE file_path = 'src/orders.py' AND symbol_name = 'RenamedService'")
+            == 1
+        )
+        assert count("SELECT COUNT(*) FROM file_line_fts WHERE file_path = ?", untouched) == untouched_lines
+
+
+def test_incremental_index_scans_each_fts_table_once_per_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """The FTS5 tables are rowid-keyed, so a delete filtered on file_path or symbol_id
+    scans the whole table. Issued per file, an incremental run cost one full scan per
+    changed file -- ~2 s each on a 12M-line index -- and a run after a pull outlived
+    the autosync subprocess's 600 s timeout, so it never committed.
+    """
+    _write_fixture_repo(tmp_path)
+    for i in range(6):
+        (tmp_path / "src" / f"extra{i}.py").write_text(f"def extra{i}():\n    return {i}\n", encoding="utf-8")
+    engine = CodeContextEngine(tmp_path, db_path=tmp_path / "code.sqlite", autosync_enabled=False)
+    engine.index_repo()
+    for i in range(6):
+        (tmp_path / "src" / f"extra{i}.py").write_text(f"def extra{i}():\n    return {i + 100}\n", encoding="utf-8")
+    (tmp_path / "src" / "checkout.py").unlink()
+
+    statements: list[str] = []
+    connect = engine._connect
+
+    def traced_connect(*args: object, **kwargs: object) -> sqlite3.Connection:
+        conn = connect(*args, **kwargs)  # type: ignore[arg-type]
+        conn.set_trace_callback(statements.append)
+        return conn
+
+    monkeypatch.setattr(engine, "_connect", traced_connect)
+    stats = engine.index_repo(force=False)
+
+    assert stats.files_indexed == 6
+    deletes = [m.group(1) for s in statements if (m := _FTS_DELETE.match(s))]
+    assert sorted(deletes) == sorted(_FTS_TABLES), f"7 stale files cost {len(deletes)} FTS scans"
+
+
+def test_dir_cached_relpath_resolves_exactly_like_safe_relpath(tmp_path: Path) -> None:
+    """Resolving each directory once must not change what any file resolves to."""
+    from lemoncrow.pro.capabilities.code_context.engine import _dir_cached_relpath, _safe_relpath
+
+    root = (tmp_path / "repo").resolve()
+    outside = (tmp_path / "outside").resolve()
+    (root / "real").mkdir(parents=True)
+    outside.mkdir()
+    (root / "real" / "a.py").write_text("x = 1\n", encoding="utf-8")
+    (outside / "b.py").write_text("y = 2\n", encoding="utf-8")
+    (root / "linked_dir").symlink_to(root / "real")
+    (root / "linked_file.py").symlink_to(root / "real" / "a.py")
+    (root / "escape.py").symlink_to(outside / "b.py")
+    (root / "escape_dir").symlink_to(outside)
+    paths = [
+        root / "real" / "a.py",
+        root / "linked_dir" / "a.py",
+        root / "linked_file.py",
+        root / "escape.py",
+        root / "escape_dir" / "b.py",
+        root / "real" / "missing.py",
+    ]
+
+    relpath = _dir_cached_relpath(root)
+
+    assert [relpath(p) for p in paths] == [_safe_relpath(root, p) for p in paths]
+
+
 def test_search_symbols_filters_with_zoekt_candidate_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     _write_fixture_repo(tmp_path)
     (tmp_path / "src" / "other.py").write_text(