diff --git a/src/lemoncrow/pro/capabilities/code_context/engine.py b/src/lemoncrow/pro/capabilities/code_context/engine.py index cb5884256..1a0cc595d 100644 --- a/src/lemoncrow/pro/capabilities/code_context/engine.py +++ b/src/lemoncrow/pro/capabilities/code_context/engine.py @@ -975,6 +975,31 @@ def _safe_relpath(repo_root: Path, path: Path) -> str: return str(resolved) +def _dir_cached_relpath(repo_root: Path) -> Callable[[Path], str]: + """``_safe_relpath`` that resolves each directory once instead of once per file. + + ``Path.resolve`` lstat()s every component of the path, so one call per file of + a 17k-file tree is ~2 s of syscalls on every index run. A file that is not + itself a symlink resolves to its parent's resolution joined with its name, and + parents repeat, so only they need resolving -- and only once each. + """ + parents: dict[Path, Path] = {} + + def relpath(path: Path) -> str: + if path.is_symlink(): + return _safe_relpath(repo_root, path) + parent = parents.get(path.parent) + if parent is None: + parent = parents[path.parent] = path.parent.resolve() + resolved = parent / path.name + try: + return str(resolved.relative_to(repo_root)) + except ValueError: + return str(resolved) + + return relpath + + # Binary/non-text extensions to skip when walking non-source files for the # Python-text-search fallback. Kept small and conservative — the walk is # only reached when rg is unavailable and the FTS index is empty. @@ -4287,9 +4312,11 @@ def _index_repo_unsafe( to_extract: list[tuple[Path, bytes]] = [] # (path, source_bytes) current_paths: set[str] = set() + stale: list[str] = [] + relpath = _dir_cached_relpath(self.repo_root) for path in all_files: - rel = _safe_relpath(self.repo_root, path) + rel = relpath(path) current_paths.add(rel) try: stat = path.stat() @@ -4297,7 +4324,7 @@ def _index_repo_unsafe( continue if stat.st_size > _MAX_FILE_BYTES: if rel in existing: - self._delete_file_index(conn, rel) + stale.append(rel) continue previous = existing.get(rel) # Fast path: a file whose (size, mtime) matches the indexed row @@ -4328,12 +4355,11 @@ def _index_repo_unsafe( (int(stat.st_mtime_ns), self.repo_id, rel), ) continue - self._delete_file_index(conn, rel) + stale.append(rel) to_extract.append((path, source_bytes)) removed_paths = set(existing.keys()) - current_paths - for rel in sorted(removed_paths): - self._delete_file_index(conn, rel) + self._delete_files_index(conn, [*stale, *sorted(removed_paths)]) if to_extract: paths = [item[0] for item in to_extract] @@ -4468,47 +4494,47 @@ def _stamp_indexer_semantics_version(self, conn: sqlite3.Connection) -> None: (str(_CODE_INDEXER_SEMANTICS_VERSION),), ) - def _delete_file_index(self, conn: sqlite3.Connection, rel: str) -> None: - conn.execute("DELETE FROM file_line_fts WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel)) - conn.execute("DELETE FROM file_path_trigram WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel)) - conn.execute( - """ - DELETE FROM symbol_trigram - WHERE symbol_id IN ( - SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ? - ) - """, - (self.repo_id, rel), - ) + def _delete_files_index(self, conn: sqlite3.Connection, rels: list[str]) -> None: + """Remove every indexed row for the files *rels*, in one pass per table. + + The FTS5 tables are keyed by rowid alone -- ``file_path`` and ``symbol_id`` + are UNINDEXED columns -- so any delete filtered on them scans the whole + table. Issued once per file, that was ~2 s per file on a 12M-line index: + an incremental run after a pull (hundreds of changed files) took longer + than the autosync subprocess's 600 s timeout, was killed before it + committed, and the next run started over. Filtering every table on the + whole batch at once costs one scan per FTS table per run. The regular + tables are indexed on these columns, so the batch costs them nothing. + """ + if not rels: + return + batch = json.dumps(rels) + paths = "SELECT value FROM json_each(?)" + symbols = f"SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path IN ({paths})" + conn.execute(f"DELETE FROM file_line_fts WHERE repo_id = ? AND file_path IN ({paths})", (self.repo_id, batch)) conn.execute( - """ - DELETE FROM symbol_fts - WHERE symbol_id IN ( - SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ? - ) - """, - (self.repo_id, rel), + f"DELETE FROM file_path_trigram WHERE repo_id = ? AND file_path IN ({paths})", (self.repo_id, batch) ) - # Prune persisted embeddings for this file's symbols *before* the symbols + conn.execute(f"DELETE FROM symbol_trigram WHERE symbol_id IN ({symbols})", (self.repo_id, batch)) + conn.execute(f"DELETE FROM symbol_fts WHERE symbol_id IN ({symbols})", (self.repo_id, batch)) + # Prune persisted embeddings for these files' symbols *before* the symbols # themselves. symbol_id encodes the file content hash, so an edited or # removed file yields fresh ids -- without this the old vectors orphan # (never overwritten, never cleaned) and pollute semantic ranking. The # vector table is created lazily, so guard against its absence. with contextlib.suppress(sqlite3.OperationalError): conn.execute( - """ - DELETE FROM symbol_vectors - WHERE repo_id = ? AND symbol_id IN ( - SELECT symbol_id FROM symbols WHERE repo_id = ? AND file_path = ? - ) - """, - (self.repo_id, self.repo_id, rel), + f"DELETE FROM symbol_vectors WHERE repo_id = ? AND symbol_id IN ({symbols})", + (self.repo_id, self.repo_id, batch), ) - conn.execute("DELETE FROM symbols WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel)) - conn.execute("DELETE FROM imports WHERE repo_id = ? AND source_file = ?", (self.repo_id, rel)) - conn.execute('DELETE FROM "references" WHERE repo_id = ? AND file_path = ?', (self.repo_id, rel)) - conn.execute("DELETE FROM call_edges WHERE repo_id = ? AND caller_file_path = ?", (self.repo_id, rel)) - conn.execute("DELETE FROM files WHERE repo_id = ? AND file_path = ?", (self.repo_id, rel)) + for table, column in ( + ("symbols", "file_path"), + ("imports", "source_file"), + ('"references"', "file_path"), + ("call_edges", "caller_file_path"), + ("files", "file_path"), + ): + conn.execute(f"DELETE FROM {table} WHERE repo_id = ? AND {column} IN ({paths})", (self.repo_id, batch)) def tool_index( self, @@ -8746,7 +8772,7 @@ def _build_symbol_embeddings(self, conn: sqlite3.Connection, index_version: int) # Skip symbols whose vector is already current. symbol_id encodes the file # content hash, so an unchanged symbol keeps its id across reindexes and is # skipped here; an edited symbol gets a new id and its stale vector is pruned - # by _delete_file_index. index_version stays provenance-only -- gating on it + # by _delete_files_index. index_version stays provenance-only -- gating on it # would make every reindex re-embed the whole repo instead of just the delta. fresh = self._ann_symbol_index.existing_stamped_ids(conn, embedder_name=embedder.name, embedding_dim=dim) pending = [sym for sym in (_row_to_symbol(row) for row in rows) if sym.symbol_id not in fresh] @@ -11533,6 +11559,8 @@ def _ensure_indexed(self) -> None: # autosync always on in practice; index will be built by the worker def _excluded(self, path: Path, patterns: list[str]) -> bool: + if not patterns: + return False rel = _safe_relpath(self.repo_root, path) return any(fnmatch.fnmatch(rel, pattern) for pattern in patterns) @@ -13454,8 +13482,7 @@ def _reindex_locked() -> None: return with self._connect() as conn: self._init_schema(conn) - for rel in rels: - self._delete_file_index(conn, rel) + self._delete_files_index(conn, rels) results = ( self._parallel_extract(existing_paths, total=len(existing_paths)) if existing_paths else [] ) diff --git a/tests/core/test_code_context.py b/tests/core/test_code_context.py index 7d1f4b97b..4b2972ede 100644 --- a/tests/core/test_code_context.py +++ b/tests/core/test_code_context.py @@ -2,6 +2,7 @@ import json import os +import re import sqlite3 import subprocess import time @@ -2209,6 +2210,112 @@ def test_incremental_index_updates_changed_and_removed_files(tmp_path: Path) -> assert int(row["n"]) == 0 +_FTS_TABLES = ("file_line_fts", "file_path_trigram", "symbol_fts", "symbol_trigram") +_FTS_DELETE = re.compile(r"\s*DELETE FROM (" + "|".join(_FTS_TABLES) + r")\s+WHERE") + + +def test_incremental_index_leaves_no_rows_for_replaced_or_removed_files(tmp_path: Path) -> None: + """Every table drops a changed file's old rows and a removed file's rows; the rest stay.""" + _write_fixture_repo(tmp_path) + engine = CodeContextEngine(tmp_path, db_path=tmp_path / "code.sqlite", autosync_enabled=False) + engine.index_repo() + untouched = "tests/test_checkout.py" + with engine._connect() as conn: + untouched_lines = conn.execute( + "SELECT COUNT(*) FROM file_line_fts WHERE file_path = ?", (untouched,) + ).fetchone()[0] + assert untouched_lines > 0 + + (tmp_path / "src" / "orders.py").write_text("class RenamedService:\n pass\n", encoding="utf-8") + (tmp_path / "src" / "checkout.py").unlink() + engine.index_repo(force=False) + + with engine._connect() as conn: + + def count(sql: str, *args: object) -> int: + return int(conn.execute(sql, args).fetchone()[0]) + + indexed = "SELECT file_path FROM files" + assert count(f"SELECT COUNT(*) FROM file_line_fts WHERE file_path NOT IN ({indexed})") == 0 + assert count(f"SELECT COUNT(*) FROM file_path_trigram WHERE file_path NOT IN ({indexed})") == 0 + assert count(f'SELECT COUNT(*) FROM "references" WHERE file_path NOT IN ({indexed})') == 0 + assert count(f"SELECT COUNT(*) FROM call_edges WHERE caller_file_path NOT IN ({indexed})") == 0 + assert count(f"SELECT COUNT(*) FROM imports WHERE source_file NOT IN ({indexed})") == 0 + live_symbols = "SELECT symbol_id FROM symbols" + assert count(f"SELECT COUNT(*) FROM symbol_fts WHERE symbol_id NOT IN ({live_symbols})") == 0 + assert count(f"SELECT COUNT(*) FROM symbol_trigram WHERE symbol_id NOT IN ({live_symbols})") == 0 + + assert count("SELECT COUNT(*) FROM files WHERE file_path = 'src/checkout.py'") == 0 + assert ( + count("SELECT COUNT(*) FROM file_line_fts WHERE file_path = 'src/orders.py' AND text LIKE '%OrderService%'") + == 0 + ) + assert ( + count("SELECT COUNT(*) FROM symbols WHERE file_path = 'src/orders.py' AND symbol_name = 'RenamedService'") + == 1 + ) + assert count("SELECT COUNT(*) FROM file_line_fts WHERE file_path = ?", untouched) == untouched_lines + + +def test_incremental_index_scans_each_fts_table_once_per_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The FTS5 tables are rowid-keyed, so a delete filtered on file_path or symbol_id + scans the whole table. Issued per file, an incremental run cost one full scan per + changed file -- ~2 s each on a 12M-line index -- and a run after a pull outlived + the autosync subprocess's 600 s timeout, so it never committed. + """ + _write_fixture_repo(tmp_path) + for i in range(6): + (tmp_path / "src" / f"extra{i}.py").write_text(f"def extra{i}():\n return {i}\n", encoding="utf-8") + engine = CodeContextEngine(tmp_path, db_path=tmp_path / "code.sqlite", autosync_enabled=False) + engine.index_repo() + for i in range(6): + (tmp_path / "src" / f"extra{i}.py").write_text(f"def extra{i}():\n return {i + 100}\n", encoding="utf-8") + (tmp_path / "src" / "checkout.py").unlink() + + statements: list[str] = [] + connect = engine._connect + + def traced_connect(*args: object, **kwargs: object) -> sqlite3.Connection: + conn = connect(*args, **kwargs) # type: ignore[arg-type] + conn.set_trace_callback(statements.append) + return conn + + monkeypatch.setattr(engine, "_connect", traced_connect) + stats = engine.index_repo(force=False) + + assert stats.files_indexed == 6 + deletes = [m.group(1) for s in statements if (m := _FTS_DELETE.match(s))] + assert sorted(deletes) == sorted(_FTS_TABLES), f"7 stale files cost {len(deletes)} FTS scans" + + +def test_dir_cached_relpath_resolves_exactly_like_safe_relpath(tmp_path: Path) -> None: + """Resolving each directory once must not change what any file resolves to.""" + from lemoncrow.pro.capabilities.code_context.engine import _dir_cached_relpath, _safe_relpath + + root = (tmp_path / "repo").resolve() + outside = (tmp_path / "outside").resolve() + (root / "real").mkdir(parents=True) + outside.mkdir() + (root / "real" / "a.py").write_text("x = 1\n", encoding="utf-8") + (outside / "b.py").write_text("y = 2\n", encoding="utf-8") + (root / "linked_dir").symlink_to(root / "real") + (root / "linked_file.py").symlink_to(root / "real" / "a.py") + (root / "escape.py").symlink_to(outside / "b.py") + (root / "escape_dir").symlink_to(outside) + paths = [ + root / "real" / "a.py", + root / "linked_dir" / "a.py", + root / "linked_file.py", + root / "escape.py", + root / "escape_dir" / "b.py", + root / "real" / "missing.py", + ] + + relpath = _dir_cached_relpath(root) + + assert [relpath(p) for p in paths] == [_safe_relpath(root, p) for p in paths] + + def test_search_symbols_filters_with_zoekt_candidate_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _write_fixture_repo(tmp_path) (tmp_path / "src" / "other.py").write_text(