diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..ce5168c --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,25 @@ +# Workspace-level Cargo configuration. +# +# This file is read by `cargo` invocations from the repository root (where CI +# runs `cargo build --release --target x86_64-pc-windows-gnu --bin codescope`). +# The per-crate `server/.cargo/config.toml` (profile/lto tuning) is NOT +# consulted for root-level builds, so cross-cutting target flags belong here. + +[build] +# Use all available CPU cores for parallel compilation +jobs = -1 + +# Windows GNU target: statically link the MinGW runtime so `codescope.exe` is +# self-contained. This eliminates dynamic dependencies on libgcc_s_seh-1.dll, +# libstdc++-6.dll, and libwinpthread-1.dll — the root cause of: +# * ABI incompatibility between MinGW 14.0.0 (official release) and 16.1.0 +# (local dev) runtime DLLs. +# * Access-violation crashes after SQLite init (heap/exception-model +# mismatch between the statically-linked engine and a mismatched runtime +# DLL on PATH). +# * System-wide DLL pollution (PowerShell memory access violations). +# `-static` makes the gcc linker prefer static archives for every `-l` flag; +# Windows system DLLs (KERNEL32 / msvcrt / ucrtbase / ADVAPI32) only ship as +# import libraries and are linked dynamically regardless, so they are unaffected. +[target.x86_64-pc-windows-gnu] +rustflags = ["-C", "link-arg=-static"] diff --git a/.github/workflows/_ci.yml b/.github/workflows/_ci.yml index 34253f2..4f941ce 100644 --- a/.github/workflows/_ci.yml +++ b/.github/workflows/_ci.yml @@ -77,15 +77,30 @@ jobs: - name: Install deps (Windows) if: matrix.os == 'windows-2022' + shell: bash run: | - # Install MinGW-w64 toolchain (version pinned to match local dev: GCC 16.1.0) - choco install mingw --version=14.0.0 -y --no-progress - echo "C:\tools\mingw64\bin" >> $GITHUB_PATH + # WinLibs portable MinGW-w64: GCC 16.1.0 + MinGW-w64 14.0.0 + UCRT + + # SEH + POSIX threads (x86_64), pinned to match local dev toolchain. + # + # Why not chocolatey: the `mingw` chocolatey package version number + # does NOT correspond to the GCC version (package 14.0.0 ships a + # different GCC), so it cannot reliably pin GCC 16.1.0. WinLibs + # portable builds give a deterministic, exact toolchain. Combined + # with the static-link rustflags in .cargo/config.toml + # ([target.x86_64-pc-windows-gnu] -static), codescope.exe becomes + # self-contained and no longer depends on MinGW runtime DLLs, so + # the official release and local builds can no longer clash on + # libgcc_s_seh-1.dll / libstdc++-6.dll / libwinpthread-1.dll. + WINLIBS_URL="https://github.com/brechtsanders/winlibs_mingw/releases/download/16.1.0posix-14.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-16.1.0-mingw-w64ucrt-14.0.0-r3.7z" + curl -fsSL -o winlibs.7z "$WINLIBS_URL" + 7z x winlibs.7z -o"C:/winlibs" -y -bd + echo "C:\winlibs\mingw64\bin" >> $GITHUB_PATH # Add Rust Windows GNU target (Rust version pinned to 1.95.0 by dtolnay/rust-toolchain@1.95.0) rustup target add x86_64-pc-windows-gnu - echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV - echo "CXX=x86_64-w64-mingw32-g++" >> $GITHUB_ENV + echo "CC=gcc" >> $GITHUB_ENV + echo "CXX=g++" >> $GITHUB_ENV # CMake 4.4.0 and Ninja 1.13.2 are pre-installed on GitHub Actions Windows runners + /C/winlibs/mingw64/bin/gcc --version - name: Cache Cargo registry uses: actions/cache@v6 # v4.2.3 @@ -279,12 +294,17 @@ jobs: LBUG_FILE="liblbug.0.dylib" ;; *-windows) - # Windows: bundle lbug_shared.dll next to the .exe - LBUG_LIB="engine/third_party/ladybug/lib/windows/lbug_shared.dll" - LBUG_FILE="lbug_shared.dll" + # Windows: LadybugDB is intentionally disabled (see + # engine/CMakeLists.txt — HAS_LADYBUG undefined). No shared + # library to bundle; graph storage uses SQLite only. + LBUG_LIB="" ;; esac - cp -L "$LBUG_LIB" "package/$LBUG_FILE" + # Bundle LadybugDB shared lib (Linux/macOS only). Windows builds + # without LadybugDB, so LBUG_LIB is empty there — skip the copy. + if [ -n "$LBUG_LIB" ]; then + cp -L "$LBUG_LIB" "package/$LBUG_FILE" + fi # Set rpath to $ORIGIN (Linux) / @executable_path (macOS) so the # dynamic linker finds the bundled library in the same directory. if [[ "${{ matrix.os }}" == ubuntu-* ]]; then diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml new file mode 100644 index 0000000..5358ac5 --- /dev/null +++ b/.github/workflows/dev.yml @@ -0,0 +1,51 @@ +# Dev branch validation: cross-compile Windows binary from macOS runner +# to catch regressions in build.rs cross-compilation logic early. +name: Dev CI (Windows Cross-Compile) + +on: + workflow_dispatch: + push: + branches: [hhh] + +permissions: + contents: read + +concurrency: + group: dev-ci-${{ github.sha }} + cancel-in-progress: true + +jobs: + cross-compile-windows: + name: Cross-compile codescope.exe (macOS → Windows) + # Use macOS runner to test the macOS→Windows cross-compilation path + # (build.rs host-OS detection + MinGW cross-compiler selection). + # Native Windows builds are covered by the main CI on PR/main. + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 # v4.2.2 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.95.0 # pinned to match local dev + + - name: Install MinGW cross-compiler + run: | + brew install mingw-w64 + # Verify the cross-compiler is on PATH + x86_64-w64-mingw32-gcc --version + + - name: Add Windows GNU target + run: rustup target add x86_64-pc-windows-gnu + + - name: Cross-compile codescope.exe + run: cargo build --release --target x86_64-pc-windows-gnu --bin codescope + + - name: Verify binary + run: | + ls -lh target/x86_64-pc-windows-gnu/release/codescope.exe + file target/x86_64-pc-windows-gnu/release/codescope.exe + + - name: Check binary is PE (Windows executable) + run: | + file target/x86_64-pc-windows-gnu/release/codescope.exe | grep -i "PE32" + echo "✅ Windows cross-compile succeeded" diff --git a/CHANGELOG.md b/CHANGELOG.md index ceeb8b3..7457df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,27 @@ # Changelog -## Unreleased +## v0.2.4 (2026-07-24) + +Windows compilation stability — fully static-linked `codescope.exe` (zero MinGW runtime DLLs), LadybugDB disabled on Windows (SQLite-only), and critical cross-compilation bug fixes. + +### 🚀 New Features + +- **Fully static Windows binary**: `codescope.exe` statically links `libstdc++`, `libgcc`, and `libwinpthread` via `.cargo/config.toml` `-static` rustflag. Zero MinGW runtime DLL dependencies at runtime — no more `libstdc++-6.dll` / `libgcc_s_seh-1.dll` / `libwinpthread-1.dll` version conflicts. (`engine/CMakeLists.txt`, `server/build.rs`) +- **Dev branch CI for Windows**: New `.github/workflows/dev.yml` validates Windows cross-compilation on push to `dev` branch (manual `workflow_dispatch` also supported). (`dev.yml`) + +### 🐛 Bug Fixes + +- **Cross-compile host OS detection**: `build.rs` previously used `CARGO_CFG_TARGET_OS` to detect the build host, but during cross-compilation this returns the *target* OS ("windows"), causing `-DCMAKE_SYSTEM_NAME=Windows` to never be passed to CMake. Fixed to use `std::env::consts::OS` for the actual build host. (`server/build.rs`) +- **Cross-compile compiler selection**: `platform_default_compiler("windows")` returned `gcc`/`g++` on macOS, which resolves to native clang, not the MinGW cross-compiler. Fixed to detect cross-compilation and use `x86_64-w64-mingw32-gcc`/`x86_64-w64-mingw32-g++`. (`server/build.rs`) +- **Stale LadybugDB CMake cache on cross-compile**: The shared `build-release/` directory retained LadybugDB cache entries from a previous native macOS build. When cross-compiling to Windows, the stale macOS `.dylib` path was passed to the MinGW linker. Fixed by `unset(LADYBUG_LIBRARY CACHE)` in the Windows CMake branch and a Rust-side guard that skips all LadybugDB cache reading on Windows. (`engine/CMakeLists.txt`, `server/build.rs`) +- **LadybugDB now skipped on Windows**: The vendored `lbug_shared.lib` is a static archive of unverified MinGW ABI with no corresponding `.dll`. Windows builds now compile with `HAS_LADYBUG` undefined and use SQLite as the sole graph store. (`engine/CMakeLists.txt`) + +### 🧹 Chores + +- **Version bump**: 0.2.3 → 0.2.4 +- **Windows support now explicitly documented as "beta"** in both README.md and README.zh.md. + +--- ## v0.2.3 (2026-07-21) diff --git a/Cargo.lock b/Cargo.lock index 0cad009..2b3c302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "codescope" -version = "0.2.3" +version = "0.2.4" dependencies = [ "libc", "once_cell", diff --git a/README.md b/README.md index 004d2a0..dcbf10b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ It transforms source code into verifiable facts, understandable models, and inspectable evidence — enabling AI to validate claims against reality instead of hallucinating. -**Version**: v0.2.3 | **License**: Apache 2.0 +**Version**: v0.2.4 | **License**: Apache 2.0 --- @@ -231,6 +231,7 @@ codescope cli force_index_files '{"paths":["/path/to/test/file.rs"]}' |----------|-------------| | **macOS** | Xcode CLT, cmake, Rust (1.85+) | | **Linux** | build-essential, cmake, Rust (1.85+) | +| **Windows** ⚠️ **Beta** | MinGW-w64 14.0.0+, Rust `x86_64-pc-windows-gnu` target, cmake. LadybugDB/Cypher not available (SQLite-only). | ### Install Pre-built Binary @@ -277,7 +278,7 @@ codescope index-parallel /path/to/large/project --- -## 4. MCP Tools (42 Tools) +## 5. MCP Tools (42 Tools) ### Indexing @@ -401,7 +402,7 @@ Change impact → detect_changes --- -## 5. Knowledge Graph +## 6. Knowledge Graph CodeScope builds a **module-level knowledge graph** as a side product of the verification pipeline. It learns structured metadata about how the project is organized, what's important, what's redundant, and what it promises: @@ -426,7 +427,7 @@ Supported tables: `entity`, `relation`, `architecture_edge`, `module_edge`, `cap --- -## 6. Benchmark +## 7. Benchmark All benchmarks measured on **Apple M3 Max (36 GB RAM)**. Other hardware will produce different results — expect slower performance on less capable machines. @@ -501,7 +502,7 @@ Using code graphs instead of raw source files saves **~98.9% tokens** on average --- -## 7. Skills (Shell Wrappers) +## 8. Skills (Shell Wrappers) The `skills/` directory provides shell scripts that wrap common CodeScope queries so you don't need to remember the JSON schema: @@ -534,7 +535,7 @@ Each script calls `codescope cli ''` internally. See `ski --- -## 8. Environment Variables +## 9. Environment Variables | Variable | Default | Description | |----------|---------|-------------| @@ -552,8 +553,8 @@ Each script calls `codescope cli ''` internally. See `ski --- -## 9. License +## 10. License Apache 2.0 — see [LICENSE](LICENSE). -**CodeScope v0.2.3** — Built with Rust 2024 + C++23 + tree-sitter + SQLite. \ No newline at end of file +**CodeScope v0.2.4** — Built with Rust 2024 + C++23 + tree-sitter + SQLite. \ No newline at end of file diff --git a/README.zh.md b/README.zh.md index 14f689f..18ff1af 100644 --- a/README.zh.md +++ b/README.zh.md @@ -4,7 +4,7 @@ 它将源代码转化为可验证的事实、可理解的模型和可检查的证据 — 让 AI 能够根据现实验证断言，而非凭空编造。 -**版本**: v0.2.3 | **许可证**: Apache 2.0 +**版本**: v0.2.4 | **许可证**: Apache 2.0 --- @@ -231,6 +231,7 @@ codescope cli force_index_files '{"paths":["/path/to/test/file.rs"]}' |------|------| | **macOS** | Xcode CLT, cmake, Rust (1.85+) | | **Linux** | build-essential, cmake, Rust (1.85+) | +| **Windows** ⚠️ **Beta** | MinGW-w64 14.0.0+，Rust `x86_64-pc-windows-gnu` 目标，cmake。不支持 LadybugDB/Cypher（仅 SQLite）。| ### 安装预编译二进制 @@ -277,7 +278,7 @@ codescope index-parallel /path/to/large/project --- -## 4. MCP 工具（37 个工具） +## 5. MCP 工具（37 个工具） ### 索引 @@ -385,7 +386,7 @@ HTTP 路由 → get_routes --- -## 5. 知识图谱 +## 6. 知识图谱 CodeScope 在验证管线的副产品中构建了一个**模块级知识图谱**。它学习的是结构化的元数据 — 项目如何组织、什么重要、什么冗余、承诺了什么： @@ -410,7 +411,7 @@ get_knowledge_graph {"table":"capability","limit":10} --- -## 6. 性能基准 +## 7. 性能基准 所有基准测试在 **Apple M3 Max（36 GB RAM）** 上测得。其他硬件会产生不同的结果 — 性能较低的机器上预期会慢一些。 @@ -484,7 +485,7 @@ get_knowledge_graph {"table":"capability","limit":10} --- -## 7. Skills（Shell 封装脚本） +## 8. Skills（Shell 封装脚本） `skills/` 目录提供了一系列 Shell 脚本，封装了常用的 CodeScope 查询，无需记忆 JSON 参数格式： @@ -517,7 +518,7 @@ cd CodeScope --- -## 8. 环境变量 +## 9. 环境变量 | 变量 | 默认值 | 说明 | |------|--------|------| @@ -535,8 +536,8 @@ cd CodeScope --- -## 9. 许可证 +## 10. 许可证 Apache 2.0 — 详见 [LICENSE](LICENSE)。 -**CodeScope v0.2.3** — 使用 Rust 2024 + C++23 + tree-sitter + SQLite 构建。 \ No newline at end of file +**CodeScope v0.2.4** — 使用 Rust 2024 + C++23 + tree-sitter + SQLite 构建。 \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md index b6e5fee..af907e0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,52 +1,36 @@ -## v0.2.3 (2026-07-21) +## v0.2.4 (2026-07-24) -Windows beta support — cross-compilation from macOS to `x86_64-pc-windows-gnu` (MinGW), vendored LadybugDB Windows DLL, and CI pipeline. Plus critical bug fixes for the v0.2.2 LadybugDB migration that broke all query tools. +Windows compilation stability — fully static-linked `codescope.exe` (zero MinGW runtime DLLs), LadybugDB disabled on Windows (SQLite-only), and critical cross-compilation bug fixes. ### What changed | Area | Before | After | |------|--------|-------| -| **Windows support** | Not supported | Beta: cross-compiled `codescope.exe` (16MB PE32+), `lbug_shared.dll` bundled, CI `windows-2022` | -| **`search` tool** | Async FTS on `graph_nodes` (empty), returned no results | Synchronous `MATCH (n) WHERE n.name CONTAINS 'query'` via LadybugDB, ~1ms | -| **`isGraphReady()`** | Process-in-memory flag only (broken across worker/CLI processes) | Probes LadybugDB directly via `MATCH (n) RETURN count(*)` | -| **All query tools** | `findSymbolJson` prepare failed, `find_callers`/`find_callees` etc. "graph not ready" | All queries go through `entity`/`relation` tables or LadybugDB | -| **`graph_nodes`/`graph_edges`** | Still referenced by >20 SQL queries (returning empty) | Fully removed from all query paths. Tables still exist but unused | -| **`verify_integrity`** | Hangs indefinitely (no timeout) | 10s `QueryDeadlineGuard` | -| **`make fmt`** | Only checked recently modified files | Full project lint check (`lint-cpp-full`) | -| **Cross-compilation** | N/A | `build.rs` detects cross-compile vs native Windows, sets `CMAKE_SYSTEM_NAME` only when needed | +| **Windows runtime deps** | Depended on `libstdc++-6.dll`, `libgcc_s_seh-1.dll`, `libwinpthread-1.dll` — crash if MinGW version mismatched | Fully static via `-static` rustflag — single `codescope.exe` with zero MinGW DLL deps | +| **Windows LadybugDB** | Vendored `lbug_shared.lib` + `lbug_shared.dll` of unverified MinGW ABI | Disabled entirely — SQLite-only on Windows (`HAS_LADYBUG` undefined) | +| **Cross-compile host detection** | `build.rs` compared `CARGO_CFG_TARGET_OS` (returns *target* = "windows" during cross-compile) → `-DCMAKE_SYSTEM_NAME=Windows` never set | Uses `std::env::consts::OS` for actual build host | +| **Cross-compile compiler** | `platform_default_compiler("windows")` returned `gcc`/`g++` (macOS native clang) | Returns `x86_64-w64-mingw32-gcc`/`x86_64-w64-mingw32-g++` when cross-compiling | +| **Stale CMake cache** | macOS LadybugDB path persisted in shared `build-release/`, passed to MinGW linker | `unset(LADYBUG_LIBRARY CACHE)` on Windows branch + build.rs skips cache reading on Windows | +| **Dev branch CI** | No automated Windows validation on `dev` | New `.github/workflows/dev.yml`: triggers on push to `dev` (or manual dispatch) | +| **Windows support** | Unmarked | Documented as **beta** in README | ### Upgrade notes -- **Windows**: Use `--target x86_64-pc-windows-gnu` for Rust builds. Requires MinGW-w64 14.0.0+. `lbug_shared.dll` must be in the same directory as `codescope.exe` (bundled in the package). +- **Windows**: The single `codescope.exe` is now fully self-contained — no DLLs to bundle. LadybugDB/Cypher queries are unavailable on Windows; graph storage uses SQLite only. - **No breaking API changes**: All MCP tools maintain the same JSON response schema. -- **`graph_nodes`/`graph_edges` tables are no longer written**: If you have custom scripts that query these tables, migrate to `entity`/`relation`. -- **LadybugDB is now required for graph queries**: Without LadybugDB, all query tools return "graph not ready" or "LadybugDB not compiled". -### Bug fixes (v0.2.2 migration fallout) +### Bug fixes | # | Bug | Root cause | Fix | |---|-----|------------|-----| -| 1 | `findSymbolJson` prepare failed | SQL query referenced `graph_nodes` (empty) | Query `entity` instead | -| 2 | All tools "graph not ready" | `isGraphReady()` in-memory flag not shared across processes | `probeGraphReady()` queries LadybugDB directly | -| 3 | `project_overview` total_symbols:0 | Counted `graph_nodes` rows | Count `entity` rows | -| 4 | `search` returns empty | FTS built on `graph_nodes` (empty), no fallback | `searchLadybugJson()` via LadybugDB Cypher | -| 5 | `buildFTSFromGraph` empty | FTS tables built from `graph_nodes` | Build from `entity` | -| 6 | `engine_get_project_node_count` returns 0 | Queried `graph_nodes` | Query `entity` | -| 7 | `getLatestProjectId` wrong project | Queried `graph_nodes` for node count | Query `entity` | -| 8 | `verify_integrity` hangs | No query timeout | 10s `QueryDeadlineGuard` | -| 9 | `buildCSR` reverse query fails | `ORDER BY target_node_id` (wrong column name) | `ORDER BY target_id` | -| 10 | Version string still 0.2.1 | Hardcoded in `engine_ffi.cpp` | Updated to 0.2.3 | - -### Performance - -| Metric | Value | -|--------|-------| -| CodeScope self-index (212 files) | 899ms | -| Windows binary size | 16MB (PE32+ executable) | -| LadybugDB search latency | ~1ms (Cypher CONTAINS) | -| Query latency (all graph tools) | ~1ms | -| `make check` | 85 Rust tests + all C++ tests pass | +| 1 | Cross-compile build.rs ignored cmake system name | `CARGO_CFG_TARGET_OS` returns target during cross-compile | `std::env::consts::OS` for build host | +| 2 | Wrong compiler used for cross-compile | `platform_default_compiler` returned native `gcc` on macOS | Detect cross-compile → use MinGW cross-compiler | +| 3 | Stale LadybugDB cache breaks Windows link | macOS `.dylib` path persisted in shared build dir | `unset()` + Rust-side Windows guard | +| 4 | Windows crash at startup (runtime DLL mismatch) | MinGW libstdc++/libgcc/libwinpthread version conflict | `-static` rustflag bakes all runtime into .exe | +| 5 | LadybugDB ABI risk on Windows | Vendored `.lib` of unverified MinGW version | Disable LadybugDB on Windows entirely | ### Full changelog -See [CHANGELOG.md](./CHANGELOG.md) for the complete list of changes, bug fixes, and cross-platform fixes. +See [CHANGELOG.md](./CHANGELOG.md) for the complete list of changes. + +--- diff --git a/docs/articles/en/codescope-architecture-01-intro.md b/docs/articles/en/codescope-architecture-01-intro.md deleted file mode 100644 index bd886bf..0000000 --- a/docs/articles/en/codescope-architecture-01-intro.md +++ /dev/null @@ -1,264 +0,0 @@ -# CodeScope Architecture (1): Introduction — When an MCP Tool Returns 56KB and You Only Need 629 Bytes - -> I didn't set out to build something "X times better than CBM." -> I was using codebase-memory-mcp and got frustrated by one thing: -> **AI asks for a symbol, it returns the entire project.** -> -> 56KB of JSON, 70% of which is fingerprints, AST profiles, and body tokens that AI doesn't need. The truly useful parts — struct fields, constant values, switch-case branches — were missing. -> So I decided to build a tool that only answers what was asked. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 1 | **Introduction** (this article) | Why rewrite a code understanding tool, and the overall architecture | -| 2 | Progressive Readiness | Millisecond-level understanding | -| 3 | Worker Isolation | Why indexing won't slow down your MCP Server | -| 4 | Zero-Redundancy Responses | Minimal responses, on-demand | -| 5 | C++ Engine Pipeline | From source code to multi-dimensional code graph | -| 6 | MCP Protocol Layer | Design philosophy of 35+ tools | -| 7 | Language Translators | 10 languages unified into one IR | -| 8 | SQLite Storage Layer | The secret of 270KB replacing 64MB | -| 9 | Adaptive Queries | When data isn't ready yet | -| 10 | Performance Truth | From small projects to 60,000 files | - -## I. Starting Point: A 56KB Response - -Let's start with data so you understand what I'm talking about. - -Same machine, same codebase (GoAgent, ~24K lines of Go), same question ("How does Chaos work, and what patterns does it have"), two tools with different design philosophies: - -| Dimension | codebase-memory-mcp v0.8.1 | CodeScope | Note | -|-----------|:--------------------------:|:---------:|------| -| Minimum useful response | **56,183 bytes** | **629 bytes** | CBM returns full info, CodeScope returns minimal on-demand | -| Equivalent tokens | ~14,046 | ~157 | CodeScope uses fewer tokens, CBM responses contain more native info | -| First query wait time | After full index (minutes) | **Query immediately after create** | CodeScope's progressive model vs CBM's full-index model | - -Both designs have trade-offs. CBM's full-index model provides lower query latency and richer information after indexing completes. CodeScope's progressive model lets AI start interacting faster, but deep queries wait for background indexing. - -### What's in 56KB - -Using CBM's `search_graph` response as an example: - -``` -┌─────────────────────────────────────────────┐ -│ search_graph response 56,183 bytes breakdown │ -│ │ -│ ████████████████████████████████ 70% noise │ -│ (test functions, doc sections, route nodes) │ -│ │ -│ ████████████ 20% needed │ -│ (node names, labels, files) │ -│ │ -│ ████ 10% useful │ -│ (ChaosExecutor, ChaosAction, etc.) │ -└─────────────────────────────────────────────┘ -``` - -This is a design choice. CBM returns as much information as possible in one shot, suitable for deep analysis. CodeScope returns on-demand to minimize unnecessary token consumption — at the cost of potentially requiring multiple queries to get full context. - -It's worth noting that [codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) is an excellent open-source code understanding tool that outperforms CodeScope in some scenarios: -- **Full Linux kernel indexing**: CBM can index the entire Linux kernel source (~65K files) in minutes; CodeScope's full-index stability for large projects still needs improvement -- **Response richness**: CBM's `search_graph` returns more comprehensive fields, suitable for one-shot full information retrieval -- **Community ecosystem**: CBM has a more mature open-source community and documentation - -CodeScope's goal is not to replace CBM, but to offer a different design approach — progressive readiness and minimal responses. If you need fast startup interaction and are token-sensitive, CodeScope may suit you better. If you need deep analysis after full indexing, CBM is the better choice. - -### What's in 629 Bytes - -```json -{ - "results": [ - { - "id": 2851, - "kind": "struct", - "name": "ChaosExecutor", - "signature": "type ChaosExecutor struct {", - "visibility": "default", - "language": "go", - "file_path": "~/go/src/goagent/.../chaos.go", - "line": 72, - "column": 1 - } - ] -} -``` - -Every field is useful. No fingerprints, no AST profiles, no body tokens. AI knows it's a struct, which file, which line, and its signature. If it wants to know more, it will ask again — not get everything dumped at once. - ---- - -## II. Architecture Overview: Three Layers + One DB - -CodeScope's overall architecture is a three-layer structure: - -```mermaid -flowchart TB - Client["MCP Client
(Claude Desktop / Cursor / CLI)"] - - subgraph Server["Rust MCP Server"] - MCP["JSON-RPC 2.0
35+ tool dispatcher"] - FFI["FFI Bridge
Rust ↔ C++ extern \"C\""] - Tokio["Tokio Runtime
async tasks"] - end - - subgraph Worker["C++ Worker Subprocess"] - Scanner["Scanner
ms-level regex scan"] - Parser["Parallel Parser
14 tree-sitter workers"] - Graph["Graph Builder
call graph + symbol refs"] - Store["SQLite Writer
WAL mode"] - end - - DB["SQLite
.codescope/codescope.db"] - - Client <-->|"stdio"| MCP - MCP -->|"spawn"| Worker - Worker -->|"write"| DB - MCP -->|"poll/query"| FFI - FFI -->|"read"| DB - Tokio -->|"async enhance"| DB -``` - -- **Rust MCP Server**: Handles protocol, tool dispatch, async tasks. Doesn't touch source parsing. -- **C++ Worker Subprocess**: Does the heavy lifting — scanning files, parsing ASTs, building graphs, writing to DB. Exits when done, RSS fully returned to OS. -- **SQLite**: The sole persistence layer. WAL mode, read-write separation. - -Why C++ engine as subprocess instead of threads? Memory isolation. When a Worker indexes 60,000 files of Linux Kernel, RSS can hit several GB, but it's an independent process — memory is fully released on exit, never polluting the Server process. - ---- - -## III. Progressive Readiness Model - -This is CodeScope's core design decision, and its biggest philosophical difference from codebase-memory-mcp. - -CBM's model: queries only after full indexing. This means users must wait for full indexing to complete before interacting, but once indexed, query latency is low and information is complete. - -CodeScope's model: progressive readiness — **deepen gradually, available at any time.** - -```mermaid -flowchart LR - PA["Phase A (ms-level)
scan_project
regex symbol scan"] -->|"270KB DB"| Ready["✓ symbols/modules/entry points"] - PB["Phase B (async)
enhance_project
tree-sitter full parse"] -->|"background"| Enhanced - PC["Phase C (on-demand)
index_project
graph build + FTS"] -->|"full index"| Full - Ready -.->|"auto-trigger"| PB - Ready -->|"queryable"| Query1["find_symbol
get_module_tree
get_entry_points"] - PB -.->|"auto-trigger"| PC - Enhanced -->|"queryable"| Query2["find_callers
get_hotspots ❌
search(FTS)"] - Full -->|"queryable"| Query3["codescope_trace
get_communities ❌
semantic search"] -``` - -| Phase | Time (small project ~200 files) | DB Size | Capabilities | -|:-----:|:-------------------------------:|:-------:|--------------| -| Phase A | **~9ms** | 270 KB | Symbol names, module tree, entry points | -| Phase B | Background 1-5 min | A few MB | Call graph, FTS search | -| Phase C | On-demand (minutes for large projects) | Full | Call chain trace, community detection | - -The result: `scan_project` returns immediately and queries are available. Phase B/C run in the background without affecting normal usage. - ---- - -## IV. Data Flow: From Folder to AI Answer - -```mermaid -sequenceDiagram - participant AI as AI Agent - participant Server as Rust Server - participant Worker as C++ Worker - participant DB as SQLite - - AI->>Server: scan_project("/path/to/project") - Server->>Worker: spawn worker subprocess - Worker->>Worker: Phase 1: scan files (filter_policy) - Worker->>Worker: Phase 2: regex symbol scan (ms-level) - Worker->>DB: write semantic_records - Worker-->>Server: stdout JSON result - Server-->>AI: {symbols: 14366, files: 200} - - Note over AI,DB: AI can query now - - AI->>Server: find_symbol("ChaosExecutor") - Server->>DB: SELECT * FROM semantic_records WHERE name="ChaosExecutor" - DB-->>Server: 2 results - Server-->>AI: {kind: "struct", file: "chaos.go", line: 72} - - Note over Server: Auto-trigger Phase B in background - - Server->>Server: RUNTIME.spawn(enhance_project) - Server->>Worker: spawn worker (tree-sitter full parse) - Worker->>DB: write graph_nodes, graph_edges, FTS - Worker-->>Server: done - - AI->>Server: find_callees("ChaosExecutor.Execute") - Server->>DB: SELECT * FROM graph_edges WHERE source="Execute" - DB-->>Server: 7 callees - Server-->>AI: {methods: [executeStaleData, ...]} -``` - -Key design point: **After Phase A returns, AI can immediately start querying. Phase B runs in the background without affecting AI's normal workflow.** - ---- - -## V. Tech Stack: Why C++ + Rust + SQLite - -This combination wasn't decided on day one. I took some detours. - -### Why C++ for the Engine - -tree-sitter's C API is a first-class citizen. C++ can directly call tree-sitter parsers without an extra FFI layer. And tree-sitter parsing itself is C-based, so wrapping in C++ is natural. - -Additionally, C++ has advantages in memory layout and performance. 14 worker threads parsing hundreds of files in parallel — C++'s pthread + arena allocator is more controllable than GC'd languages. - -### Why Rust for the Server - -The MCP protocol is essentially a long-running JSON-RPC 2.0 server over stdio transport. Rust's Tokio async runtime is perfect for this — handling multiple tool calls, background tasks, and polling simultaneously. - -More importantly, Rust's FFI ecosystem (`extern "C"` bindings) integrates with C++ very directly. 40+ FFI functions, each exported as `extern "C"`, callable from Rust via `#[link(name = "engine")]`. - -### Why SQLite for Storage - -No distributed setup needed. No high concurrency needed. One DB file per project in `.codescope/` directory. SQLite's WAL mode supports read-write separation — Worker writes data, Server reads data, no blocking. - -Plus, SQLite's FTS5 full-text search and vec0 vector extensions work out of the box, no need for Elasticsearch or Milvus. - -### Honest Reflection: Why Not Pure Rust - -If I were to do it again, I would seriously consider rewriting the entire engine in Rust. C++'s build system (CMake) plus cross-platform compatibility (Windows .dll, macOS .dylib, Linux .so) brings maintenance costs far beyond expectations. - -But the choice of C++ was reasonable at the time: tree-sitter's ecosystem is C/C++ based, the team had C++ background, and C++ is indeed more flexible in low-level memory control. **There's no perfect tech stack, only the right one for the current stage.** - ---- - -## VI. Technical Metrics - -| Metric | Value | -|--------|:-----:| -| Supported languages | 13 (C/C++/Go/Rust/Java/Python/TS/JS etc.) | -| Phase A scan | **ms-level** for Linux Kernel 60,000 files | -| Index speed | 1,167 Go files → **3.28s** | -| Max graph nodes | 263,614 (GoAgent project) | -| Max graph edges | 245,849 | -| Query latency | Most queries **<1ms**, full graph Label Propagation **<500ms** | -| Index DB size | 270 KB (Phase A) → full (MB-tens of MB) | -| Process isolation | Worker subprocess, RSS 100% returned to OS | -| Worker timeout | 300s + 3 retries | -| MCP tools | 35+ | - ---- - -## VII. Series Preview - -| # | Title | Core Content | -|:--:|------|--------------| -| **2** | Progressive Readiness | How Phase A scans ms-level, Phase B async enhancement, Phase C full index | -| **3** | Worker Isolation | Why subprocesses over threads, cost and benefit of memory isolation | -| **4** | Zero-Redundancy Responses | Why each field exists or doesn't, comparison with CBM's 56KB | -| **5** | C++ Engine Pipeline | From filter_policy to graph_builder, a data's complete journey | -| **6** | MCP Protocol Layer | How 35+ tools are designed, registration and routing | -| **7** | Language Translators | How 10 languages unify into IR, tree-sitter visitor design | -| **8** | SQLite Storage Layer | WAL mode, FTS5, vec0, incremental indexing | -| **9** | Adaptive Queries | Graceful degradation when data isn't ready | -| **10** | Performance Truth | From 200 files to 60,000 files, measured data | - -Every article follows the same pattern: **Problem → Design Journey → Trade-offs → Honest Reflection.** No marketing. No "10x faster than X." Just engineers talking about engineering. - -Next: [CodeScope Architecture (2): Progressive Readiness](codescope-architecture-02-progressive-readiness.md) diff --git a/docs/articles/en/codescope-architecture-02-progressive-readiness.md b/docs/articles/en/codescope-architecture-02-progressive-readiness.md deleted file mode 100644 index 886fe85..0000000 --- a/docs/articles/en/codescope-architecture-02-progressive-readiness.md +++ /dev/null @@ -1,129 +0,0 @@ -# CodeScope Architecture (2): Progressive Readiness — Millisecond-Level Code Understanding - -> The first time I used CBM to index the Linux Kernel, I waited 12 minutes. -> Later I realized this isn't a technical problem — it's a design philosophy problem. CBM's model requires full indexing before querying. CodeScope's model is "get you usable first, then deepen gradually." - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 1 | [Introduction](codescope-architecture-01-intro.md) | Why rewrite a code understanding tool | -| **2** | **Progressive Readiness** (this article) | Millisecond-level code understanding | -| 3 | Worker Isolation | Why indexing won't slow down MCP Server | -| 4 | Zero-Redundancy Responses | Minimal responses, on-demand | -| 5 | C++ Engine Pipeline | From source code to multi-dimensional code graph | - -## I. The Problem: 1-12 Minutes of Waiting - -MCP code understanding tools share a common problem: **full indexing required before querying.** - -You have to wait for the tool to scan all files, parse all ASTs, and build the complete call graph before asking anything. This takes from 1 minute (small project) to 12 minutes (Linux Kernel). - -The issues with this wait: - -1. **AI doesn't know if it's worth it.** It just wants to check the project's module structure, but has to wait for full indexing. -2. **Most queries don't need full data.** "What entry points does this project have?" — does this need full AST parsing? No. File names and symbol names are enough. -3. **Waste.** You waited 12 minutes for indexing, and AI's first query is "what language is this project written in." - -### First-Principles Question - -> In a code understanding tool, what's the minimum data needed to answer the top 80% of AI's most common questions? - -I analyzed the typical query distribution of AI agents in code understanding scenarios: - -| Query Type | Frequency | Minimum Data Needed | -|-----------|:---------:|-------------------| -| Find symbol definition | ~40% | Symbol name + file + line number | -| View project structure/modules | ~20% | Directory structure + file names | -| Find entry points | ~10% | Symbol name + kind (main/init) | -| Query call relationships | ~15% | Graph edges | -| Search code (keywords) | ~10% | FTS index | -| Full analysis | ~5% | Complete graph + complexity | - -### Design Implications - -- AI can start querying symbols after **milliseconds** -- Level 2 auto-triggers in background, no manual action needed -- Query engine auto-detects readiness, falls back when data isn't ready - ---- - -## II. Five Levels of Readiness - -The progressive readiness model has five levels, each adding one capability: - -| Level | What's Ready | How | Time (60K files) | -|:-----:|-------------|:---:|:----------------:| -| 1 | File scan + filter | filter_policy reading `.gitignore` | ~5ms | -| 2 | Symbol scan | RapidJSON regex scan | **~ms** | -| 3 | Module tree | Directory hierarchy | **~ms** | -| 4 | Call graph | tree-sitter parsing + IR | Background 1-5 min | -| 5 | FTS + Vector | Full-text search + semantics | Background 5-15 min | - ---- - -## III. Phase A: Millisecond-Level Scan of 60,000 Files - -Phase A is the first layer of progressive readiness, and CodeScope's most "counter-intuitive" design. - -### 3.1 What Makes It Fast - -### 3.2 Filter Policy - -### 3.3 Regex Scanner - -| Project | Files | Phase A Time | -|---------|:-----:|:------------:| -| ARES Agent | 95 | **<100ms** | -| GoAgent | 1,167 | **<500ms** | -| memscope-rs | 238 | **<200ms** | - -### 3.4 What Phase A Can Answer - -- Symbol name search (find_symbol) -- Module tree (get_module_tree) -- Entry points (get_entry_points) -- Project overview - -### 3.5 What Phase A Cannot Answer - -- Call relationships (no call graph built yet) -- Full-text search (no FTS index) -- Call chain tracing -- Community detection - ---- - -## IV. Phase B: Async Enhancement - -### 4.1 tree-sitter Full Parse - -### 4.2 IR Translation - -### 4.3 Call Graph Construction - ---- - -## V. Phase C: On-Demand Full Index - ---- - -## VI. Measured Data: GoAgent Project - -Rustc project benchmark: - -| Phase | Time | Cumulative | -|:-----:|:----:|:----------:| -| Phase A scan | **<500ms** | <500ms | -| Phase B parse | **~2s** | ~2.5s | -| Phase C graph | **~26s** | ~29.5s | -| FTS post-process | **~2s** | ~31.5s | - -Comparing with CBM on the same project: - -| Tool | First Query Time | Full Completion | -|------|:---------------:|:---------------:| -| CBM | After full index | **3.94s** | -| **CodeScope** | **<500ms** | **31.5s** | - -Of course, CodeScope's full indexing takes longer because it builds more nodes and edges (263,614 nodes vs 24,658 nodes, **10.7x**). But AI doesn't need to wait — it starts working from <500ms. diff --git a/docs/articles/en/codescope-architecture-03-worker-isolation.md b/docs/articles/en/codescope-architecture-03-worker-isolation.md deleted file mode 100644 index 63137db..0000000 --- a/docs/articles/en/codescope-architecture-03-worker-isolation.md +++ /dev/null @@ -1,144 +0,0 @@ -# CodeScope Architecture (3): Worker Isolation — Why Indexing Won't Crash Your MCP Server - -> The original version didn't have Worker subprocesses. The Server called engine functions directly. -> Then one day, indexing a large project, RSS hit 3.5 GB and the MCP Server OOM'd. Claude Desktop showed "tool call timeout" with no error message. -> It took me three days to find the cause — not a memory leak, but thread stacks that were too large. -> -> From that day on I realized: **in a long-running Server process doing heavy computation, either isolate the computation or be prepared to be dragged down by it.** - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 1 | [Introduction](codescope-architecture-01-intro.md) | Why rewrite a code understanding tool | -| 2 | [Progressive Readiness](codescope-architecture-02-progressive-readiness.md) | Millisecond-level code understanding | -| **3** | **Worker Isolation** (this article) | Why indexing won't crash your MCP Server | - -## I. The Problem: Heavy Computation Can't Run in the Server Process - -An MCP Server is a long-running process. Its job is to: -1. Receive JSON-RPC requests -2. Route to the correct tool -3. Return JSON-RPC response - -It should not do heavy computation. But source code indexing IS heavy computation — reading hundreds of files, parsing ASTs, building graphs, writing to DB. - -If all this happens in the Server process, problems arise: - -### 1.1 Memory Pollution - -C++ engine uses arena allocators and various local buffers. After indexing a project, this memory isn't 100% returned to the OS — C++'s free list retains some fragments. The Server process's RSS only grows. - -### 1.2 Thread Stack Issues - -The original version allocated **256MB stacks** per worker thread. 14 threads × 256MB = 3.5GB. This isn't physical memory — it's virtual address space reservation — but it still stresses the OS. - -``` -14 threads × 256MB stack = 3.5 GB virtual address space - ↓ after fix -14 threads × 8MB stack = 112 MB virtual address space - ↓ 256MB → 8MB is a 32x reduction -``` - -I discovered this bug in an embarrassing way — I started by looking for memory leaks with valgrind, found nothing, then used `/usr/bin/time -l` to check peak RSS, and realized it was thread stacks. - -### 1.3 Long Hangs - -Indexing Linux Kernel takes minutes. If indexing runs in the main process, the MCP Server can't respond to any other requests during those minutes — including `get_index_progress` polling. The AI sees "tool unresponsive." - ---- - -## II. Solution: Worker Subprocess - -The solution is straightforward: **put indexing in a subprocess. When it finishes, the subprocess exits and RSS is fully returned to the OS.** - -### 2.1 Architecture - -``` -┌─────────────────────────────────────────────┐ -│ Rust MCP Server (long-running, ~12MB RSS) │ -│ │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ Tool │ │ FFI │ │ Tokio │ │ -│ │ Dispatch │ │ Bridge │ │ Scheduler│ │ -│ └──────────┘ └──────────┘ └──────────┘ │ -│ │ ▲ │ │ -│ ▼ │ ▼ │ -│ ┌─────────────────────────────────────────┐ │ -│ │ Subprocess Manager │ │ -│ │ spawn + pipe stdout + timeout + reap │ │ -│ └─────────────────────────────────────────┘ │ -│ │ │ -│ ▼ spawn (RSS 0 → 3GB → 0) │ -│ ┌─────────────────────────────────────────┐ │ -│ │ C++ Worker (short-lived, a few min) │ │ -│ │ scan → parse → build → write → exit │ │ -│ └─────────────────────────────────────────┘ │ -└─────────────────────────────────────────────┘ -``` - -### 2.2 Process Lifecycle - -1. Server spawns Worker subprocess via `fork() + execvp()` -2. Worker receives project path via command-line argument -3. Worker scans, parses, builds graph, writes to DB -4. Worker writes JSON result to stdout -5. Server reads stdout via pipe -6. Worker exits, RSS fully returned to OS (confirmed by `/usr/bin/time -l`) - -### 2.3 Timeout Protection - -```cpp -// server/src/ffi/worker.rs -let child = Command::new(worker_path) - .arg(project_path) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - -// 300s timeout + 3 retries -let result = tokio::time::timeout( - Duration::from_secs(300), - child.wait_with_output() -).await; -``` - -If the Worker doesn't complete within 300 seconds, it's killed. The Server retries up to 3 times. If all retries fail, the error is returned to AI. - ---- - -## III. Cost: Serialization Overhead - -Worker isolation isn't free. The cost is serialization. - -### 3.1 What Needs to Be Serialized - -Arguments passed via command line: -- `project_id` (uint64): 8 bytes -- `project_path` (string): N bytes -- `db_path` (string): N bytes - -Result returned via stdout JSON: -- `symbols` count, `files` count, `errors` array, etc. - -### 3.2 What Doesn't Need Serialization - -The Worker and Server share the SQLite database file. The Worker writes, the Server reads — no data copy needed. - -### 3.3 Measured Overhead - -Process spawn overhead is ~2-5ms on Linux/macOS (measured with 100 iterations). This is negligible compared to indexing time (seconds to minutes). - ---- - -## IV. Trade-offs Summary - -| Aspect | Thread Model | Worker Subprocess | -|--------|:------------:|:-----------------:| -| Memory isolation | Poor — RSS never returns | **Perfect** — 100% returned on exit | -| Crash isolation | Poor — one thread brings down all | **Perfect** — Worker crash ≠ Server crash | -| Spawn overhead | None | **~2-5ms** | -| Data sharing | Direct memory access | Via SQLite file | -| Complexity | Lower | Higher (IPC, timeout, retry) | - -For an MCP Server that needs to stay responsive for hours or days, Worker isolation is clearly the right choice. diff --git a/docs/articles/en/codescope-architecture-04-zero-redundancy.md b/docs/articles/en/codescope-architecture-04-zero-redundancy.md deleted file mode 100644 index 66d78ed..0000000 --- a/docs/articles/en/codescope-architecture-04-zero-redundancy.md +++ /dev/null @@ -1,150 +0,0 @@ -# CodeScope Architecture (4): Zero-Redundancy Responses — Minimal Responses, On-Demand Return - -> The first time I used codebase-memory-mcp, a `search_graph` response returned 56KB. I asked myself: does AI really need all this? - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 1 | [Introduction](codescope-architecture-01-intro.md) | Why rewrite a code understanding tool | -| 2 | [Progressive Readiness](codescope-architecture-02-progressive-readiness.md) | Millisecond-level code understanding | -| 3 | [Worker Isolation](codescope-architecture-03-worker-isolation.md) | Why indexing won't crash your MCP Server | -| **4** | **Zero-Redundancy Responses** (this article) | Minimal responses, on-demand return | -| 5 | C++ Engine Pipeline | From source code to multi-dimensional code graph | - -## I. See the Data First - -Same machine, same project (GoAgent, ~24K lines of Go), same question. Two tools, different design philosophies, different response formats: - -### CBM v0.8.1 — search_graph Response (56,183 bytes) - -```json -[ - { - "id": "2851", - "name": "ChaosExecutor", - "kind": "STRUCT", - "file": "internal/ares_quant/.../chaos.go", - "line": 72, - "column": 1, - "fingerprint": "abcdef1234567890", - "ast_profile": { - "node_count": 427, - "depth": 12, - "token_count": 3842, - "body_tokens": 2891, - "doc_tokens": 0 - }, - "route": "/api/v1/...", - "dataType": "code_symbol", - "body": "type ChaosExecutor struct { ... }", - "doc": "", - "children": ["...", "..."], - "parent_id": "2845", - "siblings": ["...", "..."], - "metadata": { ... } - } -] -``` - -56KB, 64 results. CBM returns complete information in one shot — fields like fingerprint and ast_profile are valuable for dedup and performance analysis, but in a "quick answer a simple question" scenario, they may not be needed. - -### CodeScope — find_symbol Response (629 bytes) - -```json -{ - "results": [ - { - "id": 2851, - "kind": "struct", - "name": "ChaosExecutor", - "signature": "type ChaosExecutor struct {", - "visibility": "default", - "language": "go", - "file_path": "~/go/src/goagent/.../chaos.go", - "line": 72, - "column": 1 - } - ] -} -``` - -629 bytes, 2 results (exact match). Every field is one AI needs. - -| Dimension | CBM | CodeScope | -|-----------|:---:|:---------:| -| Response size | **56,183 bytes** | **629 bytes** | -| Equivalent tokens (ASCII×0.3) | ~16,855 | ~189 | -| Results | 64 | 2 | -| Useful field ratio | ~30% (depends on scenario) | ~100% | -| Unnecessary fields | 7+ | 0 | - -Both designs are valid. CBM's response is field-rich, suitable for one-shot deep analysis. CodeScope's is minimal, suitable for token-sensitive and fast interaction scenarios. - -## II. Design Philosophy: Every Field Must Answer "Why" - -CodeScope's tool response field design follows a simple rule: - -> **Every field must answer "what can AI do with this field."** - -### Field Existence Analysis - -| Field | Exists? | Why Exists / Why Not | -|-------|:-------:|---------------------| -| `id` | ✅ | AI needs id for follow-up queries (e.g., `get_complexity(id)`) | -| `kind` | ✅ | AI needs to know if this is struct / func / var / const | -| `name` | ✅ | Symbol name, basic identifier | -| `signature` | ✅ | AI doesn't need full body, but needs function signature (param types, return types) | -| `visibility` | ✅ | AI needs to know if externally callable | -| `language` | ✅ | AI needs to know which language this symbol belongs to | -| `file_path` | ✅ | AI needs file location for citation | -| `line` | ✅ | AI needs line number for context reference | -| `column` | ✅ | Precise position | -| `fingerprint` | ❌ | **AI doesn't need this.** It's for dedup algorithms, not AI | -| `ast_profile` | ❌ | **AI doesn't need this.** Node count, depth, token count — AI can compute these itself | -| `body_tokens` | ❌ | **AI doesn't need this.** Full body is for human reading; AI needs the signature | -| `doc_tokens` | ❌ | **AI doesn't need this.** FTS search is sufficient | -| `route` | ❌ | **AI doesn't need this.** A REST API artifact | - -## III. The Balance: How Much Is Enough? - -### 3.1 The Risk of Over-Minimalism - -Minimal responses have trade-offs: - -**Low error tolerance.** If CodeScope's `find_symbol` returns wrong results, AI has no extra context to detect the error. CBM's 56KB contains redundancy that can help with cross-validation. - -**AI uncertainty.** Some AI agents repeatedly call the same tool to confirm — "are you sure this struct has no other fields?" — because the response lacks `children`, and AI doesn't know if it's "no children" or "children not returned." - -### 3.2 Token Savings Are Real - -| Scenario | CBM | CodeScope | Savings | -|----------|:---:|:---------:|:-------:| -| Find symbol | 19,632 tokens | 552 tokens | **97.2%** | -| Check project overview | Full index 12 min | ms-level + 629 bytes | Time and token savings | -| Check call relationships | 56KB + extra source reading | Call graph direct return | No source reading needed | -| Index DB size | 64 MB | 270 KB | **99.6%** | - -### 3.3 Visual Comparison - -``` -CBM approach: - ┌────────────────────────────────────────────────┐ - │ 56KB JSON (70% noise) │ Actually analyzing │ - │ ██████████████████████████████░░░░░░░░░░░░░░░ │ - └────────────────────────────────────────────────┘ - Tokens: ~19,632 Result: answered - -CodeScope approach: - ┌────────────────────────────────────────────────┐ - │ 629B JSON (100% useful) │ Actually analyzing │ - │ ████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ │ - └────────────────────────────────────────────────┘ - Tokens: ~552 Result: answered -``` - -AI's limited context window means every token matters. But there's no free lunch. - -**Minimalism's cost: low error tolerance.** If CodeScope returns wrong results, AI has no buffer to detect the error. CBM's redundancy can help with cross-validation. - -**Overly minimal responses may confuse AI.** Some agents repeatedly call to confirm — "are you sure?" — because they can't distinguish "no data" from "data not returned." diff --git a/docs/articles/en/codescope-architecture-05-engine-pipeline.md b/docs/articles/en/codescope-architecture-05-engine-pipeline.md deleted file mode 100644 index 8f67c61..0000000 --- a/docs/articles/en/codescope-architecture-05-engine-pipeline.md +++ /dev/null @@ -1,165 +0,0 @@ -# CodeScope Architecture (5): C++ Engine Pipeline — From Source Code to Multi-Dimensional Code Graph - -> Once I was debugging a cross-file call chain and found a missing edge in the call graph. A function was clearly being called, but it wasn't in the graph. After an afternoon of debugging, I found the reason: **the call happened inside a macro expansion, and our visitor doesn't expand macros.** This wasn't a bug — it was a trade-off. But it made me realize: from source code to code graph, every step involves trade-offs. - -## Three Questions - -Any code understanding tool's core task is to turn "text" into "structure." This can be broken into three questions: - -1. **How to quickly get code structure without parsing AST?** (key to ms-level result return) -2. **How to unify 10 languages' ASTs into one intermediate representation?** (foundation of cross-language analysis) -3. **How to build a queryable multi-dimensional code graph from unified IR?** (final deliverable) - -CodeScope's C++ engine is designed around these three questions. - -## Overall Pipeline - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ C++ Engine Pipeline │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ Phase A (Fast Scan) Phase B/C (Full Index) │ -│ ┌───────────────┐ ┌─────────────────────────┐ │ -│ │ Line-by-line │ │ File Read + Language │ │ -│ │ Regex Scan │ │ Detection │ │ -│ │ (no AST!) │ │ │ │ │ -│ └───────┬───────┘ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ ▼ │ │ tree-sitter │ │ │ -│ ┌───────────────┐ │ │ Parse │ │ │ -│ │ Modules │ │ └──────┬───────┘ │ │ -│ │ Symbols │ │ ▼ │ │ -│ │ EntryPoints │ │ ┌──────────────┐ │ │ -│ │ Stubs │ │ │ IR Translator │ │ │ -│ └───────────────┘ │ │ (CST → IR) │ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ ▼ │ ┌──────────────┐ │ │ -│ ┌────────────┐ │ │ Linker Passes │ │ │ -│ │ JSON │ │ │ ┌────────────┐│ │ │ -│ │ Response │ │ │ │ BuildSymbol││ │ │ -│ └────────────┘ │ │ │ IndexPass ││ │ │ -│ │ │ │ ├────────────┤│ │ │ -│ │ │ │ │ ResolveCall││ │ │ -│ │ │ │ │ Pass ││ │ │ -│ │ │ │ ├────────────┤│ │ │ -│ │ │ │ │ EmitGraph ││ │ │ -│ │ │ │ │ Pass ││ │ │ -│ │ │ │ └────────────┘│ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ │ │ │ Complexity │ │ │ -│ │ │ │ Analyzer │ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ │ │ │ SQLite │ │ │ -│ │ │ │ Persistence │ │ │ -│ │ │ └──────────────┘ │ │ -│ │ └─────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Phase A: Understanding Code Without Building an AST - -Phase A's goal is to **let AI know what's in the project at millisecond level.** Full AST parsing is impossible — tree-sitter parsing 24K lines takes seconds, and IR translation and graph building take even longer. - -CodeScope's Phase A uses a different approach: **line-by-line regex scanning.** - -### Filter Policy - -Phase A starts by reading the project's `.gitignore`, `.hgignore`, and common ignore patterns. This filters out non-source files early. - -- Ignored patterns: `node_modules/`, `.git/`, `build/`, `dist/`, `*.pyc`, etc. -- Accepted extensions: `.c`, `.h`, `.cpp`, `.go`, `.rs`, `.py`, `.java`, `.ts`, `.js`, `.swift`, etc. -- Max file size: 1MB by default (configurable) - -### Regex Scanner - -The scanner matches language-specific patterns line by line: - -``` -Go: func | type | struct | interface | const | var -Rust: fn | struct | enum | trait | impl | mod | pub -Python: def | class | import | from -C/C++: int|void|char|struct|class|#define|#include -Java: class | interface | enum | public | private -``` - -### Output - -Phase A outputs structured data written to SQLite `semantic_records` table, including: -- Symbol name -- Kind (function/struct/class/interface/enum/const/var) -- File path and line number -- Visibility (public/private/internal) -- Language - ---- - -## Phase B/C: Full Index Pipeline - -### File Read + Language Detection - -Reading a file involves: - -1. `open()` the file, `fstat()` to get size -2. `read()` entire file content -3. Detect language by file extension -4. Pass to tree-sitter for parsing - -### tree-sitter Parse - -Each language has a dedicated tree-sitter parser. Parsing produces a Concrete Syntax Tree (CST). - -### IR Translator - -The CST is language-specific. The IR translator converts it into a language-neutral Intermediate Representation. - -### ResolverPipeline: Constraint-Chain Resolution - -The link between per-file IR and cross-file call graph is the **ResolverPipeline** (`engine/src/resolver/pipeline.h`). It replaced the old 3-pass Linker with a constraint-chain, multi-factor scoring system: - -``` -Step 0: Pre-load entity table → in-memory HashMap> - (avoids one SQL query per reference — the former bottleneck) - -Step 1: Iterate reference table → for each reference: - a. HashMap lookup by name - b. No candidates? → FuzzyResolver fallback (case-insensitive / prefix / suffix) - c. applyConstraints() → 9-factor weighted scoring - d. Language visibility hard filter (Go lowercase = unexported, etc.) - e. Best score above 0.4 threshold → write to relation + graph_edges -``` - -#### 9 Scoring Factors - -| Factor | Weight | Purpose | -|--------|:------:|---------| -| ImportMatch | **0.80** | Dominant for cross-module resolution | -| ModuleMatch | 0.15 | Same module bonus | -| CallKindMatch | 0.15 | Direct/Method/Interface/Constructor awareness | -| ReceiverMatch | 0.15 | Go/Python receiver method matching | -| SignatureMatch | 0.10 | Parameter count matching | -| NamespaceMatch | 0.10 | Shared namespace | -| ConstructorMatch | 0.10 | Constructor call boost | -| CommonNamePenalty | 0.10 | Penalize common names (Len, Init, Run) cross-module | -| DistanceMatch | 0.05 | File path distance | - -#### Language Visibility (Hard Filter) - -- **Go**: lowercase names → not exported, cannot be called cross-package -- **Python**: `_`-prefixed → private -- **Java**: no `public` modifier → package-private - -### Complexity Analyzer - -Computes cyclomatic and cognitive complexity for each function. - -### SQLite Persistence - -Writes the final graph to SQLite tables. diff --git a/docs/articles/en/codescope-architecture-06-mcp-layer.md b/docs/articles/en/codescope-architecture-06-mcp-layer.md deleted file mode 100644 index 8ccb8f8..0000000 --- a/docs/articles/en/codescope-architecture-06-mcp-layer.md +++ /dev/null @@ -1,110 +0,0 @@ -# CodeScope Architecture (6): MCP Protocol Layer — Design Philosophy of 35+ Tools - -> The first version had only 8 tools. As I added more, I realized tool design isn't just about adding functions — it's about how AI discovers and uses them. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 1 | [Introduction](codescope-architecture-01-intro.md) | Why rewrite a code understanding tool | -| 2 | [Progressive Readiness](codescope-architecture-02-progressive-readiness.md) | Millisecond-level code understanding | -| 3 | [Worker Isolation](codescope-architecture-03-worker-isolation.md) | Why indexing won't crash your MCP Server | -| 4 | [Zero-Redundancy Responses](codescope-architecture-04-zero-redundancy.md) | Minimal responses, on-demand | -| 5 | [C++ Engine Pipeline](codescope-architecture-05-engine-pipeline.md) | From source code to code graph | -| **6** | **MCP Protocol Layer** (this article) | Design of 35+ tools | -| 7 | Language Translators | 10 languages unified into one IR | -| 8 | SQLite Storage Layer | The secret of 270KB replacing 64MB | -| 9 | Adaptive Queries | When data isn't ready | -| 10 | Performance Truth | Measured data across project scales | - -## I. Tool Classification - -35+ tools are organized into 4 categories by readiness level: - -### Phase A Tools (Available Immediately) - -| Tool | Function | Response Time | -|------|----------|:------------:| -| `scan_project` | Scan project, extract declarations | ~ms-level | -| `find_symbol` | Exact symbol match | Instant | -| `get_module_tree` | Module tree | Instant | -| `get_entry_points` | Entry points (main, init) | Instant | -| `get_project_overview` | Project summary | Instant | -| `search_symbol` | Fuzzy symbol search | Instant | - -### Phase B Tools (After Async Enhancement) - -| Tool | Function | Response Time | -|------|----------|:------------:| -| `find_callers` | Find callers of a symbol | ~5ms | -| `find_callees` | Find callees of a symbol | ~5ms | -| `get_call_graph` | Get subgraph around a function | ~10ms | -| `search_semantic` | Semantic code search | ~50ms | -| `get_complexity` | Cyclomatic/cognitive complexity | ~5ms | -| `get_hotspots` | ❌ Not implemented — no MCP tool | — | - -### Phase C Tools (After Full Index) - -| Tool | Function | Response Time | -|------|----------|:------------:| -| `codescope_trace` | Call chain trace | ~5ms | -| `get_communities` | ❌ Engine impl exists, no MCP tool | — | -| `get_module_map` | Module dependency map | ~10ms | -| `trace_call_chain` | Cross-file call chain | ~6ms | -| `export_artifact` | Export project artifact | ~1s | -| `import_artifact` | Import project artifact | ~1s | - -### Utility Tools - -| Tool | Function | -|------|----------| -| `detect_changes` | Detect file changes for incremental indexing | -| `enhance_project` | Trigger background full analysis | -| `codescope_build_context` | Build AI query context | -| `codescope_capabilities` | Check feature readiness status | - -## II. Tool Design Principles - -### 2.1 One Question, One Tool - -Each tool answers exactly one question. If a tool tries to answer two, it's split. - -### 2.2 Response Structure - -Every response follows the same pattern: -```json -{ - "ok": true, - "data": { /* tool-specific */ }, - "readiness": { // optional: data readiness info - "phase_a": 1.0, - "phase_b": 0.85, - "phase_c": 0.0 - } -} -``` - -### 2.3 Error Handling - -Errors return structured JSON, never exceptions: -```json -{ - "ok": false, - "error": "project not found: /path/to/project" -} -``` - -## III. Tool Registration - -Tools are registered in `server/src/tools/mod.rs` via a handler table: - -```rust -pub struct ToolHandler { - pub name: &'static str, - pub description: &'static str, - pub handler: fn(&mut ToolContext, &Value) -> Result, - pub min_readiness: f64, -} -``` - -The `min_readiness` field ensures a tool is only available when the project's data readiness meets the threshold. diff --git a/docs/articles/en/codescope-architecture-07-language-translators.md b/docs/articles/en/codescope-architecture-07-language-translators.md deleted file mode 100644 index e63b9c6..0000000 --- a/docs/articles/en/codescope-architecture-07-language-translators.md +++ /dev/null @@ -1,118 +0,0 @@ -# CodeScope Architecture (7): Language Translators — 10 Languages Unified into One IR - -> tree-sitter gives us 10 different AST structures. We need to turn them into one. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| 6 | [MCP Protocol Layer](codescope-architecture-06-mcp-layer.md) | Design of 35+ tools | -| **7** | **Language Translators** (this article) | 10 languages unified into one IR | -| 8 | [SQLite Storage Layer](codescope-architecture-08-storage-layer.md) | WAL, FTS5, vec0, incremental indexing | -| 9 | [Adaptive Queries](codescope-architecture-09-adaptive-queries.md) | When data isn't ready | -| 10 | [Performance Truth](codescope-architecture-10-performance-truth.md) | Measured data across project scales | - -## I. Why Unified IR Matters - -Each language has its own AST structure: -- Go's AST has `FuncDecl`, `TypeSpec`, `GenDecl` -- Rust's AST has `ItemFn`, `ItemStruct`, `ItemImpl` -- Python's AST has `FunctionDef`, `ClassDef`, `AsyncFunctionDef` - -Without a unified IR, cross-language analysis is impossible. With a unified IR, code understanding becomes language-agnostic. - -## II. IR Design - -### Node Types - -``` -NodeType enum: - TranslationUnit // root node for a file - Function // function/method definition - Struct // struct/class definition - Interface // interface/trait/protocol - Enum // enum definition - Variable // variable/constant - Field // struct field - Parameter // function parameter - Call // function call expression - TypeRef // type reference - Import // import/include/use statement - ... -``` - -### Node Structure - -```cpp -struct SemanticNode { - uint64_t id; - NodeKind kind; // the type enum above - std::string name; // symbol name - std::string signature; // full signature - std::string doc_comment; // documentation - uint32_t line_start, line_end; - uint32_t col_start, col_end; - std::vector children; // child nodes - std::vector edges; // semantic relationships -}; -``` - -## III. Translator Implementation - -Each language has a dedicated translator class: - -``` -Translator base class -├── GoTranslator -├── RustTranslator -├── PythonTranslator -├── CTranslator -├── CppTranslator -├── JavaTranslator -├── JavaScriptTranslator -├── TypeScriptTranslator -├── SwiftTranslator -└── TsxTranslator -``` - -Each translator implements a `translate(tree, source, file_path)` method that walks the tree-sitter CST and produces `SemanticNode` trees. - -### Example: Go Function Translation - -```cpp -SemanticNode* GoTranslator::translateFunction(TSNode node) { - auto* fn = new SemanticNode(); - fn->kind = NodeKind::Function; - fn->name = nodeText(node, "name"); - fn->signature = extractSignature(node); - fn->doc_comment = extractComment(node); - // ... recurse into body - return fn; -} -``` - -## IV. Cross-Language Mapping - -| Language | Function | Struct | Interface | Enum | Import | -|----------|:--------:|:------:|:---------:|:----:|:------:| -| Go | `func` | `struct` | `interface` | iota | `import` | -| Rust | `fn` | `struct` | `trait` | `enum` | `use` | -| Python | `def` | `class` | `ABC` | `Enum` | `import` | -| C | func decl | `struct` | — | `enum` | `#include` | -| C++ | func/method | `class/struct` | `virtual` | `enum` | `#include` | -| Java | method | `class` | `interface` | `enum` | `import` | -| JS/TS | `function` | `class` | `interface` | — | `import/require` | - -## V. Trade-offs - -### Visitor Pattern vs. Translator Pattern - -The original design used a visitor pattern (walk the tree, emit nodes). The new design uses a translator pattern (walk the tree, return IR nodes). The translator pattern gives each language more control over node structure. - -### Depth vs. Breadth - -Deep nesting (e.g., Go's anonymous functions inside struct methods) complicates the IR. CodeScope flattens deeply nested structures where possible, at the cost of losing some nesting context. - -### Macro Expansion - -C/C++ macros are not expanded. This means some symbols defined in macros are invisible to the graph. This is a known limitation. diff --git a/docs/articles/en/codescope-architecture-08-storage-layer.md b/docs/articles/en/codescope-architecture-08-storage-layer.md deleted file mode 100644 index c7764e1..0000000 --- a/docs/articles/en/codescope-architecture-08-storage-layer.md +++ /dev/null @@ -1,90 +0,0 @@ -# CodeScope Architecture (8): Storage Layer — SQLite WAL + FTS5 + vec0 - -> I know it sounds counter-intuitive: a code analysis engine using SQLite for storage? -> -> I've tried other options. Neo4j's call graph queries are elegant, but asking users to install JDK + Neo4j for a CLI tool is absurd. RocksDB has excellent write performance, but a `LIKE '%query%'` search requires building your own Bloom filter. As for storing `HashMap>` in memory — queries are fast, 300ms for a call chain. But when the process dies, everything is lost, let alone incremental indexing. -> -> SQLite's advantage: no deployment needed. No separate process, no configuration, no user installation. Just `sqlite3_open()` one line. The downside: you have to manually orchestrate 22 tables + FTS5 + vec0 + WAL to solve a problem that should use a multi-model database. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| **8** | **SQLite Storage Layer** (this article) | WAL, FTS5, vec0, incremental indexing | - -## I. 22 Tables Design - -Phase A (Fast Scan — ms-level ready): -- projects: root table, stores project root path and name -- files: file metadata (path, language, content hash) -- modules: module/directory hierarchy -- symbols: symbol definitions (core) -- symbol_status: readiness flags per symbol -- entry_points: entry point markers -- file_scan_state: incremental indexing mtime/hash tracking -- index_tasks: Tokio background task status - -Phase B (Knowledge Enhancement — async background): -- call_edges: call graph edges (caller → callee) -- dependency_edges: inter-module dependencies -- metrics: cyclomatic and cognitive complexity -- search_index: FTS5 with summary and body - -Phase C (Deep Index): -- ir_nodes: legacy IR nodes (old pipeline) -- ir_semantic_edges: legacy semantic edges -- graph_nodes: graph nodes (final product) -- graph_edges: graph edges (call/include/inherit) -- semantic_records: new pipeline flat records (DB-first) -- node_complexity: complexity scores -- node_vectors: n-gram hash vector BLOBs -- code_fts: FTS5 full-text search index -- fts_node_map: FTS node ID mapping -- embeddings: vec0 384-dim embedding table (optional) - -## II. WAL Mode - -SQLite's WAL (Write-Ahead Log) mode allows concurrent reads during writes: - -- Worker writes to the database (indexing) -- Server reads from the database (queries) -- WAL checkpoint runs automatically when the WAL file grows large - -### Checkpoint Behavior - -When the WAL file reaches several hundred MB, checkpoint can block writes for ~0.5-2s. This is imperceptible in normal use but shows up in benchmarks. - -## III. FTS5 Full-Text Search - -FTS5 provides: -- Prefix queries (e.g., `func*`) -- Phrase queries -- Ranking by relevance -- Column-specific search (name vs body vs comment) - -## IV. vec0 Vector Search - -vec0 is an experimental SQLite extension for vector similarity search: -- 384-dimensional embedding vectors -- Cosine similarity search -- Optional — falls back to n-gram hash search when unavailable - -## V. Incremental Indexing - -`file_scan_state` tracks: -- Last modified time (mtime) per file -- Content hash -- File size - -On re-index, unchanged files are skipped. Changed files are re-parsed and re-indexed. The graph is updated incrementally. - -## VI. DB Size Comparison - -| Index Mode | Files | DB Size | -|-----------|:-----:|:-------:| -| Phase A (GoAgent) | 1,167 | **270 KB** | -| Full (GoAgent) | 1,167 | **~64 MB** (CBM: 64 MB) | -| Full (Bun) | 9,641 | ~300 MB (estimated) | -| Full (rustc) | 36,807 | ~1.2 GB (estimated) | - -Note: Phase A's 270KB is particularly notable — it provides immediate query capability for symbol/module/entry point queries without waiting for full indexing. diff --git a/docs/articles/en/codescope-architecture-09-adaptive-queries.md b/docs/articles/en/codescope-architecture-09-adaptive-queries.md deleted file mode 100644 index 62dee8a..0000000 --- a/docs/articles/en/codescope-architecture-09-adaptive-queries.md +++ /dev/null @@ -1,99 +0,0 @@ -# CodeScope Architecture (9): Adaptive Queries — When AI Asks a Question That Only Half the Data Can Answer - -> I spent a week writing the query engine. It failed during the first integration test. -> -> The scenario: indexing was mid-Phase B (symbols 60% written, call_edges untouched). The user asked "who calls this function." The engine only had one query path — through the call_edges table. Result: it queried an empty table, returned `[]`. The user thought the project had no call relationships, when in fact the indexing just hadn't finished yet. -> -> The fix was two options: either wait for indexing to finish before responding (which violates the entire "progressive readiness" design), or teach the query engine to say "what I know isn't complete yet, but use it for now." I chose the latter. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| **9** | **Adaptive Queries** (this article) | When data isn't ready yet | - -## I. Three-Layer Query Architecture - -``` -┌─────────────────────────────────────────────┐ -│ Rust Tool Layer (server/src/tools/mod.rs) │ -│ Receives query → calls FFI → returns JSON │ -└──────────────────┬──────────────────────────┘ - │ FFI: extern "C" - ▼ -┌─────────────────────────────────────────────┐ -│ C++ Adaptive Dispatch (engine_queries.cpp) │ -│ Check readiness → choose optimal path → fallback │ -│ Graceful degradation: readiness > 0.5 → 0.1 → 0.0 │ -│ getReadyRatio() + project_readiness combined check │ -└──────────────────┬──────────────────────────┘ - │ C++ function call - ▼ -┌─────────────────────────────────────────────┐ -│ C++ Data Access (store.cpp + query_engine) │ -│ New pipeline: symbols → call_edges → search_index │ -│ Old pipeline: graph_nodes → graph_edges → code_fts │ -│ Triple layer search: new FTS5 → old FTS5 → LIKE fallback │ -└─────────────────────────────────────────────┘ -``` - -## II. Readiness Calculation - -The core of adaptive queries is `getReadyRatio()`: - -```cpp -double GraphStore::getReadyRatio(uint64_t project_id, const char *ready_field) { - std::string sql = "SELECT CASE WHEN COUNT(*) > 0 THEN " - "CAST(SUM(ss." + ready_field + - ") AS REAL) / COUNT(*) ELSE 0 END " - "FROM symbol_status ss " - "JOIN files f ON ss.file_id = f.id " - "WHERE f.project_id = ?"; - // ... execute and return ratio -} -``` - -### Readiness Thresholds - -| Readiness | Meaning | Behavior | -|:---------:|---------|----------| -| 1.0 | Phase complete | Full query capability | -| > 0.5 | Partial data | Return results + readiness info | -| > 0.1 | Minimal data | Return best-effort results + warning | -| 0.0 | No data | Return "data not ready" error | - -## III. Graceful Degradation - -### Example: find_callers - -1. **Phase C complete**: Query graph_edges for precise caller/callee pairs -2. **Phase B complete**: Query call_edges (new pipeline) -3. **Phase A only**: Query semantic_records for symbol references (best-effort matching) -4. **No data**: Return descriptive error message - -### Response with Readiness Info - -```json -{ - "ok": true, - "data": { - "callers": [ /* ... */ ], - "readiness": { - "phase_a": 1.0, - "phase_b": 0.6, - "phase_c": 0.0, - "note": "Call edges are 60% indexed — results may be incomplete" - } - } -} -``` - -## IV. Triple-Layer Search - -For full-text search, three layers provide progressive coverage: - -1. **FTS5 (new pipeline)**: Fastest, structured search -2. **FTS5 (old pipeline)**: Fallback if new pipeline not built -3. **LIKE query**: Last resort, slow but always available - -This ensures search always returns something, even when the project is mid-indexing. diff --git a/docs/articles/en/codescope-architecture-10-performance-truth.md b/docs/articles/en/codescope-architecture-10-performance-truth.md deleted file mode 100644 index faaf4c6..0000000 --- a/docs/articles/en/codescope-architecture-10-performance-truth.md +++ /dev/null @@ -1,182 +0,0 @@ -# CodeScope Architecture (10): Performance Truth — Measured from 95 to 89,465 Files - -> Before I ran proper benchmarks, I told people "our engine is fast." -> -> That was vague confidence — demo projects returned instantly, but I hadn't tested real large projects. Until one day I ran the Linux kernel full index, stared at the terminal for 15 seconds, and thought it had deadlocked. It was just the indexing thread doing a WAL checkpoint. After that, I made a decision: all performance claims must come from at least three projects of different sizes, and must be timed phase-by-phase. - -## Series Index - -| # | Title | One-liner | -|:--:|------|-----------| -| **10** | **Performance Truth** (this article) | Measured data across project scales | - -## Three Questions - -1. **From small to large projects, how does indexing time scale? Linear, super-linear, or explosive after a threshold?** -2. **Where's the bottleneck? CPU (parsing), IO (SQLite writes), or something else?** -3. **When the project grows from 100 to 100,000 files, can CodeScope's architecture hold up? Where does it break first?** - -## Full Picture: Measured Data from 7 Projects - -| Project | Language | Files | Parse Time | Index Time (Total) | Graph Nodes | Node Density | -|---------|----------|:-----:|:----------:|:-----------------:|:-----------:|:------------:| -| ARES Agent | Go | 95 | 9ms | **0.3s** | 24,924 | 262/file | -| CodeScope self | C++/Rust | 174 | — | **1s** | — | — | -| memscope-rs | Rust | 238 | 31ms | **2s** | 123,270 | 518/file | -| CPython | Python | 1,022 | 0.5s | **6s** | 446,618 | 437/file | -| Bun | Zig/C++/JS | 9,641 | 1.0s | **61s** | 2,951,664 | 306/file | -| rustc | Rust | 36,807 | 3.2s | **104s** | 4,130,017 | 112/file | -| JDK | Java/C++ | 19,821 | 2.4s | **211s** | 8,047,762 | 405/file | - -From 95 to 36,807 files (388x), indexing time goes from 0.3s to 104s (347x). Roughly linear, with two anomalies. - -### Anomaly: JDK vs rustc - -JDK has 19,821 files (54% of rustc), but takes 211s (203% of rustc). Why? - -The answer is **node density**. JDK produces 405 graph nodes per file, 3.6x rustc's 112 nodes/file. Total nodes 8.05M is 1.95x rustc's 4.13M. JDK is slower not because of file count, but because **Java code has inherently higher symbol density than Rust.** - -This finding is important: **CodeScope's indexing time is O(symbols), not O(files).** File count is a proxy metric; the real driver is semantic unit count. - -### Scaling Pattern - -- **Small (<1K files)**: ~0.3-6s, near-instant -- **Medium (1K-10K files)**: 6-61s, ~1 minute -- **Large (10K-40K files)**: 61-211s, 2-3.5 minutes -- **Extra-large (>60K files)**: Linux kernel full indexing stability and performance still under optimization. - -> Note: codebase-memory-mcp performs excellently on Linux kernel full indexing, completing ~65K files in minutes. CodeScope's support for very large projects is still being improved and cannot yet reliably index Linux-scale codebases. This data will be updated after optimization. - ---- - -## Bottleneck Analysis - -Breaking indexing into four phases: - -``` -Parse → SQLite insert → buildGraph → FTS build -``` - -Phase time distribution (averaged across projects): - -``` -Parse: 2.6% ← tree-sitter parsing, CPU-bound -SQLite: 37.4% ← semantic record writes, IO-bound -buildGraph: 33.2% ← graph building, SQL+JOIN-bound -FTS: 18.9% ← full-text index build -Other: 7.9% ← Vector, complexity, etc. -``` - -The key finding: **CodeScope is SQLite-bound, not CPU-bound.** Parsing (tree-sitter) takes less than 3% of time. Over 75% of time is spent on SQLite writes and graph building. - -### Phase Breakdown by Project - -| Project | Parse | SQLite | buildGraph | FTS | -|---------|:----:|:------:|:---------:|:---:| -| ARES (95 files) | 3.0% | 32.0% | 27.0% | 10.0% | -| Bun (9,641 files) | 1.7% | 43.6% | 34.4% | 19.6% | -| rustc (36,807 files) | 3.0% | 37.4% | 29.5% | 22.1% | -| JDK (19,821 files) | 1.2% | 41.7% | 38.1% | 19.1% | - -Parse ratio is always low (1-3%). FTS grows with project size (10% → 22%). SQLite is always the largest component. - ---- - -## Single-File Performance - -Benchmark: parsing `kernel/latencytop.c` (7,763 bytes): - -| Metric | Value | -|--------|:-----:| -| engine_init | 14.6 ms (75% SQLite schema, 15% grammar load) | -| Single file index | **4.9 ms** | -| Throughput | 1,533 KB/s | -| 9 queries total | **0.17 ms** | - ---- - -## Phase A: Fastest Mode - -| Project | Files | Phase A Time | -|---------|:-----:|:------------:| -| ARES Agent | 95 | **9ms** | -| GoAgent | 1,167 | **<500ms** | -| memscope-rs | 238 | **<200ms** | - ---- - -## Token Efficiency - -Comparing `codescope_trace` (find callers and callees): - -**CBM approach**: Bring full call chain source into context (~19,632 tokens), let LLM analyze. -**CodeScope approach**: Return structured caller/callee list (~552 tokens). - -``` -CBM: 19,632 tokens → LLM analyzes and responds -CodeScope: 552 tokens → directly usable structured data - -Savings: 97.2% -``` - -### E2E Pipeline Token Comparison - -| Step | CBM | CodeScope | Savings | -|------|:---:|:---------:|:-------:| -| Find symbol `Symbol` | 828 tokens | 138 tokens | 83% | -| Get detailed definition | 5,012 tokens | 17 tokens | 99.7% | -| Resolve type reference | 598 tokens | 18 tokens | 97% | -| Find callers | 9,342 tokens | 207 tokens | 97.8% | -| Trace call chain | 3,852 tokens | 172 tokens | 95.5% | -| **Total** | **19,632 tokens** | **552 tokens** | **97.2%** | - ---- - -## Scalability Limits - -### Verified Scale - -| Size | Project | Index Time | -|:----:|---------|:----------:| -| 100 files | ARES Agent | 0.3s | -| 1,000 files | CPython | 6s | -| 10,000 files | Bun | 61s | -| 20,000 files | JDK | 211s | -| 35,000 files | rustc | 104s | -| >60K files | Linux kernel | ⏳ Under optimization | - -### Expected Bottlenecks - -1. **SQLite write contention** (first bottleneck): WAL mode allows concurrent reads but writes are serialized. Multiple workers would need serialized writes — currently implemented as single-worker index_project. - -2. **buildGraph ROW_NUMBER() OVER()**: This window function is sub-millisecond at million-node scale, but becomes a bottleneck at tens of millions (JDK's r2n phase ~14.5s). Expected to need sharding or incremental buildGraph at hundred-million scale. - -3. **Brute-force vector search**: `searchSemantic()` does full table scan on `node_vectors`. Currently microsecond-level at <100K nodes, second-level at million nodes. vec0 extension solves this but requires user installation. - -4. **FTS5 index rebuild**: `buildFTSFromGraph()` uses batch INSERT-SELECT, performs well at tens of millions of nodes (rustc 23s). Full rebuild is the slowest phase (JDK 38.3s). - -### Practical Limits - -Based on measured data: - -- **Files: <50,000 files**. Beyond this, buildGraph + FTS combined time exceeds 5 minutes. -- **Nodes: <10M nodes**. Beyond this, SQLite WAL checkpoint time becomes noticeable. -- **DB size: <5 GB**. Beyond this, mmap I/O page fault rate increases. - ---- - -## Honest Reflection - -**Benchmark data is more honest than I initially assumed.** - -Several things I believed before benchmarking turned out to be wrong: - -**"Parsing is the bottleneck"** — Wrong. Tree parsing takes 1-3%. The bottleneck is SQLite. - -**"File count determines speed"** — Wrong. Node density is the dominant factor. JDK's 19K files is 2x slower than rustc's 36K files. - -**"Querying symbols directly is always fast"** — Mostly true, but Phase A's regex scan took 1.8s on the fs/ directory. Regex scanning is slower than expected on files with heavy macro expansion. - -**"New architecture is always faster than old"** — True, but not by the expected margin. The new architecture was proven better not because it's faster, but because it produces higher quality output. - -**WAL checkpoint is a hidden pitfall.** When the WAL file grows to hundreds of MB, checkpoint can block writes for ~0.5-2s. This is imperceptible in normal use but shows up in benchmarks. diff --git a/docs/articles/en/codescope-architecture-11-verification-layer.md b/docs/articles/en/codescope-architecture-11-verification-layer.md deleted file mode 100644 index 1ce8fb3..0000000 --- a/docs/articles/en/codescope-architecture-11-verification-layer.md +++ /dev/null @@ -1,117 +0,0 @@ -# CodeScope Architecture (11): Verification Layer — Making AI Accountable - -> Before writing this document, I ran `verify_integrity`. It checks whether features claimed in the README actually exist in the code. It reported one missing capability: `CommunityDetection` was in the capability list, but the MCP Server has no tool for it. An awkward but honest discovery. - ---- - -## Three Questions - -Code analysis tools can answer "what's in the code", but not "does the code actually do what the README claims?" — that's the verification layer's job. - -1. **How to turn natural-language "promises" into executable checks?** (Claim parsing) -2. **How to verify a promise against code facts?** (Verifier dispatch) -3. **How to persist findings so AI can see them next time?** (Evidence chain) - ---- - -## Architecture - -The verification layer lives in `engine/src/verify/` (23 source files): - -``` -Natural language / README / PR description - │ - ▼ - ClaimParser ───→ Claim (structured assertion) - │ │ - │ ▼ - │ VerifierRegistry - │ │ - │ ┌──────┼──────┐ - │ ▼ ▼ ▼ - │ Capability Contract Architecture - │ Verifier Verifier Verifier - │ │ │ │ - │ ▼ ▼ ▼ - │ Finding (evidence + confidence) - │ │ - │ ▼ - │ SQLite findings table (persistent) -``` - ---- - -## ClaimParser: Natural Language → Structured Assertion - -```cpp -struct Claim { - ClaimType type; // capability_exists / contract_holds / architecture_follows - std::string subject; // subject: module name / function name - std::string predicate; // predicate: supports / implements / ensures - std::string object; // object: feature / interface / property - double confidence; // parse confidence (0-1) -}; -``` - -Supported sentence patterns: - -| Pattern | Example | ClaimType | -|---------|---------|-----------| -| X supports Y | "Login module supports JWT" | capability_exists | -| X implements Y | "Server implements Service interface" | contract_holds | -| X ensures Y | "System ensures data consistency" | architecture_follows | - ---- - -## VerifierRegistry: Dispatch to the Right Verifier - -Three built-in verifiers in priority order: - -1. **CapabilityVerifier**: Check "does module X actually support feature Y?" -2. **ContractVerifier**: Check "does entity X implement interface Y?" -3. **ArchitectureVerifier**: Check layer constraints (Controller→Service→Repository) - ---- - -## Drift Detection (Batch Scan) - -| Tool | Scope | Detects | -|------|-------|---------| -| `documentation_drift` | README | Claimed languages vs actual entities | -| `capability_drift` | Capability list | Claimed capabilities vs actual code | -| `architecture_drift` | All CALLS edges | Layer violations | -| `drift` (full) | All above | Aggregate report | - -### Finding Data Structure - -```cpp -struct Finding { - uint64_t id; - FindingSeverity severity; - std::string category; - std::string title; - std::string detail; - double confidence; - std::string source_kind; -}; -``` - -Findings are written to the SQLite `findings` table and support filtering by `source_kind`. - ---- - -## Series Navigation - -| # | Article | Topic | -|---|---------|-------| -| (1) | Intro | 56KB vs 629 bytes, what problem CodeScope solves | -| (2) | Progressive Readiness | ms-level project understanding | -| (3) | Worker Isolation | Indexing won't crash MCP Server | -| (4) | Zero-Redundancy Responses | Lean responses, on-demand return | -| (5) | C++ Engine Pipeline | Source code to multi-dimensional code graph | -| (6) | MCP Protocol Layer | Tool design philosophy | -| (7) | Language Translators | 10 languages → unified IR | -| (8) | Storage Layer | SQLite WAL + FTS5 + vec0 | -| (9) | Adaptive Queries | Fallback mechanism & readiness detection | -| (10) | Performance Truth | 200 to 60,000 files measured | -| **(11)** | **Verification Layer** | **Making AI Accountable ← this article** | diff --git a/docs/articles/en/codescope-architecture-12-model-engine.md b/docs/articles/en/codescope-architecture-12-model-engine.md deleted file mode 100644 index dbc07d9..0000000 --- a/docs/articles/en/codescope-architecture-12-model-engine.md +++ /dev/null @@ -1,92 +0,0 @@ -# CodeScope Architecture (12): Model Engine & State Builder — From Facts to Understanding - -> After Phase B completes, the database has entity, relation, and graph_edges — raw facts. But facts aren't understanding. You need to compose them into higher-level models: what capabilities does this module have? What's its workflow? Is the architecture layering correct? That's the Model Engine's job. - ---- - -## Overview - -Model Engine runs after the Resolver Pipeline, building high-level models from facts and resolution results. - -``` -Resolver Pipeline ──→ entity + relation + call_edges - │ - ▼ - ModelEngine::runAll() - │ - ┌───────┼───────┐ - ▼ ▼ ▼ - Workflow Capability Architecture - Plugin Plugin Plugin -``` - ---- - -## ModelEngine: Plugin Manager - -```cpp -// engine/src/model/engine.h -class ModelEngine { -public: - void addPlugin(std::unique_ptr plugin); - int64_t runAll(uint64_t project_id); - ModelResult run(const std::string &name, uint64_t project_id); -}; -``` - -Each plugin implements the `ModelPlugin` interface and produces output into its own SQLite table. - ---- - -## Built-in Plugins - -| Plugin | Output Table | Description | -|--------|-------------|-------------| -| Workflow | workflow | Cross-module execution paths | -| Capability | capability | Module capability declarations & verification | -| Architecture | architecture | Layer classification & inter-layer dependencies | -| Contract | contract | Interface implementation relationships | - ---- - -## StateBuilder: Project State Recovery - -```cpp -// engine/src/model/state_builder.h -// Reads from built models and assembles AI-readable module -// summaries and project overviews. -``` - -Used by `project_overview` and `explain_module` tools. - ---- - -## Relationship with Verification Layer - -| | Model Engine | Verification Layer | -|---|---|---| -| Input | entity + relation + graph_edges | entity + relation + README | -| Output | Model tables (workflow/capability/architecture) | Findings (evidence chain) | -| Timing | During index (Phase B) | On-demand query | -| Used by | `project_overview`, `explain_module` | `verify_claim`, `detect_drift` | - -Model Engine **builds models**; Verification Layer **uses models to check claims**. - ---- - -## Series Navigation - -| # | Article | Topic | -|---|---------|-------| -| (1) | Intro | 56KB vs 629 bytes, what problem CodeScope solves | -| (2) | Progressive Readiness | ms-level project understanding | -| (3) | Worker Isolation | Indexing won't crash MCP Server | -| (4) | Zero-Redundancy Responses | Lean responses, on-demand return | -| (5) | C++ Engine Pipeline | Source code to multi-dimensional code graph | -| (6) | MCP Protocol Layer | Tool design philosophy | -| (7) | Language Translators | 10 languages → unified IR | -| (8) | Storage Layer | SQLite WAL + FTS5 + vec0 | -| (9) | Adaptive Queries | Fallback mechanism & readiness detection | -| (10) | Performance Truth | 200 to 60,000 files measured | -| (11) | Verification Layer | Making AI Accountable | -| **(12)** | **Model Engine** | **From Facts to Understanding ← this article** | diff --git a/docs/articles/en/codescope-architecture-13-parser-graph.md b/docs/articles/en/codescope-architecture-13-parser-graph.md deleted file mode 100644 index 1be6f23..0000000 --- a/docs/articles/en/codescope-architecture-13-parser-graph.md +++ /dev/null @@ -1,142 +0,0 @@ -# CodeScope Architecture (13): Parser & GraphBuilder — Parsing to Graph Construction - -> tree-sitter was one of the best technology choices in this project. Not because it's fast — because it **never errors**. No matter how malformed the source code, tree-sitter always produces an AST. This is critical when indexing millions of lines: one file parse failure should not block the entire index. - ---- - -## Parser: tree-sitter Wrapper - -Located at `engine/src/parser/parser.cpp`, a C++ wrapper around the tree-sitter C API. - -### Flow - -1. Select language parser by file extension -2. Call tree-sitter to generate a CST -3. Pass CST to the appropriate IR Translator -4. Handle parse errors (primarily memory limits) - -```cpp -TranslationUnit *parseFile(const char *path, const char *source, - size_t source_len, const char *language) { - TSParser *parser = ts_parser_new(); - ts_parser_set_language(parser, getLanguage(language)); - TSTree *tree = ts_parser_parse_string(parser, nullptr, source, source_len); - Translator *t = createTranslator(language); - return t->translate(tree, source, path); -} -``` - -### Why tree-sitter? - -| Aspect | tree-sitter | Traditional LALR/LR | -|--------|-------------|-------------------| -| Error tolerance | ✅ Always produces AST (with error nodes) | ❌ Fails on syntax errors | -| Incremental parsing | ✅ Supported | ❌ Full re-parse required | -| Multi-language | ✅ 100+ languages built-in | ❌ Separate toolchain per language | - -The downside: tree-sitter's CST includes **all syntactic details** — parentheses, semicolons, keywords. The IR Translator's job is to extract semantic structure and discard syntactic noise. - ---- - -## Grammar Management - -```cpp -// engine/src/codescope_grammars.h -// Manages 8 language tree-sitter grammars (.so files) -``` - -Grammars are auto-downloaded and compiled via CMake FetchContent at build time. Each language maps to a `.so` file loaded at runtime. - -| Language | Grammar Source | AST Node Types | -|----------|---------------|:--------------:| -| C | tree-sitter-c | ~50 | -| C++ | tree-sitter-cpp | ~100 | -| Go | tree-sitter-go | ~60 | -| Python | tree-sitter-python | ~40 | -| Rust | tree-sitter-rust | ~80 | -| JavaScript | tree-sitter-javascript | ~70 | -| TypeScript | tree-sitter-typescript | ~80 | -| Java | tree-sitter-java | ~70 | - ---- - -## GraphBuilder: IR → Graph - -`GraphBuilder` at `engine/src/graph/graph_builder.h` bridges IR translation and the Resolver. - -### Two APIs - -``` -Old API: TranslationUnit + Node* tree - └── Used for: C, C++, Go, Python, Rust, Java - └── Visitor pattern recursively walks IR tree - Each NodeKind → GraphNode - Each Relation → GraphEdge - -New API: SemanticUnit + flat records - └── Used for: JavaScript, TypeScript - └── Accepts flat records directly (no tree traversal) - Because JS/TS visitor output is already flat -``` - -### Graph Types - -```cpp -enum class NodeType : uint8_t { - Function, Method, Class, Struct, Interface, - Variable, Macro, Module, File -}; - -enum class EdgeType : uint8_t { - References, Calls, Defines, Contains, Imports, Inherits -}; -``` - -### Parent Chain Cache - -Key optimization: caches parent node IDs to avoid repeated SQLite lookups during deep AST traversal. Reduces database queries by **O(depth)** for deeply nested expressions. - -### Write Flow - -``` -GraphBuilder::build(translation_unit) - │ - ├── Walk IR node tree - ├── Walk Relation list - └── Batch write to SQLite - ├── graph_nodes (individual insert) - └── graph_edges (batch insert) -``` - ---- - -## Performance - -| Phase | Small project (200 files) | Linux kernel (64K files) | -|-------|:------------------------:|:-----------------------:| -| tree-sitter parse | ~seconds | ~2 min | -| IR translation | ~ms | ~30s | -| GraphBuilder write | ~ms | ~20s | -| ResolverPipeline | ~ms | ~1 min (bottleneck) | - -The **ResolverPipeline is the only super-linear phase** — each reference must search all entities for candidates. All other phases are O(N). - ---- - -## Series Navigation - -| # | Article | Topic | -|---|---------|-------| -| (1) | Intro | 56KB vs 629 bytes, what problem CodeScope solves | -| (2) | Progressive Readiness | ms-level project understanding | -| (3) | Worker Isolation | Indexing won't crash MCP Server | -| (4) | Zero-Redundancy Responses | Lean responses, on-demand return | -| (5) | C++ Engine Pipeline | Source code to multi-dimensional code graph | -| (6) | MCP Protocol Layer | Tool design philosophy | -| (7) | Language Translators | 10 languages → unified IR | -| (8) | Storage Layer | SQLite WAL + FTS5 + vec0 | -| (9) | Adaptive Queries | Fallback mechanism & readiness detection | -| (10) | Performance Truth | 200 to 60,000 files measured | -| (11) | Verification Layer | Making AI Accountable | -| (12) | Model Engine | From Facts to Understanding | -| **(13)** | **Parser & GraphBuilder** | **Parsing to Graph Construction ← this article** | diff --git a/docs/articles/en/codescope-deepdive-01-dual-language.md b/docs/articles/en/codescope-deepdive-01-dual-language.md new file mode 100644 index 0000000..fc99de9 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-01-dual-language.md @@ -0,0 +1,221 @@ +# CodeScope Deep Dive (1): Dual-Language Architecture — Why We Wrote a C++ Project in Rust + +> *"Sometimes the best tool for the job isn't the one you'd like to use."* +> 有时最好的工具并不是你喜欢的那个。 + +## Why Two Languages? + +If I had to explain CodeScope's architecture in one sentence, I'd say: **Rust writes an MCP server, C++ writes an entire engine, FFI glues them together, and each index task runs in its own subprocess.** + +This is not an elegant design. It's ugly, it's messy, and debugging it touches two language runtimes simultaneously. But it solves two problems I couldn't solve otherwise: + +1. **Rust's ecosystem** has MCP SDKs, serde serialization, tokio async — these are the most comfortable way to write a JSON-RPC server. +2. **C++'s ecosystem** has tree-sitter parsers, SQLite's native C API, and syntax parsers for various languages — these are the most direct path to building a code analysis engine. + +I initially tried a pure Rust approach. Using the `tree-sitter` crate to parse code, writing my own IR and storage layer. But I quickly discovered that when you need to cover 8 languages, handle repositories in the hundreds of GB range, and still deliver millisecond-level responses, the overhead of Rust's abstraction layers becomes non-negligible. On the C++ side, tree-sitter's C API can be called directly, SQLite's C API is zero-overhead, and every memory layout can be precisely controlled. + +So it ended up like this: **Rust writes the frontend (MCP server), C++ writes the backend (engine core), with a nasty but necessary FFI layer in between.** + +## Core Files + +``` +server/src/ffi/mod.rs ← Rust-side FFI declarations (~700 lines) +engine/src/engine_ffi.cpp ← C++-side FFI implementation +engine/src/engine.h ← C++ public API +``` + +## The Harsh Reality of FFI + +Let's look at the Rust side first. Here's what the FFI declarations look like: + +```rust +// server/src/ffi/mod.rs +unsafe extern "C" { + fn engine_init(db_path: *const c_char) -> i32; + fn engine_shutdown(); + + fn engine_create_project(root_path: *const c_char, name: *const c_char) -> u64; + fn engine_index_project( + project_id: u64, + dir_path: *const c_char, + language_filter: *const c_char, + ) -> *mut c_char; + fn engine_find_definition( + project_id: u64, + symbol_name: *const c_char, + file_filter: *const c_char, + ) -> *mut c_char; + // ... 40+ similar declarations +} +``` + +Every `unsafe extern "C"` block is saying: **"Rust doesn't know what's going on out there, but I trust it."** And that trust is easy to break. + +And the C++ side implementation looks like this: + +```cpp +// engine/src/engine_ffi.cpp +extern "C" char* engine_index_project( + uint64_t project_id, + const char* dir_path, + const char* language_filter) +{ + std::string result = // ... actual indexing logic + return dupString(result); // heap-allocated, Rust side must free +} +``` + +This code hides a classic FFI trap: **Who is responsible for freeing the memory?** + +After the Rust side calls `engine_index_project` and gets back a `*mut c_char`, it must call `engine_free_string` to release it. Forget to free = memory leak. Free twice = crash. This is what cross-language memory management is like — you're always worried the other side forgot something. + +## The Wrapper Layer Around FFI + +On the Rust side, each FFI call is wrapped in a safe Rust function that converts C pointers into Rust `String` or `Result`: + +```rust +// server/src/ffi/mod.rs (simplified) +pub fn index_project(project_id: u64, dir_path: &str, language_filter: &str) -> Result { + let dir_c = CString::new(dir_path).unwrap(); + let lang_c = CString::new(language_filter).unwrap(); + + let ptr = unsafe { engine_index_project(project_id, dir_c.as_ptr(), lang_c.as_ptr()) }; + if ptr.is_null() { + return Err("engine returned null".into()); + } + let result = unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned(); + unsafe { engine_free_string(ptr) }; + Ok(result) +} +``` + +This pattern is repeated 40+ times. Every time it's: +1. `CString::new` — Rust string → C string +2. unsafe call +3. Null pointer check +4. `CStr::from_ptr` — C string → Rust string +5. `engine_free_string` — Free C-side memory + +This process is so mechanical that I've written several bugs: forgetting to check for null pointers, forgetting to free, and even using a pointer after freeing it. + +```mermaid +flowchart LR + Rust["Rust MCP Server"] -->|"CString::new"| C["C String (heap)"] + Rust -->|"unsafe extern C"| FFI["FFI Boundary"] + FFI --> Cpp["C++ Engine"] + Cpp -->|"dupString"| Ret["C String (heap)"] + Ret -->|"CStr::from_ptr"| Rust + Rust -->|"engine_free_string"| Free["Free C heap"] + Free --> Cpp +``` + +## Global Singleton: C++'s "Implicit State" + +Inside the engine, there are three global singletons: + +```cpp +// engine/src/engine_internal.h +extern std::unique_ptr g_store; +extern std::unique_ptr g_query; +extern std::unique_ptr g_parser; +``` + +They are initialized by `engine_init()` and destroyed by `engine_shutdown()`. The Rust-side MCP server calls `engine_init` once at startup and `engine_shutdown` once at shutdown. + +But the thread-safety model here is fragile: + +```cpp +// engine/src/engine_internal.h +// Thread-safety model: +// - The Rust MCP server calls FFI functions SEQUENTIALLY from a single +// thread (the main event loop in server/src/mcp/server.rs). +// - Index worker SUBPROCESSES have their own engine instance (fork+exec). +// - engine_shutdown() must only be called when no FFI query is in flight. +``` + +In plain English: **"Don't call FFI from multiple threads at the same time."** + +This constraint was never an issue in v0.1 — the MCP server was a single-threaded event loop, processing all requests serially. But when the parallel scheduler was introduced in v0.2, the problem emerged: **The scheduler forks subprocesses, each with its own engine instance, but the parent process's engine is still handling query requests simultaneously.** + +The solution: within subprocesses, `fork+exec` restarts a `codescope worker` process, with a completely independent memory space that won't interfere with the parent process's global singletons. + +```mermaid +flowchart TB + subgraph "Main Process (Rust MCP Server)" + FFI_Call["FFI Call
engine_find_definition"] + Global["Global Singletons
g_store / g_query / g_parser"] + end + + subgraph "Worker 1 (Subprocess)" + Worker1["Independent engine instance
fork+exec fresh process"] + end + + subgraph "Worker 2 (Subprocess)" + Worker2["Independent engine instance
fork+exec fresh process"] + end + + FFI_Call --> Global + Scheduler["Parallel Scheduler"] -->|"fork+exec"| Worker1 + Scheduler -->|"fork+exec"| Worker2 +``` + +## A Lesson That Made My Blood Run Cold + +During v0.1.2 development, I encountered a baffling crash: on CI, `engine_shutdown` would occasionally trigger a `double free`. + +After three days of debugging, I found the root cause was a **dangling pointer**. After a certain Rust-side FFI call returned, the C++ side was holding a pointer to data on Rust's stack. When the Rust `CString` was dropped, the C++ side was still using it. On a single thread this might occasionally work, but under CI's high load it would crash. + +The fix was to change all cross-boundary strings to **deep copies** — the C++ side immediately copies the received C string into a `std::string`, never holding any pointer to Rust memory. + +The lesson: **Pointers across the FFI boundary must either be const and not freed during their lifetime, or be copied immediately. There is no middle ground.** + +## Why Endure This Pain? + +If FFI is this much trouble, why not use pure Rust or pure C++? + +**Pure C++ approach**: C++ has no native MCP SDK. I would need to manually implement the JSON-RPC 2.0 protocol, HTTP transport (or stdio transport), and routing dispatch for all tools. That's roughly 2000-3000 lines of extra code, and every new tool requires manual registration. + +**Pure Rust approach**: Rust's `tree-sitter` crate is a wrapper around the C API, with minimal performance loss, but the problem is this: a large amount of code in the C++ engine (especially the IR translator and graph builder) is written using C++17/C++20 features. Porting it all to Rust would require reimplementing a complete IR system and graph algorithms — that's months of work. + +So FFI isn't the optimal solution; it's the **optimal solution under the current constraints**. Rust handles what it's good at: networking, serialization, process management, type safety. C++ handles what it's good at: parsing, graph construction, memory control, performance-critical paths. + +```mermaid +flowchart LR + subgraph "Rust Layer (MCP Server)" + MCP["MCP Protocol Handling
JSON-RPC 2.0 / stdio"] + Tools["42+ Tool Routes
TOOL_HANDLERS Registry"] + Sched["Parallel Scheduler
Three Scheduling Modes"] + WorkerMgmt["Worker Subprocess Management"] + end + + subgraph "FFI Boundary" + FFI["unsafe extern C
~40 Functions"] + end + + subgraph "C++ Layer (Engine)" + Parser["tree-sitter Parser
8 Languages"] + IR["Unified IR Translator"] + Graph["Graph Builder"] + Store["SQLite Storage Layer"] + Verify["Verification Pipeline"] + end + + Worker["codescope worker
Independent Subprocess"] + + MCP --> Tools + Tools --> FFI + Sched --> WorkerMgmt + WorkerMgmt -->|"fork+exec"| Worker + Worker --- FFI + FFI --> Parser + Parser --> IR --> Graph --> Store + Store --> Verify +``` + +## Summary + +CodeScope's dual-language architecture wasn't designed — it was forced. We couldn't give up Rust's MCP ecosystem or C++'s parsing ecosystem, so FFI became the cost we had to bear. + +40+ `unsafe extern "C"` declarations, manually managing the lifetime of every string, thread-safety constraints on global singletons — these are all real technical debt. But so far, it runs reliably enough. + +In the next article, I'll break down **Worker Subprocess Isolation** — why every index task must run in its own subprocess, and how to handle repositories with millions of lines of code without crashing. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-02-worker-isolation.md b/docs/articles/en/codescope-deepdive-02-worker-isolation.md new file mode 100644 index 0000000..247b365 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-02-worker-isolation.md @@ -0,0 +1,260 @@ +# CodeScope Deep Dive (2): Worker Subprocess Isolation — Why Indexing Must Run in Subprocesses + +> *"The only way to make a C++ program crash-proof is to run it in a separate process."* +> 让 C++ 程序不崩溃的唯一方法，是把它跑在单独的进程里。 + +## Starting with an OOM Crash + +In the early versions of CodeScope (v0.1), indexing ran directly in the main process. After calling `engine_index_project`, the entire MCP server would freeze — single-threaded event loop, unable to handle any new requests until indexing completed. + +But that wasn't the most serious problem. The most serious problem was: **When indexing a large repository (like the Linux kernel or JDK), the C++ engine's memory usage would spike to several GB, and then the OOM killer would take down the entire process.** + +The MCP server crashes, the LLM client receives a `connection closed` error, and the user probably mutters "what garbage software." + +The solution was straightforward: **Run indexing tasks in subprocesses.** If a subprocess crashes, the main process is unaffected and can reschedule or gracefully degrade. + +## Core Files + +``` +server/src/scheduler/worker.rs ← Worker subprocess management (~576 lines) +server/src/scheduler/mod.rs ← Scheduler main logic (~1382 lines) +server/src/main.rs ← worker subcommand entry point +``` + +## fork + exec: A Simple Approach + +The Rust scheduler spawns subprocesses in a straightforward way: + +```rust +// server/src/scheduler/worker.rs +pub(super) fn run_module_worker( + exe: &str, + project_dir: &str, + module_name: &str, + files_estimate: u64, + workers: u32, + grammars_dir: &str, + db_prefix: &str, + project_id: u64, + quarantine_exclude: Option<&str>, +) -> ModuleResult { + let module_dir = Path::new(project_dir).join(module_name); + let module_db = format!("{}_{}.db", db_prefix, module_name); + + // Each worker starts from a clean DB file + let _ = std::fs::remove_file(&module_db); + + let mut cmd = Command::new(exe); + cmd.args([ + "worker", + &module_db, + module_dir.to_str().unwrap_or(""), + "", // empty language filter + &project_name, + &project_id_str, + ]); + cmd.env("GRAMMARS_DIR", grammars_dir); + cmd.env("CODESCOPE_DB_PATH", &module_db); + cmd.env("CODESCOPE_INDEX_MODE", "fast"); + // ... +} +``` + +Each module launches an independent `codescope worker` subprocess. This subprocess is a completely independent process instance — it has its own address space, its own global variables, its own `g_store` singleton. This means: + +- Subprocess crashes → main process is unaffected +- Subprocess memory leaks → OS automatically reclaims on process exit +- Subprocess OOM → only the subprocess itself is killed, the main process keeps running + +```mermaid +flowchart LR + subgraph "Main Process" + Scheduler["Parallel Scheduler"] + MCP["MCP Server
(Handles Queries)"] + MCP --->|"Queries indexed DB"| DB["Main SQLite DB"] + end + + subgraph "Worker 1" + W1["codescope worker
Module: src/"] + W1 -->|"Writes to"| DB1["Module DB: main_src.db"] + end + + subgraph "Worker 2" + W2["codescope worker
Module: lib/"] + W2 -->|"Writes to"| DB2["Module DB: main_lib.db"] + end + + subgraph "Worker 3" + W3["codescope worker
Module: tests/"] + W3 -->|"Writes to"| DB3["Module DB: main_tests.db"] + end + + Scheduler -->|"fork+exec"| W1 + Scheduler -->|"fork+exec"| W2 + Scheduler -->|"fork+exec"| W3 + DB1 --> Merge["DB Merge"] + DB2 --> Merge + DB3 --> Merge + Merge --> DB +``` + +## Timeout and Isolation + +Each worker has a hard timeout: + +```rust +const DEFAULT_WORKER_TIMEOUT_SECS: u64 = 600; +``` + +If it doesn't finish within 10 minutes, the subprocess is `kill`ed. This prevents an infinite loop in a parser from hogging the entire scheduler. + +But there's a question: **How do we know if a worker exited normally or crashed?** + +```rust +// Wait for the subprocess to exit +match child.wait_with_output() { + Ok(output) => { + if output.status.success() { + // Normal exit, parse stdout JSON + } else { + // Crash or non-zero exit code + // Enter isolation flow + } + } + Err(e) => { + // wait itself failed + } +} +``` + +## Isolation Mechanism: Binary Search to Locate the Crashing File + +When a worker crashes, the scheduler doesn't simply retry the entire module — it uses **binary search** to locate the file causing the crash: + +```rust +// server/src/scheduler/mod.rs (simplified) +fn quarantine_crashing_file( + module: &Module, + files: &[FileEntry], + crash_index: usize, +) -> Option { + let mut lo = 0; + let mut hi = files.len(); + + while lo < hi { + let mid = (lo + hi) / 2; + let test_files = &files[lo..=mid]; + + // Index only these files + if run_worker_with_files(module, test_files).is_ok() { + lo = mid + 1; // Crash is on the right + } else { + hi = mid; // Crash is on the left (including mid) + } + } + + // Found the crashing file + Some(files[lo].path.clone()) +} +``` + +Once the crashing file is found, the scheduler adds it to `CODESCOPE_EXCLUDE_PATHS` and then **retries the entire module** — skipping this file. + +```mermaid +flowchart TD + Start["Worker Crashed"] --> BinarySearch["Binary search to locate crashing file"] + BinarySearch --> Found["Crashing file found"] + Found --> Exclude["Add to CODESCOPE_EXCLUDE_PATHS"] + Exclude --> Retry["Retry module (skip crashing file)"] + Retry --> Success["Success"] + Retry --> Fail["Crashed again"] + Fail --> BinarySearch + + BinarySearch -.->|"Note"| Note["Max 10 isolation iterations
Abandon module if exceeded"] +``` + +This mechanism has located tree-sitter parser crashes on specific C++ template code several times in my testing. After isolation, the entire project can still be indexed, just missing that one file — which is 100x better than a total indexing failure. + +## One DB Per Worker to Avoid SQLite Concurrent Writes + +Another key design decision: **Each worker writes to its own SQLite file.** + +```rust +let module_db = format!("{}_{}.db", db_prefix, module_name); +``` + +SQLite's WAL mode supports concurrent reads, but concurrent writes require locking. If multiple workers write to the same DB file simultaneously, lock contention causes severe performance degradation, or even deadlocks. + +Each worker having its own DB file means: + +- No lock contention +- No deadlock risk +- After indexing completes, merge via `ATTACH + INSERT OR IGNORE` + +```mermaid +flowchart LR + subgraph "Parallel Write Phase" + W1 -->|"Exclusive write"| DB1 + W2 -->|"Exclusive write"| DB2 + W3 -->|"Exclusive write"| DB3 + end + + subgraph "Merge Phase" + DB1 -->|"ATTACH"| Main + DB2 -->|"ATTACH"| Main + DB3 -->|"ATTACH"| Main + Main["Main DB
ATTACH + INSERT OR IGNORE"] + end +``` + +## A Lesson That Made My Blood Run Cold + +During v0.2.0 development, I discovered that **entity IDs from different modules conflicted in the merged DB.** + +Each worker's `project_id` was assigned by the scheduler (1, 2, 3, ...), but entity IDs auto-incremented starting from 1. Two modules could both have an entity with `id=42`, and `INSERT OR IGNORE` during the merge would lose the second module's entity. + +The fix was: **Apply offset remapping to each module's entity IDs during merge.** + +```rust +// server/src/scheduler/merge.rs (simplified) +// For module i > 0: +// 1. Calculate offset = MAX(id) of current main table +// 2. Copy module data to a temp table +// 3. Add offset to temp table's id and FK columns +// 4. INSERT OR IGNORE into the main table +// 5. DROP the temp table + +const TABLE_SPECS: &[TableSpec] = &[ + TableSpec { + name: "entity", + remap_cols: &[("id", "self")], // id += offset + skip_rowid: false, + }, + TableSpec { + name: "relation", + remap_cols: &[ + ("id", "self"), // Primary key remap + ("source_id", "entity"), // FK follows entity's offset + ("target_id", "entity"), // FK follows entity's offset + ], + skip_rowid: false, + }, + // ... +]; +``` + +This remapping needs to know the foreign key relationships between tables: `relation.source_id` references `entity.id`, so `source_id`'s offset must match `entity.id`'s offset. + +**Table order matters**: referenced tables (entity) must be merged first, referencing tables (relation) later. That's why the order of `TABLE_SPECS` is carefully arranged. + +## Summary + +Worker subprocess isolation is the key to CodeScope running stably on production-grade repositories. It solves three problems: + +1. **Crash isolation**: C++ engine crashes don't bring down the MCP server +2. **Memory reclamation**: All memory is automatically freed when the process exits +3. **Concurrent writes**: Each worker has its own DB file, no lock contention + +The cost is: process startup overhead (~10ms), the extra step of DB merging, and those annoying ID remappings. But compared to the pain of the main process being killed by OOM, this cost is negligible. + +In the next article, I'll break down the **Parallel Scheduler** — the design and trade-offs of three scheduling modes, and why a simple "static allocation" approach fails on large repositories. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-03-scheduler.md b/docs/articles/en/codescope-deepdive-03-scheduler.md new file mode 100644 index 0000000..74edbdd --- /dev/null +++ b/docs/articles/en/codescope-deepdive-03-scheduler.md @@ -0,0 +1,258 @@ +# CodeScope Deep Dive (3): Parallel Scheduler — Design and Trade-offs of Three Scheduling Modes + +> *"A scheduler that doesn't know when to give up is worse than no scheduler at all."* + +## The Starting Point + +CodeScope's indexing needs to process a massive number of files. The Linux kernel has ~70,000 source files, and JDK has ~40,000. If parsed serially, a single file's tree-sitter parsing + IR construction + SQLite writing takes 50-200ms, meaning 70,000 files would take 1-4 hours. + +Users can't wait 4 hours. So parallelism is essential. + +But parallel scheduling isn't as simple as "spawn N threads." The problems are: + +1. **Huge disparity in module sizes**: `src/` might have 10,000 files, while `docs/` has only 50 +2. **Subsystem resource contention**: SQLite writes, memory allocation, file I/O +3. **Crash handling**: A crash in one module must not affect other modules +4. **CPU core limits**: Unlimited parallelism is not possible + +## Core Files + +``` +server/src/scheduler/mod.rs ← Scheduler main logic (~1382 lines) +server/src/scheduler/chunk_plan.rs ← Chunk planning (~524 lines) +server/src/scheduler/dyn_config.rs ← Dynamic scheduling config (~220 lines) +server/src/scheduler/shm.rs ← Shared memory +server/src/scheduler/chunk_queue.rs← Work-stealing queue +``` + +## Mode 1: Static Proportional Allocation + +This is the most straightforward approach and the default scheduling mode of CodeScope. + +```rust +// server/src/scheduler/mod.rs (conceptual) +fn allocate_workers_static(modules: &[Module], total_workers: u32) -> Vec { + let total_files: u64 = modules.iter().map(|m| m.file_count).sum(); + + modules.iter().map(|m| { + std::cmp::max( + 1, // At least 1 worker per module + (m.file_count as f64 / total_files as f64 * total_workers as f64).ceil() as u32 + ) + }).collect() +} +``` + +The core logic is simple: **allocate CPU cores proportionally by file count.** + +- Module A has 10,000 files (50% of total) → 4 cores allocated (out of 8 total) +- Module B has 5,000 files (25%) → 2 cores allocated +- Module C has 500 files (2.5%) → 1 core allocated (minimum 1) + +```mermaid +flowchart LR + subgraph "Module Discovery" + Discover["discover_modules()
scan top-level dirs + file count"] + end + + subgraph "Static Allocation" + Alloc["allocate workers proportionally by file count"] + W1["Module A: 4 workers"] + W2["Module B: 2 workers"] + W3["Module C: 1 worker"] + W4["Module D: 1 worker"] + end + + subgraph "Parallel Execution" + WA["Worker subprocess
CODESCOPE_SKIP_ASYNC=1"] + WB["Worker subprocess"] + WC["Worker subprocess"] + WD["Worker subprocess"] + end + + subgraph "Merge" + Merge["DB Merge
ATTACH + INSERT OR IGNORE"] + end + + Discover --> Alloc + Alloc --> W1 --> WA + Alloc --> W2 --> WB + Alloc --> W3 --> WC + Alloc --> W4 --> WD + WA --> Merge + WB --> Merge + WC --> Merge + WD --> Merge +``` + +**Pros**: Simple, predictable, no runtime monitoring needed. + +**Cons**: If Module A's 10,000 files are all small files while Module B's 5,000 files are all large files, the allocation no longer reflects the actual workload. Once a small module finishes, its cores sit idle until all modules are complete. + +## Mode 2: CPU-Dynamic (Chunked Work-Stealing) + +The "core waste" problem of static allocation becomes unacceptable on large repositories (>50,000 files). This leads to the second mode: **work-stealing**. + +```rust +// server/src/scheduler/dyn_config.rs +pub fn should_enable(&self, total_modules: usize, total_files: u64) -> bool { + match self.force_on { + Some(b) => b, + None => total_modules > 4 && total_files > 10000, + } +} +``` + +This mode is **opt-in** by default—it only activates via the `CODESCOPE_CPU_DYNAMIC=1` environment variable or when a project exceeds 10,000 files and 4 modules. + +Its core idea: **split files into smaller chunks, then have multiple workers dynamically fetch tasks from a shared queue.** + +```rust +// server/src/scheduler/chunk_plan.rs +/// Default: 8 MB per chunk +pub const TARGET_BYTES: u64 = 8 * 1024 * 1024; +/// Maximum: 32 MB +pub const MAX_BYTES: u64 = 32 * 1024 * 1024; +/// Cluster by first 2 path levels +pub const CLUSTER_DEPTH: usize = 2; +``` + +```mermaid +flowchart TD + subgraph "File Chunking" + Files["70,000 source files"] + Cluster["Cluster by directory
(first 2 path levels)"] + Split["Split at 8MB"] + Chunks["100~250 chunks"] + end + + subgraph "Work Stealing" + Q["Shared Task Queue"] + W1["Worker 1"] -->|"fetch"| Q + W2["Worker 2"] -->|"fetch"| Q + W3["Worker 3"] -->|"fetch"| Q + W4["Worker 4"] -->|"fetch"| Q + W1 -->|"after completion"| Q + W2 -->|"after completion"| Q + end + + subgraph "Shared Memory Coordination" + SHM["SchedShm
Shared Memory State"] + SHM -->|"idle worker count"| Q + SHM -->|"RSS monitoring"| Q + end + + Files --> Cluster --> Split --> Chunks --> Q +``` + +**Chunking algorithm**: + +1. Cluster by first 2 path levels (`src/compiler/`, `src/parser/`) +2. Sort descending by byte size +3. Greedy packing: accumulate files until `TARGET_BYTES` (8MB) is reached +4. Force-split oversized directories (>32MB) + +This way, files in the same directory are likely to end up in the same chunk, preserving **directory locality**—related files are parsed together, reducing cross-module context switching. + +**Memory monitoring**: The dynamic scheduler also has a memory limit mechanism. When RSS exceeds the threshold (default 4GB), no new chunks are allocated to prevent OOM. + +```rust +// server/src/scheduler/dyn_config.rs +pub fn from_env() -> Self { + let mem_limit_mb = env::var("CODESCOPE_MEM_LIMIT_MB") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(4096); + // ... +} +``` + +## Mode 3: Memory Bulk Indexing (Membulk) + +For small modules (<= file count threshold), the scheduler uses a third mode: **memory bulk indexing**. + +```cpp +// engine/src/engine_internal.h +/// For small modules (<= kMemBulkFileThreshold files) the parse workers +/// aggregate FileResult in memory instead of pushing through BoundedQueue, +/// then flush once via insertFileResultBatch. +char *engine_index_project_membulk( + uint64_t project_id, const std::string &dir, uint64_t max_file_size, + const FilterPolicy &filter, + const std::vector> &job_lang, + const std::unordered_map &lang_ptrs, + bool is_reindex, bool mode_fast, bool mode_deep); +``` + +Why is this mode needed? Because the **BoundedQueue** (thread-safe producer-consumer queue) overhead is too high for small modules. If a module has only 50 files, each file must be pushed to the queue after parsing, then popped by another thread and written to SQLite—this synchronization overhead can account for up to 30% of total time. + +For small modules, a better strategy is: **all parsing threads aggregate results in memory, then flush everything in a single batch write after parsing completes.** + +```mermaid +flowchart LR + subgraph "Streaming Path (large modules)" + F1["File 1"] -->|"parse"| Queue["BoundedQueue"] + F2["File 2"] -->|"parse"| Queue + F3["File N"] -->|"parse"| Queue + Queue -->|"pop"| Writer["Single-threaded SQLite write"] + end + + subgraph "Memory Bulk Path (small modules)" + MF1["File 1"] -->|"parse"| Mem["In-memory aggregation"] + MF2["File 2"] -->|"parse"| Mem + MF3["File N"] -->|"parse"| Mem + Mem -->|"single batch write"| BatchWriter["insertFileResultBatch"] + end +``` + +## Selection Criteria for Scheduling Modes + +The three modes correspond to three scenarios: + +```mermaid +flowchart TD + Start["Start Indexing"] --> ModuleCount{"Modules > 4
and files > 10,000?"} + ModuleCount -->|"No"| Static["Static Proportional Allocation
Simple & reliable"] + ModuleCount -->|"Yes"| Dynamic["CPU Dynamic Scheduling
Work-stealing"] + + Start --> FileCount{"Single module
file count < threshold?"} + FileCount -->|"Yes"| Membulk["Memory Bulk Indexing
Reduce queue overhead"] + FileCount -->|"No"| Streaming["Streaming Indexing
BoundedQueue"] +``` + +## A Lesson That Made My Blood Run Cold + +In v0.2.2, I encountered a **deadlock**. + +In dynamic scheduling mode, multiple workers coordinate task allocation through shared memory (`SchedShm`). When all chunks have been claimed, the last worker sends a "done" signal, and the scheduler begins merging databases. + +But there was an edge case: **if a worker is scheduled out by the OS after claiming a chunk but before starting work** (e.g., due to CPU contention from other processes), other workers might have already completed all work and started merging. When the delayed worker wakes up, it's still writing to its own DB, but the merge thread is already reading from it. Under SQLite's WAL mode, reads and writes can proceed concurrently, but if we're using `INSERT OR IGNORE` for merging while the worker is still writing, we get a **partial merge** problem. + +The fix: **ensure all workers have exited before merging.** The scheduler no longer relies on a "done" signal in shared memory; instead, it waits for each child process's `wait()` to return. + +```rust +// Fixed logic +for handle in worker_handles { + let result = handle.join().unwrap_or_else(|_| { + // Thread panic or worker crash + ModuleResult::crashed(module_name) + }); + results.push(result); +} +// Only start merging after all workers have confirmed exit +merge_results(results, &db_path); +``` + +## Summary + +The three scheduling modes weren't designed upfront; they were added incrementally as problems emerged: + +1. **Static Proportional Allocation**: Simple, predictable, suitable for most projects +2. **CPU Dynamic Scheduling**: Solves the core waste problem for large repositories, but requires opt-in +3. **Memory Bulk Indexing**: Solves the queue overhead problem for small modules, a performance optimization + +The core design principle of the scheduler is: **the scheduler only manages CPU core allocation; it does not participate in file discovery, parsing, or graph construction.** It neither knows nor cares about the content of each file; it only cares about "which module has how many files" and "which core is idle." + +This separation of concerns makes it easy to replace the scheduler in the future—for example, with a Kubernetes cluster scheduler or a priority-based queue. + +In the next article, I'll take apart the **Two-Phase Index**—how Fast Scan returns results in milliseconds, and what Background Enhance does behind the scenes. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-04-two-phase-index.md b/docs/articles/en/codescope-deepdive-04-two-phase-index.md new file mode 100644 index 0000000..09cff80 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-04-two-phase-index.md @@ -0,0 +1,179 @@ +# CodeScope Deep Dive (4): Two-Phase Index — The Trade-off Between Millisecond Response and Deep Analysis + +> *"The user doesn't care about your graph database. They care about getting an answer before they forget the question."* + +## The Problem: Indexing Is Too Slow + +CodeScope's full indexing pipeline includes: file discovery → tree-sitter parsing → IR construction → metric computation → graph construction → graph enhancement (community detection, clustering). For a repository the size of the Linux kernel, full indexing takes 5-15 minutes. + +An LLM's patience is limited. If a user initiates a `codescope_index_project` request and then has to wait 10 minutes before they can query, the tool becomes unusable. + +But here's the good news: **most queries only need basic symbol information** (definitions, references, call relationships), not graph analysis. If we can return a basic symbol index within seconds, and let graph analysis quietly complete in the background, the user won't perceive any latency. + +That's the origin of the two-phase index. + +## Core Files + +``` +server/src/scheduler/worker.rs ← CODESCOPE_SKIP_ASYNC in Worker +server/src/engine_ffi.cpp ← engine_index_project implementation +engine/src/engine_internal.h ← engine_index_post_parse declaration +engine/src/engine_helpers.cpp ← Index helper functions +``` + +## Phase A: Fast Scan (milliseconds to seconds) + +The goal of Fast Scan is: **return a queryable index as quickly as possible.** + +It only does the essential work: + +1. File discovery + filtering (8-layer filter strategy) +2. tree-sitter parsing (AST generation) +3. IR translation (unified intermediate representation) +4. Symbol extraction (entities, relations) +5. SQLite writing + +```mermaid +flowchart LR + subgraph "Phase A: Fast Scan (ms~s)" + FD["File Discovery
FilterPolicy 8-layer filtering"] + Parse["tree-sitter Parsing
AST Construction"] + IR["IR Translation
Unified Intermediate Representation"] + SQL["SQLite Write
entity + relation"] + end + + subgraph "Phase B: Background Enhance (async seconds)" + Graph["Graph Construction
Symbol Graph + Call Graph"] + Metric["Metric Computation
Cyclomatic Complexity, etc."] + Enhance["Graph Enhancement
Community Detection, etc."] + FTS["FTS5 Full-Text Index
Construction"] + end + + FD --> Parse --> IR --> SQL + SQL -->|"CODESCOPE_SKIP_ASYNC=1"| Skip["Skip Phase B"] + SQL -->|"Normal mode"| Graph --> Metric --> Enhance --> FTS +``` + +During the parallel worker indexing phase, each worker sets `CODESCOPE_SKIP_ASYNC=1`: + +```rust +// server/src/scheduler/worker.rs +cmd.env("CODESCOPE_INDEX_MODE", "fast"); +``` + +This means workers only do Phase A, skipping Phase B. The merged DB can be ready within seconds, and **users can start querying immediately.** + +## Phase B: Background Enhance (async, seconds to minutes) + +Phase B starts after Phase A completes, executing a series of **compute-intensive** operations: + +1. **Graph construction**: Build a symbol graph from the `entity` and `relation` tables +2. **Metric computation**: Cyclomatic complexity, cognitive complexity, nesting depth, etc. +3. **Graph enhancement**: Community detection, cluster analysis +4. **FTS5 full-text index**: Build an inverted index for code search + +```cpp +// engine/src/engine_internal.h +/// Index Project: in-memory bulk path + shared post-parse +/// +/// For small modules (<= kMemBulkFileThreshold files) the parse workers +/// aggregate FileResult in memory instead of pushing through BoundedQueue, +/// then flush once via insertFileResultBatch. The post-parse graph-building +/// sequence is shared with the streaming path via engine_index_post_parse. +char *engine_index_project_membulk( + uint64_t project_id, const std::string &dir, uint64_t max_file_size, + const FilterPolicy &filter, + const std::vector> &job_lang, + const std::unordered_map &lang_ptrs, + bool is_reindex, bool mode_fast, bool mode_deep); +``` + +The `mode_fast` parameter controls whether Phase B is skipped. The `mode_deep` parameter controls whether additional deep analysis is performed (e.g., preprocessing for drift detection). + +## The 8-Layer Filtering Strategy + +During the file discovery phase of Phase A, FilterPolicy applies 8 layers of filtering to reduce the number of files that need to be parsed: + +```cpp +// engine/src/filter_policy.h +class FilterPolicy { + // ... + bool shouldSkipEntry(const std::string &rel_path, bool is_dir) const; +}; +``` + +```mermaid +flowchart TD + F["36,919 files"] --> L1["Layer 1: Path component skip
node_modules, .venv, target..."] + L1 --> L2["Layer 2: .gitignore matching"] + L2 --> L3["Layer 3: .codescopeignore matching"] + L3 --> L4["Layer 4: Exact filename skip
package-lock.json, .DS_Store..."] + L4 --> L5["Layer 5: Filename prefix skip
.env.*, yarn-error.log.*"] + L5 --> L6["Layer 6: Extension skip
.exe, .zip, .min.js..."] + L6 --> L7["Layer 7: Non-source file extensions
binary, archive, media files"] + L7 --> L8["Layer 8: File size check
too large / too small"] + L8 --> Result["6,029 candidate files
(~83% filtered out)"] +``` + +On real-world projects, these 8 layers of filtering can eliminate about 83% of files. The effect is especially significant for repositories like the Linux kernel, which contain a large number of documentation files, scripts, configuration files, and binaries. + +## Progressive Readiness UX + +The core value of the two-phase index lies in **progressive readiness**: + +```mermaid +sequenceDiagram + participant User as User + participant MCP as MCP Server + participant Sched as Scheduler + participant Worker as Worker Subprocess + + User->>MCP: codescope_index_project + MCP->>Sched: Start indexing + Sched->>Worker: fork+exec (SKIP_ASYNC=1) + Worker-->>Sched: Phase A complete (2s) + Sched-->>MCP: Return index result (fast) + + User->>MCP: codescope_find_definition + MCP-->>User: Immediate result ✓ + + Note over Sched,Worker: Phase B continues in background + Worker->>Worker: Graph construction + Metric computation + FTS5 + Worker-->>Sched: Phase B complete (30s later) + + User->>MCP: codescope_graph_query + MCP-->>User: Graph query available ✓ +``` + +After the user sends an indexing request, they receive an "indexing complete" response within 2 seconds, and can immediately query definitions and references. Graph analysis and full-text search results become available 30 seconds later. + +If the user doesn't perceive that work is still happening in the background, then the design is successful. + +## A Lesson That Made My Blood Run Cold + +In v0.2.0, I discovered that the **merged DB was missing the FTS5 index**. + +The reason was: each worker skipped FTS5 construction during Phase A (`CODESCOPE_SKIP_ASYNC=1`), and the merged DB did not trigger FTS5 reconstruction either. As a result, the `search_code` tool returned empty results, even though the user was certain the code contained a particular keyword. + +The fix: **perform a full FTS5 rebuild on the main DB after the merge step.** + +But this introduced another problem: FTS5 rebuild time can be long (5-10 seconds for large repositories). If we rebuild FTS5 after every index merge, the user's wait time for the first query increases. + +The final solution: **rebuild FTS5 immediately after the merge, but make the rebuild process incremental—only index newly inserted rows, not re-index existing ones.** + +```sql +-- Rebuild FTS5 index +INSERT INTO code_fts(code_fts) +VALUES('rebuild'); +``` + +## Summary + +The two-phase index is CodeScope's trade-off between "response speed" and "analysis depth": + +- **Phase A (Fast Scan)**: Only does symbol extraction, skips graph analysis, allowing users to get a queryable index in milliseconds to seconds +- **Phase B (Background Enhance)**: Performs graph construction, metric computation, and FTS5 indexing in the background, automatically upgrading query capabilities upon completion + +This approach is not perfect—if a user initiates a graph query before graph construction is complete, they'll receive a "data not yet ready" prompt. But most of the time, the user's first few queries are symbol lookups; graph queries come later. The two-phase index aligns perfectly with this usage pattern. + +In the next article, I'll take apart the **Unified IR and Language Translators**—how 8 different languages share the same code model. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-05-parser-ir.md b/docs/articles/en/codescope-deepdive-05-parser-ir.md new file mode 100644 index 0000000..f74c975 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-05-parser-ir.md @@ -0,0 +1,244 @@ +# CodeScope Deep Dive (5): tree-sitter Parser & Unified IR — 8 Languages, One Model + +> *"A programming language is a tool. Code analysis should not care which tool you used."* +> 编程语言是工具。代码分析不应该关心你用了哪个工具。 + +## The Problem: 8 Languages, One Format + +CodeScope supports 8 languages: C, C++, Rust, Go, JavaScript, TypeScript, Python, Java. + +Each language has its own AST structure, syntax rules, and scoping rules. Writing a separate set of analysis code for each language would cause maintenance costs to spiral out of control. + +The solution is: **use a unified IR (Intermediate Representation) to abstract across all languages**. + +## Core Files + +``` +engine/src/parser/parser.cpp ← tree-sitter parser wrapper +engine/src/parser/parser.h ← Parser interface +engine/src/ir/ir.h ← Unified IR definition +engine/src/ir/ir.cpp ← IR construction +engine/src/ir/ir_translator.h ← IR translator base class +engine/src/ir/ir_translator.cpp ← IR translator implementation +``` + +## tree-sitter: One Parser, Many Languages + +tree-sitter is an incremental parsing library that supports grammar definitions for many languages. CodeScope uses tree-sitter as its underlying parser — each language needs only one `.so` grammar file. + +```cpp +// engine/src/parser/parser.h (conceptual) +class Parser { + std::unordered_map languages_; + // ... +}; +``` + +Parsing flow: + +1. Detect the file's language (by extension or shebang) +2. Load the corresponding tree-sitter grammar (`.so` file) +3. Parse the file to generate an AST (CST is technically a concrete syntax tree) +4. Traverse the AST to extract symbol information + +```mermaid +flowchart LR + subgraph "Source Files" + C["main.c"] + RS["lib.rs"] + PY["utils.py"] + JS["app.js"] + end + + subgraph "tree-sitter Grammars" + C_G["tree-sitter-c.so"] + RS_G["tree-sitter-rust.so"] + PY_G["tree-sitter-python.so"] + JS_G["tree-sitter-javascript.so"] + end + + subgraph "Unified IR" + IR["ir::Record
Name | Type | Location | Relations"] + end + + C --> C_G --> IR + RS --> RS_G --> IR + PY --> PY_G --> IR + JS --> JS_G --> IR +``` + +## Unified IR Definition + +The core data structure of IR is `ir::Record`: + +```cpp +// engine/src/ir/ir.h (conceptual) +namespace ir { + +struct Record { + std::string name; // Symbol name + std::string kind; // Symbol type (function, class, variable, ...) + std::string file_path; // File path + int line; // Start line + int col; // Start column + int end_line; // End line + int end_col; // End column + std::vector relations; // Relations to other symbols + std::vector children; // Nested symbols +}; + +struct Relation { + std::string kind; // Relation type (call, reference, inherit, contain, ...) + std::string target_name; // Target symbol name + int target_line; // Target location + int target_col; +}; + +} // namespace ir +``` + +The key to this IR design: **abstract enough to express the structure of all languages, yet concrete enough to retain sufficient information for queries.** + +## IR Translators + +The conversion from each language's tree-sitter AST to the unified IR is implemented by `IrTranslator`: + +```cpp +// engine/src/ir/ir_translator.h (conceptual) +class IrTranslator { +public: + virtual std::vector translate(const TSNode &root, + const std::string &source) = 0; + virtual ~IrTranslator() = default; +}; +``` + +Each language has a translator subclass: + +```mermaid +flowchart TD + IrTranslator["IrTranslator (Base Class)"] + C_Trans["CTranslator"] + CPP_Trans["CppTranslator"] + Rust_Trans["RustTranslator"] + Go_Trans["GoTranslator"] + JS_Trans["JsTranslator"] + TS_Trans["TsTranslator"] + Python_Trans["PythonTranslator"] + Java_Trans["JavaTranslator"] + + IrTranslator --> C_Trans + IrTranslator --> CPP_Trans + IrTranslator --> Rust_Trans + IrTranslator --> Go_Trans + IrTranslator --> JS_Trans + IrTranslator --> TS_Trans + IrTranslator --> Python_Trans + IrTranslator --> Java_Trans +``` + +Each translator knows how to extract function definitions, class definitions, variable declarations, function calls, etc., from the AST of its specific language. + +## Relation Extraction + +The `Relation` structure in IR is the foundation of CodeScope's graph queries. As the translator traverses the AST, it identifies the following relations: + +- **call**: Function call relationship +- **reference**: Symbol reference +- **inherit**: Inheritance relationship +- **contain**: Containment relationship (class contains method, function contains variable) +- **implement**: Implementation relationship (interface implementation) + +```mermaid +flowchart LR + subgraph "C Code" + C_CODE["void foo() { bar(); }"] + end + + subgraph "tree-sitter AST" + C_AST["translation_unit + └── function_definition + ├── type: void + ├── name: foo + └── body + └── call_expression + └── name: bar"] + end + + subgraph "Unified IR" + IR_REC["Record: foo + kind: function + file: main.c + line: 1 + relations: + - call -> bar"] + end + + C_CODE --> C_AST --> IR_REC +``` + +## Incremental Parsing + +One key feature of tree-sitter is **incremental parsing**. When a file is modified, tree-sitter can re-parse only the modified portion instead of the entire file. + +CodeScope leverages this feature for incremental indexing: + +```cpp +// Conceptual: Incremental indexing pseudocode +TSTree *old_tree = cache.get(file_path); +TSTree *new_tree = ts_parser_parse_incremental( + parser, old_tree, new_input); +``` + +In incremental mode, re-indexing a modified file takes only a few milliseconds, rather than tens of milliseconds. + +## A Lesson That Made My Blood Run Cold + +While developing the IR translator, I ran into a **C++ template parsing issue**. + +```cpp +template +T add(T a, T b) { + return a + b; +} +``` + +In this simple template function, tree-sitter's C++ grammar parses `T` as a `type_identifier`, and `a` and `b` as `declaration`. The IR translator needs to correctly identify: + +- The function name is `add`, not `T` +- `T` is a template parameter, not a function parameter +- `a` and `b` are function parameters with type `T` +- The return type is `T` + +But the initial translator implementation recognized `T` as the function name (because it was the first `identifier` node in the AST), resulting in the indexed function name being `T` instead of `add`. + +The fix: **In the translator, check the `template` node first, then process the function definition.** + +```cpp +// Fixed C++ translator logic (conceptual) +if (node_kind == "template_declaration") { + // First process template parameters + auto template_params = extract_template_params(node); + // Then process the inner function/class definition + for (auto child : ts_node_named_children(node)) { + if (is_function_definition(child)) { + return translate_function(child, source, template_params); + } + } +} +``` + +This bug exposed a deeper issue: **The unified IR design assumes that all languages have similar structures, but C++ templates, Java generics, and Rust generics differ fundamentally at the AST level.** Translators must handle these language-specific details while maintaining IR abstraction. + +## Summary + +The unified IR is the core abstraction that enables CodeScope to support multiple languages: + +- **tree-sitter** provides the underlying parsing capability, one grammar file per language +- **IR translators** convert language-specific ASTs into a unified data structure +- **Relation extraction** is done during AST traversal, building the symbol relationship graph +- **Incremental parsing** makes re-indexing efficient + +This design is not perfect — language-specific edge cases (C++ templates, Rust macros, Python decorators) make the translators complex. But compared to writing a separate analysis tool for each language, the unified IR has much lower maintenance costs. + +In the next article, I'll break down the **SQLite Graph Storage** — how to store code graphs using a relational database. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-06-sqlite-storage.md b/docs/articles/en/codescope-deepdive-06-sqlite-storage.md new file mode 100644 index 0000000..17261f5 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-06-sqlite-storage.md @@ -0,0 +1,335 @@ +# CodeScope Deep Dive (6): SQLite Graph Storage — Schema Design & FTS + +> *"A graph database is a great idea. A graph database that requires a server process is a deployment nightmare."* +> 图数据库是个好主意。但需要启动一个服务进程的图数据库，就是部署噩梦了。 + +## The Problem: Where to Store the Code Graph? + +The core output of CodeScope is a **code symbol graph**: nodes are functions, classes, variables; edges are calls, references, inheritance relationships. This graph must be persisted to support subsequent queries. + +There were three options: + +1. **In-memory graph** — Fast, but gone after indexing, no incremental reuse +2. **Dedicated graph database** — Neo4j, KuzuDB are powerful, but users need to install a service, configure ports, handle authentication +3. **SQLite** — Zero configuration, single file, embedded in-process + +CodeScope chose the third path: **SQLite as the primary store, with LadybugDB (embedded graph database) as an optional acceleration layer.** + +## Core Files + +``` +engine/src/store/store.h ← GraphStore interface +engine/src/store/store_schema.cpp ← All CREATE TABLE / INDEX DDL +engine/src/store/store_core.cpp ← SQLite operation implementation +engine/src/store/store_ladybug_core.cpp ← LadybugDB graph storage integration +``` + +## Schema Design: Three-Layer Storage + +CodeScope's database schema is organized into three layers: + +```mermaid +flowchart TD + subgraph "Layer 1: Raw Data" + SR["semantic_records
flat parse output"] + FILES["files
file metadata + hash"] + end + + subgraph "Layer 2: Graph Structure" + GN["graph_nodes
symbol nodes"] + GE["graph_edges
call/reference edges"] + ADJ["adjacency
CSR BLOB (O(1) query)"] + end + + subgraph "Layer 3: Knowledge Layer" + CAP["capability / contract
project claims"] + EVI["evidence / finding
verification results"] + FTS["code_fts / name_trgm
full-text search"] + end + + SR --> GN --> GE --> ADJ + FILES --> GN + GN --> CAP --> EVI + GN --> FTS +``` + +### Layer 1: Raw Parse Data + +The most central table is `semantic_records`: + +```sql +CREATE TABLE IF NOT EXISTS semantic_records ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + original_id INTEGER NOT NULL, -- In-file ID (starting from 1) + project_id INTEGER NOT NULL, + kind INTEGER NOT NULL, -- Symbol type (function/class/variable/...) + name TEXT, + qualified_name TEXT DEFAULT '', + parent_id INTEGER DEFAULT 0, -- Parent symbol ID (containment relationship) + arity INTEGER DEFAULT 0, -- Number of function parameters (for overload disambiguation) + is_static INTEGER DEFAULT 0, + type_name TEXT DEFAULT '', + call_kind INTEGER DEFAULT 0, -- 0=direct, 1=method, 2=interface, ... + visibility INTEGER NOT NULL DEFAULT 0, + start_row INTEGER DEFAULT 0, start_col INTEGER DEFAULT 0, + end_row INTEGER DEFAULT 0, end_col INTEGER DEFAULT 0, + file_path TEXT NOT NULL, + language TEXT DEFAULT '' +); +``` + +There is one key design decision in this table: **flat storage, no graph construction**. All parse results are laid out flat in a single table, with `parent_id` representing containment relationships. The graph structure is built later from `semantic_records` during the `buildGraph()` phase. + +Why this design? Because the parsing phase runs in parallel — multiple workers write to the same table simultaneously. If we built the graph structure directly, we'd need cross-file ID coordination, which is far too complex. Flat storage allows each worker to write independently without interference. + +### Layer 2: Graph Structure + +`buildGraph()` constructs `graph_nodes` and `graph_edges` from `semantic_records`: + +```sql +CREATE TABLE IF NOT EXISTS graph_nodes ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL, + ir_node_id INTEGER NOT NULL, + node_type INTEGER NOT NULL, + name TEXT NOT NULL, + qualified_name TEXT, + file_path TEXT NOT NULL, + language TEXT NOT NULL, + start_row INTEGER NOT NULL, start_col INTEGER NOT NULL, + end_row INTEGER NOT NULL, end_col INTEGER NOT NULL, + is_stub INTEGER DEFAULT 0, + visibility INTEGER NOT NULL DEFAULT 0, + ... +); + +CREATE TABLE IF NOT EXISTS graph_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + source_node_id INTEGER NOT NULL, + target_node_id INTEGER NOT NULL, + edge_type INTEGER NOT NULL, + graph_type TEXT NOT NULL DEFAULT 'symbol_reference', + call_site_file TEXT DEFAULT '', + call_site_line INTEGER DEFAULT 0, + ... +); +``` + +`graph_edges` has a `UNIQUE(project_id, source_node_id, target_node_id, edge_type, graph_type)` constraint. This constraint looks simple, but we've been bitten by it — the initial version didn't have the `graph_type` field, causing conflicts between `symbol_reference` and `call_graph` graphs on the same pair of nodes. + +#### Query Optimization: CSR BLOB + +For high-frequency queries like `getCallers` / `getCallees`, CodeScope uses **CSR (Compressed Sparse Row) BLOB** optimization: + +```sql +CREATE TABLE IF NOT EXISTS adjacency ( + src_id INTEGER PRIMARY KEY, -- Caller node ID + project_id INTEGER NOT NULL, + tgt_blob BLOB -- packed u32[] callee node ID list +); + +CREATE TABLE IF NOT EXISTS adjacency_rev ( + tgt_id INTEGER PRIMARY KEY, -- Callee node ID + project_id INTEGER NOT NULL, + src_blob BLOB -- packed u32[] caller node ID list +); +``` + +At query time, a single B-tree lookup retrieves the entire BLOB, then pointer arithmetic traverses it directly: + +``` +getCallees(id): + 1. B-tree lookup on adjacency WHERE src_id = id + 2. Get tgt_blob (packed u32 array) + 3. Cast pointer: u32* arr = (u32*)blob.data + 4. Return arr[0..len/4] +``` + +**O(1) B-tree lookup + O(n) traversal**, which is two orders of magnitude faster than a JOIN on `graph_edges`. + +### Layer 3: Full-Text Search + +CodeScope uses SQLite FTS5 for full-text search: + +```sql +-- Unicode tokenizer, supports Chinese +CREATE VIRTUAL TABLE IF NOT EXISTS code_fts USING fts5( + name, qualified_name, file_path, content, + project_id UNINDEXED, + node_id UNINDEXED, + node_kind UNINDEXED, + tokenize='unicode61' +); + +-- Trigram tokenizer, supports fuzzy search +CREATE VIRTUAL TABLE IF NOT EXISTS name_trgm USING fts5( + name, qualified_name, + project_id UNINDEXED, + node_id UNINDEXED, + node_type UNINDEXED, + tokenize='trigram' +); +``` + +Two FTS indexes serve different use cases: + +- **`code_fts`**: Standard full-text search, matching symbol names, qualified names, file paths +- **`name_trgm`**: Trigram substring search, supporting fuzzy matching in the `LIKE '%substr%'` pattern + +The Trigram index was specifically designed for million-node projects. Without it, `LIKE '%foo%'` queries would require a full table scan, easily exceeding the MCP timeout threshold of 30 seconds on large projects. + +## Batch Processing Optimization + +The initial implementation inserted one row at a time, which had poor performance. After optimization, we switched to batch inserts: + +```cpp +// store.h (conceptual) +void insertGraphNodes(uint64_t project_id, + const std::vector &nodes); +void insertFileResultBatch(uint64_t project_id, + const std::vector &batch, + bool is_reindex = true); +``` + +The core idea of batch inserts: **prepare once, bind/step/reset in a loop**. + +```cpp +// Conceptual: Batch insert pseudocode +sqlite3_stmt *stmt; +sqlite3_prepare_v2(db, "INSERT INTO graph_nodes VALUES (...)", -1, &stmt, NULL); + +for (const auto &node : nodes) { + sqlite3_bind_int64(stmt, 1, node.id); + sqlite3_bind_text(stmt, 2, node.name.c_str(), -1, SQLITE_TRANSIENT); + // ... + sqlite3_step(stmt); + sqlite3_reset(stmt); +} +sqlite3_finalize(stmt); +``` + +Performance improved by about **10x**. Not because SQLite got faster, but because we reduced the overhead of `prepare`/`finalize` — each `prepare` goes through SQL parsing and VDBE compilation. + +## LadybugDB: Optional Graph Storage Acceleration + +SQLite is a relational database. Graph queries (like k-hop traversal, shortest path) require recursive CTEs or multi-table JOINs, which don't perform well enough. + +LadybugDB is an embedded graph database (similar to KuzuDB) that CodeScope uses as an optional acceleration layer: + +```cpp +// store.h +bool initLadybugDB(); // Create .lbug file +bool hasLadybugDB() const; // Check if available +bool isGraphReady() const; // Check if graph data has been populated +``` + +Data flow: + +```mermaid +flowchart LR + subgraph "Indexing Phase" + PARSE["Parse Worker"] --> SQLITE["SQLite
semantic_records"] + SQLITE --> BUILD["buildGraph()"] + BUILD --> GN["graph_nodes + graph_edges"] + end + + subgraph "Sync Phase" + GN --> LB["LadybugDB
.lbug file"] + LB --> SYNC["lbug_sync_state
incremental sync tracking"] + end + + subgraph "Query Phase" + Q["Query Request"] --> LB_QUERY{"LadybugDB
Available?"} + LB_QUERY -->|Yes| LB_FAST["LadybugDB Query
O(log n) graph traversal"] + LB_QUERY -->|No| SQLITE_Q["SQLite Query
JOIN + CTE"] + end +``` + +LadybugDB synchronization is incremental. The `lbug_sync_state` table tracks the last-synced node and edge IDs, avoiding full rebuilds. + +## A Lesson That Made My Blood Run Cold + +**SQLite's concurrent write issue.** + +In the initial design, multiple parse workers wrote to the same SQLite database file in parallel. The result was frequent `SQLITE_BUSY (5)` errors. + +SQLite's WAL mode supports concurrent reads, but writes are still serialized. Multiple workers writing simultaneously inevitably cause lock contention. + +```mermaid +sequenceDiagram + participant W1 as Worker 1 + participant W2 as Worker 2 + participant SQL as SQLite + + W1->>SQL: BEGIN + W2->>SQL: BEGIN + SQL->>W1: OK + SQL->>W2: OK + W1->>SQL: INSERT + SQL->>W1: OK + W2->>SQL: INSERT + SQL->>W2: SQLITE_BUSY + W2->>SQL: RETRY... + SQL->>W2: SQLITE_BUSY + W2->>SQL: RETRY... + SQL->>W2: OK (after W1 commits) +``` + +The fix: **Introduce a BoundedQueue, single-threaded writes.** + +```mermaid +flowchart LR + W1["Parse Worker 1"] --> QUEUE + W2["Parse Worker 2"] --> QUEUE + W3["Parse Worker 3"] --> QUEUE + W4["Parse Worker 4"] --> QUEUE + + subgraph QUEUE["BoundedQueue (in-memory pipe)"] + Q["FIFO Queue"] + end + + Q --> WRITER["Single-threaded Writer
insertFileResultBatch()"] + WRITER --> DB["SQLite"] +``` + +Parse workers are only responsible for parsing — they push results into an in-memory queue. A dedicated writer thread pulls data from the queue and writes to SQLite in batches. This way: + +- Writes are serialized, no lock contention +- Batch writes can merge results from multiple files, making full use of `insertFileResultBatch()` +- Parse workers don't wait for I/O and can immediately process the next file + +## Index Design + +The schema's many indexes are not all created at once. CodeScope's approach is: **load data in bulk first, then build indexes**. + +```cpp +// Conceptual: Two-phase index creation +bool GraphStore::createSchema() { + // 1. Create tables (no indexes) + exec("CREATE TABLE IF NOT EXISTS graph_nodes (...);"); + // 2. Load data... +} + +bool GraphStore::createIndexesAfterBulkLoad() { + // 3. Create indexes after data loading is complete + exec("CREATE INDEX IF NOT EXISTS idx_gn_file_row_type ON ..."); + exec("CREATE INDEX IF NOT EXISTS idx_ge_callers ON ..."); + // ... +} +``` + +Why? Because SQLite's index maintenance overhead during data insertion is significant. Inserting data first and then bulk-creating indexes is faster overall. + +## Summary + +CodeScope's storage layer design is a pragmatic choice: + +- **SQLite as the primary store**, zero configuration, single file, embedded in-process +- **Flat storage**, no graph construction during parsing, reducing concurrency complexity +- **CSR BLOB** optimizes high-frequency queries, avoiding JOIN overhead +- **FTS5 + Trigram** dual indexes, supporting full-text search and fuzzy matching +- **LadybugDB** as optional acceleration, without changing the core architecture +- **BoundedQueue** resolves the concurrent write issue + +In the next article, I'll break down the **Verification Pipeline** — how ClaimParser extracts assertions from documentation, how VerifierRegistry dispatches verifiers, and how Evidence generates a Verdict. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-07-verification-pipeline.md b/docs/articles/en/codescope-deepdive-07-verification-pipeline.md new file mode 100644 index 0000000..ad4fe6e --- /dev/null +++ b/docs/articles/en/codescope-deepdive-07-verification-pipeline.md @@ -0,0 +1,259 @@ +# CodeScope Deep Dive (7): Verification Pipeline — Claim → Evidence → Verdict + +> *"The most dangerous assumption in software engineering is that the code matches the docs."* +> 软件工程中最危险的假设，就是认为代码和文档是一致的。 + +## The Problem: How to Keep Code and Docs in Sync? + +As a project grows to a certain scale, deviation between code and documentation is inevitable. The README says "supports incremental indexing," but the code may only implement full rebuilds. The API docs say "thread-safe," but the implementation uses bare locks. + +This deviation is not a bug — it's **drift** — code evolves continuously, documentation is forgotten, and the two gradually grow apart. + +The traditional solution is manual review, but humans are not good at this kind of mechanical comparison work. CodeScope's Verification Pipeline is designed for exactly this: **automatically extract assertions from documentation, find evidence in the code, and deliver a verdict.** + +## Core Files + +``` +engine/src/verify/claim.h ← Claim/EvidenceRecord data structures +engine/src/verify/claim_parser.h ← Extract Claims from documentation +engine/src/verify/verifier.h ← Verifier base class +engine/src/verify/registry.h ← VerifierRegistry (scheduler) +engine/src/verify/capability_verifier.h ← Capability verifier +engine/src/verify/contract_verifier.h ← Contract verifier +engine/src/verify/architecture_verifier.h ← Architecture verifier +engine/src/verify/finding.h ← Legacy Finding structure +``` + +## Pipeline Architecture + +The verification pipeline is divided into three phases: + +```mermaid +flowchart LR + subgraph "Phase 1: Parse" + DOC["README / Documentation"] --> CP["ClaimParser"] + CP --> CLAIM1["Claim{type: CapabilityExists
subject: 'IncrementalIndex'}"] + CP --> CLAIM2["Claim{type: ContractHolds
subject: 'ThreadSafe'}"] + end + + subgraph "Phase 2: Verify" + CLAIM1 --> VR["VerifierRegistry"] + VR --> V1["CapabilityVerifier"] + CLAIM2 --> VR + VR --> V2["ContractVerifier"] + V1 --> ER1["EvidenceRecord{verdict: Supported
confidence: 0.85}"] + V2 --> ER2["EvidenceRecord{verdict: Unknown
confidence: 0.0}"] + end + + subgraph "Phase 3: Attribution" + ER1 --> FIND["Finding{type: 'ActiveCapability'
evidence: [file, line, ...]}"] + ER2 --> FIND2["Finding{type: 'BrokenContract'
evidence: []}"] + end +``` + +### Phase 1: ClaimParser + +`ClaimParser` extracts structured assertions from free-form text: + +```cpp +// claim_parser.h +class ClaimParser { +public: + std::vector parse( + const std::string &text, + const std::string &source_kind, + const std::string &source_ref + ) const; +}; +``` + +The input is README, AI summary, or PR description; the output is a `Claim` struct: + +```cpp +// claim.h +struct Claim { + ClaimType type; // CapabilityExists / ContractHolds / ArchitectureFollows + std::string subject; // Subject, e.g. "IncrementalIndex" + std::string predicate; // Predicate, e.g. "implemented_by" + std::string object; // Object, e.g. "Runtime" + std::string scope; // "repository" by default + std::string source_kind; // "readme" / "ai_summary" / "pr" / "manual" + std::string source_ref; // File path, traceable +}; +``` + +Design principle: **Conservative extraction**. Only match high-confidence patterns (e.g., "supports X", "thread-safe"). Ambiguous text produces no Claim — because Unknown is a legitimate verdict, but a wrong Claim poisons the entire verification chain. + +### Phase 2: VerifierRegistry + +`VerifierRegistry` is a Meyers singleton that holds all registered `Verifier` instances: + +```cpp +// registry.h +class VerifierRegistry { +public: + static VerifierRegistry &instance(); + + void register_verifier(std::unique_ptr v); + void register_default_verifiers(GraphStore *store, uint64_t project_id); + void clear(); + Verifier *match(const Claim &claim) const; +}; +``` + +The core logic is in `match()`: iterate the Claim through all registered verifiers and return the first one whose `accepts()` returns true. + +```cpp +// Concept: match implementation +Verifier *VerifierRegistry::match(const Claim &claim) const { + for (const auto &v : verifiers_) { + if (v->accepts(claim)) { + return v.get(); + } + } + return nullptr; +} +``` + +Each verifier declares the types of Claims it can handle: + +```cpp +// verifier.h +class Verifier { +public: + virtual std::string name() const = 0; + virtual bool accepts(const Claim &claim) const = 0; + virtual EvidenceRecord verify(const Claim &claim) = 0; +}; +``` + +### Phase 3: EvidenceRecord + +The evidence record output by the verifier: + +```cpp +// claim.h +struct EvidenceRecord { + int64_t claim_id = 0; + Verdict verdict = Verdict::Unknown; // Supported / Contradicted / Unknown + double confidence = 0.0; + std::string verifier_name; + std::string detail; + std::vector> facts; // (fact_kind, ref_id) +}; +``` + +`facts` is the chain of evidence. Each `(fact_kind, ref_id)` pair points to a specific line in `graph_nodes`, `graph_edges`, or documentation. This way, users can not only see that "code is inconsistent with docs," but also see **which specific function, which specific line of code caused the inconsistency.** + +## Verifier Implementations + +### CapabilityVerifier + +Verifies "whether a capability claimed by the project is actually implemented in the code." + +Logic: +1. Look up the symbol corresponding to the Claim's subject in `graph_nodes` +2. Check if the symbol has callers (is invoked) +3. If it has callers → Supported (capability is implemented) +4. If it has no callers and is a stub → Contradicted (capability not implemented) +5. Otherwise → Unknown + +```cpp +// Concept: CapabilityVerifier::verify() pseudocode +EvidenceRecord CapabilityVerifier::verify(const Claim &claim) { + auto nodes = store_->findNodesByName(claim.subject); + if (nodes.empty()) { + return {.verdict = Unknown, .confidence = 0.0}; + } + + for (auto &node : nodes) { + auto callers = store_->getCallers(node.id); + if (!callers.empty()) { + return {.verdict = Supported, .confidence = 0.85}; + } + } + return {.verdict = Contradicted, .confidence = 0.7}; +} +``` + +### ContractVerifier + +Verifies "whether architectural contracts are being followed" (e.g., "thread-safe"). + +The logic is more complex and requires checking multiple dimensions: +- Whether shared mutable state is protected by locks +- Whether global variables are properly encapsulated +- Whether code blocks marked as `unsafe` conflict with the contract + +### ArchitectureVerifier + +Verifies "whether actual call relationships follow architectural conventions" (e.g., "Controller → Service → Repository"). + +Extracts all call relationships from `graph_edges` and compares them against the allowed directions defined in the architectural specification. Relationships that violate the direction are flagged as Contradicted. + +## Persistence + +Verification results are written to the SQLite `evidence` and `evidence_fact` tables: + +```sql +CREATE TABLE IF NOT EXISTS evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + claim_id INTEGER NOT NULL, + verdict INTEGER NOT NULL, -- 0=Supported, 1=Contradicted, 2=Unknown + confidence REAL NOT NULL, + verifier_name TEXT NOT NULL, + detail TEXT DEFAULT '', + created_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (claim_id) REFERENCES claim(id) +); + +CREATE TABLE IF NOT EXISTS evidence_fact ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + evidence_id INTEGER NOT NULL, + fact_kind INTEGER NOT NULL, -- 0=node, 1=edge, 2=document + fact_ref INTEGER NOT NULL, -- graph_nodes.id / graph_edges.id / document.rowid + FOREIGN KEY (evidence_id) REFERENCES evidence(id) +); +``` + +## A Lesson That Made Me Break Out in a Cold Sweat + +**Singleton lifecycle issues in multi-project scenarios.** + +`VerifierRegistry` is a Meyers singleton — globally unique per process. However, `Verifier` instances are constructed bound to a specific `(store, project_id)`. + +When a user switches projects within an MCP session, the old verifiers remain in the registry while new verifiers are appended. The result: + +1. User switches project → calls `register_default_verifiers(new_store, new_pid)` +2. New verifiers are appended to the end of the list +3. Old verifiers still hold references to the old project's `(store, project_id)` +4. If `match()` hits an old verifier, it queries the new project's data using the old project's store — with unpredictable results + +```cpp +// Problem: during project switching +VerifierRegistry::instance().register_default_verifiers(store1, pid1); +// ... user switches project ... +VerifierRegistry::instance().register_default_verifiers(store2, pid2); +// Old verifiers are still there! If match() hits an old verifier, it uses store1 to query pid2's data +``` + +The fix: + +```cpp +// Must call clear() before switching projects +VerifierRegistry::instance().clear(); +VerifierRegistry::instance().register_default_verifiers(store2, pid2); +``` + +This bug was documented in the comments, but nobody remembered to call `clear()` during the first implementation. The result was that the data retrieved was completely nonsensical. + +## Summary + +The Verification Pipeline is a key step in CodeScope's evolution from a "code search engine" to a "code truth engine": + +- **ClaimParser** extracts assertions from documentation, conservatively but precisely +- **VerifierRegistry** dispatches assertions to the appropriate verifier +- **Verifier** queries the knowledge graph, collects evidence, and delivers a verdict +- **EvidenceRecord** contains a complete chain of evidence traceable back to the source code + +In the next article, I'll break down **Drift Detection** — how to automatically discover discrepancies between code and documentation across three dimensions: DocumentationDrift, CapabilityDrift, and ArchitectureDrift. \ No newline at end of file diff --git a/docs/articles/en/codescope-deepdive-08-drift-detection.md b/docs/articles/en/codescope-deepdive-08-drift-detection.md new file mode 100644 index 0000000..3ecb5d5 --- /dev/null +++ b/docs/articles/en/codescope-deepdive-08-drift-detection.md @@ -0,0 +1,309 @@ +# CodeScope Deep Dive (8): Drift Detection — The Trinity of Documentation, Capability, and Architecture + +> *"Documentation is a love letter to your future self. Drift is the heartbreak when you realize it was never updated."* +> 文档是写给未来自己的情书。漂移是当你发现它从未被更新时的心碎。 + +## The Problem: When Does Documentation Start to Diverge from Code? + +In the previous article, I broke down the Verification Pipeline — ClaimParser extracts assertions from documentation, Verifier finds evidence in the code, and ultimately delivers a verdict of Supported / Contradicted / Unknown. + +But the Verification Pipeline has a prerequisite: **someone must provide the Claims first.** Either the user inputs them manually, or the AI extracts them automatically. + +Drift Detection is the "self-driven mode" of the Verification Pipeline — it requires no user input, automatically scans three dimensions, and proactively discovers discrepancies between code and documentation: + +1. **Documentation Drift**: The README says it supports C++, but no C++ entities are found in the code +2. **Capability Drift**: The documentation claims "supports incremental indexing," but the implementing function has no callers +3. **Architecture Drift**: Actual call relationships violate the Controller → Service → Repository layering convention + +## Core Files + +``` +engine/src/verify/documentation_drift.h ← Documentation drift detection +engine/src/verify/documentation_drift.cpp ← Implementation +engine/src/verify/capability_drift.h ← Capability drift detection +engine/src/verify/capability_drift.cpp ← Implementation +engine/src/verify/architecture_drift.h ← Architecture drift detection +engine/src/verify/architecture_drift.cpp ← Implementation +engine/src/verify/finding.h ← DriftItem data structure +``` + +## The Trinity + +```mermaid +flowchart TB + subgraph "Inputs" + README["README.md"] + CODE["Source Code"] + CAP["capability table
(declared capabilities)"] + end + + subgraph "Detectors" + DD["DocumentationDrift
severity=1, confidence=0.8"] + CD["CapabilityDrift
severity=2, confidence=0.9"] + AD["ArchitectureDrift
severity=1, confidence=0.7"] + end + + subgraph "Outputs" + ITEM1["DriftItem{type: 'DocumentationDrift'
subject: 'go'
detail: 'README says Go but no entities'}"] + ITEM2["DriftItem{type: 'CapabilityDrift'
subject: 'IncrementalIndex'
detail: 'has 0 callers'}"] + ITEM3["DriftItem{type: 'ArchitectureDrift'
subject: 'Repo.call() → Controller'
detail: 'layer violation'}"] + end + + README --> DD + CAP --> CD + CODE --> CD + CODE --> AD + + DD --> ITEM1 + CD --> ITEM2 + AD --> ITEM3 +``` + +### 1. Documentation Drift (severity=1, confidence=0.8) + +The most intuitive drift detection: **Does the code actually implement the languages claimed in the README?** + +```cpp +// documentation_drift.h +struct DriftItem { + std::string type; // "DocumentationDrift" + int severity; // 1 = warning + std::string subject; // The missing language name + std::string detail; // Human-readable description +}; + +struct LanguageClaim { + std::string canonical; // "cpp", "rust", "go", ... + std::string display; // "C++", "Rust", "Go", ... + size_t mention_count; // Number of occurrences in the README +}; +``` + +Detection flow: + +1. Read the project README content from the `document` table +2. `extractLanguageClaims()` scans the README using regex to extract language names (C++, Python, Go, Rust, JavaScript, TypeScript, Java) +3. `countEntitiesByLanguage()` queries the `entity` table to count the actual number of entities per language +4. If the README claims support for a language but the `entity` table shows 0 entities for that language → report Drift + +```cpp +// Concept: detectDocumentationDrift pseudocode +std::vector detectDocumentationDrift(store &store, uint64_t pid) { + auto readme = store.getReadme(pid); + auto claims = extractLanguageClaims(readme); + std::vector items; + + for (const auto &claim : claims) { + int64_t count = countEntitiesByLanguage(store, pid, claim.canonical); + if (count == 0) { + items.push_back({ + .type = "DocumentationDrift", + .severity = kDriftSeverityDoc, // 1 + .subject = claim.display, + .detail = "README claims " + claim.display + + " but no entities found", + }); + } + } + return items; +} +``` + +**Why is severity only 1?** Because the README may be describing an external dependency, not the project's own implementation language. For example, "CodeScope supports C++ projects" might mean users can analyze C++ projects with CodeScope, not that the project itself is written in C++. + +### 2. Capability Drift (severity=2, confidence=0.9) + +This is the most severe type of drift: **a feature documented in the docs is not implemented in the code.** + +```cpp +// capability_drift.h +inline constexpr int kDriftSeverityCapability = 2; // error +inline constexpr double kDriftConfidenceCapability = 0.9; +``` + +Detection flow: + +1. Read the list of capabilities claimed by the project from the `capability` table +2. For each capability, look up entities with matching names in the `entity` table +3. For matched entities, check whether they have callers (are invoked) +4. If an entity has no callers → the capability is "dead" — the code exists but is unused +5. If the entity does not exist → the capability is entirely missing + +```cpp +// Concept: detectCapabilityDrift pseudocode +std::vector detectCapabilityDrift(store &store, uint64_t pid) { + auto capabilities = store.getCapabilities(pid); + std::vector items; + + for (const auto &cap : capabilities) { + int64_t count = countImplementingEntities(store, pid, cap.name); + if (count == 0) { + items.push_back({ + .type = "CapabilityDrift", + .severity = kDriftSeverityCapability, // 2 + .subject = cap.name, + .detail = "Declared capability '" + cap.name + + "' has 0 implementing entities with callers", + }); + } + } + return items; +} +``` + +**Why is confidence as high as 0.9?** Because this detection is deterministic — a row in the `capability` table either has a corresponding implementing entity or it doesn't. There is no ambiguity. + +### 3. Architecture Drift (severity=1, confidence=0.7) + +The most complex drift detection: **whether actual call relationships violate architectural layering conventions.** + +```cpp +// architecture_drift.h +inline constexpr const char *kLayerController = "Controller"; +inline constexpr const char *kLayerService = "Service"; +inline constexpr const char *kLayerRepository = "Repository"; +``` + +Detection flow: + +1. `classifyEntityLayer()` classifies an entity into the Controller / Service / Repository layer based on its name and file path + +```cpp +// Concept: classifyEntityLayer pseudocode +std::string classifyEntityLayer(const std::string &name, + const std::string &file_path) { + if (name.ends_with("Controller") || + file_path.contains("/controllers/")) { + return "Controller"; + } + if (name.ends_with("Service") || + file_path.contains("/services/")) { + return "Service"; + } + if (name.ends_with("Repository") || + file_path.contains("/repositories/")) { + return "Repository"; + } + return ""; // Unknown +} +``` + +2. Read the `relation` table and extract all call edges (type=1) +3. For each call edge, check whether the flow from source layer to target layer is valid + +Valid flow: **Controller → Service → Repository** + +```mermaid +flowchart LR + CTRL["Controller Layer"] -->|valid| SVC["Service Layer"] + SVC -->|valid| REPO["Repository Layer"] + CTRL -->|invalid: skip layer| REPO + REPO -->|invalid: reverse| SVC + SVC -->|invalid: reverse| CTRL + REPO -->|invalid: reverse| CTRL +``` + +**Why is confidence only 0.7?** Because layer classification is a heuristic based on naming conventions and file paths. Not all projects follow the Controller/Service/Repository naming convention. False positives are expected. + +## Scheduler Integration + +Drift detection does not run once. CodeScope's scheduler automatically dispatches all three detectors after indexing completes: + +```cpp +// Concept: Drift detection scheduling +schedule_drift_detection(project_id) { + // Run three detectors in parallel + auto doc_drifts = detectDocumentationDrift(store, project_id); + auto cap_drifts = detectCapabilityDrift(store, project_id); + auto arch_drifts = detectArchitectureDrift(store, project_id); + + // Merge results + auto all_drifts = concat(doc_drifts, cap_drifts, arch_drifts); + + // Write to finding table + for (auto &drift : all_drifts) { + store.insertFinding(project_id, drift); + } +} +``` + +## A Lesson That Made Me Break Out in a Cold Sweat + +**The false positive problem with Capability Drift.** + +The original Capability Drift detection logic was simple: if `capability.name` could not be found in `entity.name`, report a drift. + +But in real projects, there is a **naming discrepancy** between capabilities and implementing entities. For example, the capability table declares "Incremental Index," while the implementing entity is called "incrementalIndex" or "incremental_index." + +```mermaid +flowchart LR + CAP["capability: 'Incremental Index'"] + E1["entity: 'incrementalIndex'"] + E2["entity: 'IncrementalIndexBuilder'"] + E3["entity: 'buildIncrementalIndex'"] + CAP -->|"name exact match
❌ not found"| MISS["False positive: capability missing"] + CAP -->|"name fuzzy match
✅ found"| FOUND["Correct: capability implemented"] +``` + +The fix: **changed from exact match to containment match.** `countImplementingEntities()` was changed to use a `LIKE '%cap_name%'` fuzzy query, and comparisons are performed after stripping underscores and spaces. + +This bug exposed a deeper issue: **the mapping between capability names and entity names is not 1:1.** A single capability may be implemented by multiple entities, and a single entity may implement multiple capabilities. Perfect mapping requires semantic understanding; fuzzy matching is a pragmatic but imperfect compromise. + +## The Relationship Between the Three + +```mermaid +flowchart TB + subgraph "Drift Pyramid" + DOC["Documentation Drift
severity=1
README says supported but code doesn't have it"] + CAP["Capability Drift
severity=2
docs claim but code doesn't implement"] + ARCH["Architecture Drift
severity=1
call relationships violate layering"] + end + + DOC --> CAP + CAP --> ARCH + + subgraph "Increasing Scope of Impact" + D1["Single file
(README inaccurate)"] + D2["Single feature
(capability not implemented)"] + D3["Entire system
(architecture decay)"] + end + + DOC --> D1 + CAP --> D2 + ARCH --> D3 +``` + +- **Documentation Drift** is a surface-level issue, affecting a single file +- **Capability Drift** is a functional issue, affecting a single capability +- **Architecture Drift** is a structural issue, affecting the entire system + +The three also form a progressive relationship: inaccurate documentation → untrustworthy capability claims → architectural conventions being ignored. + +## Summary + +Drift Detection is the core capability that distinguishes CodeScope from traditional code search tools: + +- **Documentation Drift**: README semantics vs. code entities, severity=1, confidence=0.8 +- **Capability Drift**: Declared capabilities vs. implementation code, severity=2, confidence=0.9 +- **Architecture Drift**: Layering conventions vs. actual calls, severity=1, confidence=0.7 + +The three detectors cover all levels from documentation to code, from features to architecture. They require no user input, run automatically after indexing completes, and continuously track the health of the codebase. + +--- + +## Postscript + +With this, the eight-article CodeScope Deep Dive series is complete: + +| # | Article | Core Module | +|---|---------|-------------| +| 1 | Dual-Language Architecture | Rust MCP Server + C++ Engine FFI | +| 2 | Worker Subprocess Isolation | fork+exec + DB merging | +| 3 | Parallel Scheduler | Three scheduling modes | +| 4 | Two-Phase Indexing | Fast Scan + Background Enhance | +| 5 | tree-sitter Parser & Unified IR | 8 languages, one model | +| 6 | SQLite Graph Storage | Schema + FTS + batch processing | +| 7 | Verification Pipeline | Claim → Evidence → Verdict | +| 8 | Drift Detection | Trinity of documentation, capability, and architecture | + +CodeScope's evolution from a "code search engine" to a "project truth engine" centers on the fact that it no longer just **indexes code**, but rather **understands code** — from the symbol level to the semantic level, from static analysis to verification and reasoning. Every step involved engineering trade-offs and compromises, and those trade-offs and compromises are precisely what this series has attempted to document. \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-01-intro.md b/docs/articles/zh/codescope-architecture-01-intro.md deleted file mode 100644 index 260cfaa..0000000 --- a/docs/articles/zh/codescope-architecture-01-intro.md +++ /dev/null @@ -1,278 +0,0 @@ -# CodeScope 架构拆解（一）：开篇——当 MCP 工具返回 56KB 而你只需要 629 bytes - -> 我一开始不是要造一个"比 CBM 好 X 倍"的东西。 -> 我是在用 codebase-memory-mcp 的时候，被一个问题折磨得受不了了： -> **AI 问一个符号，它返回整个项目。** -> -> 56KB 的 JSON 里，70% 是 AI 不需要的指纹、AST profile、body tokens。而真正有用的——struct 字段、常量值、switch-case 分支——它一个都没告诉 AI。 -> 所以我决定写一个只回答"被问到的问题"的工具。 - ---- - -## 系列目录 - -| 篇 | 标题 | 一句话 | -|:--:|------|--------| -| 一 | **开篇**（本文） | 为什么重写一个代码理解工具，以及整体架构 | -| 二 | 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| 三 | Worker 隔离 | 为什么索引不会拖垮你的 MCP Server | -| 四 | 零冗余响应 | 精简响应，按需返回 | -| 五 | C++ 引擎拆解 | 从源码到多维代码图的管线 | -| 六 | MCP 协议层 | 35+ 工具的设计哲学 | -| 七 | 语言翻译器 | 10 种语言统一为一种 IR | -| 八 | SQLite 存储层 | 270KB 替代 64MB 的秘密 | -| 九 | 自适应查询 | 当数据还没准备好时 | -| 十 | 性能真相 | 从 GoAgent 到 Linux Kernel | - ---- - -## 一、起点：一个 56KB 的那不勒斯 - -先上数据，这样你理解我在说什么。 - -同一台机器、同一份代码（GoAgent，~24K 行 Go）、同一个问题（"Chaos 是如何工作的，有哪些模式"），两个工具的设计哲学不同，响应也不同： - -| 维度 | codebase-memory-mcp v0.8.1 | CodeScope | 说明 | -|------|:--------------------------:|:---------:|------| -| 最小可用响应大小 | **56,183 bytes** | **629 bytes** | CBM 倾向一次返回完整信息，CodeScope 倾向精简按需 | -| 等价 tokens | ~14,046 | ~157 | CodeScope token 更少，但 CBM 的响应包含更多原生信息 | -| 第一次查询等待时间 | 等全量索引完（几分钟） | **创建项目即可查** | CodeScope 的渐进式模型 vs CBM 的全量模型 | - -两种设计各有优劣。CBM 的全量模型在索引完成后查询延迟更低、信息更丰富。CodeScope 的渐进式模型能让 AI 更快开始交互，但深度查询需要等待后台索引完成。 - -技术选型没有对错，取决于使用场景。 - -### 56KB 里有什么 - -以 CBM 的 `search_graph` 响应为例，我拆解了一下它的构成： - -``` -┌─────────────────────────────────────────────┐ -│ search_graph 响应 56,183 bytes 的构成 │ -│ │ -│ ████████████████████████████████ 70% 无关数据 │ -│ （test 函数、doc sections、route 节点等） │ -│ │ -│ ████████████ 20% 必要元数据 │ -│ （node names, labels, files） │ -│ │ -│ ████ 10% 实际有用的 │ -│ （ChaosExecutor, ChaosAction 等） │ -└─────────────────────────────────────────────┘ -``` - -这是一个设计取舍。CBM 选择了一次性返回尽可能多的信息，适合深度分析场景。CodeScope 选择按需返回，减少不必要的 token 消耗——但代价是 AI 可能需要多次查询才能获取完整上下文。 - -值得一提的是，[codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) 是一个非常优秀的开源代码理解工具，在某些场景下表现优于 CodeScope： -- **Linux 内核全量索引**：CBM 可以索引整个 Linux 内核源码（~65K 文件）并在数分钟内完成；CodeScope 目前对超大型项目的全量索引稳定性还需改进 -- **查询响应丰富度**：CBM 的 `search_graph` 返回的字段更全面，适合一次获取完整信息 -- **社区生态**：CBM 有更成熟的开源社区和文档体系 - -CodeScope 的定位不是替代 CBM，而是提供了一个不同的设计思路——渐进式就绪、零冗余响应。如果你需要快速启动交互、对 token 消耗敏感，CodeScope 可能更适合。如果你需要全量索引后的深度分析能力，CBM 是更好的选择。 - -### 629 bytes 里有什么 - -```json -{ - "results": [ - { - "id": 2851, - "kind": "struct", - "name": "ChaosExecutor", - "signature": "type ChaosExecutor struct {", - "visibility": "default", - "language": "go", - "file_path": "~/go/src/goagent/.../chaos.go", - "line": 72, - "column": 1 - } - ] -} -``` - -每个字段都有用。没有指纹、没有 AST profile、没有 body tokens。AI 知道这是个 struct，在哪个文件、第几行、signature 是什么。如果它想知道更多，它会再问——而不是一次全塞给它。 - ---- - -## 二、架构总览：三层 + 一个 DB - -既然你知道了"为什么"，我们来看"是什么"。 - -CodeScope 的整体架构不算复杂，三层结构： - -```mermaid -flowchart TB - Client["MCP Client
(Claude Desktop / Cursor / CLI)"] - - subgraph Server["Rust MCP Server"] - MCP["JSON-RPC 2.0
35+ 工具调度"] - FFI["FFI Bridge
Rust ↔ C++ extern \"C\""] - Tokio["Tokio Runtime
异步任务"] - end - - subgraph Worker["C++ Worker 子进程"] - Scanner["扫描器
ms 级正则扫描"] - Parser["并行解析器
14 tree-sitter workers"] - Graph["图构建器
调用图 + 符号引用"] - Store["SQLite 写入器
WAL 模式"] - end - - DB["SQLite
.codescope/codescope.db"] - - Client <-->|"stdio"| MCP - MCP -->|"spawn"| Worker - Worker -->|"write"| DB - MCP -->|"poll/query"| FFI - FFI -->|"read"| DB - Tokio -->|"async enhance"| DB -``` - -解释一下各层干什么： - -- **Rust MCP Server**：负责协议、工具调度、异步任务。不碰源码解析。 -- **C++ Worker 子进程**：干脏活累活——扫描文件、解析 AST、建图、写库。跑完就退，RSS 全量归还 OS。 -- **SQLite**：唯一的数据持久化层。WAL 模式，读写分离。 - -为什么 C++ 引擎用子进程而不是线程？——内存隔离。Worker 索引 60,000 个文件的 Linux Kernel 时 RSS 冲到几个 GB，但它是独立的进程，出口即释放，不会污染 Server 进程的内存。 - ---- - -## 三、渐进式就绪模型 - -这是 CodeScope 最核心的设计决策，也是它与 codebase-memory-mcp 最大的哲学差异。 - -CBM 的设计模型是全量索引完成后才能查询。这意味着用户需要等待全量索引完成才能开始交互，但索引完成后查询延迟低、信息完整。 - -CodeScope 的模型是渐进式就绪：**逐步加深，随时可用。** - -```mermaid -flowchart LR - PA["Phase A (ms 级)
scan_project
正则扫描符号"] -->|"270KB DB"| Ready["✓ 符号/模块/入口点"] - PB["Phase B (异步)
enhance_project
tree-sitter 全量解析"] -->|"后台执行"| Enhanced - PC["Phase C (按需)
index_project
图构建 + FTS"] -->|"完整索引"| Full - Ready -.->|"自动触发"| PB - Ready -->|"可查询"| Query1["find_symbol
get_module_tree
get_entry_points"] - PB -.->|"自动触发"| PC - Enhanced -->|"可查询"| Query2["find_callers
get_hotspots ❌
search(FTS)"] - Full -->|"可查询"| Query3["codescope_trace
get_communities ❌
语义搜索"] -``` - -| 阶段 | 耗时（小项目 ~200 文件） | DB 大小 | 能做什么 | -|:----:|:------------------------:|:-------:|----------| -| Phase A | **~9ms** | 270 KB | 符号名、模块树、入口点 | -| Phase B | 后台 1-5 min | 几 MB | 调用图、FTS 搜索 | -| Phase C | 按需（大项目几分钟） | 完整 | 调用链追踪、社区检测 | - -实际效果是：`scan_project` 返回后立即可以查询符号和模块结构。Phase B/C 在后台执行，不影响正常使用。 - ---- - -## 四、数据流：从文件夹到 AI 回答 - -一条查询从"项目路径"到"AI 组织答案"的完整路径： - -```mermaid -sequenceDiagram - participant AI as AI Agent - participant Server as Rust Server - participant Worker as C++ Worker - participant DB as SQLite - - AI->>Server: scan_project("/path/to/project") - Server->>Worker: spawn worker subprocess - Worker->>Worker: Phase 1: 扫描文件（filter_policy） - Worker->>Worker: Phase 2: 正则扫描符号（ms 级） - Worker->>DB: 写入 semantic_records - Worker-->>Server: stdout JSON result - Server-->>AI: {symbols: 14366, files: 200} - - Note over AI,DB: 此时 AI 就可以查询了 - - AI->>Server: find_symbol("ChaosExecutor") - Server->>DB: SELECT * FROM semantic_records WHERE name="ChaosExecutor" - DB-->>Server: 2 results - Server-->>AI: {kind: "struct", file: "chaos.go", line: 72} - - Note over Server: 后台自动触发 Phase B - - Server->>Server: RUNTIME.spawn(enhance_project) - Server->>Worker: spawn worker (tree-sitter 全量解析) - Worker->>DB: 写入 graph_nodes, graph_edges, FTS - Worker-->>Server: done - - AI->>Server: find_callees("ChaosExecutor.Execute") - Server->>DB: SELECT * FROM graph_edges WHERE source="Execute" - DB-->>Server: 7 callees - Server-->>AI: {methods: [executeStaleData, ...]} -``` - -关键设计点：**Phase A 返回后，AI 可以立即开始查询。Phase B 在后台跑，不影响 AI 的正常工作流。** - ---- - -## 五、技术选型：为什么是 C++ + Rust + SQLite - -这个组合不是第一天就定下来的。我走过一些弯路。 - -### 为什么 C++ 做引擎 - -tree-sitter 的 C API 是第一公民。C++ 能直接调用 tree-sitter 的解析器，不需要额外的 FFI 层。而且 tree-sitter 的解析本身是 C 实现的，C++ 包装一层很自然。 - -另外，C++ 在内存布局和性能上有优势。14 个 worker 线程并行解析几百个文件，C++ 的 pthread + arena 分配器比带 GC 的语言更可控。 - -### 为什么 Rust 做 Server - -MCP 协议本质是一个长期运行的 JSON-RPC 2.0 服务器，stdio 传输。Rust 的 Tokio 异步运行时非常适合这种场景——同时处理多个工具调用、后台任务、轮询。 - -更重要的是，Rust 的 FFI 生态（`extern "C"` bindings）跟 C++ 对接非常直接。40+ 个 FFI 函数，每个都是 `extern "C"` 导出，Rust 这边 `#[link(name = "engine")]` 就能调用。 - -### 为什么 SQLite 做存储 - -不需要分布式。不需要高并发。每个项目一个 DB 文件，放在 `.codescope/` 目录下。SQLite 的 WAL 模式支持读写分离——Worker 写数据，Server 读数据，互不阻塞。 - -而且 SQLite 的 FTS5 全文搜索和 vec0 向量扩展开箱即用，不需要额外搭 Elasticsearch 或 Milvus。 - -### 坦诚反思：为什么不是纯 Rust - -说实话，如果重新来一次，我可能会认真考虑用 Rust 重写整个引擎。C++ 的构建系统（CMake）加上多平台兼容（Windows .dll、macOS .dylib、Linux .so）带来的维护成本远超预期。 - -但当时选择 C++ 的原因也合理：tree-sitter 的生态是 C/C++ 的，团队有 C++ 背景，而且 C++ 在底层内存控制上确实更灵活。**技术选型没有完美的，只有适合当前阶段的。** - ---- - -## 六、技术指标 - -一些硬数字，这样你有一个直观的感知： - -| 指标 | 数值 | -|------|:----:| -| 支持语言 | 13 种（含 C/C++/Go/Rust/Java/Python/TS/JS 等） | -| RapidJSON 扫描 | **毫秒级** 扫描 Linux Kernel 60,000 文件 | -| 索引速度 | 1,167 个 Go 文件 → **3.28s** | -| 图节点数 | 最大 263,614（GoAgent 项目） | -| 图边数 | 最大 245,849 | -| 查询延迟 | 多数查询 **<1ms**，全图 Label Propagation **<500ms** | -| 索引 DB 大小 | 270 KB（Phase A）→ 完整（几 MB-几十 MB） | -| 进程隔离 | Worker 子进程，RSS 100% 归还 OS | -| Worker 超时保护 | 300s + 3 次重试 | -| MCP 工具数 | 35+ | - ---- - -## 七、系列预告 - -| 篇 | 标题 | 核心内容 | -|:--:|------|----------| -| **二** | 渐进式就绪 | Phase A 怎么在 毫秒级 内完成扫描，Phase B 异步增强，Phase C 完整索引 | -| **三** | Worker 隔离 | 为什么用子进程而不是线程，内存隔离的代价和收益 | -| **四** | 零冗余响应 | 每个字段为什么存在，为什么不存在，对比 CBM 的 56KB | -| **五** | C++ 引擎拆解 | 从 filter_policy 到 graph_builder，一条数据在引擎里的完整旅程 | -| **六** | MCP 协议层 | 35+ 工具如何设计，注册和路由机制 | -| **七** | 语言翻译器 | 10 种语言如何统一为 IR，tree-sitter visitor 设计 | -| **八** | SQLite 存储层 | WAL 模式、FTS5、vec0、增量索引 | -| **九** | 自适应查询 | 当数据还没准备好时，如何优雅降级 | -| **十** | 性能真相 | 从 200 文件到 60,000 文件，实测数据 | - -每篇文章遵循同一个模式：**问题 → 设计旅程 → 权衡取舍 → 坦诚反思。** 不营销。不"比 X 快 10 倍"。只有工程师聊工程。 - -下一期：[CodeScope 架构拆解（二）：渐进式就绪——毫秒级让 AI 开始理解你的代码](codescope-architecture-02-progressive-readiness.md) \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-02-progressive-readiness.md b/docs/articles/zh/codescope-architecture-02-progressive-readiness.md deleted file mode 100644 index bdc9568..0000000 --- a/docs/articles/zh/codescope-architecture-02-progressive-readiness.md +++ /dev/null @@ -1,354 +0,0 @@ -# CodeScope 架构拆解（二）：渐进式就绪——毫秒级让 AI 开始理解你的代码 - -> 第一次用 CBM 索引 Linux Kernel 的时候，我等了 12 分钟。 -> 后来我意识到这不是技术问题，是设计哲学问题。CBM 的模型是全量索引完成后才能查询。CodeScope 的模型是"先让你能用，再逐步加深"。 -> -> 这篇文章讲的就是这个"逐步加深"是怎么做到的。 - ---- - -## 系列目录 - -| 篇 | 标题 | 一句话 | -|:--:|------|--------| -| 一 | [开篇](codescope-architecture-01-intro.md) | 为什么重写一个代码理解工具 | -| **二** | **渐进式就绪**（本文） | 毫秒级让 AI 开始理解你的代码 | -| 三 | Worker 隔离 | 为什么索引不会拖垮你的 MCP Server | -| 四 | 零冗余响应 | 精简响应，按需返回 | -| 五 | C++ 引擎拆解 | 从源码到多维代码图的管线 | - ---- - -## 一、问题：1-12 分钟的等待 - -MCP 的代码理解工具有一个通病：**全量索引才能查。** - -你要理解一个项目，得先等工具扫描完所有文件、解析完所有 AST、构建完完整的调用图。这个时间从 1 分钟（小项目）到 12 分钟（Linux Kernel）不等。 - -这个等待的问题是： - -1. **AI 不知道值不值得。** 它只是想先看看这个项目的模块结构，结果要等全量索引完。 -2. **大部分查询不需要全量数据。** "这个项目有哪些入口点"——这需要全量 AST 解析吗？不需要。文件名和符号名就够了。 -3. **浪费。** 你花了 12 分钟等索引，结果 AI 的第一条查询是"这个项目是什么语言写的"。 - -### 第一性原理问题 - -> 在一个代码理解工具里，最少需要多少数据才能回答 AI 最常问的前 80% 的问题？ - -我统计了一下 AI agent 在代码理解场景下的典型查询分布： - -| 查询类型 | 占比 | 最少需要什么数据 | -|----------|:----:|------------------| -| 找符号定义在哪 | ~40% | 符号名 + 文件 + 行号 | -| 看项目结构/模块 | ~20% | 目录结构 + 文件名 | -| 找入口点 | ~10% | 符号名 + kind（main/init） | -| 查调用关系 | ~15% | 图边 | -| 搜代码（关键词） | ~10% | FTS 索引 | -| 全量分析 | ~5% | 完整图 + 复杂度 | - -前 70% 的查询（符号查找、模块结构、入口点）只需要一个东西：**符号表。** 不需要 AST、不需要图、不需要 FTS。 - -这就引出了 CodeScope 的核心设计：**如果能用 O(n) 的正则扫描在毫秒级拿到符号表，为什么一定要等 O(n+m) 的 tree-sitter 全量解析？** - ---- - -## 二、核心洞察：把复杂度拆到时间轴上 - -### 三种能力，三个时间窗口 - -我把一个代码理解工具需要的能力拆成三个级别： - -| 级别 | 能回答什么 | 技术实现 | 时间窗口 | -|:----:|-----------|----------|:--------:| -| **Level 1** | 符号在哪个文件、哪一行、什么类型 | 正则 + RapidJSON | **毫秒级** | -| **Level 2** | 调用图、FTS 搜索 | tree-sitter 全量解析 | **后台秒级** | -| **Level 3** | 调用链追踪、社区检测 | 图构建 + 图算法 | **按需分钟级** | - -关键洞察：**这三个级别不是递进依赖关系。Level 1 的回答不需要等 Level 2 和 Level 3 完成。** - -这在设计上意味着什么？ - -- AI 可以在 **毫秒级** 后开始查符号 -- Level 2 在后台自动触发，AI 不需要手动操作 -- 查询引擎自动检测 readiness，数据没准备好就用替代方案 - -这不只是技术优化——它是**设计哲学的转变**。 - ---- - -## 三、Phase A：毫秒级扫描 60,000 文件 - -Phase A 是渐进式就绪的第一层，也是 CodeScope 最"反直觉"的设计。 - -### 3.1 为什么不用 tree-sitter - -tree-sitter 的解析能力很强，但它有一个问题：**慢。** 对于一个 60,000 文件的 Linux Kernel，tree-sitter 全量解析需要几分钟。 - -但 AI 只是想先知道 `sched_init` 在哪个文件、哪一行、是什么类型。这需要 AST 吗？不需要。一个正则表达式就够了。 - -### 3.2 快速扫描器的设计 - -`engine_scanner.cpp`（~1,180 行）干的活很简单： - -``` -1. 遍历目录（FilterPolicy 过滤） -2. 逐文件读取 -3. 按语言选择正则集 -4. 扫描提取符号 -5. 写入 SQLite -``` - -```cpp -// 简化的扫描逻辑 -for (auto &file : fileJobs) { - std::string content = readFile(file.path); - for (auto &pattern : patternsForLanguage(file.lang)) { - auto matches = regex_search(content, pattern); - for (auto &m : matches) { - sqlite.insertSymbol({ - .name = m.name, - .kind = m.kind, // struct / func / var / const - .file = file.path, - .line = m.line, - .column = m.column, - .signature = m.signature - }); - } - } -} -``` - -这个扫描器不用 tree-sitter，用的是 RapidJSON 级别的快速正则匹配。它的限制很明显——不能处理嵌套结构、不能生成调用图——但它在一个指标上赢得了选择：**速度。** - -### 3.3 实际数据 - -| 项目 | 文件数 | Phase A 耗时 | -|------|:-----:|:------------:| -| ARES Agent | 95 | **\<100ms** | -| GoAgent | 1,167 | **\<500ms** | -| memscope-rs | 238 | **\<200ms** | -| **Linux Kernel** | **\~60,000** | **毫秒级** | - -60,000 个文件，毫秒级。这是 Phase A 的核心竞争力——它为 AI 提供了一个"先看看"的入口，成本低到可以忽略不计。 - -### 3.4 Phase A 能回答什么 - -``` -scan_project → AI 可以立即开始查询: - -find_symbol("ChaosExecutor") - → struct, chaos.go:72 - -get_module_tree() - → internal/ares_quant/marketmaking/ (3 files) - -get_entry_points() - → main() in cmd/ares/main.go - -get_graph_stats() - → 14,366 symbols across 200 files -``` - -这 4 个查询覆盖了 AI 在新项目上最常问的前几步。 - ---- - -## 四、Phase B：后台全量解析 - -Phase A 返回后，CodeScope 自动触发 Phase B——真正的 tree-sitter 全量解析。 - -```mermaid -sequenceDiagram - participant AI as AI Agent - participant Server as Rust Server - participant Worker as C++ Worker - participant DB as SQLite - - AI->>Server: scan_project - Server->>Worker: spawn (Phase A) - Worker-->>Server: 符号表就绪 - Server-->>AI: ✅ Phase A 完成 - - Note over Server: 自动触发 Phase B - - Server->>Worker: spawn (Phase B: tree-sitter) - Worker->>Worker: 14 workers 并行解析 - Worker->>DB: graph_nodes + graph_edges - Worker->>DB: semantic_records (AST IR) - Worker-->>Server: ✅ Phase B 完成 - - Note over Server: 自动触发 FTS 构建 - - Server->>DB: buildFTSFromGraph() - Server->>DB: fts_ready=1 -``` - -### 4.1 14 个 worker 并行解析 - -文件收集完成后，按大小降序排序，分批次喂给 14 个 worker 线程： - -```mermaid -flowchart TB - Batch["文件批次 [start..end]"] - - subgraph Workers["14 Workers (pthread, 8MB 栈)"] - W1["Worker 1
read → parse → visit"] - W2["Worker 2
read → parse → visit"] - W14["Worker 14
read → parse → visit"] - end - - Collect["collect_lock
收集结果"] - - Batch --> W1 - Batch --> W2 - Batch --> W14 - W1 --> Collect - W2 --> Collect - W14 --> Collect - Collect --> SQLite["SQLite 批量写入
beginTransaction
→ 批量 insert
→ commit"] -``` - -### 4.2 SQLite WAL 模式 - -Phase B 和 Phase A 有一个重要的设计差异：**它们写入的数据库文件是同一个，但互不冲突。** - -SQLite WAL（Write-Ahead Logging）模式允许一个写入器和一个读取器同时工作。Phase B 写入 `graph_nodes/edges`，Phase A 写入的 `semantic_records` 不受影响。Server 读 DB 时走 WAL 的快照，不阻塞。 - ---- - -## 五、渐进式就绪的查询引擎 - -当 AI 的查询进来时，CodeScope 的查询引擎需要知道：**当前的数据就绪到什么程度了？** - -### 5.1 就绪状态 - -``` -Phase A 完成: - find_symbol → ✅ 直接可用 - get_module_tree → ✅ 直接可用 - get_entry_points → ✅ 直接可用 - get_graph_stats → ✅ 直接可用 - -Phase B 完成: - find_callers → ✅ 调用图可用 - find_callees → ✅ 调用图可用 - search (FTS) → ✅ 全文搜索可用 - -Phase C 完成: - codescope_trace → ✅ 调用链追踪可用 - get_communities ❌ → ✅ 社区检测可用 -``` - -### 5.2 自适应降级 - -当查询的目标数据还没就绪时，查询引擎不报错，而是尝试替代方案： - -```cpp -// engine_queries.cpp — 自适应查询的核心逻辑 -QueryResult engine_find_callers_adaptive(const char *name) { - if (callgraph_ready > 0.5f) { - // 新 call_edges 表已经建好 50% 以上 - return query_call_edges_new(name); - } else if (callgraph_ready > 0.1f) { - // 至少有一些数据，用旧 schema - return query_call_edges_old(name); - } else { - // 还没开始建调用图，给提示 - return QueryResult::hint("仍在建图中，请稍后再查"); - } -} -``` - -这个降级逻辑不是临时补丁。它是故意的——AI 不需要知道底层数据是否就绪，它只需要知道"能查还是不能查"以及"什么时候能查"。 - ---- - -## 六、FTS 后置构建 - -FTS（Full-Text Search，全文搜索）是另一个典型例子：它很重要，但不需要阻塞索引流程。 - -```mermaid -sequenceDiagram - participant Client as MCP Client - participant Server as Rust Server - participant Worker as C++ Worker - participant DB as SQLite - - Client->>Server: index_project - Server->>Worker: spawn subprocess - Worker->>DB: 写入 semantic_records + graph_nodes - Worker-->>Server: stdout JSON result - Server-->>Client: {"ok":true, "files_indexed":N} - - Note over Client,Server: ✅ 用户收到返回，可以去查符号了 - - Server->>Server: RUNTIME.spawn(build_fts) - Server->>DB: buildFTSFromGraph() - - Note over Client,Server: 这期间 search 走 graph fallback - - Server->>DB: fts_ready=1 - - Note over Client,Server: FTS 就绪，search 切回 FTS5 - - Client->>Server: search("scheduler") - Server->>DB: FTS5 全文搜索 - DB-->>Server: 结果 - Server-->>Client: ✅ -``` - -关键设计点：**FTS 就绪前，`search` 命令不走 FTS，而是走 `graph_nodes.name LIKE` 降级搜索。** 慢一点，但能用。等你需要精确全文搜索的时候，FTS 已经建好了。 - ---- - -## 七、真实数据：三个阶段的时间分布 - -以 GoAgent 项目（1,167 个 Go 文件）的实测数据为例： - -| 阶段 | 耗时 | 累积 | 此时 AI 可以 | -|:----:|:----:|:----:|--------------| -| Phase A 扫描 | **\<500ms** | \<500ms | 查询符号/模块/入口点 | -| Phase B 解析 | **~3s** | ~3.5s | 查询调用图/FTS | -| Phase C 建图 | **~26s** | ~29.5s | 调用链追踪/社区检测 | -| FTS 后置 | **~2s** | ~31.5s | 全文搜索 | - -对比 CBM，同样的项目： - -| 工具 | 首次可查询时间 | 全量完成时间 | -|------|:-------------:|:------------:| -| CBM | 全量索引完才能查 | **3.94s** | -| **CodeScope** | **\<500ms** | **31.5s** | - -当然，CodeScope 全量索引比 CBM 时间长，因为它建了更多的节点和边（263,614 个节点 vs 24,658 个节点，**10.7x**）。但 AI 不需要等——它从 \<500ms 开始就在工作了。 - ---- - -## 八、坦诚反思 - -### 渐进式就绪的代价 - -这个模型不是没有成本。 - -**复杂度。** Phase A 和 Phase B 两条数据路径需要维护两套代码。Phase A 的正则扫描器（`engine_scanner.cpp`，1,180 行）和 Phase B 的 tree-sitter 解析器是完全不同的代码路径。bug 可能只出现在其中一条上。 - -**数据一致性。** Phase A 扫描出来的符号和 Phase B 解析出来的符号可能不完全一致。正则扫描可能漏掉一些嵌套结构里的符号，也可能把注释里的东西误认为符号。这需要两套逻辑之间的对齐。 - -**AI 对这种模型的适配。** 不是所有的 AI agent 都知道"先 scan 再 enhance"。有些 AI 习惯了一次全量索引完再查，需要额外的 prompt 指导来适配渐进式模型。 - -### 为什么还是选了这条路 - -因为我更在意**首查延迟**。 - -对于 AI agent 工作流来说，第一次查询的等待时间决定了用户体验的下限。如果 AI 问"这个项目是什么"要等 12 分钟，用户基本不会用第二次。 - -CodeScope 选择了"先回答，再深入"——前 70% 的查询在 \<500ms 内解决，剩下的 30% 在后台准备。这是**一个关于延迟分布的设计决策，不是技术能力的限制。** - ---- - -## 九、下期预告 - -渐进式就绪解决了"什么时候能查"的问题。但还有一个问题没解决——**查出来的数据有多大**。 - -56KB 的 JSON 响应里，70% 是无用数据。下一期我们讲 CodeScope 的第二个核心设计：Worker 进程隔离——为什么索引不会拖垮你的 MCP Server，以及如何控制内存风暴。 - -[CodeScope 架构拆解（三）：Worker 隔离——为什么索引不会拖垮你的 MCP Server](codescope-architecture-03-worker-isolation.md) \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-03-worker-isolation.md b/docs/articles/zh/codescope-architecture-03-worker-isolation.md deleted file mode 100644 index 4b49427..0000000 --- a/docs/articles/zh/codescope-architecture-03-worker-isolation.md +++ /dev/null @@ -1,259 +0,0 @@ -# CodeScope 架构拆解（三）：Worker 隔离——为什么索引不会拖垮你的 MCP Server - -> 最初版本是没有 Worker 子进程的。Server 直接调用引擎函数。 -> 然后有一天，索引一个很大的项目，RSS 冲到了 3.5 GB，MCP Server 直接 OOM。Claude Desktop 那边显示"工具调用超时"，没有任何错误信息。 -> 我花了三天才找到原因——不是内存泄漏，是线程栈太大了。 -> -> 从那天起我认识到一件事：**在一个长期运行的 Server 进程里做重型计算，要么你隔离计算，要么你就等着被计算拖垮。** - ---- - -## 系列目录 - -| 篇 | 标题 | 一句话 | -|:--:|------|--------| -| 一 | [开篇](codescope-architecture-01-intro.md) | 为什么重写一个代码理解工具 | -| 二 | [渐进式就绪](codescope-architecture-02-progressive-readiness.md) | 毫秒级让 AI 开始理解你的代码 | -| **三** | **Worker 隔离**（本文） | 为什么索引不会拖垮你的 MCP Server | -| 四 | 零冗余响应 | 精简响应，按需返回 | -| 五 | C++ 引擎拆解 | 从源码到多维代码图的管线 | - ---- - -## 一、问题：重型计算不能跑在 Server 进程里 - -MCP Server 是一个长期运行的进程，它的职责是： - -1. 接收 JSON-RPC 请求 -2. 路由到对应工具 -3. 返回 JSON-RPC 响应 - -它不应该做重型计算。但源代码索引恰好是重型计算——要读几百个文件、解析 AST、建图、写库。 - -如果这一切发生在 Server 进程中，问题就来了： - -### 1.1 内存污染 - -C++ 引擎用 arena 分配器和各种局部缓冲。索引完一个项目后，这些内存不会 100% 归还给 OS——C++ 的 free list 会保留一些碎片。Server 进程的 RSS 只增不减。 - -### 1.2 线程栈问题 - -最初的版本给每个 worker 线程分配了 **256MB 的栈**。14 个线程 × 256MB = 3.5GB。这还不是物理内存——这是虚拟地址空间预留，但对操作系统来说仍然是压力。 - -``` -14 threads × 256MB stack = 3.5 GB virtual address space - ↓ 修复后 -14 threads × 8MB stack = 112 MB virtual address space - ↓ 这是 256MB → 8MB 的 32x 缩减 -``` - -这个 bug 的发现过程很丢人——我开始以为是内存泄漏，用 valgrind 跑了一遍没发现问题，然后用 `/usr/bin/time -l` 看峰值 RSS，才发现是线程栈的问题。 - -### 1.3 长时间挂起 - -索引 Linux Kernel 需要几分钟。如果索引在主进程中跑，这几分钟里 MCP Server 无法响应任何其他请求——包括 `get_index_progress` 轮询。AI 端看到的就是"工具无响应"。 - ---- - -## 二、解决方案：Worker 子进程 - -解决思路很直接：**索引放到子进程里做，做完子进程退出，RSS 全量归还 OS。** - -```mermaid -flowchart TB - subgraph Server["MCP Server (Rust)"] - A["index_project 请求"] - B["spawn worker 子进程"] - C["poll progress
get_index_progress"] - D["返回结果给 AI"] - end - - subgraph Worker["Worker 子进程 (C++)"] - E["读取文件"] - F["解析 AST"] - G["建图"] - H["写入 SQLite"] - I["stdout JSON"] - end - - DB["SQLite DB"] - - A --> B - B -->|"fork + exec"| E - E --> F --> G --> H --> I - I -->|"进程退出"| C - C --> D - H -->|"WAL 写入"| DB - C -->|"WAL 读取"| DB -``` - -### 2.1 进程隔离的核心机制 - -Worker 子进程的生命周期很简单： - -``` -1. Server 收到 index_project 请求 -2. Server fork + exec worker 进程 -3. Worker 干完活 → stdout 打印 JSON 结果 -4. Worker 进程 exit → RSS 全量归还 OS -5. Server 读取 stdout → 组织 MCP 响应 -6. Worker 死了，Server 活得好好的 -``` - -关键细节：**Worker 和 Server 共享同一个 SQLite 数据库文件。** Worker 用 WAL 模式写入，Server 同时用快照读取。不需要 IPC、不需要共享内存、不需要 socket。 - -### 2.2 代码路径 - -```rust -// Rust Server — spawn worker 子进程 -fn index_project(path: &str) -> Result { - let worker_path = find_worker_binary(); - let mut child = Command::new(worker_path) - .args(&[db_path, path, "auto", "1"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - - // 轮询进度 - loop { - if let Some(status) = child.try_wait()? { - let output = child.stdout.as_ref().unwrap(); - return Ok(read_output(output)); - } - // 通过 SQLite 读进度 - let progress = get_index_progress(); - send_progress_to_client(progress); - std::thread::sleep(Duration::from_millis(200)); - } -} -``` - -```cpp -// C++ Worker — main 函数 -int main(int argc, char *argv[]) { - const char *db_path = argv[1]; - const char *project_path = argv[2]; - - engine_init(db_path); - int project_id = engine_create_project(project_path); - engine_index_project(project_id, project_path, /*sync=*/true); - engine_shutdown(); - - // 结果通过 stdout 返回 - json_result(project_id); - return 0; -} -``` - ---- - -## 三、Worker 超时保护 - -如果索引卡住了怎么办？死循环？磁盘满？网络文件系统挂起？ - -处理方式：**超时 + 重试。** - -```rust -fn run_worker_with_timeout(cmd: &mut Command, timeout: Duration) -> Result { - let mut child = cmd.spawn()?; - let start = Instant::now(); - - loop { - if start.elapsed() > timeout { - child.kill()?; // kill -9 - child.wait()?; - return Err("worker timed out".into()); - } - if let Some(status) = child.try_wait()? { - return collect_output(child); - } - std::thread::sleep(Duration::from_millis(200)); - } -} -``` - -当前配置： - -| 参数 | 值 | -|------|:---:| -| Worker 超时 | **300 秒** | -| 最大重试次数 | **3 次** | -| kill 信号 | SIGKILL | -| 重试间隔 | 立即重试 | - -300 秒对于绝大多数项目足够了。如果真的超时 3 次，那可能是项目本身有问题（比如符号链接成环、NFS 挂起），Server 会返回明确错误而不是永久挂起。 - ---- - -## 四、线程栈 256MB → 8MB 的修复 - -回到开头那个 3.5GB 的 bug。它不复杂，但很有代表性： - -```cpp -// 修复前：默认 256MB 栈 -pthread_attr_t attr; -pthread_attr_init(&attr); -// 没有设置栈大小 → 系统默认 256MB - -// 修复后：8MB 栈 -pthread_attr_t attr; -pthread_attr_init(&attr); -pthread_attr_setstacksize(&attr, 8 * 1024 * 1024); // 8MB -``` - -| 指标 | 修复前 | 修复后 | -|------|:-----:|:------:| -| 每线程栈 | 256 MB | 8 MB | -| 14 线程合计 | 3.5 GB | 112 MB | -| 峰值 RSS（GoAgent） | ~3.5 GB | ~372 MB | -| 线程栈溢出风险 | 低 | 低（已验证） | - -这个修复最让我羞愧的一点是：**bug 根本不在逻辑里，而在配置里。** - ---- - -## 五、子进程模式 vs 线程模式 - -为什么不用线程？这是一个反复被问到的问题。 - -| 维度 | 线程模式 | 子进程模式 | -|------|---------|-----------| -| 内存隔离 | ❌ 共享地址空间 | ✅ 完全隔离 | -| 内存归还 | ❌ free list 碎片残留 | ✅ 进程退出，全量归还 | -| 崩溃影响 | 🔴 Server 一起崩 | ✅ Worker 崩了，Server 无事 | -| 通信开销 | 低（共享内存） | 中（stdout + 共享 DB） | -| 启动开销 | 低 | 中（fork + exec） | -| 资源控制 | 难（ulimit 不精确） | 容易（cgroup / rlimit） | - -### 坦诚反思 - -子进程模式的核心代价是**通信复杂了**。如果是线程模式，进度追踪就是一个全局变量的事。子进程模式下，Worker 得把进度写进 SQLite，Server 去轮询读。 - -但选子进程模式的根本原因只有一个：**内存泄露可以被 OS 兜底。** 线程模式下一旦有内存泄漏，Server 的 RSS 只增不减，最终 OOM。子进程模式下，Worker 泄漏再多内存，exit 之后就全部归还了。 - -对于 C++ 代码来说，这是最实在的安全网——不管你忘写了哪个 `delete` 或 `free`，进程退出时 OS 会替你收拾。 - ---- - -## 六、实际效果 - -| 项目 | 文件数 | Worker RSS | Server RSS（索引期间） | Worker 退出后 | -|------|:-----:|:----------:|:---------------------:|:-------------:| -| ARES Agent | 95 | ~50 MB | ~10 MB | RSS 0 | -| GoAgent | 1,167 | ~372 MB | ~12 MB | RSS 0 | -| memscope-rs | 238 | ~180 MB | ~10 MB | RSS 0 | -| Linux Kernel（5min） | 6,173 | ~2.1 GB | ~15 MB | RSS 0 | - -Server 进程的 RSS 基本不受索引影响。无论 Worker 冲得多高，Server 始终保持在 \~10-15 MB。 - -这才是"隔离"的真正意义：**不是让峰值降低，而是让峰值不影响你。** - ---- - -## 七、下期预告 - -Worker 隔离解决了"索引时 Server 不卡"的问题。但还有一个更"肉疼"的问题没解决——**AI 的 token 消耗**。 - -当同一个查询，一个工具返回 56KB，另一个返回 629 bytes，AI 的 token 账单差了 35 倍。下一期我们讲 CodeScope 的零冗余响应设计——每个字段为什么存在，以及更重要的是：**每个字段为什么不存在。** - -[CodeScope 架构拆解（四）：零冗余响应——精简响应，按需返回](codescope-architecture-04-zero-redundancy.md) \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-04-zero-redundancy.md b/docs/articles/zh/codescope-architecture-04-zero-redundancy.md deleted file mode 100644 index 2d0a41e..0000000 --- a/docs/articles/zh/codescope-architecture-04-zero-redundancy.md +++ /dev/null @@ -1,326 +0,0 @@ -# CodeScope 架构拆解（四）：零冗余响应——精简响应，按需返回 - -> 我统计过 codebase-memory-mcp 一个 `search_graph` 响应里的字段： -> - `fingerprint` — 不需要 -> - `ast_profile` — 不需要 -> - `body_tokens` — 不需要 -> - `doc_tokens` — 不需要 -> - `route` — 不需要 -> - `dataType` — 不需要 -> ... -> 56KB 的 JSON 里，真正对回答"Chaos 是如何工作的"有用的字段，不到 10%。 -> -> 所以我给自己定了一个规则：**CodeScope 的每个响应字段，必须能直接回答"AI 为什么需要这个"。回答不出来的，删掉。** - ---- - -## 系列目录 - -| 篇 | 标题 | 一句话 | -|:--:|------|--------| -| 一 | [开篇](codescope-architecture-01-intro.md) | 为什么重写一个代码理解工具 | -| 二 | [渐进式就绪](codescope-architecture-02-progressive-readiness.md) | 毫秒级让 AI 开始理解你的代码 | -| 三 | [Worker 隔离](codescope-architecture-03-worker-isolation.md) | 为什么索引不会拖垮你的 MCP Server | -| **四** | **零冗余响应**（本文） | 精简响应，按需返回 | -| 五 | C++ 引擎拆解 | 从源码到多维代码图的管线 | - ---- - -## 一、先看数据 - -同一台机器，同一个项目（GoAgent，~24K 行 Go），同一个问题，两个工具的设计哲学不同，响应格式也不同： - -### CBM v0.8.1 — search_graph 返回（56,183 bytes） - -```json -[ - { - "id": "2851", - "name": "ChaosExecutor", - "kind": "STRUCT", - "file": "internal/ares_quant/.../chaos.go", - "line": 72, - "column": 1, - "fingerprint": "abcdef1234567890", - "ast_profile": { - "node_count": 427, - "depth": 12, - "token_count": 3842, - "body_tokens": 2891, - "doc_tokens": 0 - }, - "route": "/api/v1/...", - "dataType": "code_symbol", - "body": "type ChaosExecutor struct { ... }", - "doc": "", - "children": ["...", "..."], - "parent_id": "2845", - "siblings": ["...", "..."], - "metadata": { ... } - } -] -``` - -56KB，64 条结果。CBM 选择了一次返回完整信息，包括 fingerprint、ast_profile 等字段——这些字段在去重、性能分析等场景中很有价值，只是在"快速回答一个简单问题"的场景下 AI 用不上。 - -### CodeScope — find_symbol 返回（629 bytes） - -```json -{ - "results": [ - { - "id": 2851, - "kind": "struct", - "name": "ChaosExecutor", - "signature": "type ChaosExecutor struct {", - "visibility": "default", - "language": "go", - "file_path": "~/go/src/goagent/.../chaos.go", - "line": 72, - "column": 1 - } - ] -} -``` - -629 bytes，2 条结果（精确匹配）。每个字段都是 AI 需要的。 - -### 不只是大小的问题 - -| 维度 | CBM | CodeScope | -|------|:---:|:---------:| -| 响应大小 | **56,183 bytes** | **629 bytes** | -| 等价 tokens（ASCII×0.3） | ~16,855 | ~189 | -| 结果数 | 64 | 2 | -| 有用字段占比 | ~30%（取决于场景） | ~100% | -| 不需要的字段数 | 7+ | 0 | - -两种设计都是合理的选择。CBM 的响应字段丰富，适合一次获取完整信息的深度分析；CodeScope 精简字段，适合 token 敏感和快速交互场景。 - ---- - -## 二、设计哲学：每个字段必须回答"为什么" - -CodeScope 的工具响应字段设计遵循一个简单的准则： - -> **每个字段的存在，必须能回答"AI 拿到这个字段后能做什么"。** - -### 字段存在性分析 - -| 字段 | 存在吗？ | 为什么存在 / 为什么不存在 | -|------|:--------:|--------------------------| -| `id` | ✅ | AI 需要用 id 做后续查询（如 `get_complexity(id)`） | -| `kind` | ✅ | AI 需要知道这是 struct / func / var / const | -| `name` | ✅ | 符号名，最基本的标识 | -| `signature` | ✅ | AI 不需要完整 body，但需要知道函数签名（参数类型、返回类型） | -| `visibility` | ✅ | AI 需要知道是否是外部可调用的 | -| `language` | ✅ | AI 需要知道这个符号属于哪种语言 | -| `file_path` | ✅ | AI 需要知道文件位置才能引用 | -| `line` | ✅ | AI 需要行号做上下文引用 | -| `column` | ✅ | 精确位置 | -| `fingerprint` | ❌ | **AI 不需要。** 这是给去重算法用的，不是给 AI 用的 | -| `ast_profile` | ❌ | **AI 不需要。** 节点数、深度、token 数——AI 自己会算 | -| `body_tokens` | ❌ | **AI 不需要。** 完整 body 是给人类读的，AI 要的是签名 | -| `doc_tokens` | ❌ | **AI 不需要。** 有 FTS 搜索就够了 | -| `route` | ❌ | **AI 不需要。** 这是 REST API 的产物 | -| `dataType` | ❌ | **AI 不需要。** kind 字段已经够了 | -| `children` | ❌ | **AI 不需要。** 需要子树时可以单独查 | -| `siblings` | ❌ | **AI 不需要。** 同上 | -| `metadata` | ❌ | **AI 不需要。** 模糊的兜底字段 | -| `body` | ❌ | **AI 不需要完整 body。** signature 就够了 | - -### 三问法 - -每个字段设计时都要过三关： - -1. **AI 拿到这个字段后能做什么？** → 不能直接回答的，删掉。 -2. **有没有更紧凑的表示？** → 能用 `kind: "struct"` 就不要 `dataType: "code_symbol"`。 -3. **AI 真的需要知道这个吗？** → 不要替 AI 做决定。AI 不需要知道的东西，它不会问。 - ---- - -## 三、工具描述优化：给 AI 写文档 - -CodeScope 的工具描述不是给人看的，是给 AI 看的。人可能会读一两次，但 AI 每次调用都会读。 - -你会发现这些描述有几个特点： - -### 3.1 明确告诉 AI 返回什么 - -```json -// 模糊的描述 -"description": "Find symbol by name" - -// CodeScope 的描述 -"description": "Find symbol(s) by exact name match. Returns id, kind, file path, line/column for each match." -``` - -AI 不需要猜测返回格式。描述里直接告诉它。 - -### 3.2 标记哪些工具有替代品 - -```json -"description": "[DEPRECATED — use search] Full-text search across code symbols, file paths, and comments." -``` - -不删旧工具（兼容性），但明确告诉 AI 用新的。 - -### 3.3 指导 AI 的工作流 - -``` -find_symbol → 找符号定义 - ↓ -find_callers → 谁调用了它 - ↓ -codescope_trace → 递归调用链 - -project_overview → 先看整体 - ↓ -入话题 → 具体工具 -``` - -`project_overview` 的描述里写着 "Call this first after initialization"——这是在教 AI 工作流，而不是只描述功能。 - ---- - -## 四、35+ 工具的设计分类 - -CodeScope 有 35+ 个工具。它们不是随意堆叠的，而是按 AI 的工作流分成了几个类别： - -### 4.1 初始化工具（Phase A 后可用） - -| 工具 | 一句话 | -|------|--------| -| `scan_project` | 扫描项目，Phase A 入门 | -| `project_overview` | 先调这个，获取项目全貌 | -| `find_symbol` | 查符号定义位置 | -| `get_module_tree` | 看模块层级 | -| `get_entry_points` | 找入口点 | -| `get_graph_stats` | 看统计信息 | - -### 4.2 增强工具（Phase B 后可用） - -| 工具 | 一句话 | -|------|--------| -| `enhance_project` | 触发全量解析 | -| `get_enhancement_status` | 查进度 | -| `find_callers` / `find_callees` | 调用关系 | -| `search` | 统一搜索（FTS5 + 语义混合） | -| `get_complexity` | 圈复杂度 | -| `graph_query ❌` | DSL 图查询 | - -### 4.3 高级工具（Phase C 后可用） - -| 工具 | 一句话 | -|------|--------| -| `codescope_trace` | 递归调用链追踪 | -| `codescope_build_context` | 智能上下文构建 | -| `detect_changes` | 变更影响分析 | -| `get_communities ❌` | 社区检测 | -| `search_semantic` | 语义搜索 | - -### 4.4 分层设计的意义 - -工具分类不是文档装饰——它直接影响 AI 的行为： - -- **Phase A 的工具不需要等全量索引。** AI 扫描完项目就能用。 -- **Phase B/C 的工具会自适应降级。** 数据没准备好时，不报错，给提示。 -- **AI 不需要知道"当前在哪个阶段"。** 它只需要调用 `find_symbol`，底层自动判断能不能查。 - ---- - -## 五、响应格式的统一性 - -### 5.1 统一的结构 - -所有响应都遵循一个统一的格式： - -```json -{ - "results": [ - { - "id": ..., - "kind": ..., - "name": ..., - "file_path": ..., - "line": ..., - "column": ... - } - ] -} -``` - -AI 不需要为每个工具学习不同的返回格式。`find_symbol` 返回 `results` 数组，`find_callers` 也返回 `results` 数组，`search` 也返回 `results` 数组。 - -### 5.2 只在需要时增加字段 - -个别工具会多返回一些字段，但不会滥用： - -- `get_complexity` 额外返回 `cyclomatic_complexity`、`cognitive_complexity`、`nesting_depth`——这些是它独有的核心信息。 -- `search` 返回 `snippet` 字段——FTS 上下文快照，方便 AI 快速判断是否相关。 -- `codescope_trace` 返回 `path` 数组——调用链的每个跳转节点。 - -冗余的代价不是磁盘空间，是 AI 的 token 预算。**每个多余的字节都在消耗 AI 理解真正问题的能力。** - ---- - -## 六、零冗余的实际效果 - -### 6.1 Token 节省 - -| 场景 | CBM | CodeScope | 节省 | -|------|:---:|:---------:|:----:| -| 查符号 | 19,632 tokens | 552 tokens | **97.2%** | -| 查项目概况 | 全量索引 12 分钟 | 毫秒级 + 629 bytes | 时间和 token 双赢 | -| 查调用关系 | 56KB + 额外读源码 | 调用图直接返回 | 不需要读源码 | -| 索引 DB | 64 MB | 270 KB | **99.6%** | - -### 6.2 这 97.2% 的 token 省下来给谁了 - -省下来的 token 不是"省了"——是**重新分配给了真正重要的事情**。 - -AI 有限的上文窗口里： - -``` -CBM 方案： - ┌────────────────────────────────────────────────┐ - │ 56KB JSON 响应 (70% 无用) │ 真正在分析代码 │ - │ ██████████████████████████████░░░░░░░░░░░░░░░ │ - └────────────────────────────────────────────────┘ - -CodeScope 方案： - ┌────────────────────────────────────────────────┐ - │ 629 bytes JSON 响应 │ 真正在分析代码 │ - │ ██░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ │ - └────────────────────────────────────────────────┘ -``` - -同样的 token 预算，CodeScope 让 AI 有更多能力去**思考和推理代码**，而不是去**解析 JSON 里的无用字段**。 - ---- - -## 七、坦诚反思 - -### 零冗余的代价 - -够用主义不是没有代价的。 - -**信息密度大意味着容错率低。** 如果 CodeScope 的 `find_symbol` 返回了错误的结果，AI 没有富余的上下文来发现这个错误。CBM 的 56KB 里虽然大部分冗余，但一旦某个字段出错，冗余信息可能帮助 AI 交叉验证。 - -**响应过于精简可能让 AI 不确定。** 有些 AI 会反复调用同一个工具来确认——"你确定这个 struct 没有其他字段吗？"——因为响应里没有 `children` 字段，AI 不知道是"没有子节点"还是"没返回子节点"。 - -**工具描述对 AI 的适配是持续的。** 不同模型对工具描述的理解不同。Claude 和 GPT 对同一个描述的解释可能不一样。这需要持续调整。 - -### 为什么还是选了这条路 - -因为 token 是 AI 工作流中最稀缺的资源。**在一个 token 就是钱的生态里，返回 56KB 无用数据不只是浪费，是不负责任。** - -而且，CodeScope 的设计哲学是"**AI 想知道什么，就告诉它什么，不要替它把所有可能的问题都回答一遍**"。零冗余是这个哲学的自然延伸。 - ---- - -## 八、下期预告 - -前面四篇文章我们从"为什么"到"架构"到"渐进式就绪"到"零冗余"——这些都是 CodeScope 上层的东西。下一期我们真正钻到引擎里，看看 C++ 是怎么把几十万行代码变成一张可查询的多维代码图的。 - -[CodeScope 架构拆解（五）：C++ 引擎拆解——从源码到多维代码图的管线](codescope-architecture-05-engine-deep-dive.md) \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-05-engine-pipeline.md b/docs/articles/zh/codescope-architecture-05-engine-pipeline.md deleted file mode 100644 index be809e5..0000000 --- a/docs/articles/zh/codescope-architecture-05-engine-pipeline.md +++ /dev/null @@ -1,450 +0,0 @@ -# CodeScope 架构拆解（五）：C++ 引擎拆解——从源码到多维代码图的管线 - -> 有一次我 debug 一个跨文件调用链，发现调用图里少了一条边。那个函数明明被调用了，但图里就是没有。翻了一下午代码，最后发现原因很简单：**调用发生在宏展开里，而我们的 visitor 不展开宏。** 这不是 bug，是 tradeoff。但这件事让我意识到，从源码到代码图，每一个环节都在做取舍。 - ---- - -## 三个问题 - -任何代码理解工具的核心，都是把"文本"变成"结构"。这个过程可以拆解成三个问题： - -1. **如何在不解析 AST 的情况下，极速获取代码结构？**（毫秒级 返回结果的关键） -2. **如何把 10 种语言的 AST 统一成一种中间表示？**（跨语言分析的基础） -3. **如何从统一的 IR 构建出可供 AI 查询的多维代码图？**（最终交付物） - -CodeScope 的 C++ 引擎，就是围绕这三个问题设计的。 - ---- - -## 整体管线 - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ C++ Engine Pipeline │ -├─────────────────────────────────────────────────────────────────┤ -│ │ -│ Phase A (Fast Scan) Phase B/C (Full Index) │ -│ ┌───────────────┐ ┌─────────────────────────┐ │ -│ │ Line-by-line │ │ File Read + Language │ │ -│ │ Regex Scan │ │ Detection │ │ -│ │ (no AST!) │ │ │ │ │ -│ └───────┬───────┘ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ ▼ │ │ tree-sitter │ │ │ -│ ┌───────────────┐ │ │ Parse │ │ │ -│ │ Modules │ │ └──────┬───────┘ │ │ -│ │ Symbols │ │ ▼ │ │ -│ │ EntryPoints │ │ ┌──────────────┐ │ │ -│ │ Stubs │ │ │ IR Translator │ │ │ -│ └───────────────┘ │ │ (CST → IR) │ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ ▼ │ ┌──────────────┐ │ │ -│ ┌────────────┐ │ │ Linker Passes │ │ │ -│ │ JSON │ │ │ ┌────────────┐│ │ │ -│ │ Response │ │ │ │ BuildSymbol││ │ │ -│ └────────────┘ │ │ │ IndexPass ││ │ │ -│ │ │ │ ├────────────┤│ │ │ -│ │ │ │ │ ResolveCall││ │ │ -│ │ │ │ │ Pass ││ │ │ -│ │ │ │ ├────────────┤│ │ │ -│ │ │ │ │ EmitGraph ││ │ │ -│ │ │ │ │ Pass ││ │ │ -│ │ │ │ └────────────┘│ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ │ │ │ Complexity │ │ │ -│ │ │ │ Analyzer │ │ │ -│ │ │ └──────┬───────┘ │ │ -│ │ │ ▼ │ │ -│ │ │ ┌──────────────┐ │ │ -│ ├─────────────────►│ SQLite Store │ │ │ -│ │ │ (FTS5 + vec0) │ │ │ -│ │ │ + Graph Nodes │ │ │ -│ │ │ + Graph Edges │ │ │ -│ │ └───────────────┘ │ │ -│ │ │ │ │ -│ ▼ ▼ │ │ -│ ┌────────────────────────────────────────┐ │ │ -│ │ Query Engine (adaptive) │ │ │ -│ │ Phase A ready → fast scan results │ │ │ -│ │ Phase B ready → + symbol search │ │ │ -│ │ Phase C ready → + call graph / FTS │ │ │ -│ └────────────────────────────────────────┘ │ │ -│ │ -└─────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Phase A：不建 AST，怎么理解代码？ - -Phase A 的目标是：**在 毫秒级 内，让 AI 知道这个项目里有什么。** 解析 AST 是不可能的——tree-sitter 解析 24K 行代码需要秒级，而且还不是瓶颈，后续的 IR 翻译和图构建才是大头。 - -CodeScope 的 Phase A 用了另一种思路：**逐行正则扫描**。 - -实际的扫描逻辑位于 `engine/src/engine_scanner.cpp`（1194 行），核心流程是： - -```cpp -// engine/src/engine_scanner.cpp:786 -char *engine_scan_project(uint64_t project_id, const char *dir_path, - const char *language_filter) -``` - -它做的事情： - -1. **遍历目录树**，跳过 `.gitignore` 匹配的文件 -2. **逐行读取文件**，跳过注释和空行 -3. 对每一行，调用 `trimLeft()` 去掉前导空格，然后用 `detectDecl()` 判断声明类型 -4. 匹配到的声明用 `extractName()` 提取符号名 -5. 插入 SQLite，构建模块树 - -### detectDecl 的"黑魔法" - -`detectDecl()` 的实现很有意思——它没有用正则表达式，而是用了一系列前缀匹配和启发式规则，针对每种语言做了特化： - -```cpp -// engine/src/engine_scanner.cpp: 内部函数 -// 检测 C 函数声明：如 "int foo(" 或 "void *bar(" -bool looksLikeCFunction(std::string_view sv) { - // 32 种 C 类型关键字：int, void, char, size_t, gfp_t, ... - // 匹配 "type name(" 模式 -} -``` - -对于 C/C++，它检查行首是不是类型关键字 + 标识符 + 左括号。对于 Rust，它检查 `fn` 关键字。对于 Python，它检查 `def` 和 `class`。 - -这种做法的好处是**极快**——没有正则编译开销，没有 AST 构建，就是纯粹的字符串操作。代价是**不精确**——会漏掉一些声明，也会误判一些非声明。但 Phase A 的定位是"快速预览"，不要求 100% 召回。 - -### Stub 检测 - -一个有趣的细节是 `stub` 检测。Phase A 只能在单行内判断一个函数是不是 stub（空实现）： - -```cpp -// engine/src/engine_scanner.cpp: 约 1080 行 -if ((kind == "function" || kind == "method") && - line.find("{}") != std::string::npos) { - // 检查 {} 之间是否只有空白字符 - if (!has_content) { - g_store->setSymbolStub(sym_id, true); - } -} -``` - -多行 stub（左大括号在下一行）Phase A 检测不到，需要 Phase B 的 AST 解析才能覆盖。 - -### 增量扫描 - -Phase A 还支持 Git 增量扫描： - -```cpp -// engine/src/engine_scanner.cpp -std::unordered_set git_changed = getGitChangedFiles(dir); -``` - -通过 `git diff --name-only` 获取变更文件列表，只扫描有变化的文件。同时记录文件的 `mtime` 和 `size`，未变化的文件跳过。 - ---- - -## CST 到 IR：翻译器的设计 - -Phase B 的核心是 tree-sitter 解析 + IR 翻译。tree-sitter 输出的是 **CST（Concrete Syntax Tree）**——保留了所有语法细节，包括括号、分号、关键字。 - -CodeScope 的 IR 定义在 `engine/src/ir/ir.h`： - -```cpp -// engine/src/ir/ir.h -enum class NodeKind : uint16_t { - TranslationUnit, Module, // 编译单元 - FunctionDecl, MethodDecl, // 函数/方法 - ClassDecl, VariableDecl, FieldDecl, ParameterDecl, // 声明 - EnumDecl, EnumMemberDecl, TypeAliasDecl, // 枚举/类型 - MacroDecl, TemplateDecl, NamespaceDecl, // 宏/模板/命名空间 - BlockStmt, IfStmt, ForStmt, WhileStmt, // 语句 - CallExpr, BinaryExpr, MemberExpr, IdentifierExpr, // 表达式 - // ... 共约 40 种 -}; - -enum class Relation : uint8_t { - Parent, Child, TypeRef, SymbolRef, CallTarget, Receiver, BaseClass -}; -``` - -翻译器接口定义在 `engine/src/ir/ir_translator.h`： - -```cpp -// engine/src/ir/ir_translator.h -class Translator { -public: - virtual ~Translator() = default; - virtual TranslationUnit *translate(TSTree *tree, const char *source, - const char *file_path) = 0; - virtual const char *language() const = 0; -}; - -// 工厂方法 -Translator *createTranslator(const char *language); -``` - -`createTranslator()` 支持 10 种语言：Python, C++, C, Rust, JavaScript, TypeScript, Go, Java, Swift, TSX。 - -每种语言的翻译器实现在 `engine/src/ir/translators/` 目录下。它们通过 **tree-sitter visitor 模式**遍历 CST 节点，转换成统一的 IR。 - -比如 `c_visitor.h` 中的 C 语言翻译器，会把： - -```c -int foo(int x) { - return x + 1; -} -``` - -翻译成： - -``` -TranslationUnit - └─ FunctionDecl "foo" - ├─ ParameterDecl "x" (type: int) - └─ BlockStmt - └─ ReturnStmt - └─ BinaryExpr "+" - ├─ IdentifierExpr "x" - └─ LiteralExpr "1" -``` - -这个转换过程丢掉了很多 CST 细节——分号、括号、逗号——保留了语义结构。代价是**丢失了精确的源码位置信息**（比如分号的位置），但这对代码理解来说不是必须的。 - ---- - -## ResolverPipeline：约束链驱动的跨文件解析 - -IR 翻译是逐文件进行的。文件之间的调用关系怎么处理？这就是 Resolver（解析器）的工作。 - -最初版本有一个 `engine/src/linker/linker.h`，采用三趟 Pass 模式。经过重构，现已被 **`engine/src/resolver/pipeline.h`** 替代——一个基于约束链（constraint chain）的多因子评分管线。 - -``` -旧 Linker（已删除） 新 ResolverPipeline（当前） -────────────────────── ────────────────────────── -3 趟 Pass 单趟 run() + applyConstraints() -整数加分 9 因子加权评分（0.0-1.0） -无模糊匹配 FuzzyResolver 兜底 -无可见性检查 语言级可见性规则（Go/Python/Java） -无 call kind 区分 CallKind 感知（Direct/Method/Interface/Constructor） -``` - -### 架构：两步走 - -`ResolverPipeline::run()` 的核心逻辑只有两步： - -``` -Step 0: 预加载 entity 表 → HashMap> - 避免逐条 SQL 查询（之前的性能瓶颈） - -Step 1: 遍历 reference 表 → 对每个引用: - a. 从 HashMap 按名查找候选 - b. 零候选？ → FuzzyResolver 兜底（大小写/前缀/后缀） - c. applyConstraints() → 9 因子加权评分 - d. 语言可见性硬过滤（如 Go 小写名不可跨包调用） - e. 选最高分，阈值 0.4 以上写入 relation + graph_edges -``` - -### 九因子评分系统 - -```cpp -// engine/src/resolver/factors.h -constexpr double kWeightModuleMatch = 0.15; // 同模块 -constexpr double kWeightImportMatch = 0.80; // 跨模块主导因子 -constexpr double kWeightNamespaceMatch = 0.10; // 同命名空间 -constexpr double kWeightSignatureMatch = 0.10; // 参数数量匹配 -constexpr double kWeightDistanceMatch = 0.05; // 文件路径距离 -constexpr double kWeightConstructorMatch = 0.10; // 构造函数匹配 -constexpr double kWeightReceiverMatch = 0.15; // Go/Python receiver -constexpr double kWeightCommonNamePenalty = 0.10; // 常见名降权 -constexpr double kWeightCallKindMatch = 0.15; // 调用类型感知 -``` - -注意 `ImportMatch` 权重 **0.80**——这是跨模块解析的主导因子。如果目标名称在当前文件有 import 语句引用，基本可以确定。其他因子只在同模块或模糊匹配时起作用。 - -### CallKind 感知 - -从 `CallKind` 枚举看调用类型对解析的影响： - -| 类型 | 值 | 行为 | -|------|----|------| -| `Direct` | 0 | 不应用 CallKind 因子 | -| `Method` | 1 | 略降分：方法调用通常在同模块 | -| `Interface` | 2 | 降分：接口派发更难解析 | -| `Constructor` | 3 | 加分：构造函数跨模块调用是合理的 | - -### FuzzyResolver 兜底 - -当精确名称查找零候选时，`FuzzyResolver` 尝试三种模糊匹配： - -- **大小写不敏感**（`getUser` ⇢ `getuser`） -- **前缀匹配**（`parseJSON` ⇢ `parse`） -- **后缀匹配**（`initDB` ⇢ `DB`） - -每种匹配都有独立的评分衰减。找到候选后再进入 `applyConstraints()` 走完整评分流程。 - -### 语言可见性硬过滤 - -这是**硬规则**，不是加权因子——加权因子可以被其他因子覆盖，但语言规则必须绝对遵守： - -- **Go**：小写字母开头的名称不可跨包调用 -- **Python**：`_` 开头的名称视为私有 -- **Java**：无 `public` 修饰符的为包内可见 - -这个设计参考了 codebase-memory-mcp 的 `cbm_is_exported()`，但实现为绝对拒绝而非打分。 - -### 写双表：relation + graph_edges - -解析结果同时写入两张表： - -- `relation` — 关系表（`source_id`, `target_id`, `type`, `confidence`） -- `graph_edges` — 兼容 CSR 格式的图边表 - -写双表是为了不同查询路径都能走索引，同时通过 `INSERT OR IGNORE` 保证幂等。 - ---- - -## GraphBuilder：从 IR 到图 - -GraphBuilder 定义在 `engine/src/graph/graph_builder.h`，核心是两套 API： - -- 旧 API（`TranslationUnit`/`Node*` 树）：用于非 JS 语言 -- 新 API（`SemanticUnit`/flat records）：用于 JS/TS（因为 JS/TS 的 visitor 输出已经是 flat records） - -图节点的类型定义在 `engine/src/graph/graph_types.h`： - -```cpp -// engine/src/graph/graph_types.h -enum class NodeType : uint8_t { - Function, Method, Class, Struct, Interface, Variable, Macro, Module, File -}; - -enum class EdgeType : uint8_t { - References, Calls, Defines, Contains, Imports, Inherits -}; - -struct GraphNode { - uint64_t id, ir_node_id; - NodeType type; - std::string name, qualified_name, module_path, file_path, language, signature; - uint32_t start_row, start_col, end_row, end_col; - int complexity; // 圈复杂度 - bool is_entry_point; -}; - -struct GraphEdge { - uint64_t id, source_id, target_id; - EdgeType type; - std::string graph_type; // "call_graph" | "symbol_reference" - std::string call_site_file; - int call_site_line; - std::string label; // "async" / "virtual" / "override" -}; -``` - -GraphBuilder 使用 Visitor 模式遍历 IR 树，对每种 NodeKind 产生不同的图节点。一个关键优化是 **parent chain cache**——避免在插入子节点时反复查找父节点的 ID。 - ---- - -## ComplexityAnalyzer：不只是圈复杂度 - -复杂度分析实现在 `engine/src/ir/ir_complexity.cpp`： - -```cpp -// engine/src/ir/ir_complexity.h -struct ComplexityResult { - uint64_t cyclomatic = 0; // McCabe 圈复杂度 - uint64_t cognitive = 0; // 认知复杂度 (v2) - uint64_t nesting_depth = 0; // 最大嵌套深度 - uint64_t decision_points = 0; // 决策点计数 -}; -``` - -计算规则： - -- **圈复杂度** = 1 + 决策点数量（`if/for/while/do-while/switch-case/catch/ternary`） -- **认知复杂度** = 决策点数量 + 嵌套深度加权 -- **嵌套深度** = 控制流嵌套的最大层数 - -一个已知限制：当前版本**不计算 `&&` 和 `||` 运算符**作为决策点，因为需要检查运算符类型时需要查看源码文本，而纯 IR 遍历做不到。这在代码注释中有标注，是未来的改进方向。 - ---- - -## 查询层：自适应引擎 - -全部分析完成后，数据落入 SQLite。查询引擎定义在 `engine/src/engine_queries.cpp`（1052 行），提供： - -- `engine_get_module_tree()` — 获取模块树 -- `engine_find_symbol()` — 符号搜索（带智能提示） -- `engine_find_definition()` — 定义查找 -- `engine_find_callers()` / `engine_find_callees()` — 调用图查询 -- `engine_get_entry_points()` — 入口点 -- `engine_get_complexity()` — 复杂度 -- `engine_get_graph_stats()` — 图统计 -- `engine_get_index_progress()` — 索引进度 - -查询引擎具备**自适应能力**：根据当前索引完成度，决定返回的信息量。Phase A 完成后只能返回符号列表，Phase B 完成后可以返回定义和引用，Phase C 完成后才能返回完整的调用图和 FTS 搜索。 - ---- - -## 坦诚反思 - -这个引擎设计有一些明显的取舍，值得坦白说： - -**1. Phase A 的精度问题** - -逐行扫描一定会漏掉声明。比如 C 语言中返回复杂类型的函数声明跨越多行： - -```c -const struct very_long_type_name * -foo(int x, int y) -``` - -Phase A 无法识别这种声明——`detectDecl()` 看到 `const` 只是类型修饰符，下一行 `foo(` 没有类型关键字前缀，所以跳过了。Phase B 的 AST 解析能覆盖，但 Phase A 返回的数据就不完整。 - -**2. 跨文件调用的存根模式** - -`ResolveCallPass` 创建的存根（stub）IR 节点，只包含函数名和文件路径，没有参数签名。这意味着如果两个同名的函数在不同文件中，排名启发式可能选错。在我们的实测中，同名函数在大型 C 项目（如 Linux 内核）中很常见，存根的准确率在 70% 左右。 - -**3. 复杂度分析器的局限性** - -不计算 `&&`/`||` 运算符，意味着某些条件表达式的复杂度被低估了。比如： - -```c -if (a && b && c && d && e) // 实际有 5 个决策点，但只算 1 个 -``` - -这对认知复杂度的计算影响更大——因为 IR 中无法区分 `&&` 和 `||`，也就无法应用"每个条件运算符 +1"的规则。 - -**4. ResolverPipeline 的执行效率** - -当前的 `run()` 会预加载整个 entity 表到 HashMap 中。对于大型项目（百万级 entity），这个 HashMap 可能占用数百 MB 内存。虽然有 SQL 索引和批量写入优化，但全量扫描 reference 表 + 逐条 `applyConstraints()` 仍然是索引阶段最耗时的部分。实测在 Linux 内核规模下，解析耗时约占全索引时间的 40%。 - -`INSERT OR IGNORE INTO relation` 中的 `WHERE NOT EXISTS` 子查询也增加了写开销——这是为了过滤 test/bench 文件的调用边，避免被测试代码污染调用图。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| **(五) C++ 引擎拆解** | **从源码到多维代码图的管线 ← 本文** | -| (六) MCP 协议层 | 35+ 工具的设计哲学 | -| (七) 语言翻译器 | 10 种语言 → 统一 IR | -| (八) 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) 自适应查询 | Fallback 机制与就绪检测 | -| (十) 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) 验证层 | 让 AI 对自己的话负责 | -| (十二) Model Engine | 从事实到理解 | -| (十三) Parser + GraphBuilder | 解析与建图 | - ---- - -下一篇我们将拆解 **MCP 协议层**——看看 35+ 个工具是怎么组织、路由、和自描述的。如果一个工具返回错误，AI 怎么知道下一步该调用哪个？ \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-06-mcp-layer.md b/docs/articles/zh/codescope-architecture-06-mcp-layer.md deleted file mode 100644 index 6fea6e5..0000000 --- a/docs/articles/zh/codescope-architecture-06-mcp-layer.md +++ /dev/null @@ -1,331 +0,0 @@ -# CodeScope 架构拆解（六）：MCP 协议层——35+ 工具的设计哲学 - -> 我犯过一个错误。早期设计 CodeScope 的时候，我给每个 C++ 函数都配了一个 MCP 工具——一个函数对应一个工具，看起来完美映射。结果工具列表膨胀到 60+，AI 根本不知道该用哪个。用户反馈说："你工具太多了，AI 随机挑一个调用，经常选错。" -> -> 后来我做了三件事：砍掉重复的工具，用前缀分组，给每一个工具写**像人话一样**的描述。工具列表从 60+ 降到 35+，AI 的调用准确率翻了一倍。 - ---- - -## 三个问题 - -MCP 协议层的设计，本质上要回答三个问题： - -1. **AI 怎么知道有哪些工具可用？**（工具发现） -2. **AI 怎么选对工具？**（工具路由） -3. **AI 怎么理解工具返回什么？**（响应契约） - -这三个问题，每一个都在做同一个权衡：**给 AI 的信息越多，token 越贵；给的信息越少，AI 越可能犯错。** - ---- - -## 整体架构 - -``` - MCP Client (Claude Desktop / Cursor / 自定义客户端) - │ - │ JSON-RPC 2.0 over stdin/stdout - │ {"jsonrpc":"2.0","method":"tools/call", - │ "params":{"name":"find_callers","arguments":{...}}} - ▼ -┌─────────────────────────────────────┐ -│ transport.rs │ -│ stdin/stdout 逐行 JSON-RPC 2.0 │ -└──────────┬──────────────────────────┘ - ▼ -┌─────────────────────────────────────┐ -│ mcp/server.rs │ -│ dispatch(): │ -│ "initialize" → 自动索引项目 │ -│ "tools/list" → 返回 35+ 工具 │ -│ "tools/call" → 路由到 handler │ -└──────────┬──────────────────────────┘ - │ tools::execute(pid, name, args) - ▼ -┌─────────────────────────────────────┐ -│ tools/mod.rs │ -│ TOOL_HANDLERS: HashMap<&str, fn> │ -│ ┌────────────────────────────┐ │ -│ │ 35+ handler functions │ │ -│ │ h_find_callers() │ │ -│ │ h_search() │ │ -│ │ h_codescope_trace() │ │ -│ │ h_build_context() │ │ -│ └────────┬───────────────────┘ │ -└───────────┼────────────────────────┘ - │ ffi::scan_project(), ffi::index_project(), etc. - ▼ -┌─────────────────────────────────────┐ -│ ffi/mod.rs (Rust ↔ C++ FFI) │ -│ unsafe extern "C" → 安全包装 │ -│ take_string() → 自动 free C 内存 │ -└──────────┬──────────────────────────┘ - │ C ABI (动态链接) - ▼ -┌─────────────────────────────────────┐ -│ C++ Engine (engine.so) │ -│ engine_scan_project() │ -│ engine_index_project() │ -│ engine_find_callers_adaptive() │ -│ → 所有函数返回 char* (JSON) │ -└─────────────────────────────────────┘ -``` - ---- - -## 工具分类：三阶段就绪 - -35+ 个工具不做平铺，而是按**就绪阶段**分类： - -### Phase A：亚秒级工具 - -| 工具 | 做什么 | 响应时间 | -|---|---|---| -| `scan_project` | 扫描项目，提取声明 | ~毫秒级 | -| `find_symbol` | 精确符号匹配 | 即时 | -| `get_module_tree` | 模块树 | 即时 | -| `get_entry_points` | 入口点（main/initcall/probe） | 即时 | - -这些工具只依赖 Phase A 数据，永远立即可用。AI 可以**在任何时候**调用它们。 - -### Phase B：后台增强工具 - -| 工具 | 做什么 | 响应时间 | -|---|---|---| -| `enhance_project` | 全量解析 → 调用图 → 复杂度 → 向量嵌入 | ~数秒 | -| `get_enhancement_status` | 查询各能力就绪度 | 即时 | - -`enhance_project` 是唯一**异步执行**的工具。它被调用后立即返回，真正的全量解析在后台线程执行。AI 通过 `get_enhancement_status` 轮询进度。 - -### Phase C：自适应统一工具 - -| 工具 | 做什么 | 自适应逻辑 | -|---|---|---| -| `search` | 统一搜索 | Phase B 未完成 → FTS5；已完成 → 向量搜索 | -| `find_callers` | 调用者查询 | Phase B 未完成 → 旧图查询；已完成 → 新 call_edges 表 | -| `project_overview` | 项目概览 | 综合所有就绪数据 | - -这些工具**内部检查就绪度**，自动降级为可用的数据源。AI 不用关心"调用图有没有建好"，工具自己会处理。 - -### 特色工具 - -| 工具 | 描述 | -|---|---| -| `codescope_trace` | 递归调用链探索 / 最短路径 BFS | -| `codescope_build_context` | **主工具**：自然语言查询 → 智能上下文组装 | -| `codescope_capabilities` | 标准化能力就绪报告 | -| `count_tokens` | Token 估算（纯 Rust，不跨 FFI） | - -`codescope_build_context` 是唯一被标注为 **PRIMARY** 的工具——它把"需要多次调用才能完成的任务"打包成一次调用。AI 描述它为： - -> "AI 模型的首选工具。使用自然语言描述你需要的代码上下文，返回符号定义、引用、调用图、文件内容等的智能组合。" - ---- - -## 工具描述的艺术 - -MCP 协议里，工具的描述是 AI 选择工具的唯一依据。CodeScope 为每个工具撰写了高度精确的描述： - -```rust -// server/src/tools/mod.rs -Tool { - name: "codescope_build_context", - description: - "AI model's primary tool. Use natural language to describe \ - the code context you need: 'how does module X handle errors', \ - 'why is function Y returning null', etc. Returns a smart \ - assembly of symbol definitions, references, call graphs, \ - and file contents relevant to your question.", - input_schema: ..., -} -``` - -关键原则： -- **第一句说清楚"这是干什么的"**——AI 快速匹配意图 -- **给调用示例**——"how does module X handle errors" 这种自然语言示例，告诉 AI 这个工具可以理解自然语言 -- **明确输出内容**——"returns symbol definitions, references, call graphs, file contents" -- **不写实现细节**——AI 不需要知道背后是 C++ 还是 SQLite - -对比坏描述： - -``` -find_callers: "Query the function call edges table to find all caller functions - that call the specified callee function. Uses graph traversal - on the symbol call edge table." -``` - -好描述： - -``` -find_callers: "Return all functions that call a given function. - Use this when you need to know who invokes a specific function." -``` - -AI 不需要知道"call_edges table"或"graph traversal"，它只需要知道"谁调用了这个函数"。 - ---- - -## FFI 桥：Rust 如何驱动 C++ - -所有 MCP 工具最终都要调用 C++ 引擎。这个调用链的关键是 **FFI（Foreign Function Interface）桥**，定义在 `server/src/ffi/mod.rs`： - -### 三层设计 - -**第一层：C ABI 声明** - -```rust -// server/src/ffi/mod.rs -unsafe extern "C" { - fn engine_init(db_path: *const c_char) -> i32; - fn engine_scan_project(pid: u64, dir: *const c_char, - lang: *const c_char) -> *mut c_char; - fn engine_free_string(ptr: *mut c_char); - // ... 约 50 个函数 -} -``` - -每个 C++ 函数返回 `*mut c_char`——一个堆分配的 C 字符串。Rust 读取后必须用 `engine_free_string` 释放。 - -**第二层：内存管理辅助** - -```rust -fn cstr(s: &str) -> CString { - CString::new(s).unwrap_or_else(|_| CString::new("").unwrap()) -} - -fn take_string(ptr: *mut c_char) -> String { - if ptr.is_null() { return String::new(); } - let s = unsafe { CStr::from_ptr(ptr).to_string_lossy().into_owned() }; - unsafe { engine_free_string(ptr) }; // 自动释放 C 内存 - s -} -``` - -`take_string` 是关键的"内存契约"——Rust 借用 C++ 的内存，读取后立即归还。不会发生内存泄漏，也不会 double free。 - -**第三层：安全包装** - -```rust -pub fn scan_project(project_id: u64, dir_path: &str, - language_filter: Option<&str>) -> String { - let lf = language_filter.map(cstr); - take_string(unsafe { - engine_scan_project( - project_id, - cstr(dir_path).as_ptr(), - lf.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - ) - }) -} -``` - -所有 `unsafe` 调用被封装在安全函数内。`Option<&str>` 映射为 null 指针。Rust 侧看不到任何 `extern "C"` 调用。 - -### 为什么不用 bindgen？ - -手动写 FFI 包装而不是用 bindgen 自动生成，原因是： -- **极少的函数签名**（~50 个），不值得引入构建依赖 -- **需要精细控制内存释放时机**——`take_string` 模式 -- **C++ 端的 JSON 字符串是自描述的**——不需要 Rust 侧的复杂类型映射 - ---- - -## 统一的响应契约 - -所有工具遵循相同的响应格式： - -```json -// 成功 -{"ok": true, "total": 3, "results": [...]} - -// 错误 -{"error": "function not found"} -``` - -MCP Server 自动检测错误： - -```rust -// server/src/mcp/server.rs -let is_error = serde_json::from_str::(&result) - .ok() - .and_then(|v| v.get("error").cloned()) - .and_then(|e| if e.is_null() { None } else { Some(true) }); -``` - -如果响应 JSON 中包含非 null 的 `error` 字段，自动标记为 `is_error: true`，MCP 客户端会知道这次调用失败了。 - -这种设计的好处是：**C++ 侧不需要知道 MCP 协议**。它只需要返回 JSON 字符串。错误的检测和协议适配完全由 Rust 侧处理。 - ---- - -## Worker 子进程 - -唯一不在进程内执行的工具是 `index_project`。它的 handler 是特殊的： - -```rust -// server/src/tools/mod.rs (简化) -fn h_index_project(project_id: u64, args: &Value) -> String { - ffi::shutdown(); // 释放 SQLite 锁 - let child = Command::new("codescope") - .args(["worker", db_path, dir, lang, name, &pid]) - .spawn(); - // 300 秒超时监控 - // kill -9 如果超时 - // 最多重试 3 次 - ffi::init(); // 重新初始化引擎 - // ... -} -``` - -其他 34+ 个工具全是进程内 FFI 调用。这意味着： - -- **查询路径零延迟**——没有进程间通信 -- **共享 SQLite 连接**——无需序列化/反序列化 -- **唯一的风险点** —— `index_project` 意外退出会导致引擎状态不一致 - -这也是为什么 `scan_project` 之后会自动触发后台 `enhance_project`（在进程内），而 `index_project` 必须隔离到子进程。 - ---- - -## 坦诚反思 - -**1. 项目 ID 生命周期问题** - -`Server` 结构体只存了一个 `project_id`。如果 AI 先扫描项目 A，再扫描项目 B，server 的 `self.project_id` 就变成了项目 B。后续所有查询都会落到项目 B 上，项目 A 的查询自然出错。 - -**当前解法**：忽略——假设用户只同时处理一个项目。这不是好解法，但实际场景中很少出现"同时处理两个项目"的 MCP 用例。 - -**2. 重复的 enhancement 触发** - -`scan_project` 触发后台 enhancement 的逻辑被**重复实现**了两次——一次在 `execute()` 里，一次在 `handle_call_tool` 里。这意味着某些路径下可能会触发两次 enhancement。虽然 `spawn_enhancement` 内部有去重逻辑（`Mutex`），但这仍然是代码异味。 - -**3. 工具参数零校验** - -所有 handler 都是用 `args["name"].as_str().unwrap_or("")` 获取参数。如果 AI 传错了参数类型（比如传了数字而不是字符串），Rust 不会报错，只是静默地用空字符串代替。这让调试变得困难——你永远不知道是 AI 用错了工具，还是工具用错了参数。 - -**4. 初始化阻塞** - -在 `initialize` 阶段，server 会同步调用 `ffi::create_project()` 和 `ffi::index_project()`。这意味着 MCP Server 在建立连接的那个瞬间会卡住，等到索引完成才返回 `initialized` 响应。对于大型项目，这可能导致客户端超时。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| (五) C++ 引擎拆解 | 从源码到多维代码图的管线 | -| **(六) MCP 协议层** | **35+ 工具的设计哲学 ← 本文** | -| (七) 语言翻译器 | 10 种语言 → 统一 IR | -| (八) 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) 自适应查询 | Fallback 机制与就绪检测 | -| (十) 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) 验证层 | 让 AI 对自己的话负责 | -| (十二) Model Engine | 从事实到理解 | -| (十三) Parser + GraphBuilder | 解析与建图 | - ---- - -下一篇我们将拆解 **语言翻译器**——tree-sitter 为 10 种语言生成的 CST，如何通过 Visitor 模式翻译成统一的 IR。为什么 Rust 的 visitor 和 JavaScript 的 visitor 长得完全不一样？ \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-07-language-translators.md b/docs/articles/zh/codescope-architecture-07-language-translators.md deleted file mode 100644 index 36d2b4a..0000000 --- a/docs/articles/zh/codescope-architecture-07-language-translators.md +++ /dev/null @@ -1,412 +0,0 @@ -# CodeScope 架构拆解（七）：语言翻译器——10 种语言 → 统一 IR - -> 我一开始以为，"支持 10 种语言"是这个项目最难的工程挑战。一个语言一个解析器、一个 AST、一套规则——写到第 3 种的时候代码就开始失控了。到第 8 种的时候，每个新增语言的代码量不是"再加一个 if"，而是"再加一个 600 行的 monolithic 类"。 -> -> 后来我意识到：问题不在于语言数量，而在于抽象层的设计。如果每个语言的 visitor 都从同一个基类继承，共享调度逻辑和 scope 管理，那么新增一种语言的边际成本就可以降到 200 行。事实证明了——从 GoTranslator（600 行）到 GoVisitor（~200 行），4 倍的代码量缩减。 - ---- - -## 三个问题 - -1. **tree-sitter 为每种语言生成完全不同的 CST 节点名**——JavaScript 叫 `function_declaration`，C 叫 `function_definition`，Rust 叫 `function_item`。怎么统一？ -2. **不同语言的语义粒度不同**——Go 的 `short_var_decl` 没有类型信息，Java 的 `variable_declaration` 有。翻译成 IR 时信息损失多少是可接受的？ -3. **全量 AST 太大**——一个 1000 行的文件生成 200-400KB 的 AST。但分析代码关系只需要其中 1/5 的节点。有没有必要保留全部？ - ---- - -## 两条管线 - -CodeScope 内部不是一条翻译管线，而是**两条独立的管线共存**。这不是一开始规划好的——它反映了一个工程项目的演进过程。 - -``` - createTranslator() createJsVisitor() - | | - +------+--------+ +--------+--------+ - | | | | - [Old Pipeline] nullptr [New Pipeline] ScannerVisitor - TranslationUnit SemanticUnit - (Node 树) (扁平 Record) - ~60 bytes/节点 ~50-200 bytes/记录 - children 向量 parent_id 链接 - SemanticEdge 指针 无指针 - - 全量 AST 语义子集 - 39 种 NodeKind 16 种 RecordKind - 控制流完整 控制流舍弃 -``` - -### 旧管线：完整 AST 树 - -旧管线的核心是 `Translator` 接口： - -```cpp -// engine/src/ir/ir_translator.h -class Translator { -public: - virtual ~Translator() = default; - virtual TranslationUnit *translate(TSTree *tree, const char *source, - const char *file_path) = 0; - virtual const char *language() const = 0; -}; -``` - -每个语言有一个独立的 `*Translator` 类，全部定义在各自的 `.cpp` 文件中： - -| 文件 | 类名 | 行数 | -|---|---|---| -| `translators/c_translator.cpp` | `CTranslator` | ~600 | -| `translators/javascript_translator.cpp` | `JavascriptTranslator` | ~400 | -| `translators/go_translator.cpp` | `GoTranslator` | ~600 | - -它们共享一个模式： -- `translate()` 创建 `TranslationUnit`，设置根节点，递归调用 `translateChildren()` -- 每个 tree-sitter 节点类型对应一个 `if/else` 分支 -- 节点通过 `parent->children.push_back(child)` 构建树结构 -- 语义边直接在 `Node*` 指针间创建 - -输出是 `TranslationUnit` —— 一个完整的 AST 树： - -```cpp -// engine/src/ir/ir.h -struct Node { - uint64_t id; - NodeKind kind; // 39 种 - std::string name; - std::string qualified_name; - SourceLocation loc; - std::vector children; - std::vector semantic_edges; - std::string language; - std::string file_path; - bool has_error; -}; - -struct TranslationUnit { - Node *root; - std::vector all_nodes; // 持有所有权 - std::string source_content; -}; -``` - -39 种 NodeKind 覆盖了所有编程结构：声明（FunctionDecl、ClassDecl、VariableDecl）、语句（IfStmt、ForStmt、WhileStmt、SwitchStmt）、表达式（CallExpr、BinaryExpr、MemberExpr）、导入导出（ImportDecl、ExportDecl）、注释。 - -**优点**：完整。GraphBuilder 需要什么都能从树上取到。 - -**缺点**：太大。每个节点 ~60 字节 + 子向量 + 字符串堆分配。1000 行文件产生 200-400KB。而且大多数下游查询根本不关心控制流。 - -### 新管线：扁平语义记录 - -新管线的核心是 `JsVisitor` 基类和 `SemanticUnit` 输出： - -```cpp -// engine/src/ir/translators/js_visitor.h -class JsVisitor { -public: - virtual SemanticUnit *visit(TSTree *tree, const char *source, - const char *file_path); - virtual void reset(); - // scope 管理、handler 方法、Aho-Corasick 调度 -}; -``` - -输出是扁平向量： - -```cpp -// engine/src/ir/semantic_unit.h -struct Record { - uint64_t id; - RecordKind kind; // 16 种（语义子集） - std::string name; - std::string qualified_name; - uint64_t parent_id; // 0 = 顶层（无指针！） - SourceRange loc; - std::string file_path; - std::string language; -}; - -class SemanticUnit { - std::vector records_; // 扁平、连续内存 - uint64_t next_id_ = 1; -}; -``` - -关键差异： -- **无指针**：父子关系通过 `parent_id`（uint64_t）表达 -- **无向量**：每个 Record 没有自己的 `children` 向量 -- **连续内存**：`std::vector` 在堆上连续排列，cache-friendly -- **大小减半**：~50-200 bytes/record，1000 行文件 ~50KB - ---- - -## Aho-Corasick 调度 - -新管线的核心设计是 **Aho-Corasick 自动机** 替代了旧管线中的 `if/else strcmp` 链。 - -```cpp -// engine/src/ir/translators/js_visitor.cpp -void JsVisitor::visitNode(TSNode node, uint64_t parent_id) { - const char *type = ts_node_type(node); - int id = getJsAC().match(type); // O(n) 模式匹配 - switch (id) { - case 100: return visitFunctionDecl(node, parent_id); - case 101: return visitArrowFunction(node, parent_id); - case 102: return visitClassDecl(node, parent_id); - case 104: return visitCallExpr(node, parent_id); - // ... - case 200: emitter_->emitLiteral(nodeText(node), loc, parent_id); return; - case 300: visitChildren(node, parent_id); return; // 透传 - default: visitChildren(node, parent_id); return; - } -} -``` - -调度表分为三个级别： - -```cpp -// engine/src/ir/translators/js_visitor.cpp (AC 构建) -// 100-109: Handlers — 产生语义记录 -ac.addPattern("function_declaration", 100); -ac.addPattern("arrow_function", 101); -ac.addPattern("class_declaration", 102); -ac.addPattern("call_expression", 104); -ac.addPattern("variable_declaration", 106); -ac.addPattern("member_expression", 109); - -// 200: Literals — 产生 Literal 记录 -ac.addPattern("number", 200); -ac.addPattern("string", 200); -ac.addPattern("comment", 201); - -// 300: 复合语句 — 透传（不产生记录，仅递归子节点） -ac.addPattern("if_statement", 300); -ac.addPattern("for_statement", 300); -ac.addPattern("while_statement", 300); -ac.addPattern("binary_expression", 300); -ac.addPattern("ternary_expression", 300); -``` - -这意味着： -- **只有语义上有意义的节点产生记录**（声明、调用、成员访问、导入导出） -- **控制流结构被舍弃**（if/for/while/switch/try——它们的子节点被"透传"到最近的函数/类作用域） -- **操作符被舍弃**（二元/一元/赋值表达式——递归访问操作数） -- **类型注解被舍弃**（只在 TypeScript visitor 中特殊处理了 TypeAlias） - -**故意的信息损失**。这个设计的前提是：代码分析的下游任务（调用图、符号搜索、模块依赖）只关心"有什么符号"和"谁调了谁"，不关心"这个 if 的条件是什么"。 - ---- - -## Visitor 继承体系 - -所有新管线 visitor 共享 `JsVisitor` 作为基类——包括 C/C++ visitor： - -``` -JsVisitor ── 基类: Aho-Corasick 调度、scope 跟踪、helper 方法 - │ - +-- CVisitor ── 覆写 visitNode(), 增加 C handler（func/struct/enum/call/include/typedef/preproc） - │ │ - │ +-- CppVisitor ── 增加 class_specifier/namespace_definition/template_declaration - │ - +-- GoVisitor ── func/method/type/call/import/var/short_var - +-- PythonVisitor ── func/class/call/import/assignment - +-- JavaVisitor ── method/class/interface/enum/invocation/variable/import - +-- RustVisitor ── function/struct/enum/trait/impl/call/let/use - +-- SwiftVisitor ── func/class/struct/enum/protocol/call/var/import - │ - +-- TsVisitor ── 增加 interface_declaration/type_alias_declaration/enum_declaration - │ - +-- TsxVisitor ── JSX 元素跳过（无语义价值）, JSX 表达式递归 -``` - -### CVisitor：最复杂的实现 - -C 的 visitor 是最厚的一个，因为它要处理 GNU C 扩展带来的 tree-sitter 解析问题： - -```cpp -// engine/src/ir/translators/c_visitor.cpp (简化) -void CVisitor::visitFunctionDecl(TSNode node, uint64_t parent_id) { - // GNU C: __attribute__((...)) 包裹函数声明 - TSNode actual = skipAttributedDeclarator(node); - // GNU C: ERROR 节点可能包含函数定义 - if (actual_is_error) { - // tree-sitter 对 GNU C 扩展产生错误节点 - // 递归到 ERROR 子节点中找函数定义 - // ... - } - emitFunction(actual, parent_id); -} -``` - -这个 visitor 处理的边界情况： -- **`__attribute__((xxx))`**：跳过，找到实际的函数声明 -- **ERROR 节点**：tree-sitter 对 GNU C 扩展（如 `__sched`）会产生 ERROR 节点。C visitor 递归到 ERROR 的子节点中找到函数定义 -- **宏定义**：`#define`（简单和函数式）作为 Variable 记录 -- **预处理条件编译**：`#if`/`#ifdef` 内的声明仍然递归索引 - -### JS/TS：最小的增量差异 - -TypeScript 的 visitor 只比 JS 多了三个 handler： - -```cpp -// engine/src/ir/translators/ts_visitor.h -class TsVisitor : public JsVisitor { -protected: - void visitInterfaceDecl(TSNode node, uint64_t parent_id); - void visitTypeAliasDecl(TSNode node, uint64_t parent_id); - void visitEnumDecl(TSNode node, uint64_t parent_id); - // 覆写：class 名在 TS 中叫 type_identifier 而非 identifier - void visitClassDecl(TSNode node, uint64_t parent_id) override; -}; -``` - -TsxVisitor 更简单——只处理 JSX： - -```cpp -class TsxVisitor : public TsVisitor { -protected: - void visitJsxElement(TSNode node, uint64_t parent_id); - // JSX 元素本身跳过（无语义价值） - // JSX 表达式 { ... } 递归进去 -}; -``` - ---- - -## CallKind：每个调用都有"类型" - -从 v0.4 开始，所有 CallExpr 记录都带一个 `call_kind` 字段，帮助 ResolverPipeline 区分不同性质的调用： - -```cpp -// engine/src/ir/semantic_unit.h -enum class CallKind : uint8_t { - Direct = 0, // 裸函数调用: doThing() - Method = 1, // 方法调用: obj.Method() - Interface = 2, // 接口派发（当前未分配，预留） - Constructor = 3, // 构造函数调用: new Foo() -}; -``` - -每种语言的 visitor 在 emitCall() 之前用启发式规则分类： - -| 语言 | Method 判断 | Constructor 判断 | -|------|-----------|-----------------| -| Go | `selector_name` 不为空 | 名以 `New` 开头 | -| Java | 含 `.` | 名以大写字母开头 | -| JS/TS | 含 `.` | 名以大写字母开头（>3 字符）| -| Python | 含 `.` | 名以大写字母开头 | -| Rust | 含 `::` 或 `.` | `::new()` / `::from()` | -| C/C++ | `field_expression` 子节点 | 不支持（C 无构造） | - -CallKind 影响 ResolverPipeline 的 Factor 9（CallKindMatch）： -- **Constructor** (+0.3)：跨模块构造函数调用是合理的，加分 -- **Interface** (-0.3)：接口派发更难精确解析，降分 -- **Method** (-0.1)：方法调用通常在同模块，跨模块略降分 -- **Direct** (0)：不影响 - ---- - -## 语言差异的处理 - -不同语言的 visitor 处理相同的语义结构时，语法差异被吸收在 visitor 内部： - -| 语义 | JavaScript | C | Rust | Go | -|---|---|---|---|---| -| 函数 | `function_declaration` | `function_definition` | `function_item` | `function_declaration` | -| 类 | `class_declaration` | `struct_specifier` | `struct_item` | `type_spec` | -| 调用 | `call_expression` | `call_expression` | `call_expression` | `call_expression` | -| 赋值 | `variable_declaration` | `declaration` | `let_declaration` | `short_var_decl` | - -每个 visitor 的职责是把这些不同的语法映射到同一个 `RecordKind`（Function、Class、CallExpr、Variable），抹除语言差异。 - ---- - -## ScannerVisitor：10x 更快的备选 - -除了完整解析，还有一个轻量级的 `ScannerVisitor`： - -``` -完整 tree-sitter 解析: ~30-50ms/文件 (1000 行) -ScannerVisitor: ~3-5ms/文件 (10x 更快) -``` - -它的原理极其简单： - -```cpp -// engine/src/ir/translators/scanner_visitor.cpp (概念) -void ScannerVisitor::scanFile(const char *source, size_t len) { - for (each line in source) { - if (line matches " (") // function - emit(Function, name, location); - else if (line matches "class ") // class - emit(Class, name, location); - else if (line matches "struct ") // struct - emit(Class, name, location); - // ... 更多语言特定的关键词规则 - } -} -``` - -它不调用 tree-sitter，不解析 AST，不做 scope 跟踪。只按行匹配关键词模式。这对应的是整个管线中的 Phase A 快速扫描。 - -关键实现细节： -- **语言特定的关键词表**：每种语言有自己的检测模式 -- **C/C++ 的 false-positive 过滤**：`if (condition)` 会被误识别为函数，需要检查前面是不是 if/while/for/switch/catch/return 关键词 -- **只提取顶层声明**：不解析函数体 -- **输出兼容性**：结果写入相同的 `SemanticUnit` 格式 - ---- - -## 数据对比 - -| 维度 | 旧管线（Translator） | 新管线（JsVisitor） | ScannerVisitor | -|---|---|---|---| -| 输出 | TranslationUnit（Node 树） | SemanticUnit（扁平记录） | SemanticUnit | -| NodeKind/RecordKind | 39 种 | 16 种 | 16 种 | -| 单文件内存（1000 行） | ~200-400 KB | ~50 KB | ~10 KB | -| 控制流 | 完整保留 | 舍弃 | 舍弃 | -| 类型信息 | 部分保留（通过节点类型） | 极少 | 极少 | -| 跨文件解析 | Linker 后续处理 | GraphBuilder 后续处理 | N/A | -| 调度方式 | if/else strcmp 链 | Aho-Corasick 自动机 | 行匹配 | -| 新语言边际成本 | 400-600 行 | 150-250 行 | 30-50 行（关键词表） | -| 典型耗时（1000 行） | 30-50ms | 30-50ms | 3-5ms | - ---- - -## 坦诚反思 - -**1. 两条管线并存是历史遗留问题** - -旧管线是先写的，发现太臃肿才设计了新管线。但旧管线没有被淘汰——因为 GraphBuilder 的某些路径还依赖 `TranslationUnit` 的 Node 树结构。新管线只用于 JS/TS 系列的 visitor。 - -理想情况下应该只有一条管线。但从实用角度看，维护两套代码的边际成本（通过共享的 IR 类型系统）低于一次性迁移所有 visitor 的风险。 - -**2. RecordKind 是 NodeKind 的子集——但这是过度优化吗？** - -新管线舍弃了控制流信息。理论上，GraphBuilder 构建调用图不需要知道 if/for/while 语句。但有一些场景（如路径追踪——"这个调用是否在错误处理路径中"）需要控制流上下文。当这些场景出现时，就需要回退到旧管线或重新设计 RecordKind。 - -**3. ScannerVisitor 的局限性** - -扫描器模式匹配会漏掉一些声明——比如多行声明的续行、嵌套在表达式内部的函数定义、模板特化等。Phase A 的设计本就是"有损但足够快"。问题在于"足够"的标准——如果 AI 依赖 `find_symbol` 的结果做出决策，被漏掉的声明可能导致错误结论。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| (五) C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) MCP 协议层 | 35+ 工具的设计哲学 | -| **(七) 语言翻译器** | **10 种语言 → 统一 IR ← 本文** | -| (八) 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) 自适应查询 | Fallback 机制与就绪检测 | -| (十) 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) 验证层 | 让 AI 对自己的话负责 | -| (十二) Model Engine | 从事实到理解 | -| (十三) Parser + GraphBuilder | 解析与建图 | - ---- - -下一篇我们拆解 **存储层**——SQLite 如何同时承担关系存储（图节点/边）、全文搜索（FTS5）和向量检索（vec0）三种角色。一张 `call_edges` 表如何支持 300ms 的调用链查询。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-08-storage-layer.md b/docs/articles/zh/codescope-architecture-08-storage-layer.md deleted file mode 100644 index 5ec1bbc..0000000 --- a/docs/articles/zh/codescope-architecture-08-storage-layer.md +++ /dev/null @@ -1,457 +0,0 @@ -# CodeScope 架构拆解（八）：存储层——SQLite WAL + FTS5 + vec0 - -> 我知道这听起来有点反直觉：一个代码分析引擎，为什么用 SQLite 做存储？ -> -> 我试过其他方案。Neo4j 的调用图查询很优雅，但为了一个 CLI 工具让用户装 JDK + Neo4j 就太离谱了。RocksDB 的写入性能出色，但想做个 `LIKE '%query%'` 就得自己搭布隆过滤器。至于内存里裸放 `HashMap>`——查询确实快，300ms 搞定调用链。但进程一挂，全部白干，增量索引更是不用想。 -> -> SQLite 的好处是：它不需要部署。不需要单独的进程，不需要配置，不需要用户安装任何东西。代码里 `sqlite3_open()` 一行就搞定了。坏处是：你得在 22 张表 + FTS5 + vec0 + WAL 之间手工编排一个本应用多模型数据库才能解决的问题。 - ---- - -## 三个问题 - -1. **单机单文件数据库，如何同时担任关系存储（图节点/边）、全文搜索（FTS5）和向量检索（vec0）三种角色？** -2. **5000 个文件的索引过程中，插入 100 万条记录的同时还要响应查询，SQLite 的锁机制如何处理？** -3. **增量索引需要检测文件变更、只重建变更部分，而 SQLite 本身没有文件系统监听能力——这个缺口怎么填？** - ---- - -## 28 张表的设计 - -先看完整的 Schema 全景。28 张表按索引阶段划分： - -``` -Phase A（快速扫描 — 毫秒级 就绪） -├── projects // 根表，存储项目根路径和名称 -├── files // 文件元数据（路径、语言、内容哈希） -├── modules // 模块/目录层次结构 -├── symbols // 符号定义（核心） -├── symbol_status // 每个符号的就绪标志 -├── entry_points // 入口点标记 -├── file_scan_state // 增量索引的 mtime/哈希跟踪 -├── index_tasks // Tokio 后台任务状态 - -Phase B（知识增强 — 异步后台） -├── call_edges // 调用图边（caller → callee） -├── dependency_edges // 模块间依赖 -├── metrics // 圈复杂度、认知复杂度 -├── search_index // FTS5 带摘要和正文 - -Phase C（深度索引） -├── ir_nodes // 遗留 IR 节点（旧管道） -├── ir_semantic_edges // 遗留语义边 -├── graph_nodes // 图节点（最终产物） -├── graph_edges // 图边（调用/包含/继承） -├── semantic_records // 新管道扁平记录（DB-first） -├── node_complexity // 复杂度分数 -├── node_vectors // n-gram 哈希向量 BLOB -├── code_fts // FTS5 全文搜索索引 -├── fts_node_map // FTS 节点ID映射 -├── embeddings // vec0 384维嵌入表（可选） -├── type_info // 类型定义（struct/enum/trait/interface） -├── type_ref // 类型引用（变量 : 类型 映射） -├── routes // HTTP 路由注册（method + path + handler） -├── capabilities // 模块能力声明 -├── contracts // 契约/接口实现声明 -├── findings // 验证证据链持久化 -``` - -这些表不是同时创建的。`engine_init()` 在启动时创建全部 Schema，但填充数据的时机不同——Phase A 产生 `symbols` 和 `modules`，Phase B 填充 `call_edges` 和 `metrics`，Phase C 建立完整的 `graph_nodes` + `graph_edges`。 - ---- - -## 打开数据库的那几行代码 - -存储层的一切设计都围绕这几行 PRAGMA： - -```cpp -// engine/src/store/store.cpp — GraphStore::open() -bool GraphStore::open(const char *db_path) { - sqlite3_open(db_path, &db_); - - exec("PRAGMA journal_mode=WAL"); // 并发读取 + 写入 - exec("PRAGMA synchronous=OFF"); // 批量插入最大吞吐量 - exec("PRAGMA temp_store=MEMORY"); // 临时表在 RAM 中 - exec("PRAGMA cache_size=-64000"); // 64 MB 页缓存 - exec("PRAGMA mmap_size=268435456"); // 256 MB 内存映射 I/O -} -``` - -**WAL（Write-Ahead Logging）** 是整个并发方案的基础。没有 WAL，SQLite 默认的 journal 模式会在写入时锁定整个数据库——索引过程中所有查询都得等着。WAL 允许： -- 写入者在 WAL 文件中追加记录 -- 读取者继续从原数据库文件读取 -- 写入完成后，checkpoint 将 WAL 合并到主数据库 - -**synchronous=OFF** 是性能的关键。在批量索引场景中，每次 fsync 的代价是 ~2-10ms。关闭它后每次插入降为 ~0.5ms。代价是：如果机器在事务中崩溃，可能丢失最后一批数据。但对于代码索引——数据可以从源代码重新生成——这个权衡是可以接受的。 - -**mmap_size=256MB** 启用内存映射 I/O。读取时 SQLite 可以直接访问内核的页缓存，避免 `read()` 系统调用和用户态内存拷贝。对大表全表扫描（如 `node_vectors` 的暴力搜索）有显著提升。 - ---- - -## 核心模式：Batch Insert - -批量插入使用了固定的模式——prepare 一次，bind/step/reset 循环： - -```cpp -void GraphStore::insertGraphNodes(uint64_t project_id, - const std::vector &nodes) { - const char *sql = "INSERT INTO graph_nodes (...) VALUES (?, ?, ...)"; - sqlite3_stmt *stmt = nullptr; - sqlite3_prepare_v2(db_, sql, -1, &stmt, nullptr); - - for (auto &node : nodes) { - sqlite3_bind_int64(stmt, 1, node.id); - sqlite3_bind_text(stmt, 2, node.name.c_str(), -1, SQLITE_TRANSIENT); - // ... bind all fields - sqlite3_step(stmt); - sqlite3_reset(stmt); - } - sqlite3_finalize(stmt); -} -``` - -注意这里没有显式的 `BEGIN TRANSACTION` / `COMMIT`。SQLite 在 `sqlite3_step()` 返回 `SQLITE_ROW` 或 `SQLITE_DONE` 时会自动开始一个隐式事务——也就是说上面这个循环，每插入一条节点就提交一次。 - -对于批量构建场景，这种模式被显式事务替代： - -```cpp -exec("BEGIN"); -for (auto file's semantic records) { - // 复用已准备的 stmt，在一次事务中插入所有记录 -} -exec("COMMIT"); -``` - -代码注释记录了这个问题： - -``` -// Previously each storeVector auto-committed — 261K transactions. -// With explicit begin/commit, all inserts are one transaction. -``` - -261K 个事务 vs 1 个事务——差了几个数量级的性能。 - ---- - -## buildGraph()：从扁平记录到多维图 - -这是存储层中最有意思的 SQL 操作。`buildGraph()` 接受 `semantic_records`（扁平记录列表）并构建 `graph_nodes` + `graph_edges`。 - -核心是三步 SQL： - -**Step 1: 记录到节点的映射** - -```sql -CREATE TEMP TABLE _r2n AS -SELECT sr.rowid as rid, sr.original_id, sr.file_path, sr.name, - CAST(ROW_NUMBER() OVER () - + COALESCE((SELECT MAX(id) FROM graph_nodes WHERE project_id=?), 0) - AS INTEGER) as node_id -FROM semantic_records sr -WHERE sr.project_id = ? - AND sr.kind IN (0,1,2,3,4,5,6,9,10) -- 只取语义节点 - AND sr.file_path IN (SELECT file_path FROM _rf) -``` - -注释特别说明了 `ROW_NUMBER() OVER ()` 的用意： - -``` -// Note: ROW_NUMBER() OVER () avoids ORDER BY sort cost. -// Node IDs are sequential but not sorted by file_path — sorting is -// not required for correctness since JOINs use indexes, not sequential scans. -``` - -**Step 2: 从声明插入图节点** - -```sql -INSERT INTO graph_nodes (id, project_id, ..., node_type, ...) -SELECT r2n.node_id, ..., CASE sr.kind - WHEN 0 THEN 0 -- Function - WHEN 1 THEN 1 -- Method - WHEN 2 THEN 2 -- Class - WHEN 4 THEN 4 -- Interface - WHEN 9 THEN 9 -- CallExpr - ELSE 7 -- TypeAlias (fallback) -END, ... -FROM semantic_records sr -JOIN _r2n r2n ON sr.rowid = r2n.rid -``` - -**Step 3: 调用边（跨文件名称匹配）** - -```sql -INSERT INTO graph_edges (id, project_id, source_node_id, target_node_id, - edge_type, graph_type, call_site_file, call_site_line) -SELECT [...], 1, caller.node_id, callee.node_id, [...] -FROM semantic_records sr -JOIN _r2n callee ON SUBSTR(sr.name, -LENGTH(callee.name)) = callee.name -JOIN _r2n caller ON sr.parent_id = caller.original_id AND [...] -WHERE sr.kind = 9 -- CallExpr -``` - -这个跨文件调用解析的精髓在 `SUBSTR(sr.name, -LENGTH(callee.name)) = callee.name`——用后缀匹配做名称解析。不需要类型检查，不需要重载解析，甚至不需要知道函数在哪个文件定义。只要调用表达式的名称结尾等于目标函数的名称，就建立调用边。 - -**一个 42 个文件的 JPetStore 构建时间**： - -``` -buildGraph: 42 files | file_list=5ms delete=12ms rf=3ms r2n=45ms nodes=120ms edges=30ms calls=80ms total=295ms -``` - -从 42 个文件、数千条 `semantic_records` 到完整的图结构——总共 **295ms**。 - ---- - -## 三重搜索体系 - -存储层维护了三套独立的搜索索引，对应不同的查询场景： - -### 1. FTS5 全文搜索 - -```sql --- 遗留索引 -CREATE VIRTUAL TABLE code_fts USING fts5( - name, qualified_name, file_path, content, - project_id UNINDEXED, node_id UNINDEXED, node_kind UNINDEXED, - tokenize='unicode61' -); - --- 新索引（带排序的摘要/正文） -CREATE VIRTUAL TABLE search_index USING fts5( - title, summary, body, - project_id UNINDEXED, symbol_id UNINDEXED, - tokenize='unicode61' -); -``` - -`unicode61` 分词器支持 Unicode 文本，包括中文 CJK 字符的分词——这对分析包含中英文混杂的代码库很重要。 - -`searchCode()` 的查询逻辑将用户输入转为 FTS5 前缀匹配： - -``` -用户输入: "vector_store" -FTS5 查询: "vector_store*" -匹配结果: "vector_store", "vector_store_impl", "vector_store_test" -``` - -结果按节点类型排序——函数/类声明优先，然后按 FTS5 `rank`（BM25 相关性评分）。 - -### 2. n-gram 哈希向量搜索 - -这是自制的语义搜索，不需要外部嵌入模型： - -```cpp -// engine/src/query/vector_search.h -const int VECTOR_DIM = 64; - -void stringToVector(const std::string &str, float *vec) { - // 提取所有 2-gram 和 3-gram - // 哈希到 [0, VECTOR_DIM) 索引 - // 构建频率向量 → L2 归一化 -} -``` - -搜索逻辑是暴力扫描： - -```cpp -// store.cpp — searchSemantic() -for (each node_vector) { - float sim = cosineSimilarity(query_vec, node_vec); - if (sim > 0.1f) candidates.push_back({node_id, sim}); -} -partial_sort(candidates, topK); // O(N log K) -``` - -`partial_sort` 将复杂度从 O(N log N) 降到 O(N log K)。在 <100K 节点时暴力扫描是可行的。 - -### 3. vec0 嵌入搜索（可选） - -```sql -CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0( - symbol_id INTEGER PRIMARY KEY, - vector FLOAT[384] -); -``` - -这个表不是无条件创建的。`engine_init()` 尝试通过 `sqlite3_load_extension()` 加载 `vec0.dylib`——如果用户的系统上没有这个扩展（比如 Windows 或非标准安装），核心功能照样工作，只是回退到 n-gram 哈希搜索。 - ---- - -## 增量索引的设计 - -存储层通过 `file_scan_state` 表实现增量索引： - -```sql -CREATE TABLE file_scan_state ( - project_id INTEGER, - file_path TEXT, - file_mtime INTEGER, - file_size INTEGER DEFAULT 0, - content_hash TEXT, - scanned_at TEXT, - PRIMARY KEY(project_id, file_path) -); -``` - -三个关键方法： - -```cpp -// 检查文件是否未变更 -bool GraphStore::isFileUnchanged(project_id, file_path, mtime, size) { - // SELECT 1 FROM file_scan_state WHERE - // project_id=? AND file_path=? - // AND file_mtime=? AND file_size=? -} - -// 更新扫描状态 -void GraphStore::updateFileScanState(project_id, file_path, mtime, size) { - // INSERT OR REPLACE INTO file_scan_state -} - -// 清理已删除的文件 -void GraphStore::cleanupStaleFiles(project_id, active_files) { - // CREATE TEMP TABLE _active_files (path TEXT PRIMARY KEY) - // INSERT active files - // DELETE FROM file_scan_state WHERE path NOT IN _active_files -} -``` - -`buildGraph()` 接受一个可选的 `changed_files` 参数——只重建变更文件的图节点和边，而不是全量重建。 - -这个设计有个隐含的前提：**文件的 mtime 和 size 是足够可靠的变更检测器**。对于 git checkout 切换分支的场景，被恢复的文件 mtime 会变，size 不变——但仍然会触发重建（mtime 变了）。`content_hash` 字段存在但实际使用中优先级低于 mtime+size（查一次哈希需要读整个文件，代价远高于 stat）。 - ---- - -## 就绪系统与自适应查询 - -五个就绪标志分散在 `project_readiness` 和 `symbol_status` 两张表中： - -```sql --- 项目级就绪标志 -CREATE TABLE project_readiness ( - project_id INTEGER PRIMARY KEY, - fast_ready INTEGER DEFAULT 0, - normal_ready INTEGER DEFAULT 0, - deep_ready INTEGER DEFAULT 0, - fts_ready INTEGER DEFAULT 0, - vector_ready INTEGER DEFAULT 0 -); - --- 每个符号的细粒度就绪标志 -CREATE TABLE symbol_status ( - symbol_id INTEGER PRIMARY KEY, - callgraph_ready INTEGER DEFAULT 0, - metrics_ready INTEGER DEFAULT 0, - embedding_ready INTEGER DEFAULT 0, - is_stub INTEGER DEFAULT 0 -); -``` - -这种设计允许查询引擎在"项目级"和"符号级"两个粒度上做 fallback： - -- 如果项目 `fast_ready=0`：不返回结果（扫描都没完成） -- 如果项目 `fast_ready=1` 但 `callgraph_ready=0`：返回符号信息但不返回调用者/被调用者 -- 如果符号 `is_stub=1`：该函数只有声明没有实现（作为占位符标记） - -`getReadyRatio()` 计算就绪进度： - -```sql -SELECT CAST(SUM(ss.callgraph_ready) AS REAL) / COUNT(*) -FROM symbols s JOIN symbol_status ss ON ss.symbol_id = s.id -WHERE s.project_id = ? -``` - -这用来支持查询引擎的 "渐进式就绪" 特性——Phase A 完成后就告诉用户"我扫描完了，30% 的符号有调用图信息"，而不用等到 Phase B 全部完成。 - ---- - -## 导出/导入：VACUUM INTO + zstd - -共享产物的设计走了捷径——但也足够聪明： - -**导出**： -``` -1. VACUUM INTO 'temp_path' -- SQLite 3.27.0+，创建原子快照 -2. fork + execlp('zstd', '--compress', ..., temp_path) -3. 移除临时文件 -``` - -**导入**： -``` -1. fork + execlp('zstd', '-d', ..., input_path) -2. ATTACH DATABASE artifact AS import_db -3. INSERT OR IGNORE INTO semantic_records, graph_nodes, graph_edges -4. DETACH, 清理 -``` - -`VACUUM INTO` 的美妙之处在于：它创建一个原子的、去碎片化的数据库副本，不需要锁定原始数据库。整个过程中原始数据库仍然可读可写。 - -防御注入的方式是检查路径中的 shell 元字符： - -```cpp -bool pathHasMeta(const std::string &path) { - return path.find_first_of(";&|$`\\'\"") != std::string::npos; -} -``` - -并使用 `fork+execvp` 而非 `system()`——避免 shell 解释。 - ---- - -## 设计取舍 - -### 优点 - -1. **零部署依赖**。不需要 MySQL、PostgreSQL、Neo4j。`brew install sqlite` 就够了——甚至 macOS 自带 sqlite3。 -2. **单文件备份**。整个项目的数据在一个 `.db` 文件中。迁移、复制、分享都非常简单。 -3. **并发读取**。WAL 模式下，Phase B/C 在后台写入，Phase A 查询完全不受影响。 -4. **SQL 的力量**。`buildGraph()` 的核心逻辑用 5 条 SQL 语句完成，不到 100 行代码。如果用 Rust 手写同样逻辑，至少 500 行。 - -### 痛点 - -1. **没有真正的并发写入**。SQLite 的 WAL 虽然支持读取+写入并发，但不支持两个写入者。在多 Worker 场景下需要序列化写入操作。 -2. **暴力向量搜索不可扩展**。`searchSemantic()` 的全表扫描在 <100K 节点可行，但超过百万节点就会显著变慢。vec0 扩展缓解了这个问题，但它是可选依赖。 -3. **Schema 缺乏迁移机制**。当前没有版本化的 Schema 迁移。如果表结构变化，只能删库重建。这在开发阶段可以接受，但生产部署会是个问题。 -4. **22 张表的冗余**。`ir_nodes` 和 `semantic_records` 存储相似但不相同的数据——前者是旧管道的产物，后者是新管道的。从工程角度看应该只保留一个。 - ---- - -## 坦诚反思 - -**两条管道共存的问题**。 - -你可能会注意到，`ir_nodes` + `ir_semantic_edges` 和 `semantic_records` + `graph_nodes` 的功能高度重叠。前者是旧管道的产物（通过 `Translator` 接口），后者是新管道的（通过 `JsVisitor` 接口）。 - -在查询时，`engine_find_callers` 的代码需要写上： - -```cpp -// 先尝试新管道（semantic_records） -if (getCallersFromRecords(...)) return; -// 回退到旧管道（ir_nodes） -if (getCallersFromLegacy(...)) return; -``` - -这反映了从旧系统到新系统的过渡状态。理想情况下应该只有一套表结构。但现实是两条管道并行在线上运行，各自的测试都通过——重构它们的工作量大于让其共存的风险。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| (五) C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) MCP 协议层 | 35+ 工具的设计哲学 | -| (七) 语言翻译器 | 10 种语言 → 统一 IR | -| **(八) 存储层** | **SQLite WAL + FTS5 + vec0 ← 本文** | -| (九) 自适应查询 | Fallback 机制与就绪检测 | -| (十) 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) 验证层 | 让 AI 对自己的话负责 | -| (十二) Model Engine | 从事实到理解 | -| (十三) Parser + GraphBuilder | 解析与建图 | - ---- - -下一篇我们拆解 **自适应查询**——当一个查询到达时，引擎如何根据当前的就绪状态选择最优的数据源（Phase A 直查 symbols，Phase B 走 semantic_records，Phase C 走 graph_nodes），以及 fallback 链的详细设计。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-09-adaptive-queries.md b/docs/articles/zh/codescope-architecture-09-adaptive-queries.md deleted file mode 100644 index 4f83b19..0000000 --- a/docs/articles/zh/codescope-architecture-09-adaptive-queries.md +++ /dev/null @@ -1,374 +0,0 @@ -# CodeScope 架构拆解（九）：自适应查询——当 AI 问了一个只有一半数据能回答的问题 - -> 我花了一周写的查询引擎，在第一次集成测试的时候就翻车了。 -> -> 场景很简单：索引跑到 Phase B 中途（symbols 表已经写了 60%，call_edges 还没碰），用户问"这个函数的调用者是谁"。引擎那时只实现了一套查询路径——走 call_edges 表。结果是：查了一个空表，返回 `[]`。用户以为项目没有调用关系，实际上只是索引还没跑完。 -> -> 修复方案是两条：要么等索引跑完再响应用户（这违背了整个"渐进式就绪"的核心设计），要么让查询引擎学会说"我知道的还没全，你先将就用"。我选了后者。这就有了 CodeScope 的自适应查询层。 - ---- - -## 三个问题 - -1. **索引的五个就绪级别，对应五个不同粒度的查询路径——当 AI 的查询到达时，如何让引擎"恰好选到"当前可用数据中最优的那条？** -2. **同一个查询函数（比如 `find_callers`），Phase A 只能从 symbols 表提取符号名，Phase B 能做精确匹配，Phase C 能做图遍历——如何用一个函数签名覆盖这三种模式？** -3. **项目就绪度 37% 的时候，该返回数据还是返回"请稍后再试"？阈值设在多少？** - ---- - -## 查询系统的三层架构 - -要理解自适应查询，需要先看整个查询系统的层级： - -``` -┌─────────────────────────────────────────────┐ -│ Rust Tool 层 (server/src/tools/mod.rs) │ -│ 收到 query → 调用 FFI → 返回 JSON │ -│ execute(project_id, tool_name, args) │ -│ → 查找 TOOL_HANDLERS → handler() → result │ -└──────────────────┬──────────────────────────┘ - │ FFI: extern "C" - ▼ -┌─────────────────────────────────────────────┐ -│ C++ 自适应调度层 (engine_queries.cpp) │ -│ 检查 readiness → 选择最优路径 → fallback │ -│ 梯度降级策略：readiness > 0.5 → 0.1 → 0.0 │ -│ getReadyRatio() + project_readiness 联合判断 │ -└──────────────────┬──────────────────────────┘ - │ C++ 函数调用 - ▼ -┌─────────────────────────────────────────────┐ -│ C++ 数据访问层 (store.cpp + query_engine) │ -│ 新管道: symbols → call_edges → search_index │ -│ 旧管道: graph_nodes → graph_edges → code_fts│ -│ 三层查询: 新FTS5 → 旧FTS5 → LIKE 兜底 │ -└─────────────────────────────────────────────┘ -``` - -每一层都有独立的数据源。自适应的核心逻辑在**中间层**——它不是简单地"调用 store"，而是在调用前判断"应该调用 store 的哪个方法"。 - ---- - -## 就绪度计算的数学 - -自适应查询的基石是 `getReadyRatio()`： - -```cpp -// store.cpp -double GraphStore::getReadyRatio(uint64_t project_id, const char *ready_field) { - std::string sql = "SELECT CASE WHEN COUNT(*) > 0 THEN " - "CAST(SUM(ss." + std::string(ready_field) + - ") AS REAL) / COUNT(*) " - "ELSE 0.0 END " - "FROM symbols s " - "JOIN symbol_status ss ON ss.symbol_id = s.id " - "WHERE s.project_id = ?"; - // bind project_id, execute, return ratio -} -``` - -这个函数读取 `symbols` 表的总行数，然后统计 `symbol_status` 表中对应的就绪字段有多少个为 1。结果是一个 `[0.0, 1.0]` 的浮点数。 - -比如一个项目有 100 个 `symbols` 记录，其中 40 个的 `callgraph_ready=1`： - -``` -getReadyRatio(project_id, "callgraph_ready") = 40 / 100 = 0.4 -``` - -阈值体系是经验得出的： - -| 阈值 | 含义 | 行为 | -|------|------|------| -| `> 0.5` | 完全就绪 | 走精确路径（call_edges JOIN symbols） | -| `> 0.1` | 部分可用 | 返回数据但标记为不完整 | -| `< 0.1` | 不可用 | 硬阻断，返回空结果+提示 | - -三个阈值对应三种不同的用户体验策略——不提供错误数据、提供部分数据但标明风险、提供完整数据。 - ---- - -## 自适应降级的五种模式 - -读完所有查询函数后，我发现自适应策略可以归纳为五种模式： - -### 模式一：比率驱动降级 - -这是最核心的模式，以 `engine_find_callers_adaptive()` 为代表： - -```cpp -// engine_queries.cpp -char *engine_find_callers_adaptive(uint64_t project_id, const char *symbol_name) { - double cg_ratio = g_store->getReadyRatio(project_id, "callgraph_ready"); - - if (cg_ratio > 0.5) { - // Layer 1: 新管道，call_edges + symbols 精确匹配 - return dupString(g_store->findCallersJson(project_id, symbol_name)); - } - - // Layer 2: 旧管道，graph_edges + graph_nodes 模糊匹配 - return dupString(g_query->getCallers(project_id, symbol_name)); -} -``` - -关键洞察：**什么时候用比怎么查询更重要**。`findCallersJson()` 依赖 `call_edges` 表（Phase B 产物），而 `g_query->getCallers()` 依赖 `graph_edges` 表（Phase C 产物）。两者返回的数据结构相同，但数据源不同——前者在 Phase B 完成后就可用，后者要等到 Phase C。 - -### 模式二：尝试+检查降级 - -以 `findCalleesJson()` 为代表： - -```cpp -// store.cpp -std::string GraphStore::findCalleesJson(uint64_t project_id, const char *symbol_name) { - // 先尝试 call_edges JOIN symbols - std::string result = /* query call_edges table */; - - // 检查第一行是否为实际数据 - bool firstIsData = /* check if first result row has real data */; - - if (firstIsData) { - return "[" + result + "]"; // 有数据，直接返回 - } - - // 降级到 graph_edges + graph_nodes - return /* fallback to query_engine */; -} -``` - -这种模式在函数内部做降级决策，调用方无需感知。`findCallersJson()` 和 `findCalleesJson()` 内部都实现了这种双路径。 - -### 模式三：三重 FTS5 降级 - -以 `searchUnifiedJson()` 为代表： - -```cpp -// store.cpp -std::string GraphStore::searchUnifiedJson(uint64_t project_id, - const char *query, int limit) { - // Layer 1: search_index FTS5（Phase B 产物） - std::string result = trySearchIndex(project_id, query, limit); - if (!result.empty()) return result; // 标注 "method":"fts" - - // Layer 2: code_fts FTS5（Phase C 产物） - result = tryCodeFts(project_id, query, limit); - if (!result.empty()) return result; // 标注 "method":"legacy_fts" - - // Layer 3: 返回空 - return json_result(/* "method":"none" */); -} -``` - -`method` 字段的设计很有意思——它让调用方可以判断数据质量。`"method":"fts"` 表示来自新管道的精确 FTS5 索引（标题+摘要+正文），`"method":"legacy_fts"` 表示来自旧管道的代码 FTS5（只有名称+路径）。 - -### 模式四：硬阻断加提示 - -以 `engine_trace_path()` 为代表： - -```cpp -// engine_queries.cpp -char *engine_trace_path(uint64_t project_id, const char *from, const char *to) { - double cg_ratio = g_store->getReadyRatio(project_id, "callgraph_ready"); - - if (cg_ratio < 0.1) { - // 调用图就绪率太低，硬阻断 - return dupString(json_result(/* error, "调用图就绪度不足" */)); - } - - return dupString(g_store->tracePathJson(project_id, from, to)); -} -``` - -这里阈值是 `< 0.1`——比 `engine_find_callers_adaptive()` 的 `> 0.5` 宽松得多。原因是不同的操作对数据完整性的敏感度不同：路径追踪（`trace_path`）需要完整的调用链才能有意义，如果只有一半的调用图，返回的路径可能是错误的（最短路径在缺失的边里）。而"查找调用者"即使只有一半数据，返回的结果也是准确的——只是不全。 - -### 模式五：意图驱动的条件包含 - -以 `engine_build_context()` 为代表： - -```cpp -// engine_queries.cpp -char *engine_build_context(uint64_t project_id, - const char *query, const char *file_path, - const char *intent) { - // 检测用户意图类型：debug、review、understand、modify - auto features = detectFeatures(intent); - - // 只包含就绪度 > 0.1 的特性 - if (getReadyRatio("callgraph_ready") > 0.1 && features.calls) - context += getCallGraph(file_path); - if (getReadyRatio("metrics_ready") > 0.1 && features.complexity) - context += getComplexity(file_path); - if (getReadyRatio("embedding_ready") > 0.1 && features.similar) - context += getSimilarFiles(file_path); - - return dupString(context); -} -``` - -这是所有模式中最灵活的——它将查询的"答案"替换为"上下文"，根据用户的意图（`intent`）和当前的就绪度来组合返回内容。 - ---- - -## 从 tool handler 到数据源的完整路径 - -以一个具体的查询为例——AI 调用了 `find_callers` 工具。完整路径如下： - -### Step 1: Rust Tool 层调度 - -```rust -// server/src/tools/mod.rs — execute() -pub fn execute(project_id: u64, tool_name: &str, args: &Value) -> String { - let handler = TOOL_HANDLERS.get(tool_name); - // execute handler(project_id, args) - // for "find_callers" → h_find_callers(project_id, args) -} -``` - -### Step 2: FFI 跨语言边界 - -```rust -// mod.rs — handler 内部 -fn h_find_callers(project_id: u64, args: &Value) -> String { - let symbol_name = args["symbol_name"].as_str(); - let result = unsafe { - CString::from_raw( - engine_find_callers_adaptive(project_id, symbol_name.as_ptr()) - ) - }; - result.to_str() -} -``` - -### Step 3: C++ 自适应调度 - -```cpp -// engine_queries.cpp — engine_find_callers_adaptive() -double cg_ratio = g_store->getReadyRatio(project_id, "callgraph_ready"); -if (cg_ratio > 0.5) { - return g_store->findCallersJson(project_id, symbol_name); -} -return g_query->getCallers(project_id, symbol_name); -``` - -### Step 4: SQLite 查询 - -如果走新管道： -```sql -SELECT s2.name, s2.file_path, s2.line, ce.line, ce.col -FROM call_edges ce -JOIN symbols s1 ON ce.caller_symbol_id = s1.id -JOIN symbols s2 ON ce.callee_symbol_id = s2.id -WHERE s2.name = ? AND ce.project_id = ? -``` - -如果降级到旧管道： -```sql -SELECT gn.name, gn.file_path, gn.start_row -FROM graph_edges ge -JOIN graph_nodes gn ON ge.source_node_id = gn.id -WHERE ge.target_node_id = (SELECT id FROM graph_nodes - WHERE name = ? AND project_id = ?) -``` - ---- - -## 零准备查询：没有 call_edges 表怎么办 - -`buildGraph()` 的 3 步 SQL 管道需要 Phase C 完成才能运行。但如果 AI 在 Phase B 就要查调用关系呢？ - -答案在 `getCallersFromRecords()` 和 `getCalleesFromRecords()` 中——它们直接从 `semantic_records` 表做调用推断，不需要预构建的 `call_edges` 表。 - -**获取调用者**： -- 在 `semantic_records` 中找到 `kind=7`（CallExpr）的记录 -- 名称匹配目标函数的名称 -- JOIN `parent_id` 找到调用函数 - -**获取被调用者**： -- 找到 `kind=7` 的记录，`parent_id` 等于函数的 `original_id` -- 这些 CallExpr 的 name 就是被调用函数名 - -这种"零准备"查询在数据量小时性能可接受（几十毫秒），但随着 `semantic_records` 的增长会变慢。这就是为什么它在 Phase B 作为兜底方案存在，而 Phase C 后由 `call_edges` 表接手。 - ---- - -## 就绪度信息的暴露 - -自适应查询不仅在内部做降级决策，还通过 MCP 协议暴露就绪信息： - -```json -// engine_project_overview() 的返回 -{ - "ready_features": { - "fast_scan": true, - "call_graph": true, - "metrics": false, - "full_text_search": true, - "vector_search": false - }, - "ready_ratios": { - "callgraph_ready": 0.87, - "metrics_ready": 0.32, - "embedding_ready": 0.0 - }, - "total_symbols": 1523, - "indexed_symbols": 1325 -} -``` - -AI 客户端的工具选择器可以利用这些信息决定调用哪些工具。比如如果 `call_graph` 未就绪，就不调用 `find_callers` 和 `trace_path`。 - ---- - -## 设计取舍 - -### 优点 - -1. **有损可用优于无损等待**。自适应策略的核心哲学——在数据不完全时，返回"我们知道但不全"的结果，比让 AI 等待 5 分钟好得多。MCP 协议的特点决定了 AI 可以接受不完整信息并继续推理。 - -2. **分层降级优雅**。三层降级（新管道 → 旧管道 → 硬阻断）保证了在任意索引阶段，查询系统都能给出最优结果。不会有 "Phase A 完成了但查询返回空" 的尴尬。 - -3. **阈值可调且可感知**。`getReadyRatio()` 用连续的 `[0.0, 1.0]` 而非离散级别，允许精细控制。 - -### 痛点 - -1. **新旧管道双倍维护成本**。`findCallersJson()` 和 `getCallers()` 做的是同一件事，但走不同的表。Bug fix 时需要同时改两个地方。这在 Article 7 的语言翻译器中也有同样的问题——新旧 IR 管道的共存。 - -2. **阈值选择是经验而非科学**。为什么 `find_callers` 的阈值是 0.5 而 `trace_path` 是 0.1？因为没有理论依据，只是测试中"感觉合理"。不同规模的项目、不同质量的索引，最优阈值可能不同。 - -3. **getReadyRatio 的粒度太粗**。一个项目有 1000 个符号，其中 900 个调用了 `callgraph_ready=1`，只有 100 个没有——这 100 个可能是 AI 正在查询的关键函数。项目级比率无法表达这种分布不均的情况。 - ---- - -## 坦诚反思 - -**自适应层本不应该存在**。 - -从理想主义的架构角度看，自适应层是一个"索引不够快"的补丁。如果索引能在 100ms 内完成全部 Phase A/B/C，就根本不需要什么就绪度检查、比率判断、三层降级——直接查询 `graph_nodes` + `graph_edges` + `code_fts` 就好了。 - -但这个"如果"在现实工程中不成立。5000 个文件的项目，即使有 WAL + batch insert，Phase C 的 `buildGraph()` 也需要几分钟。而 AI 的对话需要"秒级"响应。 - -自适应查询层本质上是一个 **延迟隐藏** 的抽象——让用户感觉数据已经准备好了，而实际上引擎在后台"边查边就绪"。这在很多"实时"系统中是个常见的做法——第一帧渲染、渐进式图片加载、流式搜索结果——但在代码分析领域不太常见。 - -如果让我重新设计，我可能会把自适应逻辑从 `engine_queries.cpp` 中剥离成专门的 `AdaptiveQueryPlanner` 类，而不是让它散落在各个 `engine_find_x_adaptive()` 函数里。当前实现中，每个函数独立检查就绪度、独立做降级决策——导致 5 种降级模式以不同的代码风格分散在 1000+ 行的文件里。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| (五) C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) MCP 协议层 | 35+ 工具的设计哲学 | -| (七) 语言翻译器 | 10 种语言 → 统一 IR | -| (八) 存储层 | SQLite WAL + FTS5 + vec0 | -| **(九) 自适应查询** | **Fallback 机制与就绪检测 ← 本文** | -| (十) 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) 验证层 | 让 AI 对自己的话负责 | -| (十二) Model Engine | 从事实到理解 | -| (十三) Parser + GraphBuilder | 解析与建图 | - ---- - -最后一篇我们拆解 **性能真相**——从 200 行的小脚本到 60,000 文件的 Linux 内核，CodeScope 在各个规模下的实测数据、CBM 对比、以及 "3phases + WAL + vec0" 这套架构在不同场景下的真实表现。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-10-performance-truth.md b/docs/articles/zh/codescope-architecture-10-performance-truth.md deleted file mode 100644 index 528c07c..0000000 --- a/docs/articles/zh/codescope-architecture-10-performance-truth.md +++ /dev/null @@ -1,330 +0,0 @@ -# CodeScope 架构拆解（十）：性能真相——从 95 到 89,465 文件的实测 - -> 在我还没仔细跑基准测试之前，我跟别人说"我们的引擎很快"。 -> -> 这是一种模糊的自信——测试 demo 项目确实秒出结果，但没试过真正的大家伙。直到有一天我跑了一下 Linux 内核全量索引，盯着终端发呆了 15 秒，以为死锁了。实际上只是索引线程在 WAL checkpoint 而已。那次之后我做了一个决定：所有性能结论必须来自至少三个不同规模的项目，并且必须逐阶段计时。模糊的自信到此为止。 -> -> 这篇文章是我能找到的所有公开数据——基准测试 JSON 文件、文档报告、运行时日志。数据来自 7 个基线 JSON、5 份基准报告、5 份运行时日志。我不打算解释每个数据点，而是回答三个问题。 - ---- - -## 三个问题 - -1. **从小项目到大项目，索引时间会怎样增长？是线性、超线性、还是某个阈值后突然爆炸？** -2. **索引的瓶颈到底在哪里——是 CPU（解析代码）、IO（写入 SQLite）、还是其他？** -3. **当项目从 100 个文件涨到 100,000 个文件，CodeScope 的架构能撑住吗？在哪会先崩？** - ---- - -## 全景：7 个项目的实测数据 - -先看总表。这些数据来自 `benchmarks/baselines/*.json`，是最统一、可比的测量： - -| 项目 | 语言 | 文件数 | 解析耗时 | 索引耗时(总) | 图节点 | 节点密度 | -|------|------|:-----:|:--------:|:----------:|:-----:|:--------:| -| ARES Agent | Go | 95 | 9ms | **0.3s** | 24,924 | 262/文件 | -| CodeScope 自测 | C++/Rust | 174 | — | **1s** | — | — | -| memscope-rs | Rust | 238 | 31ms | **2s** | 123,270 | 518/文件 | -| CPython | Python | 1,022 | 0.5s | **6s** | 446,618 | 437/文件 | -| Bun | Zig/C++/JS | 9,641 | 1.0s | **61s** | 2,951,664 | 306/文件 | -| rustc | Rust | 36,807 | 3.2s | **104s** | 4,130,017 | 112/文件 | -| JDK | Java/C++ | 19,821 | 2.4s | **211s** | 8,047,762 | 405/文件 | - -从 95 个文件到 36,807 个文件（388x），索引时间从 0.3s 到 104s（347x）。大致线性，但有两个异常点值得深挖。 - -### 异常点：JDK vs rustc - -JDK 文件数 19,821（rustc 的 54%），但索引时间 211s（rustc 的 203%）。为什么？ - -答案是 **节点密度**。JDK 每文件产生 405 个图节点，是 rustc（112 节点/文件）的 3.6 倍。总节点 8.05M 是 rustc 4.13M 的 1.95 倍。所以 JDK 的慢不是因为文件多，而是因为 **Java 代码的符号密度天然高于 Rust**。 - -这个发现很重要：**CodeScope 的索引时间是 O(符号数) 而非 O(文件数)**。文件数只是代理指标，真正的驱动因素是语义单元的数量。 - -### 索引速度线性图 - -从数据中拟合的缩放规律： - -- **小型（<1K 文件）**：~0.3-6s，几乎是瞬间 -- **中型（1K-10K 文件）**：6-61s，约 1 分钟量级 -- **大型（10K-40K 文件）**：61-211s，2-3.5 分钟 -- **超大型（>60K 文件）**：尚未完成实测基准。Linux 内核全量索引（~65K 文件）的稳定性和性能仍在优化中。 - -> 注：codebase-memory-mcp 在 Linux 内核全量索引上表现出色，可在数分钟内完成 ~65K 文件的索引。CodeScope 目前对超大型项目的支持仍在完善，暂无法稳定索引 Linux 内核级别代码库。完成优化后会更新本数据。 - ---- - -## 瓶颈在哪里：逐阶段分解 - -把索引过程分成四个阶段： - -``` -Parse → SQLite insert → buildGraph → FTS build -``` - -各阶段耗时占比（跨项目平均）： - -``` -Parse: 2.6% ← tree-sitter 解析，纯 CPU 密集 -SQLite: 37.4% ← 语义记录写入，IO 密集 -buildGraph: 33.2% ← 图构建，SQL + JOIN 密集 -FTS: 18.9% ← 全文索引构建 -Other: 7.9% ← Vector、复杂度等 -``` - -这是最关键的发现：**CodeScope 是 SQLite-bound，不是 CPU-bound**。解析阶段（tree-sitter）只占不到 3% 的时间。75% 以上的时间花在 SQLite 写入和建图上。 - -### 不同规模下的瓶颈变化 - -| 项目 | Parse | SQLite | buildGraph | FTS | -|------|:----:|:------:|:---------:|:---:| -| ARES (95 文件) | 3.0% | 32.0% | 27.0% | 10.0% | -| Bun (9,641 文件) | 1.7% | 43.6% | 34.4% | 19.6% | -| rustc (36,807 文件) | 3.0% | 37.4% | 29.5% | 22.1% | -| JDK (19,821 文件) | 1.2% | 41.7% | 38.1% | 19.1% | - -Parse 比例始终很低（1-3%），FTS 随项目规模增长比例上升（10% → 22%），SQLite 始终是最大头。 - -### buildGraph 的子阶段分解 - -buildGraph 内部的 7 个子阶段（来自 rustc 36,807 文件）： - -``` -file_list: 339ms (1.1%) -delete: 457ms (1.5%) -rf: 21ms (0.1%) -r2n: 7,317ms (23.8%) ← ROW_NUMBER() OVER() -nodes: 8,442ms (27.5%) ← INSERT graph_nodes -edges: 14,066ms (45.8%) ← INSERT graph_edges (containment) -calls: 0ms (0.0%) ← 无 call 边（C 语言功能受限） -``` - -`edges` 子阶段占了 buildGraph 总时间的 46%。原因是 `semantic_records` 中的每个父-子关系都要生成一条 `graph_edges`——对于节点密度高的项目（JDK 405 节点/文件），这个 JOIN 的开销随着记录数二次增长。 - ---- - -## 单文件性能 - -一个单独的基准测试：解析 `kernel/latencytop.c`（7,763 bytes）： - -| 指标 | 值 | -|------|:---:| -| engine_init | 14.6 ms（75% SQLite schema 创建，15% grammar 加载） | -| 索引单文件 | **4.9 ms** | -| 吞吐量 | 1,533 KB/s | -| 9 次查询合计 | **0.17 ms** | -| 源码 Token | ~2,000 tokens | -| 索引结果 Token | ~10 tokens | -| Token 节省 | **99.6%** | - -4.9ms 索引一个文件意味着：理想情况下，1000 个文件的索引时间是 **~5 秒**。实际会更大因为经过 FilterPolicy 过滤后、跨文件解析的额外开销。 - ---- - -## Phase A：最快的模式 - -Phase A（快速扫描）是 CodeScope 最快的模式： - -| 项目 | 文件数 | Phase A 耗时 | -|------|:-----:|:----------:| -| ARES Agent | 95 | **9ms** | -| memscope-rs | 238 | **31ms** | -| CPython | 1,022 | **0.5s** | -| Bun | 9,641 | **1.0s** | -| rustc | 36,807 | **3.2s** | -| Linux kernel/kernel/ | 115 | **522ms** | -| Linux kernel/mm/ | 144 | **200ms** | -| Linux kernel/fs/ | 74 | **1,835ms** | - -最大观察：Linux kernel/fs/ 只有 74 个文件，但耗时 1,835ms。原因是 fs/ 包含大量的头文件和宏定义，导致正则扫描器需要处理更多匹配候选。 - -本质上 Phase A 是 O(源码行数) 而不是 O(文件数)。 - ---- - -## 查询延迟：数据比索引更重要 - -全量索引的性能是一个方面，**查询性能** 是另一个。因为赋能 AI 的不是索引过程，而是索引完成后的每一次查询。 - -查询延迟与项目规模的关系： - -``` -图统计 (get_graph_stats): 0.1-0.2ms（所有规模） -入口点 (get_entry_points): 0.1-0.2ms -模块树 (get_module_tree): 0.1-0.2ms -查找调用者 (find_callers): 0.1-0.3ms -查找被调用者 (find_callees): 0.1-0.3ms -热点 (get_hotspots ❌): 0.2-0.5ms -路径追踪 (codescope_trace): 0.5-2.0ms -FTS 搜索 (search_fts): 0.5-2.0ms -社区检测 (get_communities ❌): 30-200ms（最慢查询） -``` - -查询延迟与项目规模弱相关——`find_callers` 在小项目和 rustc 上都是亚毫秒级。原因是所有查询都走精确索引（`call_edges JOIN symbols`），索引的 B-tree 访问是 O(log N)。 - -社区检测是例外——它跑标签传播算法，需要遍历所有 `graph_edges`。在 rustc 规模（9M 边）上需要 ~200ms。 - ---- - -## Token 节省的具体测量 - -前面几篇文章反复提到 Token 节省，这里用真实数据说话。 - -以 `codescope_trace` 工具查询"查找调用者和被调用者"为例： - -**CBM 方案**：需要先将完整的调用链源码拉入上下文（~19,632 tokens），然后让 LLM 自己分析。 -**CodeScope 方案**：直接返回结构化的调用者/被调用者列表（~552 tokens）。 - -``` -CBM: 19,632 tokens → LLM 分析后回复 -CodeScope: 552 tokens → 直接可用的结构化数据 - -节省: 97.2% -``` - -E2E 流水线 Token 对比（GoAgent/ARES Agent 场景）： - -| 步骤 | CBM | CodeScope | 节省 | -|------|:---:|:---------:|:----:| -| 查找符号 `Symbol` | 828 tokens | 138 tokens | 83% | -| 获取详细定义 | 5,012 tokens | 17 tokens | 99.7% | -| 获取调用者+被调用者 | 13,792 tokens | 397 tokens | 97.1% | -| **总计** | **19,632 tokens** | **552 tokens** | **97.2%** | - -这种"结构化数据替代源码"的模式是 Token 节省的根本来源。 - -每个工具的典型响应大小： - -| 工具 | 响应大小 | Token 数（DeepSeek） | -|------|:--------:|:-------------------:| -| get_graph_stats | ~60 bytes | ~18 | -| get_entry_points | ~50 bytes | ~15 | -| get_module_tree | ~100 bytes | ~30 | -| find_callers | 100-500 bytes | 30-150 | -| get_hotspots ❌ | ~500 bytes | ~150 | -| search (FTS) | 1K-5K bytes | 300-1,500 | -| get_communities ❌ | 1K-200K bytes | 300-60K | - -注意 `get_communities ❌` 的范围——当社区数量很大时，输出可能达到 200KB。这是唯一需要主动限流的工具（通过 `max_members` 参数）。 - ---- - -## 数据库效率对比 - -以 GoAgent（~200 文件，Go/Python/JS 混合）项目为例： - -| 指标 | CBM | CodeScope Phase A | 差距 | -|------|:---:|:-----------------:|:----:| -| DB 大小 | **64 MB** | **270 KB** | **242x** | -| 索引耗时 | 545 ms | **307 ms** | 1.8x 更快 | -| 节点数 | 2,057 | **24,924** | 12.1x 更多 | -| 边数 | 7,025 | **23,184** | 3.3x 更多 | - -Phase A 的 270KB 可以做到「扫描完立即查询」。不需要等待全量索引完成。 - -Linux 内核的完整索引 DB 大小： - -| 索引模式 | 文件数 | DB 大小 | -|---------|:-----:|:-------:| -| Phase A 快速（kernel/ 子集） | 115 | ~3 MB | -| 全量索引（旧架构） | 89,465 | **7.06 GB** | -| 全量索引（新架构） | 64,694 | **~1.2 GB** | - -新架构的 DB 更小（1.2GB vs 7.06GB）虽然节点更多（12M vs 4.9M）。原因是新管道的 `semantic_records` + `graph_nodes` 模式比旧管道的 `ir_nodes` + `graph_nodes` 冗余更少。 - ---- - -## 内存使用 - -CodeScope 的内存占用与文件数非线性相关： - -| 项目 | 文件数 | 峰值 RSS | 节点密度 | -|------|:-----:|:--------:|:--------:| -| ffi-demo | 77 | **48 MB** | — | -| CodeScope 自测 | 174 | **62 MB** | — | -| memscope-rs | 238 | **128 MB** | 518/文件 | -| Linux kernel | 64,694 | **~3 GB** | ~190/文件 | -| Server 常驻 | — | **~12 MB** | Worker 退出后 | - -"Worker 退出后 Server 常驻仅 ~12MB"——这是 Phase 3（Worker 隔离）设计的价值：子进程索引完毕后释放全部内存，Server 只保留 ~12MB 的常驻 RSS。 - ---- - -## 可扩展性的上限在哪里 - -根据以上数据，CodeScope 的规模化瓶颈是 **SQLite 写入**，而不是解析或图构建。 - -### 已经过验证的规模 - -| 规模 | 实测项目 | 索引时间 | -|:----:|---------|:--------:| -| 100 文件 | ARES Agent | 0.3s | -| 1,000 文件 | CPython | 6s | -| 10,000 文件 | Bun | 61s | -| 20,000 文件 | JDK | 211s | -| 35,000 文件 | rustc | 104s | -| >60K 文件 | Linux kernel | ⏳ 待优化 | - -### 预期的瓶颈 - -1. **SQLite 写入竞争**（首个瓶颈）：WAL 模式允许并发读但写入是串行的。多 Worker 需要序列化写入——当前设计中这个已经实现了（index_project 是单 Worker）。 - -2. **buildGraph 的 ROW_NUMBER() OVER()**：这个窗口函数在百万节点级别上仍然毫秒级，但在千万节点上开始成为瓶颈（JDK 的 `r2n` 阶段 ~14.5s）。预计亿级节点会需要分片或增量 buildGraph。 - -3. **暴力向量搜索**：n-gram 哈希搜索的 `searchSemantic()` 对 `node_vectors` 做全表扫描。当前 <100K 节点是微秒级，但百万节点会到秒级。vec0 扩展解决了这个问题，但需要用户安装 `.dylib`。 - -4. **FTS5 索引重建**：`buildFTSFromGraph()` 使用 `INSERT-SELECT` 批量写入，在千万节点上表现良好（rustc 23s）。但全量重建是所有阶段中最慢的（JDK 38.3s）。 - -### 实用的上限 - -根据实测数据，CodeScope 在以下范围内表现最佳： - -- **文件数：<50,000 文件**。超过这个量级 buildGraph + FTS 的组合时间会超过 5 分钟。 -- **节点数：<10M 节点**。超过这个量级 SQLite 的 WAL checkpoint 时间会明显影响用户体验。 -- **DB 大小：<5 GB**。超过这个量级 mmap I/O 的 page fault 率上升。 - -对于 Linux 内核（~65K 文件, 12M 节点, 1.2GB DB），CodeScope 跑在边界上——可以工作，但不是最佳体验。 - ---- - -## 坦诚反思 - -**基准测试数据比我最初想象的更诚实**。 - -有几件事在基准测试之前我以为是真的，分析后发现不是： - -**"解析是瓶颈"** —— 错。树解析只占 1-3%。瓶颈在 SQLite。 - -**"文件数决定速度"** —— 错。节点密度才是主导因素。JDK 的 19K 文件比 rustc 的 36K 文件慢 2 倍。 - -**"直接查 symbols 表一定快"** —— 大部分情况对，但 Phase A 的 fs/ 目录花了 1.8s。正则扫描在宏展开过多的文件上比想象中更慢。 - -**"新架构一定比旧架构快"** —— 对，但程度出乎意料。新架构在同样数据量下节点更多但 DB 更小、信息更密集。新架构最后被证明更好不是因为快，而是因为质量。 - -**WAL checkpoint 是个隐藏的坑**。当 WAL 文件增长到几百 MB 时，checkpoint 操作会卡住写入 ~0.5-2s。这在普通使用中不可感知（0.5s 的卡顿相当不明显），但在基准测试中会被记录下来。 - ---- - -## 系列导航 - -| 文章 | 主题 | -|---|---| -| (一) 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) 零冗余响应 | 精简响应，按需返回 | -| (五) C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) MCP 协议层 | 35+ 工具的设计哲学 | -| (七) 语言翻译器 | 10 种语言 → 统一 IR | -| (八) 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) 自适应查询 | Fallback 机制与就绪检测 | -| **(十) 性能真相** | **从 200 到 60,000 文件的实测 ← 本文，全系列完结** | - ---- - -## 结语 - -这是一个系列。从「56KB vs 629 bytes」的开篇开始，到「毫秒级让 AI 开始理解你的代码」的渐进式就绪，到「Worker 隔离」「零冗余响应」「C++ 引擎」「MCP 协议层」「语言翻译器」「存储层」「自适应查询」，最后到「性能真相」——十篇文章，拆解了 CodeScope 这个项目的核心设计。 - -每一篇文章都遵循一个规则：不只说做了什么，还解释为什么这么做、有没有更好的方案。这是从 GoAgent 文章风格中学到的——架构拆解的价值不在于罗列功能，而在于暴露设计过程中的"如果当时选了另一条路会怎样"。 - -如果你在看这个系列的过程中有任何疑问，或者想让我深入某个主题——比如某个 Language Translator 的具体实现、SQLite 的某个查询优化细节、某个工具的 LLM prompt 设计——这个系列只是起点，而不是终点。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-architecture-11-verification-layer.md b/docs/articles/zh/codescope-architecture-11-verification-layer.md deleted file mode 100644 index a9a8b64..0000000 --- a/docs/articles/zh/codescope-architecture-11-verification-layer.md +++ /dev/null @@ -1,218 +0,0 @@ -# CodeScope 架构拆解（十一）：验证层——让 AI 对自己的话负责 - -> 在写这篇文档之前，我跑了一下 `verify_integrity`——它会检查 README 声称的功能是否真的在代码里。结果它报了一个"缺失能力"：`CommunityDetection` 在能力列表里，但 MCP Server 根本没有暴露对应的工具。这是个尴尬但真实的发现。 - ---- - -## 三个问题 - -代码分析工具能回答"代码里有什么"，但回答不了"代码真的做到了 README 里写的那些吗？"——这是验证层要解决的问题。 - -1. **如何把自然语言的"承诺"变成可执行的检查？**（Claim 解析） -2. **如何根据代码事实验证一个承诺的真假？**（Verifier 分发） -3. **如何把发现的问题持久化，让 AI 下次还能看到？**（Evidence 链） - ---- - -## 整体架构 - -验证层位于 `engine/src/verify/`，共 23 个源文件。核心流程： - -``` -自然语言 / README / PR 描述 - │ - ▼ - ClaimParser ───→ Claim (结构化的"断言") - │ │ - │ ▼ - │ VerifierRegistry - │ │ - │ ┌──────┼──────┐ - │ ▼ ▼ ▼ - │ Capability Contract Architecture - │ Verifier Verifier Verifier - │ │ │ │ - │ ▼ ▼ ▼ - │ Finding (证据 + 置信度) - │ │ - │ ▼ - │ SQLite Findings 表（持久化） -``` - ---- - -## ClaimParser：从自然语言到结构化断言 - -```cpp -// engine/src/verify/claim_parser.h -struct Claim { - ClaimType type; // capability_exists / contract_holds / architecture_follows - std::string subject; // 主语：模块名 / 函数名 - std::string predicate; // 谓语：supports / implements / ensures - std::string object; // 宾语：功能名 / 接口名 / 属性 - double confidence; // 解析置信度（0-1） -}; -``` - -`ClaimParser` 将自然语言句子解析为 Claim 结构。当前支持的句子模式： - -| 模式 | 示例 | ClaimType | -|------|------|-----------| -| X supports Y | "登录模块支持 JWT" | capability_exists | -| X implements Y | "Server implements Service interface" | contract_holds | -| X ensures Y | "系统确保数据一致性" | architecture_follows | -| X is Y | "该模块是线程安全的" | contract_holds | - -对于无法解析的句子，`confidence` 字段设为 0，表示"不验证，跳过"。 - ---- - -## VerifierRegistry：分发到正确的验证器 - -```cpp -// engine/src/verify/registry.h -class VerifierRegistry { - static VerifierRegistry &instance(); // Meyers 单例 - void register_verifier(std::unique_ptr v); - void register_default_verifiers(store::GraphStore *store, uint64_t pid); - - Verifier *match(const Claim &claim) const; // 返回第一个 accepts() 的验证器 -}; -``` - -默认注册三个验证器，按优先级排序： - -``` -1. CapabilityVerifier (最具体，优先匹配) -2. ContractVerifier -3. ArchitectureVerifier (最通用，最后兜底) -``` - ---- - -## 三个内置验证器 - -### 1. CapabilityVerifier：能力存在性验证 - -检查"某模块是否真的支持某功能"。 - -```cpp -// engine/src/verify/capability_verifier.h -// 内建能力列表： -// "FastScan", "FullIndex", "FullTextSearch", -// "CrossFileResolution", "CommunityDetection", "EmbeddingSearch" -// 通过 entity table 查询：目标实体是否存在且有调用者？ -``` - -**验证逻辑：** - -1. 查找名为 subject 的 entity -2. 如果找到，检查它是否有 callers（被调用） -3. 有调用者 → "能力存在且被使用" -4. 无调用者 → "能力存在但未被使用"（降置信度） -5. 找不到 → "能力缺失" - -### 2. ContractVerifier：契约遵守验证 - -检查某实体是否实现了某个接口/契约。 - -**验证逻辑：** - -1. 在 entity + relation 表中查找 subject → object 的 IMPLEMENTS 边 -2. 如果存在 → "契约已履行" -3. 如果不存在 → "契约未履行" - -### 3. ArchitectureVerifier：架构层约束验证 - -检查代码是否遵循预定义的分层约束（如 Controller → Service → Repository）。 - -**验证逻辑：** - -1. 通过命名约定和文件路径将实体分类到层（Controller / Service / Repository） -2. 扫描 call_edges，检查： - - 是否存在反向调用（Repository → Controller） - - 是否存在同层绕过（Controller → Controller） -3. 违规 → ArchitectureDrift finding - ---- - -## Drift Detection：批量漂移检测 - -除了单条验证，验证层还提供批量扫描工具： - -| 工具 | 扫描范围 | 检测什么 | -|------|---------|---------| -| `documentation_drift` | README 文件 | 声称的语言支持 vs 实际 entity | -| `capability_drift` | 能力列表 | 声称的能力 vs 实际实现 | -| `architecture_drift` | 全部 CALLS 边 | 层违规 vs 预期分层 | -| `drift`（全量） | 以上全部 | 汇总所有漂移 | - -### Finding 数据结构 - -每个漂移或验证结果都是一个 Finding： - -```cpp -// engine/src/verify/finding.h -struct Finding { - uint64_t id; - FindingSeverity severity; // 1=Drift, 2=CapabilityMissing, 3=BrokenContract - std::string category; // "DocumentationDrift" / "CapabilityDrift" / "ArchitectureDrift" - std::string title; // 一句话描述 - std::string detail; // 详细证据链 - double confidence; // 0-1 - std::string source_kind; // "code_review" / "integrity_scan" / "claim_verification" -}; -``` - -Finding 写入 SQLite findings 表，支持按 source_kind 过滤（例如只看 PR review 产生的 finding）。 - ---- - -## 实际案例 - -`verify_integrity`（`engine/src/engine_verify_ffi.cpp`）的输出示例： - -```json -{ - "findings": [ - { - "severity": 2, - "category": "CapabilityDrift", - "title": "MissingCapability: CommunityDetection", - "detail": "Declared in capability list but no implementing entity with callers (or entity missing entirely)", - "confidence": 0.85 - } - ], - "integrity_score": 0.92, - "total_capabilities": 6, - "fulfilled": 5, - "broken": 1 -} -``` - ---- - -## 局限 - -1. **ClaimParser 的模式有限**——目前只支持简单的"X 支持 Y"句式，无法处理条件句（"如果配置了 A，就启用 B"）或否定句（"不依赖外部服务"）。 -2. **CapabilityVerifier 的检出条件**——一个能力只要有 entity 且有调用者就算存在。这意味着一个空壳函数如果被人调用了，也会被判定为"能力存在"。需要结合 DeadCodeInspector 交叉验证。 -3. **ArchitectureVerifier 的分层规则**——目前基于文件名前缀（`Controller`、`Service`、`Repository`）做层分类。如果项目不使用这种命名约定，分层检测不生效。 -4. **Finding 没有去重机制**——多次运行 `detect_drift` 会产生重复 finding，需要客户端自行去重。 - ---- - -## 系列导航 - -| # | 文章 | 主题 | -|---|------|------| -| (一) | 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) | 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) | Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) | 零冗余响应 | 精简响应，按需返回 | -| (五) | C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) | MCP 协议层 | 工具的设计哲学 | -| (七) | 语言翻译器 | 10 种语言 → 统一 IR | -| (八) | 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) | 自适应查询 | Fallback 机制与就绪检测 | -| (十) | 性能真相 | 从 200 到 60,000 文件的实测 | -| **(十一)** | **验证层** | **让 AI 对自己的话负责 ← 本文** | diff --git a/docs/articles/zh/codescope-architecture-12-model-engine.md b/docs/articles/zh/codescope-architecture-12-model-engine.md deleted file mode 100644 index cb66134..0000000 --- a/docs/articles/zh/codescope-architecture-12-model-engine.md +++ /dev/null @@ -1,118 +0,0 @@ -# CodeScope 架构拆解（十二）：Model Engine 与 State Builder - -> Phase B 完成后，数据库里有 entity、relation、graph_edges 这些"事实"。但事实本身不是"理解"——你需要把事实组合成更高层的模型：这个模块的能力是什么？它的工作流长什么样？架构分层是否正确？这就是 Model Engine 的工作。 - ---- - -## 职责 - -Model Engine 在 Resolver Pipeline 之后运行，负责从事实（Facts）和解析结果（Resolution）构建**高层模型**。 - -``` -Resolver Pipeline ──→ entity + relation + call_edges - │ - ▼ - ModelEngine::runAll() - │ - ┌───────┼───────┐ - ▼ ▼ ▼ - Workflow Capability Architecture - Plugin Plugin Plugin - │ │ │ - ▼ ▼ ▼ - SQLite: workflow / capability / architecture 表 -``` - ---- - -## ModelEngine：插件管理器 - -```cpp -// engine/src/model/engine.h -class ModelEngine { -public: - void addPlugin(std::unique_ptr plugin); - int64_t runAll(uint64_t project_id); - ModelResult run(const std::string &name, uint64_t project_id); -}; -``` - -`ModelEngine` 不执行任何具体分析——它只管理插件的生命周期和遍历顺序。每个插件实现 `ModelPlugin` 接口： - -```cpp -// engine/src/model/plugin.h -class ModelPlugin { -public: - virtual const char *name() const = 0; - virtual ModelResult run(uint64_t project_id) = 0; -}; -``` - ---- - -## StateBuilder：项目状态恢复 - -```cpp -// engine/src/model/state_builder.h -// 从 SQLite 恢复项目当前状态（能力、工作流、架构、模块摘要） -// 每次 project_overview / explain_module 调用时运行 -``` - -`StateBuilder` 是 Model Engine 的核心消费方——它从已构建的模型中读取数据，组装成 AI 可理解的模块摘要和项目概览。 - ---- - -## 当前内置插件 - -实际项目中注册的插件列表可以从 capabilities 表查到： - -| 插件 | 输出表 | 说明 | -|------|--------|------| -| Workflow Plugin | workflow | 跨模块的工作流路径 | -| Capability Plugin | capability | 模块能力声明与验证 | -| Architecture Plugin | architecture | 架构分层与层间依赖 | -| Contract Plugin | contract | 契约（接口实现关系） | - -每个插件在 `run()` 中执行 SQL 查询 + 关系分析，结果写入对应的 SQLite 表，供查询引擎（article 09）读取。 - ---- - -## 与验证层的关系 - -Model Engine 和 Verification Layer 共享数据但职责分离： - -| | Model Engine | Verification Layer | -|---|---|---| -| 输入 | entity + relation + graph_edges | entity + relation + README | -| 输出 | 模型表（workflow/capability/architecture） | Finding（证据链） | -| 调用时机 | 索引时（Phase B） | 查询时（按需） | -| 使用者 | `project_overview` / `explain_module` | `verify_claim` / `detect_drift` | - -简单说：Model Engine **建模型**，Verification Layer **用模型做检查**。 - ---- - -## 局限 - -1. **插件的数量有限**——目前只有 4 个内置插件。架构插件依赖命名约定（Controller/Service/Repository），不在约定内的项目不产生有效输出。 -2. **没有插件热加载机制**——所有插件编译进二进制，不能动态添加。如果需要新增模型类型（如"数据流分析"），需要改 C++ 代码重新编译。 -3. **StateBuilder 的聚合粒度**——目前以模块（目录）为单位聚合。如果一个目录里混合了多个不同职责的文件，摘要可能不够精确。 - ---- - -## 系列导航 - -| # | 文章 | 主题 | -|---|------|------| -| (一) | 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) | 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) | Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) | 零冗余响应 | 精简响应，按需返回 | -| (五) | C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) | MCP 协议层 | 工具的设计哲学 | -| (七) | 语言翻译器 | 10 种语言 → 统一 IR | -| (八) | 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) | 自适应查询 | Fallback 机制与就绪检测 | -| (十) | 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) | 验证层 | 让 AI 对自己的话负责 | -| **(十二)** | **Model Engine** | **从事实到理解 ← 本文** | diff --git a/docs/articles/zh/codescope-architecture-13-parser-graph.md b/docs/articles/zh/codescope-architecture-13-parser-graph.md deleted file mode 100644 index edca411..0000000 --- a/docs/articles/zh/codescope-architecture-13-parser-graph.md +++ /dev/null @@ -1,158 +0,0 @@ -# CodeScope 架构拆解（十三）：解析器与 GraphBuilder - -> tree-sitter 是这个项目最正确的技术选型之一。不是因为它快，而是因为它**不会报错**——无论源码有多畸形，tree-sitter 总能产出一棵 AST。这在索引百万行级别的项目时至关重要：一个文件解析失败不应该阻塞整个索引。 - ---- - -## Parser：tree-sitter 封装层 - -解析器位于 `engine/src/parser/parser.cpp`，是对 tree-sitter C API 的 C++ 封装。 - -### 职责 - -1. 根据文件扩展名选择语言解析器 -2. 调用 tree-sitter 生成 CST -3. 将 CST 传给对应的 IR Translator -4. 处理解析错误（主要是内存限制） - -```cpp -// engine/src/parser/parser.cpp 核心流程 -TranslationUnit *parseFile(const char *path, const char *source, - size_t source_len, const char *language) { - TSParser *parser = ts_parser_new(); - ts_parser_set_language(parser, getLanguage(language)); - TSTree *tree = ts_parser_parse_string(parser, nullptr, source, source_len); - // 将 TSTree 交给 IR Translator - Translator *t = createTranslator(language); - return t->translate(tree, source, path); -} -``` - -### tree-sitter 的选择理由 - -| 对比项 | tree-sitter | 传统 LALR/LR 解析器 | -|--------|-------------|-------------------| -| 容错性 | ✅ 永远产出 AST（含 error 节点） | ❌ 遇到语法错误就挂 | -| 增量解析 | ✅ 支持 | ❌ 需要重新解析全部 | -| 多语言 | ✅ 原生支持 100+ 语言 | ❌ 每种语言一套工具链 | -| 速度 | ✅ C 实现，足够快 | ✅ 更快，但不够鲁棒 | - -### 代价 - -tree-sitter 的 CST 包含**所有语法细节**——括号、分号、关键字。IR Translator 的工作就是从中提取语义结构，丢弃语法噪音。 - ---- - -## Grammar 管理 - -```cpp -// engine/src/codescope_grammars.h -// 管理 8 种语言的 tree-sitter grammar 编译产物（.so 文件） -``` - -Grammars 在构建时通过 CMake FetchContent 自动下载并编译。每种语言对应一个 `.so` 文件，运行时按需加载。当前支持的 8 种语言和其 grammar 状态： - -| 语言 | Grammar 源 | AST 节点类型数 | -|------|-----------|:-------------:| -| C | tree-sitter-c | ~50 | -| C++ | tree-sitter-cpp | ~100 | -| Go | tree-sitter-go | ~60 | -| Python | tree-sitter-python | ~40 | -| Rust | tree-sitter-rust | ~80 | -| JavaScript | tree-sitter-javascript | ~70 | -| TypeScript | tree-sitter-typescript | ~80 | -| Java | tree-sitter-java | ~70 | - ---- - -## GraphBuilder：从 IR 到图 - -`GraphBuilder` 位于 `engine/src/graph/graph_builder.h`，是 IR 翻译和 Resolver 之间的桥梁。 - -### 两套 API - -``` -旧 API: TranslationUnit + Node* 树 - ├── 用于: C, C++, Go, Python, Rust, Java - └── Visitor 模式递归遍历 IR 树 - 对每个 NodeKind → 生成 GraphNode - 对每个 Relation → 生成 GraphEdge - -新 API: SemanticUnit + flat records - ├── 用于: JavaScript, TypeScript - └── 直接接受 flat records（不需要遍历树） - 因为 JS/TS visitor 输出已经是扁平格式 -``` - -### 图类型定义 - -```cpp -// engine/src/graph/graph_types.h -enum class NodeType : uint8_t { - Function, Method, Class, Struct, Interface, - Variable, Macro, Module, File -}; - -enum class EdgeType : uint8_t { - References, Calls, Defines, Contains, Imports, Inherits -}; -``` - -### Parent Chain Cache - -GraphBuilder 的关键优化：在插入子节点时，通过 `parent_chain_cache` 缓存父节点 ID，避免反复查询 SQLite。对于深层嵌套的 AST（如复杂表达式），这个缓存减少了 **O(depth)** 的数据库查询。 - -### 写入流程 - -``` -GraphBuilder::build(translation_unit) - │ - ├── 遍历 IR 节点树 - │ ├── FunctionDecl → NodeType::Function - │ ├── ClassDecl → NodeType::Class - │ └── VariableDecl → NodeType::Variable - │ - ├── 遍历 Relation 列表 - │ ├── CallTarget → EdgeType::Calls - │ ├── BaseClass → EdgeType::Inherits - │ └── SymbolRef → EdgeType::References - │ - └── 批量写入 SQLite - ├── graph_nodes (逐条 insert) - └── graph_edges (批量 insert) -``` - ---- - -## 性能 - -从实际基准测试看： - -| 阶段 | 小项目（200 文件） | Linux 内核（6.4 万文件） | -|------|:-----------------:|:----------------------:| -| tree-sitter 解析 | ~秒级 | ~2 分钟 | -| IR 翻译 | ~毫秒级 | ~30 秒 | -| GraphBuilder 写入 | ~毫秒级 | ~20 秒 | -| ResolverPipeline | ~毫秒级 | ~1 分钟（最耗时） | - -**ResolverPipeline 是唯一跟项目规模超线性相关的阶段**——因为每个引用都需要在所有 entity 中查找候选。其他阶段基本是 O(N) 线性扩展。 - ---- - -## 系列导航 - -| # | 文章 | 主题 | -|---|------|------| -| (一) | 开篇 | 56KB vs 629 bytes，CodeScope 要解决什么问题 | -| (二) | 渐进式就绪 | 毫秒级让 AI 开始理解你的代码 | -| (三) | Worker 隔离 | 为什么索引不会拖垮 MCP Server | -| (四) | 零冗余响应 | 精简响应，按需返回 | -| (五) | C++ 引擎拆解 | 从源码到多维代码图的管线 | -| (六) | MCP 协议层 | 工具的设计哲学 | -| (七) | 语言翻译器 | 10 种语言 → 统一 IR | -| (八) | 存储层 | SQLite WAL + FTS5 + vec0 | -| (九) | 自适应查询 | Fallback 机制与就绪检测 | -| (十) | 性能真相 | 从 200 到 60,000 文件的实测 | -| (十一) | 验证层 | 让 AI 对自己的话负责 | -| (十二) | Model Engine | 从事实到理解 | -| **(十三)** | **Parser + GraphBuilder** | **解析与建图 ← 本文** | diff --git a/docs/articles/zh/codescope-deepdive-01-dual-language.md b/docs/articles/zh/codescope-deepdive-01-dual-language.md new file mode 100644 index 0000000..449c740 --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-01-dual-language.md @@ -0,0 +1,221 @@ +# CodeScope 拆解 (一)：双语言架构 — 为什么我们用 Rust 写了一个 C++ 项目 + +> *"Sometimes the best tool for the job isn't the one you'd like to use."* +> 有时最好的工具并不是你喜欢的那个。 + +## 为什么是两种语言？ + +如果让我用一句话解释 CodeScope 的架构，我会说：**Rust 写了一个 MCP 服务器，C++ 写了一整个引擎，中间用 FFI 粘起来，每个索引任务跑在独立的子进程里。** + +这不是一个优雅的设计。它很丑，很麻烦，而且调试的时候会同时触及两种语言的运行时。但它解决了两个我解决不了的问题： + +1. **Rust 的生态**里有 MCP SDK、serde 序列化、tokio 异步——这些是写一个 JSON-RPC 服务器最舒服的姿势。 +2. **C++ 的生态**里有 tree-sitter 解析器、SQLite 的原生 C API、以及各种语言的语法解析器——这些是构建代码分析引擎最直接的路径。 + +我最初试过纯 Rust 方案。用 `tree-sitter` crate 解析代码，写自己的 IR 和存储层。但很快发现，当需要覆盖 8 种语言、处理百 GB 级别的仓库、同时还要做到毫秒级响应时，Rust 的抽象层带来的开销开始变得不可忽略。而 C++ 这边，tree-sitter 的 C API 可以直接调用，SQLite 的 C API 是零开销的，所有内存布局都可以精确控制。 + +所以最终变成了：**Rust 写前端（MCP 服务器），C++ 写后端（引擎核心），中间隔着一层讨厌但必要的 FFI。** + +## 核心文件 + +``` +server/src/ffi/mod.rs ← Rust 侧的 FFI 声明 (~700 行) +engine/src/engine_ffi.cpp ← C++ 侧的 FFI 实现 +engine/src/engine.h ← C++ 公开 API +``` + +## FFI 的惨痛现实 + +先看 Rust 这边。这是 FFI 声明的样子： + +```rust +// server/src/ffi/mod.rs +unsafe extern "C" { + fn engine_init(db_path: *const c_char) -> i32; + fn engine_shutdown(); + + fn engine_create_project(root_path: *const c_char, name: *const c_char) -> u64; + fn engine_index_project( + project_id: u64, + dir_path: *const c_char, + language_filter: *const c_char, + ) -> *mut c_char; + fn engine_find_definition( + project_id: u64, + symbol_name: *const c_char, + file_filter: *const c_char, + ) -> *mut c_char; + // ... 40+ 个类似声明 +} +``` + +每个 `unsafe extern "C"` 块都在说：**"Rust 不知道外面在干什么，但我信任它。"** 这种信任很容易出问题。 + +而 C++ 侧的实现长这样： + +```cpp +// engine/src/engine_ffi.cpp +extern "C" char* engine_index_project( + uint64_t project_id, + const char* dir_path, + const char* language_filter) +{ + std::string result = // ... 实际索引逻辑 + return dupString(result); // 堆分配，Rust 侧必须 free +} +``` + +这段代码里埋了一个典型的 FFI 陷阱：**谁负责释放内存？** + +Rust 侧调用 `engine_index_project` 后得到一个 `*mut c_char`，它必须调用 `engine_free_string` 来释放。忘记释放 = 内存泄漏。释放两次 = 崩溃。跨语言边界的内存管理就是这样——你永远在担心对面是不是忘了什么。 + +## 围绕 FFI 的包装层 + +在 Rust 侧，每个 FFI 调用都包了一层安全的 Rust 函数，把 C 指针转换成 Rust 的 `String` 或 `Result`： + +```rust +// server/src/ffi/mod.rs (简化) +pub fn index_project(project_id: u64, dir_path: &str, language_filter: &str) -> Result { + let dir_c = CString::new(dir_path).unwrap(); + let lang_c = CString::new(language_filter).unwrap(); + + let ptr = unsafe { engine_index_project(project_id, dir_c.as_ptr(), lang_c.as_ptr()) }; + if ptr.is_null() { + return Err("engine returned null".into()); + } + let result = unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned(); + unsafe { engine_free_string(ptr) }; + Ok(result) +} +``` + +这个模式重复了 40 多次。每次都是： +1. `CString::new` — Rust 字符串 → C 字符串 +2. unsafe 调用 +3. 检查空指针 +4. `CStr::from_ptr` — C 字符串 → Rust 字符串 +5. `engine_free_string` — 释放 C 侧的内存 + +这个过程如此机械，以至于我写过好几个 bug：忘记检查空指针、忘记释放、甚至在释放后继续使用指针。 + +```mermaid +flowchart LR + Rust["Rust MCP Server"] -->|"CString::new"| C["C String (heap)"] + Rust -->|"unsafe extern C"| FFI["FFI Boundary"] + FFI --> Cpp["C++ Engine"] + Cpp -->|"dupString"| Ret["C String (heap)"] + Ret -->|"CStr::from_ptr"| Rust + Rust -->|"engine_free_string"| Free["释放 C 堆"] + Free --> Cpp +``` + +## 全局单例：C++ 的"隐式状态" + +引擎内部有三个全局单例： + +```cpp +// engine/src/engine_internal.h +extern std::unique_ptr g_store; +extern std::unique_ptr g_query; +extern std::unique_ptr g_parser; +``` + +它们由 `engine_init()` 初始化，`engine_shutdown()` 销毁。Rust 侧的 MCP 服务器在启动时调用一次 `engine_init`，在关闭时调用一次 `engine_shutdown`。 + +但这里的线程安全模型是脆弱的： + +```cpp +// engine/src/engine_internal.h +// Thread-safety model: +// - The Rust MCP server calls FFI functions SEQUENTIALLY from a single +// thread (the main event loop in server/src/mcp/server.rs). +// - Index worker SUBPROCESSES have their own engine instance (fork+exec). +// - engine_shutdown() must only be called when no FFI query is in flight. +``` + +译成人话：**"别在多个线程里同时调 FFI。"** + +这个约束在 v0.1 版本里从来不是问题——MCP 服务器是单线程事件循环，所有请求串行处理。但到了 v0.2 引入并行调度器后，问题来了：**调度器会 fork 子进程，每个子进程有自己的 engine 实例，但父进程的 engine 还在同时处理查询请求。** + +解决方案是：子进程里 `fork+exec` 重新启动一个 `codescope worker` 进程，完全独立的内存空间，不会干扰父进程的全局单例。 + +```mermaid +flowchart TB + subgraph "主进程 (Rust MCP Server)" + FFI_Call["FFI 调用
engine_find_definition"] + Global["全局单例
g_store / g_query / g_parser"] + end + + subgraph "子进程 1 (Worker)" + Worker1["独立的 engine 实例
fork+exec 全新进程"] + end + + subgraph "子进程 2 (Worker)" + Worker2["独立的 engine 实例
fork+exec 全新进程"] + end + + FFI_Call --> Global + Scheduler["并行调度器"] -->|"fork+exec"| Worker1 + Scheduler -->|"fork+exec"| Worker2 +``` + +## 一个让我冷汗直流的教训 + +在 v0.1.2 的开发中，我遇到了一个诡异的崩溃：在 CI 上，`engine_shutdown` 偶尔会触发 `double free`。 + +debug 了三天后，发现原因是一个**野指针**。Rust 侧的某个 FFI 调用返回后，C++ 侧持有了一个指向 Rust 栈上数据的指针。当 Rust 的 `CString` 被 drop 后，C++ 侧还在用它。这在单线程上偶尔能跑对，但在 CI 的高负载下就崩溃了。 + +修复方案是把所有跨边界的字符串都改成**深拷贝**——C++ 侧收到 C 字符串后立即拷贝到 `std::string`，不再持有任何指向 Rust 内存的指针。 + +教训是：**跨 FFI 边界的指针，要么是 const 的且生命周期内不会释放，要么就立刻拷贝。没有中间地带。** + +## 为什么要承受这种痛苦？ + +如果 FFI 这么麻烦，为什么不用纯 Rust 或纯 C++？ + +**纯 C++ 方案**：C++ 没有原生的 MCP SDK。我需要手动实现 JSON-RPC 2.0 协议、HTTP 传输（或 stdio 传输）、以及所有工具的路由分发。这大约需要 2000-3000 行额外代码，而且每增加一个工具，都要手动注册。 + +**纯 Rust 方案**：Rust 的 `tree-sitter` crate 是对 C API 的封装，性能损耗很小，但问题在于：C++ 引擎里的大量代码（尤其是 IR 翻译器和图构建器）是用 C++17/C++20 的特性写的，如果全部移植到 Rust，需要重新实现一套完整的 IR 系统和图算法——这是几个月的工作量。 + +所以 FFI 不是最优解，它是**当前约束下的最优解**。Rust 负责它擅长的：网络、序列化、进程管理、类型安全。C++ 负责它擅长的：解析、图构建、内存控制、性能关键路径。 + +```mermaid +flowchart LR + subgraph "Rust 层 (MCP Server)" + MCP["MCP 协议处理
JSON-RPC 2.0 / stdio"] + Tools["42+ 工具路由
TOOL_HANDLERS 注册表"] + Sched["并行调度器
三种调度模式"] + WorkerMgmt["Worker 子进程管理"] + end + + subgraph "FFI Boundary" + FFI["unsafe extern C
~40 个函数"] + end + + subgraph "C++ 层 (Engine)" + Parser["tree-sitter 解析器
8 种语言"] + IR["统一 IR 翻译器"] + Graph["图构建器"] + Store["SQLite 存储层"] + Verify["验证管线"] + end + + Worker["codescope worker
独立子进程"] + + MCP --> Tools + Tools --> FFI + Sched --> WorkerMgmt + WorkerMgmt -->|"fork+exec"| Worker + Worker --- FFI + FFI --> Parser + Parser --> IR --> Graph --> Store + Store --> Verify +``` + +## 总结 + +CodeScope 的双语言架构不是设计出来的，是被逼出来的。Rust 的 MCP 生态和 C++ 的解析生态谁都不愿放弃，FFI 就成了不得不承受的代价。 + +40 多个 `unsafe extern "C"` 声明、每个字符串都要手动管理生命周期、全局单例的线程安全约束——这些都是真实存在的技术债务。但到目前为止，它跑得还算稳。 + +在下一篇文章中，我会拆解**Worker 子进程隔离**——为什么每个索引任务都要跑在独立的子进程里，以及如何在不崩溃的情况下处理百万行代码的仓库。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-02-worker-isolation.md b/docs/articles/zh/codescope-deepdive-02-worker-isolation.md new file mode 100644 index 0000000..dd5e65b --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-02-worker-isolation.md @@ -0,0 +1,260 @@ +# CodeScope 拆解 (二)：Worker 子进程隔离 — 为什么索引必须跑在子进程里 + +> *"The only way to make a C++ program crash-proof is to run it in a separate process."* +> 让 C++ 程序不崩溃的唯一方法，是把它跑在单独的进程里。 + +## 从 OOM 崩溃说起 + +在 CodeScope 的早期版本（v0.1）里，索引是直接在主进程中跑的。调用 `engine_index_project` 后，整个 MCP 服务器就卡住了——单线程事件循环，不能处理任何新请求，直到索引完成。 + +但这不是最严重的问题。最严重的问题是：**当索引一个大型仓库（比如 Linux kernel 或 JDK）时，C++ 引擎的内存占用会飙升到几个 GB，然后 OOM killer 直接杀掉整个进程。** + +MCP 服务器崩溃了，LLM 客户端收到一个 `connection closed` 错误，用户可能会骂一句"什么垃圾软件"。 + +解决方案很直接：**把索引任务放到子进程里跑。** 如果子进程崩溃，主进程不受影响，可以重新调度或优雅降级。 + +## 核心文件 + +``` +server/src/scheduler/worker.rs ← Worker 子进程管理 (~576 行) +server/src/scheduler/mod.rs ← 调度器主逻辑 (~1382 行) +server/src/main.rs ← worker 子命令入口 +``` + +## fork + exec：一个简单的方案 + +Rust 的调度器生成子进程的方式简单粗暴： + +```rust +// server/src/scheduler/worker.rs +pub(super) fn run_module_worker( + exe: &str, + project_dir: &str, + module_name: &str, + files_estimate: u64, + workers: u32, + grammars_dir: &str, + db_prefix: &str, + project_id: u64, + quarantine_exclude: Option<&str>, +) -> ModuleResult { + let module_dir = Path::new(project_dir).join(module_name); + let module_db = format!("{}_{}.db", db_prefix, module_name); + + // 每个 worker 从干净的 DB 文件开始 + let _ = std::fs::remove_file(&module_db); + + let mut cmd = Command::new(exe); + cmd.args([ + "worker", + &module_db, + module_dir.to_str().unwrap_or(""), + "", // empty language filter + &project_name, + &project_id_str, + ]); + cmd.env("GRAMMARS_DIR", grammars_dir); + cmd.env("CODESCOPE_DB_PATH", &module_db); + cmd.env("CODESCOPE_INDEX_MODE", "fast"); + // ... +} +``` + +每个模块启动一个独立的 `codescope worker` 子进程。这个子进程是一个完全独立的进程实例——它有自己的地址空间、自己的全局变量、自己的 `g_store` 单例。这意味着： + +- 子进程崩溃 → 主进程不受影响 +- 子进程内存泄漏 → 进程退出后 OS 自动回收 +- 子进程 OOM → 只有它自己被 kill，主进程继续运行 + +```mermaid +flowchart LR + subgraph "主进程" + Scheduler["并行调度器"] + MCP["MCP 服务器
(处理查询)"] + MCP --->|"查询已索引的 DB"| DB["主 SQLite DB"] + end + + subgraph "Worker 1" + W1["codescope worker
模块: src/"] + W1 -->|"写入"| DB1["模块 DB: main_src.db"] + end + + subgraph "Worker 2" + W2["codescope worker
模块: lib/"] + W2 -->|"写入"| DB2["模块 DB: main_lib.db"] + end + + subgraph "Worker 3" + W3["codescope worker
模块: tests/"] + W3 -->|"写入"| DB3["模块 DB: main_tests.db"] + end + + Scheduler -->|"fork+exec"| W1 + Scheduler -->|"fork+exec"| W2 + Scheduler -->|"fork+exec"| W3 + DB1 --> Merge["DB 合并"] + DB2 --> Merge + DB3 --> Merge + Merge --> DB +``` + +## 超时与隔离 + +每个 worker 都有一个硬超时： + +```rust +const DEFAULT_WORKER_TIMEOUT_SECS: u64 = 600; +``` + +如果 10 分钟内没跑完，子进程会被 `kill`。这防止了某个死循环的解析器占用整个调度器。 + +但有个问题：**怎么知道 worker 是正常退出还是崩溃了？** + +```rust +// 等待子进程退出 +match child.wait_with_output() { + Ok(output) => { + if output.status.success() { + // 正常退出，解析 stdout JSON + } else { + // 崩溃或非零退出码 + // 进入隔离流程 + } + } + Err(e) => { + // wait 本身失败 + } +} +``` + +## 隔离机制：二进制搜索定位崩溃文件 + +当 worker 崩溃时，调度器不会简单地重试整个模块——它会用**二进制搜索**来定位导致崩溃的文件： + +```rust +// server/src/scheduler/mod.rs (简化) +fn quarantine_crashing_file( + module: &Module, + files: &[FileEntry], + crash_index: usize, +) -> Option { + let mut lo = 0; + let mut hi = files.len(); + + while lo < hi { + let mid = (lo + hi) / 2; + let test_files = &files[lo..=mid]; + + // 只索引这些文件 + if run_worker_with_files(module, test_files).is_ok() { + lo = mid + 1; // 崩溃在右侧 + } else { + hi = mid; // 崩溃在左侧（含 mid） + } + } + + // 找到崩溃文件 + Some(files[lo].path.clone()) +} +``` + +找到崩溃文件后，调度器把这个文件加入 `CODESCOPE_EXCLUDE_PATHS`，然后**重试整个模块**——跳过这个文件。 + +```mermaid +flowchart TD + Start["Worker 崩溃"] --> BinarySearch["二进制搜索定位崩溃文件"] + BinarySearch --> Found["找到崩溃文件"] + Found --> Exclude["加入 CODESCOPE_EXCLUDE_PATHS"] + Exclude --> Retry["重试模块（跳过崩溃文件）"] + Retry --> Success["成功"] + Retry --> Fail["又崩溃了"] + Fail --> BinarySearch + + BinarySearch -.->|"注意"| Note["最多 10 次隔离迭代
超过则放弃模块"] +``` + +这个机制在我的测试中定位过几次 tree-sitter 解析器对特定 C++ 模板代码的崩溃。隔离后，整个项目仍然可以索引，只是缺失了那个文件——这比整个索引失败好了 100 倍。 + +## 每个 worker 一个 DB，避免 SQLite 并发写入 + +另一个关键设计是：**每个 worker 写自己的 SQLite 文件。** + +```rust +let module_db = format!("{}_{}.db", db_prefix, module_name); +``` + +SQLite 的 WAL 模式支持并发读取，但并发写入需要锁。如果多个 worker 同时写同一个 DB 文件，锁竞争会导致性能急剧下降，甚至死锁。 + +每个 worker 拥有独立的 DB 文件，意味着： + +- 没有锁竞争 +- 没有死锁风险 +- 索引完成后通过 `ATTACH + INSERT OR IGNORE` 合并 + +```mermaid +flowchart LR + subgraph "并行写入阶段" + W1 -->|"独占写入"| DB1 + W2 -->|"独占写入"| DB2 + W3 -->|"独占写入"| DB3 + end + + subgraph "合并阶段" + DB1 -->|"ATTACH"| Main + DB2 -->|"ATTACH"| Main + DB3 -->|"ATTACH"| Main + Main["主 DB
ATTACH + INSERT OR IGNORE"] + end +``` + +## 一个让我冷汗直流的教训 + +在 v0.2.0 的开发中，我发现**合并后的 DB 中，来自不同模块的实体 ID 冲突了**。 + +每个 worker 的 `project_id` 是调度器分配的（1, 2, 3, ...），但实体 ID 从 1 开始自增。两个模块可能都有 `id=42` 的实体，合并时 `INSERT OR IGNORE` 会丢失第二个模块的实体。 + +修复方案是：**合并时对每个模块的实体 ID 做偏移量重映射。** + +```rust +// server/src/scheduler/merge.rs (简化) +// 对于模块 i > 0： +// 1. 计算 offset = 当前主表的 MAX(id) +// 2. 复制模块数据到临时表 +// 3. 对临时表的 id 和 FK 列加 offset +// 4. INSERT OR IGNORE 到主表 +// 5. DROP 临时表 + +const TABLE_SPECS: &[TableSpec] = &[ + TableSpec { + name: "entity", + remap_cols: &[("id", "self")], // id += offset + skip_rowid: false, + }, + TableSpec { + name: "relation", + remap_cols: &[ + ("id", "self"), // 主键重映射 + ("source_id", "entity"), // FK 跟随 entity 的偏移量 + ("target_id", "entity"), // FK 跟随 entity 的偏移量 + ], + skip_rowid: false, + }, + // ... +]; +``` + +这个重映射需要知道表之间的外键关系：`relation.source_id` 引用 `entity.id`，所以 `source_id` 的偏移量必须和 `entity.id` 的偏移量一致。 + +**表顺序很重要**：被引用的表（entity）必须先合并，引用它的表（relation）后合并。这就是为什么 `TABLE_SPECS` 的顺序是精心排好的。 + +## 总结 + +Worker 子进程隔离是 CodeScope 能在生产级仓库上稳定运行的关键。它解决了三个问题： + +1. **崩溃隔离**：C++ 引擎崩溃不会拖垮 MCP 服务器 +2. **内存回收**：进程退出后所有内存自动释放 +3. **并发写入**：每个 worker 独立的 DB 文件，无锁竞争 + +代价是：进程启动开销（~10ms）、DB 合并的额外步骤、以及那些恼人的 ID 重映射。但相比于主进程被 OOM 杀掉的痛苦，这点代价不值一提。 + +在下一篇文章中，我会拆解**并行调度器**——三种调度模式的设计与取舍，以及为什么一个简单的"静态分配"方案在大型仓库上会失效。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-03-scheduler.md b/docs/articles/zh/codescope-deepdive-03-scheduler.md new file mode 100644 index 0000000..4b284a0 --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-03-scheduler.md @@ -0,0 +1,259 @@ +# CodeScope 拆解 (三)：并行调度器 — 三种调度模式的设计与取舍 + +> *"A scheduler that doesn't know when to give up is worse than no scheduler at all."* +> 不知道什么时候该放弃的调度器，还不如没有调度器。 + +## 问题的起点 + +CodeScope 的索引需要处理大量文件。Linux kernel 有 ~70,000 个源文件，JDK 有 ~40,000 个。如果串行解析，一个文件的 tree-sitter 解析 + IR 构建 + SQLite 写入需要 50-200ms，70,000 个文件就是 1-4 小时。 + +用户等不了 4 小时。所以需要并行。 + +但并行调度不是简单地"开 N 个线程"就能解决的。问题在于： + +1. **模块大小差异巨大**：`src/` 可能有 10,000 个文件，`docs/` 只有 50 个 +2. **子系统资源竞争**：SQLite 写入、内存分配、文件 I/O +3. **崩溃处理**：一个模块崩溃不能影响其他模块 +4. **CPU 核心数限制**：不能无限并行 + +## 核心文件 + +``` +server/src/scheduler/mod.rs ← 调度器主逻辑 (~1382 行) +server/src/scheduler/chunk_plan.rs ← 分块规划 (~524 行) +server/src/scheduler/dyn_config.rs ← 动态调度配置 (~220 行) +server/src/scheduler/shm.rs ← 共享内存 +server/src/scheduler/chunk_queue.rs← 工作窃取队列 +``` + +## 模式一：静态比例分配 (Static Proportional) + +这是最直接的方案，也是 CodeScope 的默认调度模式。 + +```rust +// server/src/scheduler/mod.rs (概念) +fn allocate_workers_static(modules: &[Module], total_workers: u32) -> Vec { + let total_files: u64 = modules.iter().map(|m| m.file_count).sum(); + + modules.iter().map(|m| { + std::cmp::max( + 1, // 每个模块至少 1 个 worker + (m.file_count as f64 / total_files as f64 * total_workers as f64).ceil() as u32 + ) + }).collect() +} +``` + +核心逻辑很简单：**按文件数比例分配 CPU 核心。** + +- 模块 A 有 10,000 个文件（占总文件的 50%）→ 分配 4 个核心（总 8 核） +- 模块 B 有 5,000 个文件（25%）→ 分配 2 个核心 +- 模块 C 有 500 个文件（2.5%）→ 分配 1 个核心（至少 1 个） + +```mermaid +flowchart LR + subgraph "模块发现" + Discover["discover_modules()
扫描顶级目录 + 文件计数"] + end + + subgraph "静态分配" + Alloc["按文件数比例分配 worker"] + W1["模块 A: 4 workers"] + W2["模块 B: 2 workers"] + W3["模块 C: 1 worker"] + W4["模块 D: 1 worker"] + end + + subgraph "并行执行" + WA["Worker 子进程
CODESCOPE_SKIP_ASYNC=1"] + WB["Worker 子进程"] + WC["Worker 子进程"] + WD["Worker 子进程"] + end + + subgraph "合并" + Merge["DB 合并
ATTACH + INSERT OR IGNORE"] + end + + Discover --> Alloc + Alloc --> W1 --> WA + Alloc --> W2 --> WB + Alloc --> W3 --> WC + Alloc --> W4 --> WD + WA --> Merge + WB --> Merge + WC --> Merge + WD --> Merge +``` + +**优点**：简单、可预测、不需要运行时监控。 + +**缺点**：如果模块 A 的 10,000 个文件都是小文件，而模块 B 的 5,000 个文件都是大文件，分配就偏离了实际工作量。小模块完成后，它的核心闲置，直到所有模块都跑完。 + +## 模式二：CPU 动态调度 (CPU-Dynamic / Chunked Work-Stealing) + +静态分配的"核心浪费"问题，在大型仓库（>50,000 文件）上变得不可接受。于是有了第二种模式：**工作窃取**。 + +```rust +// server/src/scheduler/dyn_config.rs +pub fn should_enable(&self, total_modules: usize, total_files: u64) -> bool { + match self.force_on { + Some(b) => b, + None => total_modules > 4 && total_files > 10000, + } +} +``` + +这个模式默认是**opt-in**的——只有通过环境变量 `CODESCOPE_CPU_DYNAMIC=1` 或项目超过 10,000 文件 + 4 个模块时才会启用。 + +它的核心思想是：**把文件切分成更小的块（chunk），然后让多个 worker 动态地从共享队列中领取任务。** + +```rust +// server/src/scheduler/chunk_plan.rs +/// 默认每个 chunk 8 MB +pub const TARGET_BYTES: u64 = 8 * 1024 * 1024; +/// 最大 32 MB +pub const MAX_BYTES: u64 = 32 * 1024 * 1024; +/// 按前 2 级路径进行目录聚类 +pub const CLUSTER_DEPTH: usize = 2; +``` + +```mermaid +flowchart TD + subgraph "文件分块" + Files["70,000 个源文件"] + Cluster["按目录聚类
(前 2 级路径)"] + Split["按 8MB 切分"] + Chunks["100~250 个 chunk"] + end + + subgraph "工作窃取" + Q["共享任务队列"] + W1["Worker 1"] -->|"领取"| Q + W2["Worker 2"] -->|"领取"| Q + W3["Worker 3"] -->|"领取"| Q + W4["Worker 4"] -->|"领取"| Q + W1 -->|"完成后"| Q + W2 -->|"完成后"| Q + end + + subgraph "共享内存协调" + SHM["SchedShm
共享内存状态"] + SHM -->|"空闲 worker 计数"| Q + SHM -->|"RSS 监控"| Q + end + + Files --> Cluster --> Split --> Chunks --> Q +``` + +**分块算法**： + +1. 按前 2 级路径聚类（`src/compiler/`、`src/parser/`） +2. 按字节大小降序排序 +3. 贪心打包：文件累积到 `TARGET_BYTES`（8MB）为止 +4. 巨大目录（>32MB）强制拆分 + +这样同一个目录的文件大概率在同一个 chunk 里，保持了**目录局部性**——相关文件一起解析，减少了跨模块的上下文切换。 + +**内存监控**：动态调度器还有一个内存限制机制。当 RSS 超过阈值（默认 4GB）时，不再分配新的 chunk，防止 OOM。 + +```rust +// server/src/scheduler/dyn_config.rs +pub fn from_env() -> Self { + let mem_limit_mb = env::var("CODESCOPE_MEM_LIMIT_MB") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(4096); + // ... +} +``` + +## 模式三：内存批量索引 (Memory Bulk / Membulk) + +对于小模块（<= 文件数阈值），调度器采用第三种模式：**内存批量索引**。 + +```cpp +// engine/src/engine_internal.h +/// For small modules (<= kMemBulkFileThreshold files) the parse workers +/// aggregate FileResult in memory instead of pushing through BoundedQueue, +/// then flush once via insertFileResultBatch. +char *engine_index_project_membulk( + uint64_t project_id, const std::string &dir, uint64_t max_file_size, + const FilterPolicy &filter, + const std::vector> &job_lang, + const std::unordered_map &lang_ptrs, + bool is_reindex, bool mode_fast, bool mode_deep); +``` + +为什么需要这个模式？因为 **BoundedQueue**（线程安全的生产者-消费者队列）的开销对小型模块来说太大了。如果模块只有 50 个文件，每个文件解析后都要 push 到队列、由另一个线程 pop 然后写入 SQLite，这个同步开销可能占到总时间的 30%。 + +对于小模块，更好的策略是：**所有解析线程把结果聚合在内存中，解析完成后一次性批量写入。** + +```mermaid +flowchart LR + subgraph "流式路径 (大模块)" + F1["文件 1"] -->|"解析"| Queue["BoundedQueue"] + F2["文件 2"] -->|"解析"| Queue + F3["文件 N"] -->|"解析"| Queue + Queue -->|"pop"| Writer["单线程写入 SQLite"] + end + + subgraph "内存批量路径 (小模块)" + MF1["文件 1"] -->|"解析"| Mem["内存聚合"] + MF2["文件 2"] -->|"解析"| Mem + MF3["文件 N"] -->|"解析"| Mem + Mem -->|"一次性批量写入"| BatchWriter["insertFileResultBatch"] + end +``` + +## 调度模式的选择标准 + +三种模式对应三种场景： + +```mermaid +flowchart TD + Start["开始索引"] --> ModuleCount{"模块数 > 4
且文件数 > 10,000?"} + ModuleCount -->|"否"| Static["静态比例分配
简单可靠"] + ModuleCount -->|"是"| Dynamic["CPU 动态调度
工作窃取"] + + Start --> FileCount{"单个模块
文件数 < 阈值?"} + FileCount -->|"是"| Membulk["内存批量索引
减少队列开销"] + FileCount -->|"否"| Streaming["流式索引
BoundedQueue"] +``` + +## 一个让我冷汗直流的教训 + +在 v0.2.2 中，我遇到了一个**死锁**。 + +动态调度模式下，多个 worker 通过共享内存（`SchedShm`）协调任务分配。当所有 chunk 都被领取后，最后一个 worker 会发送"完成"信号，然后调度器开始合并 DB。 + +但有一个边缘情况：**如果某个 worker 在领取 chunk 后、开始工作前被 OS 调度出去**（比如因为其他进程的 CPU 竞争），其他 worker 可能已经完成了所有工作并开始合并。当这个被延迟的 worker 醒来后，它还在写自己的 DB，但合并线程已经在读它的 DB 了——SQLite 的 WAL 模式下，读取和写入可以并发，但如果我们用的是 `INSERT OR IGNORE` 合并，而 worker 还在写入，就会出现**部分合并**的问题。 + +修复方案：**在合并前，确保所有 worker 都已退出。** 调度器不再依赖共享内存中的"完成"信号，而是等待每个子进程的 `wait()` 返回。 + +```rust +// 修复后的逻辑 +for handle in worker_handles { + let result = handle.join().unwrap_or_else(|_| { + // 线程 panic 或 worker 崩溃 + ModuleResult::crashed(module_name) + }); + results.push(result); +} +// 所有 worker 都确认退出后，才开始合并 +merge_results(results, &db_path); +``` + +## 总结 + +三种调度模式不是一开始就设计好的，而是随着问题逐个出现而逐步加入的： + +1. **静态比例分配**：简单、可预测，适合大多数项目 +2. **CPU 动态调度**：解决大型仓库的核心浪费问题，但需要 opt-in +3. **内存批量索引**：解决小模块的队列开销问题，是性能优化 + +调度器的核心设计原则是：**调度器只管理 CPU 核心分配，不参与文件发现、解析或图构建。** 它不知道也不关心每个文件的内容，它只关心"哪个模块有多少文件"和"哪个核心空闲"。 + +这种关注点分离，让调度器可以在未来轻松替换——比如换成一个 Kubernetes 集群调度器，或者一个基于优先级的队列。 + +在下一篇文章中，我会拆解**两阶段索引**——Fast Scan 如何在毫秒级返回结果，以及 Background Enhance 在后台做了什么。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-04-two-phase-index.md b/docs/articles/zh/codescope-deepdive-04-two-phase-index.md new file mode 100644 index 0000000..7f04d69 --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-04-two-phase-index.md @@ -0,0 +1,180 @@ +# CodeScope 拆解 (四)：两阶段索引 — 毫秒级响应与深度分析的权衡 + +> *"The user doesn't care about your graph database. They care about getting an answer before they forget the question."* +> 用户不在乎你的图数据库。他们在乎的是在忘记问题之前得到答案。 + +## 问题：索引太慢了 + +CodeScope 的完整索引流程包括：文件发现 → tree-sitter 解析 → IR 构建 → 指标计算 → 图构建 → 图增强（社区检测、聚类）。对于 Linux kernel 这种规模的仓库，完整索引需要 5-15 分钟。 + +LLM 的等待耐心是有限的。如果用户发起一个 `codescope_index_project` 请求，然后等 10 分钟才能查询，这个工具就没法用。 + +但有个好消息：**大多数查询只需要基本的符号信息**（定义、引用、调用关系），而不需要图分析。如果能在几秒内返回基本的符号索引，然后让图分析在后台悄悄完成，用户就感知不到延迟。 + +这就是两阶段索引的由来。 + +## 核心文件 + +``` +server/src/scheduler/worker.rs ← Worker 中的 CODESCOPE_SKIP_ASYNC +server/src/engine_ffi.cpp ← engine_index_project 实现 +engine/src/engine_internal.h ← engine_index_post_parse 声明 +engine/src/engine_helpers.cpp ← 索引辅助函数 +``` + +## Phase A：Fast Scan（毫秒 ~ 秒级） + +Fast Scan 的目标是：**尽可能快地返回可查询的索引。** + +它只做必要的工作： + +1. 文件发现 + 过滤（8 层过滤策略） +2. tree-sitter 解析（生成 AST） +3. IR 翻译（统一中间表示） +4. 符号提取（实体、关系） +5. SQLite 写入 + +```mermaid +flowchart LR + subgraph "Phase A: Fast Scan (毫秒~秒)" + FD["文件发现
FilterPolicy 8 层过滤"] + Parse["tree-sitter 解析
AST 构建"] + IR["IR 翻译
统一中间表示"] + SQL["SQLite 写入
entity + relation"] + end + + subgraph "Phase B: Background Enhance (异步秒级)" + Graph["图构建
符号图 + 调用图"] + Metric["指标计算
圈复杂度等"] + Enhance["图增强
社区检测等"] + FTS["FTS5 全文索引
构建"] + end + + FD --> Parse --> IR --> SQL + SQL -->|"CODESCOPE_SKIP_ASYNC=1"| Skip["跳过 Phase B"] + SQL -->|"正常模式"| Graph --> Metric --> Enhance --> FTS +``` + +在 worker 并行索引阶段，每个 worker 设置 `CODESCOPE_SKIP_ASYNC=1`： + +```rust +// server/src/scheduler/worker.rs +cmd.env("CODESCOPE_INDEX_MODE", "fast"); +``` + +这意味着 worker 只做 Phase A，跳过 Phase B。合并后的 DB 可以在几秒内准备好，**用户可以立即开始查询。** + +## Phase B：Background Enhance（异步，秒~分钟级） + +Phase B 在 Phase A 完成后启动，执行一系列**计算密集型**操作： + +1. **图构建**：从 `entity` 和 `relation` 表构建符号图 +2. **指标计算**：圈复杂度、认知复杂度、嵌套深度等 +3. **图增强**：社区检测、聚类分析 +4. **FTS5 全文索引**：为代码搜索构建倒排索引 + +```cpp +// engine/src/engine_internal.h +/// Index Project: in-memory bulk path + shared post-parse +/// +/// For small modules (<= kMemBulkFileThreshold files) the parse workers +/// aggregate FileResult in memory instead of pushing through BoundedQueue, +/// then flush once via insertFileResultBatch. The post-parse graph-building +/// sequence is shared with the streaming path via engine_index_post_parse. +char *engine_index_project_membulk( + uint64_t project_id, const std::string &dir, uint64_t max_file_size, + const FilterPolicy &filter, + const std::vector> &job_lang, + const std::unordered_map &lang_ptrs, + bool is_reindex, bool mode_fast, bool mode_deep); +``` + +`mode_fast` 参数控制是否跳过 Phase B。`mode_deep` 参数控制是否执行额外的深度分析（如漂移检测的预处理）。 + +## 8 层过滤策略 + +在 Phase A 的文件发现阶段，FilterPolicy 应用了 8 层过滤来减少需要解析的文件数量： + +```cpp +// engine/src/filter_policy.h +class FilterPolicy { + // ... + bool shouldSkipEntry(const std::string &rel_path, bool is_dir) const; +}; +``` + +```mermaid +flowchart TD + F["36,919 个文件"] --> L1["层 1: 路径组件跳过
node_modules, .venv, target..."] + L1 --> L2["层 2: .gitignore 匹配"] + L2 --> L3["层 3: .codescopeignore 匹配"] + L3 --> L4["层 4: 文件名精确跳过
package-lock.json, .DS_Store..."] + L4 --> L5["层 5: 文件名前缀跳过
.env.*, yarn-error.log.*"] + L5 --> L6["层 6: 后缀名跳过
.exe, .zip, .min.js..."] + L6 --> L7["层 7: 非源文件后缀
二进制, 归档, 媒体文件"] + L7 --> L8["层 8: 文件大小检查
太大/太小"] + L8 --> Result["6,029 个候选文件
(~83% 被过滤)"] +``` + +在真实项目上，这 8 层过滤可以过滤掉约 83% 的文件。对于 Linux kernel 这种包含大量文档、脚本、配置、二进制文件的仓库，过滤效果尤其显著。 + +## 渐进式就绪的用户体验 + +两阶段索引的核心价值在于**渐进式就绪**： + +```mermaid +sequenceDiagram + participant User as 用户 + participant MCP as MCP 服务器 + participant Sched as 调度器 + participant Worker as Worker 子进程 + + User->>MCP: codescope_index_project + MCP->>Sched: 开始索引 + Sched->>Worker: fork+exec (SKIP_ASYNC=1) + Worker-->>Sched: Phase A 完成 (2s) + Sched-->>MCP: 返回索引结果 (fast) + + User->>MCP: codescope_find_definition + MCP-->>User: 立即返回结果 ✓ + + Note over Sched,Worker: Phase B 在后台继续 + Worker->>Worker: 图构建 + 指标计算 + FTS5 + Worker-->>Sched: Phase B 完成 (30s later) + + User->>MCP: codescope_graph_query + MCP-->>User: 图查询可用 ✓ +``` + +用户发出索引请求后，在 2 秒内就能收到"索引完成"的响应，然后可以立即查询定义和引用。而图分析和全文搜索的结果在 30 秒后可用。 + +如果用户感知不到后台还在工作，那这个设计就是成功的。 + +## 一个让我冷汗直流的教训 + +在 v0.2.0 中，我发现**合并后的 DB 缺少 FTS5 索引**。 + +原因是：每个 worker 在 Phase A 时跳过了 FTS5 构建（`CODESCOPE_SKIP_ASYNC=1`），合并后的 DB 也没有触发 FTS5 重建。结果 `search_code` 工具返回空结果，而用户确信代码中确实包含了某个关键字。 + +修复方案：**在合并步骤后，对主 DB 执行一次完整的 FTS5 重建。** + +但这又带来了另一个问题：FTS5 重建的时间可能很长（对于大型仓库，需要 5-10 秒）。如果每次索引合并后都重建 FTS5，用户等待第一次查询的时间就延长了。 + +最终的方案是：**在合并后立即重建 FTS5，但重建过程是增量式的——只对新插入的行建立索引，不对已有行重新索引。** + +```sql +-- 重建 FTS5 索引 +INSERT INTO code_fts(code_fts) +VALUES('rebuild'); +``` + +## 总结 + +两阶段索引是 CodeScope 在"响应速度"和"分析深度"之间的权衡方案： + +- **Phase A (Fast Scan)**：只做符号提取，跳过图分析，让用户在毫秒~秒级获得可查询的索引 +- **Phase B (Background Enhance)**：在后台执行图构建、指标计算、FTS5 索引，完成后自动提升查询能力 + +这个方案不是完美的——如果用户在图构建完成前就发起图查询，会收到"数据尚未就绪"的提示。但大多数时候，用户的前几个查询都是符号查找，图查询在之后才用到。两阶段索引正好符合这个使用模式。 + +在下一篇文章中，我会拆解**统一 IR 与语言翻译器**——8 种语言如何共享同一个代码模型。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-05-parser-ir.md b/docs/articles/zh/codescope-deepdive-05-parser-ir.md new file mode 100644 index 0000000..abe989a --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-05-parser-ir.md @@ -0,0 +1,244 @@ +# CodeScope 拆解 (五)：tree-sitter 解析器与统一 IR — 8 种语言一个模型 + +> *"A programming language is a tool. Code analysis should not care which tool you used."* +> 编程语言是工具。代码分析不应该关心你用了哪个工具。 + +## 问题：8 种语言，一个格式 + +CodeScope 支持 8 种语言：C、C++、Rust、Go、JavaScript、TypeScript、Python、Java。 + +每种语言都有自己的 AST 结构、语法规则、作用域规则。如果为每种语言写一套独立的分析代码，维护成本会迅速失控。 + +解决方案是：**用统一的 IR（Intermediate Representation）来抽象所有语言**。 + +## 核心文件 + +``` +engine/src/parser/parser.cpp ← tree-sitter 解析器封装 +engine/src/parser/parser.h ← Parser 接口 +engine/src/ir/ir.h ← 统一 IR 定义 +engine/src/ir/ir.cpp ← IR 构建 +engine/src/ir/ir_translator.h ← IR 翻译器基类 +engine/src/ir/ir_translator.cpp ← IR 翻译器实现 +``` + +## tree-sitter：一个解析器，多种语言 + +tree-sitter 是一个增量解析库，支持多种语言的语法定义。CodeScope 用 tree-sitter 作为底层解析器，每种语言只需要一个 `.so` 语法文件。 + +```cpp +// engine/src/parser/parser.h (概念) +class Parser { + std::unordered_map languages_; + // ... +}; +``` + +解析流程： + +1. 检测文件语言（根据扩展名或 shebang） +2. 加载对应的 tree-sitter 语法（`.so` 文件） +3. 解析文件生成 AST（CST 实际上是 concrete syntax tree） +4. 遍历 AST 提取符号信息 + +```mermaid +flowchart LR + subgraph "源文件" + C["main.c"] + RS["lib.rs"] + PY["utils.py"] + JS["app.js"] + end + + subgraph "tree-sitter 语法" + C_G["tree-sitter-c.so"] + RS_G["tree-sitter-rust.so"] + PY_G["tree-sitter-python.so"] + JS_G["tree-sitter-javascript.so"] + end + + subgraph "统一 IR" + IR["ir::Record
名称 | 类型 | 位置 | 关系"] + end + + C --> C_G --> IR + RS --> RS_G --> IR + PY --> PY_G --> IR + JS --> JS_G --> IR +``` + +## 统一 IR 定义 + +IR 的核心数据结构是 `ir::Record`： + +```cpp +// engine/src/ir/ir.h (概念) +namespace ir { + +struct Record { + std::string name; // 符号名称 + std::string kind; // 符号类型 (function, class, variable, ...) + std::string file_path; // 文件路径 + int line; // 起始行 + int col; // 起始列 + int end_line; // 结束行 + int end_col; // 结束列 + std::vector relations; // 与其他符号的关系 + std::vector children; // 嵌套符号 +}; + +struct Relation { + std::string kind; // 关系类型 (call, reference, inherit, contain, ...) + std::string target_name; // 目标符号名称 + int target_line; // 目标位置 + int target_col; +}; + +} // namespace ir +``` + +这个 IR 设计的关键是：**足够抽象，能表达所有语言的结构；又足够具体，能保留足够的信息用于查询。** + +## IR 翻译器 + +每种语言的 tree-sitter AST 到统一 IR 的转换由 `IrTranslator` 实现： + +```cpp +// engine/src/ir/ir_translator.h (概念) +class IrTranslator { +public: + virtual std::vector translate(const TSNode &root, + const std::string &source) = 0; + virtual ~IrTranslator() = default; +}; +``` + +每种语言有一个翻译器子类： + +```mermaid +flowchart TD + IrTranslator["IrTranslator (基类)"] + C_Trans["CTranslator"] + CPP_Trans["CppTranslator"] + Rust_Trans["RustTranslator"] + Go_Trans["GoTranslator"] + JS_Trans["JsTranslator"] + TS_Trans["TsTranslator"] + Python_Trans["PythonTranslator"] + Java_Trans["JavaTranslator"] + + IrTranslator --> C_Trans + IrTranslator --> CPP_Trans + IrTranslator --> Rust_Trans + IrTranslator --> Go_Trans + IrTranslator --> JS_Trans + IrTranslator --> TS_Trans + IrTranslator --> Python_Trans + IrTranslator --> Java_Trans +``` + +每个翻译器知道如何从特定语言的 AST 中提取函数定义、类定义、变量声明、函数调用等。 + +## 关系提取 + +IR 的 `Relation` 结构是 CodeScope 图查询的基础。翻译器在遍历 AST 时，会识别以下关系： + +- **call**: 函数调用关系 +- **reference**: 符号引用 +- **inherit**: 继承关系 +- **contain**: 包含关系（类包含方法，函数包含变量） +- **implement**: 实现关系（接口实现） + +```mermaid +flowchart LR + subgraph "C 代码" + C_CODE["void foo() { bar(); }"] + end + + subgraph "tree-sitter AST" + C_AST["translation_unit + └── function_definition + ├── type: void + ├── name: foo + └── body + └── call_expression + └── name: bar"] + end + + subgraph "统一 IR" + IR_REC["Record: foo + kind: function + file: main.c + line: 1 + relations: + - call -> bar"] + end + + C_CODE --> C_AST --> IR_REC +``` + +## 增量解析 + +tree-sitter 的一个关键特性是**增量解析**。当文件被修改时，tree-sitter 可以只重新解析被修改的部分，而不是整个文件。 + +CodeScope 利用这个特性来实现增量索引： + +```cpp +// 概念: 增量索引伪代码 +TSTree *old_tree = cache.get(file_path); +TSTree *new_tree = ts_parser_parse_incremental( + parser, old_tree, new_input); +``` + +在增量模式下，重新索引一个已修改的文件只需要几毫秒，而不是几十毫秒。 + +## 一个让我冷汗直流的教训 + +在开发 IR 翻译器时，我遇到了一个**C++ 模板的解析问题**。 + +```cpp +template +T add(T a, T b) { + return a + b; +} +``` + +这个简单的模板函数，在 tree-sitter 的 C++ 语法中，`T` 被解析为 `type_identifier`，`a` 和 `b` 被解析为 `declaration`。IR 翻译器需要正确识别： + +- 函数名是 `add`，不是 `T` +- `T` 是模板参数，不是函数参数 +- `a` 和 `b` 是函数参数，类型是 `T` +- 返回类型是 `T` + +但最初的翻译器实现把 `T` 识别为了函数名（因为它是 AST 中第一个 `identifier` 节点），导致索引后的函数名是 `T` 而不是 `add`。 + +修复方案：**在翻译器中，先检查 `template` 节点，再处理函数定义。** + +```cpp +// 修复后的 C++ 翻译器逻辑 (概念) +if (node_kind == "template_declaration") { + // 先处理模板参数 + auto template_params = extract_template_params(node); + // 然后处理内部的函数/类定义 + for (auto child : ts_node_named_children(node)) { + if (is_function_definition(child)) { + return translate_function(child, source, template_params); + } + } +} +``` + +这个 bug 暴露了一个更深层次的问题：**统一 IR 设计假定所有语言的结构是相似的，但 C++ 模板与 Java 泛型、Rust 泛型在 AST 层面有本质差异。** 翻译器必须在保持 IR 抽象的同时，处理这些语言特定的细节。 + +## 总结 + +统一 IR 是 CodeScope 能够支持多语言的核心抽象： + +- **tree-sitter** 提供底层解析能力，每种语言一个语法文件 +- **IR 翻译器** 将语言特定的 AST 转换为统一的数据结构 +- **Relation 提取** 在遍历 AST 时完成，构建符号关系图 +- **增量解析** 让重新索引变得高效 + +这个设计不是完美的——每种语言的特殊情况（C++ 模板、Rust 宏、Python 装饰器）让翻译器变得复杂。但相比于为每种语言写一套独立的分析工具，统一 IR 的维护成本要低得多。 + +在下一篇文章中，我会拆解**SQLite 图谱存储**——如何用关系型数据库存储代码图。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-06-sqlite-storage.md b/docs/articles/zh/codescope-deepdive-06-sqlite-storage.md new file mode 100644 index 0000000..d978fd4 --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-06-sqlite-storage.md @@ -0,0 +1,335 @@ +# CodeScope 拆解 (六)：SQLite 图谱存储 — Schema 设计与 FTS + +> *"A graph database is a great idea. A graph database that requires a server process is a deployment nightmare."* +> 图数据库是个好主意。但需要启动一个服务进程的图数据库，就是部署噩梦了。 + +## 问题：代码图存哪里？ + +CodeScope 的核心输出是**代码符号图**：节点是函数、类、变量，边是调用、引用、继承关系。这个图必须持久化，支持后续查询。 + +选项有三个： + +1. **内存图** — 快，但索引完就没了，没法增量复用 +2. **专用图数据库** — Neo4j、KuzuDB 功能强大，但用户要装服务、配端口、处理认证 +3. **SQLite** — 零配置，单文件，进程内嵌入 + +CodeScope 选的是第三条路：**SQLite 为主，LadybugDB（嵌入式图数据库）为可选加速层。** + +## 核心文件 + +``` +engine/src/store/store.h ← GraphStore 接口 +engine/src/store/store_schema.cpp ← 所有 CREATE TABLE / INDEX DDL +engine/src/store/store_core.cpp ← SQLite 操作实现 +engine/src/store/store_ladybug_core.cpp ← LadybugDB 图存储集成 +``` + +## Schema 设计：三层存储 + +CodeScope 的数据库 Schema 分三层： + +```mermaid +flowchart TD + subgraph "Layer 1: 原始数据" + SR["semantic_records
flat parse output"] + FILES["files
file metadata + hash"] + end + + subgraph "Layer 2: 图结构" + GN["graph_nodes
symbol nodes"] + GE["graph_edges
call/reference edges"] + ADJ["adjacency
CSR BLOB (O(1) query)"] + end + + subgraph "Layer 3: 知识层" + CAP["capability / contract
project claims"] + EVI["evidence / finding
verification results"] + FTS["code_fts / name_trgm
full-text search"] + end + + SR --> GN --> GE --> ADJ + FILES --> GN + GN --> CAP --> EVI + GN --> FTS +``` + +### Layer 1: 原始解析数据 + +最核心的表是 `semantic_records`： + +```sql +CREATE TABLE IF NOT EXISTS semantic_records ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + original_id INTEGER NOT NULL, -- 文件内 ID（从 1 开始） + project_id INTEGER NOT NULL, + kind INTEGER NOT NULL, -- 符号类型（function/class/variable/...） + name TEXT, + qualified_name TEXT DEFAULT '', + parent_id INTEGER DEFAULT 0, -- 父符号 ID（containment 关系） + arity INTEGER DEFAULT 0, -- 函数参数个数（用于重载消歧） + is_static INTEGER DEFAULT 0, + type_name TEXT DEFAULT '', + call_kind INTEGER DEFAULT 0, -- 0=direct, 1=method, 2=interface, ... + visibility INTEGER NOT NULL DEFAULT 0, + start_row INTEGER DEFAULT 0, start_col INTEGER DEFAULT 0, + end_row INTEGER DEFAULT 0, end_col INTEGER DEFAULT 0, + file_path TEXT NOT NULL, + language TEXT DEFAULT '' +); +``` + +这个表的设计有一个关键决策：**flat 存储，不建图**。所有解析结果平铺在一个表里，`parent_id` 表示 containment 关系。图结构是后续 `buildGraph()` 阶段从 `semantic_records` 构建出来的。 + +为什么这样设计？因为解析阶段是并行的，多个 worker 同时写同一个表。如果直接构建图结构，需要跨文件的 ID 协调，复杂度太高。Flat 存储让每个 worker 可以独立写入，互不干扰。 + +### Layer 2: 图结构 + +`buildGraph()` 从 `semantic_records` 构建出 `graph_nodes` 和 `graph_edges`： + +```sql +CREATE TABLE IF NOT EXISTS graph_nodes ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL, + ir_node_id INTEGER NOT NULL, + node_type INTEGER NOT NULL, + name TEXT NOT NULL, + qualified_name TEXT, + file_path TEXT NOT NULL, + language TEXT NOT NULL, + start_row INTEGER NOT NULL, start_col INTEGER NOT NULL, + end_row INTEGER NOT NULL, end_col INTEGER NOT NULL, + is_stub INTEGER DEFAULT 0, + visibility INTEGER NOT NULL DEFAULT 0, + ... +); + +CREATE TABLE IF NOT EXISTS graph_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + source_node_id INTEGER NOT NULL, + target_node_id INTEGER NOT NULL, + edge_type INTEGER NOT NULL, + graph_type TEXT NOT NULL DEFAULT 'symbol_reference', + call_site_file TEXT DEFAULT '', + call_site_line INTEGER DEFAULT 0, + ... +); +``` + +`graph_edges` 有一个 `UNIQUE(project_id, source_node_id, target_node_id, edge_type, graph_type)` 约束。这个约束看起来简单，但踩过坑——最初版本没有 `graph_type` 字段，导致 `symbol_reference` 和 `call_graph` 两种图在同一对节点之间产生冲突。 + +#### 查询优化：CSR BLOB + +对于 `getCallers` / `getCallees` 这类高频查询，CodeScope 用了 **CSR（Compressed Sparse Row）BLOB** 优化： + +```sql +CREATE TABLE IF NOT EXISTS adjacency ( + src_id INTEGER PRIMARY KEY, -- 调用者节点 ID + project_id INTEGER NOT NULL, + tgt_blob BLOB -- packed u32[] 被调用者节点 ID 列表 +); + +CREATE TABLE IF NOT EXISTS adjacency_rev ( + tgt_id INTEGER PRIMARY KEY, -- 被调用者节点 ID + project_id INTEGER NOT NULL, + src_blob BLOB -- packed u32[] 调用者节点 ID 列表 +); +``` + +查询时，一次 B-tree 查找拿到整个 BLOB，然后指针算术直接遍历： + +``` +getCallees(id): + 1. B-tree lookup on adjacency WHERE src_id = id + 2. Get tgt_blob (packed u32 array) + 3. Cast pointer: u32* arr = (u32*)blob.data + 4. Return arr[0..len/4] +``` + +**O(1) B-tree 查找 + O(n) 遍历**，比 `graph_edges` 上的 JOIN 快两个数量级。 + +### Layer 3: 全文搜索 + +CodeScope 使用 SQLite FTS5 实现全文搜索： + +```sql +-- Unicode 分词器，支持中文 +CREATE VIRTUAL TABLE IF NOT EXISTS code_fts USING fts5( + name, qualified_name, file_path, content, + project_id UNINDEXED, + node_id UNINDEXED, + node_kind UNINDEXED, + tokenize='unicode61' +); + +-- Trigram 分词器，支持模糊搜索 +CREATE VIRTUAL TABLE IF NOT EXISTS name_trgm USING fts5( + name, qualified_name, + project_id UNINDEXED, + node_id UNINDEXED, + node_type UNINDEXED, + tokenize='trigram' +); +``` + +两个 FTS 索引服务不同的场景： + +- **`code_fts`**：标准全文搜索，匹配符号名、限定名、文件路径 +- **`name_trgm`**：Trigram 子串搜索，支持 `LIKE '%substr%'` 模式的模糊匹配 + +Trigram 索引是专门为百万节点项目设计的。没有它，`LIKE '%foo%'` 查询要走全表扫描，在大型项目上轻松超过 30 秒的 MCP 超时阈值。 + +## 批处理优化 + +最初的实现是一条一条 INSERT，性能很差。优化后改为批量插入： + +```cpp +// store.h (概念) +void insertGraphNodes(uint64_t project_id, + const std::vector &nodes); +void insertFileResultBatch(uint64_t project_id, + const std::vector &batch, + bool is_reindex = true); +``` + +批量插入的核心思想：**prepare 一次，bind/step/reset 循环**。 + +```cpp +// 概念: 批量插入伪代码 +sqlite3_stmt *stmt; +sqlite3_prepare_v2(db, "INSERT INTO graph_nodes VALUES (...)", -1, &stmt, NULL); + +for (const auto &node : nodes) { + sqlite3_bind_int64(stmt, 1, node.id); + sqlite3_bind_text(stmt, 2, node.name.c_str(), -1, SQLITE_TRANSIENT); + // ... + sqlite3_step(stmt); + sqlite3_reset(stmt); +} +sqlite3_finalize(stmt); +``` + +性能提升约 **10 倍**。原因不是 SQLite 变快了，而是减少了 `prepare`/`finalize` 的开销——每个 `prepare` 都要走 SQL 解析和 VDBE 编译。 + +## LadybugDB：可选的图存储加速 + +SQLite 是关系型数据库，图查询（如 k-hop 遍历、最短路径）需要递归 CTE 或多表 JOIN，性能不够理想。 + +LadybugDB 是一个嵌入式图数据库（类似 KuzuDB），CodeScope 将其作为可选的加速层： + +```cpp +// store.h +bool initLadybugDB(); // 创建 .lbug 文件 +bool hasLadybugDB() const; // 检查是否可用 +bool isGraphReady() const; // 检查图数据是否已填充 +``` + +数据流： + +```mermaid +flowchart LR + subgraph "索引阶段" + PARSE["Parse Worker"] --> SQLITE["SQLite
semantic_records"] + SQLITE --> BUILD["buildGraph()"] + BUILD --> GN["graph_nodes + graph_edges"] + end + + subgraph "同步阶段" + GN --> LB["LadybugDB
.lbug 文件"] + LB --> SYNC["lbug_sync_state
增量同步追踪"] + end + + subgraph "查询阶段" + Q["查询请求"] --> LB_QUERY{"LadybugDB
可用?"} + LB_QUERY -->|是| LB_FAST["LadybugDB 查询
O(log n) 图遍历"] + LB_QUERY -->|否| SQLITE_Q["SQLite 查询
JOIN + CTE"] + end +``` + +LadybugDB 同步是增量式的，`lbug_sync_state` 表追踪最后同步的节点和边 ID，避免全量重建。 + +## 一个让我冷汗直流的教训 + +**SQLite 的并发写入问题。** + +在最初的设计中，多个解析 worker 并行写入同一个 SQLite 数据库文件。结果频繁出现 `SQLITE_BUSY (5)` 错误。 + +SQLite 的 WAL 模式支持读并发，但写操作仍然是串行的。多个 worker 同时写，必然产生锁冲突。 + +```mermaid +sequenceDiagram + participant W1 as Worker 1 + participant W2 as Worker 2 + participant SQL as SQLite + + W1->>SQL: BEGIN + W2->>SQL: BEGIN + SQL->>W1: OK + SQL->>W2: OK + W1->>SQL: INSERT + SQL->>W1: OK + W2->>SQL: INSERT + SQL->>W2: SQLITE_BUSY + W2->>SQL: RETRY... + SQL->>W2: SQLITE_BUSY + W2->>SQL: RETRY... + SQL->>W2: OK (after W1 commits) +``` + +修复方案：**引入 BoundedQueue，单线程写。** + +```mermaid +flowchart LR + W1["Parse Worker 1"] --> QUEUE + W2["Parse Worker 2"] --> QUEUE + W3["Parse Worker 3"] --> QUEUE + W4["Parse Worker 4"] --> QUEUE + + subgraph QUEUE["BoundedQueue (内存管道)"] + Q["FIFO 队列"] + end + + Q --> WRITER["单线程 Writer
insertFileResultBatch()"] + WRITER --> DB["SQLite"] +``` + +解析 worker 只负责解析，结果推入内存队列。一个专门的 writer 线程从队列中取出数据，批量写入 SQLite。这样： + +- 写入是串行的，没有锁冲突 +- 批量写入可以合并多个文件的结果，充分利用 `insertFileResultBatch()` +- 解析 worker 不需要等待 I/O，可以立刻处理下一个文件 + +## 索引设计 + +Schema 里大量索引不是一次性创建的。CodeScope 的做法是：**先批量加载数据，再建索引**。 + +```cpp +// 概念: 两阶段建索引 +bool GraphStore::createSchema() { + // 1. 建表（无索引） + exec("CREATE TABLE IF NOT EXISTS graph_nodes (...);"); + // 2. 加载数据... +} + +bool GraphStore::createIndexesAfterBulkLoad() { + // 3. 数据加载完成后建索引 + exec("CREATE INDEX IF NOT EXISTS idx_gn_file_row_type ON ..."); + exec("CREATE INDEX IF NOT EXISTS idx_ge_callers ON ..."); + // ... +} +``` + +为什么？因为 SQLite 在插入数据时维护索引的开销很大。先插入数据、再批量建索引，整体速度更快。 + +## 总结 + +CodeScope 的存储层设计是一个务实的选择： + +- **SQLite 为主**，零配置、单文件、进程内嵌入 +- **Flat 存储**，解析阶段不做图构建，降低并发复杂度 +- **CSR BLOB** 优化高频查询，避免 JOIN 开销 +- **FTS5 + Trigram** 双索引，支持全文搜索和模糊匹配 +- **LadybugDB** 可选加速，不改动核心架构 +- **BoundedQueue** 解决并发写入问题 + +下一篇文章，我会拆解**验证管线**——ClaimParser 如何从文档中提取断言，VerifierRegistry 如何调度验证器，以及 Evidence 如何生成 Verdict。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-07-verification-pipeline.md b/docs/articles/zh/codescope-deepdive-07-verification-pipeline.md new file mode 100644 index 0000000..3e038aa --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-07-verification-pipeline.md @@ -0,0 +1,259 @@ +# CodeScope 拆解 (七)：验证管线 — Claim → Evidence → Verdict + +> *"The most dangerous assumption in software engineering is that the code matches the docs."* +> 软件工程中最危险的假设，就是认为代码和文档是一致的。 + +## 问题：代码和文档怎么对得上？ + +当项目增长到一定规模，代码和文档之间必然产生偏差。README 里说"支持增量索引"，但代码里可能只实现了全量重建。API 文档说"线程安全"，但实现里全是裸锁。 + +这种偏差不是 bug，而是**漂移（drift）**——代码不断演进，文档被遗忘，最终两者渐行渐远。 + +传统的解决方案是人工审查，但人类不擅长做这种机械化的比对工作。CodeScope 的验证管线（Verification Pipeline）就是为此设计的：**自动从文档中提取断言，到代码中找证据，最后给出判定。** + +## 核心文件 + +``` +engine/src/verify/claim.h ← Claim/EvidenceRecord 数据结构 +engine/src/verify/claim_parser.h ← 从文档中提取 Claims +engine/src/verify/verifier.h ← Verifier 基类 +engine/src/verify/registry.h ← VerifierRegistry（调度器） +engine/src/verify/capability_verifier.h ← 能力验证器 +engine/src/verify/contract_verifier.h ← 合约验证器 +engine/src/verify/architecture_verifier.h ← 架构验证器 +engine/src/verify/finding.h ← 旧版 Finding 结构 +``` + +## 管线架构 + +验证管线分三个阶段： + +```mermaid +flowchart LR + subgraph "Phase 1: 提取 (Parse)" + DOC["README / 文档"] --> CP["ClaimParser"] + CP --> CLAIM1["Claim{type: CapabilityExists
subject: 'IncrementalIndex'}"] + CP --> CLAIM2["Claim{type: ContractHolds
subject: 'ThreadSafe'}"] + end + + subgraph "Phase 2: 验证 (Verify)" + CLAIM1 --> VR["VerifierRegistry"] + VR --> V1["CapabilityVerifier"] + CLAIM2 --> VR + VR --> V2["ContractVerifier"] + V1 --> ER1["EvidenceRecord{verdict: Supported
confidence: 0.85}"] + V2 --> ER2["EvidenceRecord{verdict: Unknown
confidence: 0.0}"] + end + + subgraph "Phase 3: 归因 (Attribution)" + ER1 --> FIND["Finding{type: 'ActiveCapability'
evidence: [file, line, ...]}"] + ER2 --> FIND2["Finding{type: 'BrokenContract'
evidence: []}"] + end +``` + +### Phase 1: ClaimParser + +`ClaimParser` 从自由文本中提取结构化断言： + +```cpp +// claim_parser.h +class ClaimParser { +public: + std::vector parse( + const std::string &text, + const std::string &source_kind, + const std::string &source_ref + ) const; +}; +``` + +输入是 README、AI 摘要或 PR 描述，输出是 `Claim` 结构体： + +```cpp +// claim.h +struct Claim { + ClaimType type; // CapabilityExists / ContractHolds / ArchitectureFollows + std::string subject; // 主语，如 "IncrementalIndex" + std::string predicate; // 谓语，如 "implemented_by" + std::string object; // 宾语，如 "Runtime" + std::string scope; // "repository" 默认 + std::string source_kind; // "readme" / "ai_summary" / "pr" / "manual" + std::string source_ref; // 文件路径，可追溯 +}; +``` + +设计原则：**保守提取**。只匹配高置信度模式（如"supports X"、"thread-safe"）。模糊文本不产生 Claim——因为 Unknown 是合法的判定结果，但错误的 Claim 会污染整个验证链。 + +### Phase 2: VerifierRegistry + +`VerifierRegistry` 是一个 Meyers 单例，持有所有注册的 `Verifier` 实例： + +```cpp +// registry.h +class VerifierRegistry { +public: + static VerifierRegistry &instance(); + + void register_verifier(std::unique_ptr v); + void register_default_verifiers(GraphStore *store, uint64_t project_id); + void clear(); + Verifier *match(const Claim &claim) const; +}; +``` + +核心逻辑在 `match()`：把 Claim 遍历所有注册的验证器，返回第一个 `accepts()` 返回 true 的。 + +```cpp +// 概念: match 实现 +Verifier *VerifierRegistry::match(const Claim &claim) const { + for (const auto &v : verifiers_) { + if (v->accepts(claim)) { + return v.get(); + } + } + return nullptr; +} +``` + +每个验证器声明它能处理的 Claim 类型： + +```cpp +// verifier.h +class Verifier { +public: + virtual std::string name() const = 0; + virtual bool accepts(const Claim &claim) const = 0; + virtual EvidenceRecord verify(const Claim &claim) = 0; +}; +``` + +### Phase 3: EvidenceRecord + +验证器输出的证据记录： + +```cpp +// claim.h +struct EvidenceRecord { + int64_t claim_id = 0; + Verdict verdict = Verdict::Unknown; // Supported / Contradicted / Unknown + double confidence = 0.0; + std::string verifier_name; + std::string detail; + std::vector> facts; // (fact_kind, ref_id) +}; +``` + +`facts` 是证据链，每个 `(fact_kind, ref_id)` 对指向 `graph_nodes`、`graph_edges` 或文档中的具体行。这样，用户不仅能看到"代码与文档不一致"，还能看到"具体是哪个函数、哪一行代码导致了不一致"。 + +## 验证器实现 + +### CapabilityVerifier + +验证"项目声称的能力是否在代码中实现"。 + +逻辑： +1. 在 `graph_nodes` 中查找 Claim 的 subject 对应的符号 +2. 检查该符号是否有 caller（被调用） +3. 如果有 caller → Supported（能力已实现） +4. 如果无 caller 且是 stub → Contradicted（能力未实现） +5. 其他情况 → Unknown + +```cpp +// 概念: CapabilityVerifier::verify() 伪代码 +EvidenceRecord CapabilityVerifier::verify(const Claim &claim) { + auto nodes = store_->findNodesByName(claim.subject); + if (nodes.empty()) { + return {.verdict = Unknown, .confidence = 0.0}; + } + + for (auto &node : nodes) { + auto callers = store_->getCallers(node.id); + if (!callers.empty()) { + return {.verdict = Supported, .confidence = 0.85}; + } + } + return {.verdict = Contradicted, .confidence = 0.7}; +} +``` + +### ContractVerifier + +验证"架构合约是否被遵守"（如"线程安全"）。 + +逻辑更复杂，需要检查多个维度： +- 共享可变状态是否有锁保护 +- 全局变量是否被正确封装 +- 标注为 `unsafe` 的代码块是否与合约冲突 + +### ArchitectureVerifier + +验证"实际调用关系是否遵循架构约定"（如"Controller → Service → Repository"）。 + +从 `graph_edges` 中提取所有调用关系，与架构规范中定义的允许方向做比对。违反方向的关系会被标记为 Contradicted。 + +## 持久化 + +验证结果写入 SQLite 的 evidence 和 evidence_fact 表： + +```sql +CREATE TABLE IF NOT EXISTS evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + claim_id INTEGER NOT NULL, + verdict INTEGER NOT NULL, -- 0=Supported, 1=Contradicted, 2=Unknown + confidence REAL NOT NULL, + verifier_name TEXT NOT NULL, + detail TEXT DEFAULT '', + created_at TEXT DEFAULT (datetime('now')), + FOREIGN KEY (claim_id) REFERENCES claim(id) +); + +CREATE TABLE IF NOT EXISTS evidence_fact ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + evidence_id INTEGER NOT NULL, + fact_kind INTEGER NOT NULL, -- 0=node, 1=edge, 2=document + fact_ref INTEGER NOT NULL, -- graph_nodes.id / graph_edges.id / document.rowid + FOREIGN KEY (evidence_id) REFERENCES evidence(id) +); +``` + +## 一个让我冷汗直流的教训 + +**多项目场景下的单例生命期问题。** + +`VerifierRegistry` 是 Meyer 单例——进程全局唯一。但 `Verifier` 实例在构造时绑定了 `(store, project_id)`。 + +当用户在 MCP 会话中切换项目时，旧的验证器还在注册表中，新的验证器被追加。结果： + +1. 用户切换项目 → 调用 `register_default_verifiers(new_store, new_pid)` +2. 新验证器被追加到列表末尾 +3. 旧验证器仍然持有旧项目的 `(store, project_id)` 引用 +4. 如果 `match()` 命中旧验证器，它用旧项目的 store 去查新项目的数据——结果不可预测 + +```cpp +// 问题: 多项目切换时 +VerifierRegistry::instance().register_default_verifiers(store1, pid1); +// ... 用户切换项目 ... +VerifierRegistry::instance().register_default_verifiers(store2, pid2); +// 旧的验证器还在！如果 match() 匹配到旧验证器，它用 store1 查 pid2 的数据 +``` + +修复方案： + +```cpp +// 切换项目前必须 clear() +VerifierRegistry::instance().clear(); +VerifierRegistry::instance().register_default_verifiers(store2, pid2); +``` + +这个 bug 在注释里写明了，但第一次实现时没人记得调用 `clear()`。结果就是查到的数据牛头不对马嘴。 + +## 总结 + +验证管线是 CodeScope 从"代码搜索引擎"走向"代码真相引擎"的关键一步： + +- **ClaimParser** 从文档中提取断言，保守但精确 +- **VerifierRegistry** 将断言分派给对应的验证器 +- **Verifier** 查询知识图谱，收集证据，给出判定 +- **EvidenceRecord** 包含完整的证据链，可追溯回源码 + +下一篇文章，我将拆解**漂移检测**——如何用 DocumentationDrift、CapabilityDrift 和 ArchitectureDrift 三个维度，自动发现代码与文档之间的偏差。 \ No newline at end of file diff --git a/docs/articles/zh/codescope-deepdive-08-drift-detection.md b/docs/articles/zh/codescope-deepdive-08-drift-detection.md new file mode 100644 index 0000000..94bc80b --- /dev/null +++ b/docs/articles/zh/codescope-deepdive-08-drift-detection.md @@ -0,0 +1,309 @@ +# CodeScope 拆解 (八)：漂移检测 — 文档、能力、架构的三位一体 + +> *"Documentation is a love letter to your future self. Drift is the heartbreak when you realize it was never updated."* +> 文档是写给未来自己的情书。漂移是当你发现它从未被更新时的心碎。 + +## 问题：文档什么时候开始偏离代码？ + +在上一篇文章中，我拆解了验证管线——ClaimParser 从文档中提取断言，Verifier 到代码中找证据，最后给出 Supported / Contradicted / Unknown 的判定。 + +但验证管线有一个前提：**有人先提供 Claim**。要么是用户手动输入，要么是 AI 自动提取。 + +漂移检测（Drift Detection）是验证管线的"自驱模式"——不需要用户输入，自动扫描三个维度，主动发现代码与文档之间的偏差： + +1. **文档漂移（Documentation Drift）**：README 里说支持 C++，但代码里找不到 C++ 实体 +2. **能力漂移（Capability Drift）**：文档声称"支持增量索引"，但实现函数没有调用者 +3. **架构漂移（Architecture Drift）**：实际的调用关系违反了 Controller → Service → Repository 的分层约定 + +## 核心文件 + +``` +engine/src/verify/documentation_drift.h ← 文档漂移检测 +engine/src/verify/documentation_drift.cpp ← 实现 +engine/src/verify/capability_drift.h ← 能力漂移检测 +engine/src/verify/capability_drift.cpp ← 实现 +engine/src/verify/architecture_drift.h ← 架构漂移检测 +engine/src/verify/architecture_drift.cpp ← 实现 +engine/src/verify/finding.h ← DriftItem 数据结构 +``` + +## 三位一体 + +```mermaid +flowchart TB + subgraph "Inputs" + README["README.md"] + CODE["源码"] + CAP["capability 表
（声明的能力）"] + end + + subgraph "Detectors" + DD["DocumentationDrift
severity=1, confidence=0.8"] + CD["CapabilityDrift
severity=2, confidence=0.9"] + AD["ArchitectureDrift
severity=1, confidence=0.7"] + end + + subgraph "Outputs" + ITEM1["DriftItem{type: 'DocumentationDrift'
subject: 'go'
detail: 'README says Go but no entities'}"] + ITEM2["DriftItem{type: 'CapabilityDrift'
subject: 'IncrementalIndex'
detail: 'has 0 callers'}"] + ITEM3["DriftItem{type: 'ArchitectureDrift'
subject: 'Repo.call() → Controller'
detail: 'layer violation'}"] + end + + README --> DD + CAP --> CD + CODE --> CD + CODE --> AD + + DD --> ITEM1 + CD --> ITEM2 + AD --> ITEM3 +``` + +### 1. Documentation Drift（severity=1, confidence=0.8） + +最直观的漂移检测：**README 里声称支持的语言，代码里有没有实现？** + +```cpp +// documentation_drift.h +struct DriftItem { + std::string type; // "DocumentationDrift" + int severity; // 1 = warning + std::string subject; // 缺失的语言名称 + std::string detail; // 人类可读的描述 +}; + +struct LanguageClaim { + std::string canonical; // "cpp", "rust", "go", ... + std::string display; // "C++", "Rust", "Go", ... + size_t mention_count; // 在 README 中出现的次数 +}; +``` + +检测流程： + +1. 从 `document` 表中读取项目 README 内容 +2. `extractLanguageClaims()` 用正则扫描 README，提取语言名（C++、Python、Go、Rust、JavaScript、TypeScript、Java） +3. `countEntitiesByLanguage()` 查询 `entity` 表，统计每种语言的实际实体数 +4. 如果 README 声称支持某语言，但 `entity` 表中该语言的实体数为 0 → 报告 Drift + +```cpp +// 概念: detectDocumentationDrift 伪代码 +std::vector detectDocumentationDrift(store &store, uint64_t pid) { + auto readme = store.getReadme(pid); + auto claims = extractLanguageClaims(readme); + std::vector items; + + for (const auto &claim : claims) { + int64_t count = countEntitiesByLanguage(store, pid, claim.canonical); + if (count == 0) { + items.push_back({ + .type = "DocumentationDrift", + .severity = kDriftSeverityDoc, // 1 + .subject = claim.display, + .detail = "README claims " + claim.display + + " but no entities found", + }); + } + } + return items; +} +``` + +**为什么 severity 只有 1？** 因为 README 可能是在描述外部依赖，而不是项目自身的实现语言。比如"CodeScope 支持 C++ 项目"可能意味着用户可以用 CodeScope 分析 C++ 项目，而不是项目本身用 C++ 写。 + +### 2. Capability Drift（severity=2, confidence=0.9） + +这是最严重的漂移类型：**文档声称的功能，代码中没有实现。** + +```cpp +// capability_drift.h +inline constexpr int kDriftSeverityCapability = 2; // error +inline constexpr double kDriftConfidenceCapability = 0.9; +``` + +检测流程： + +1. 从 `capability` 表读取项目声称的能力列表 +2. 对每个能力，在 `entity` 表中查找匹配名称的实体 +3. 对匹配的实体，检查是否有 caller（被调用者） +4. 如果实体没有 caller → 该能力已经"死亡"——代码存在但未被使用 +5. 如果实体不存在 → 能力完全缺失 + +```cpp +// 概念: detectCapabilityDrift 伪代码 +std::vector detectCapabilityDrift(store &store, uint64_t pid) { + auto capabilities = store.getCapabilities(pid); + std::vector items; + + for (const auto &cap : capabilities) { + int64_t count = countImplementingEntities(store, pid, cap.name); + if (count == 0) { + items.push_back({ + .type = "CapabilityDrift", + .severity = kDriftSeverityCapability, // 2 + .subject = cap.name, + .detail = "Declared capability '" + cap.name + + "' has 0 implementing entities with callers", + }); + } + } + return items; +} +``` + +**为什么 confidence 高达 0.9？** 因为这个检测是确定性的——`capability` 表中的行要么有对应的实现实体，要么没有。不存在模糊空间。 + +### 3. Architecture Drift（severity=1, confidence=0.7） + +最复杂的漂移检测：**实际调用关系是否违反了架构分层约定。** + +```cpp +// architecture_drift.h +inline constexpr const char *kLayerController = "Controller"; +inline constexpr const char *kLayerService = "Service"; +inline constexpr const char *kLayerRepository = "Repository"; +``` + +检测流程： + +1. `classifyEntityLayer()` 根据实体名称和文件路径，将其分类到 Controller / Service / Repository 层 + +```cpp +// 概念: classifyEntityLayer 伪代码 +std::string classifyEntityLayer(const std::string &name, + const std::string &file_path) { + if (name.ends_with("Controller") || + file_path.contains("/controllers/")) { + return "Controller"; + } + if (name.ends_with("Service") || + file_path.contains("/services/")) { + return "Service"; + } + if (name.ends_with("Repository") || + file_path.contains("/repositories/")) { + return "Repository"; + } + return ""; // Unknown +} +``` + +2. 读取 `relation` 表，提取所有调用边（type=1） +3. 对每条调用边，检查源层和目标层的流向是否合法 + +合法流向：**Controller → Service → Repository** + +```mermaid +flowchart LR + CTRL["Controller Layer"] -->|合法| SVC["Service Layer"] + SVC -->|合法| REPO["Repository Layer"] + CTRL -->|非法: 跳层| REPO + REPO -->|非法: 反向| SVC + SVC -->|非法: 反向| CTRL + REPO -->|非法: 反向| CTRL +``` + +**为什么 confidence 只有 0.7？** 因为分层分类是基于命名约定和文件路径的启发式方法。不是所有项目都遵循 Controller/Service/Repository 的命名规范。False positive 是预期内的。 + +## 调度器集成 + +漂移检测不是一次性运行的。CodeScope 的调度器在索引完成后自动调度所有三个检测器： + +```cpp +// 概念: 漂移检测调度 +schedule_drift_detection(project_id) { + // 并行执行三个检测器 + auto doc_drifts = detectDocumentationDrift(store, project_id); + auto cap_drifts = detectCapabilityDrift(store, project_id); + auto arch_drifts = detectArchitectureDrift(store, project_id); + + // 合并结果 + auto all_drifts = concat(doc_drifts, cap_drifts, arch_drifts); + + // 写入 finding 表 + for (auto &drift : all_drifts) { + store.insertFinding(project_id, drift); + } +} +``` + +## 一个让我冷汗直流的教训 + +**Capability Drift 的误报问题。** + +最初的 Capability Drift 检测逻辑很简单：如果 `capability.name` 在 `entity.name` 中找不到，就报告漂移。 + +但实际项目中，能力和实现实体之间存在**命名差异**。比如能力表中声明的是"Incremental Index"，而实现实体叫"incrementalIndex"或"incremental_index"。 + +```mermaid +flowchart LR + CAP["capability: 'Incremental Index'"] + E1["entity: 'incrementalIndex'"] + E2["entity: 'IncrementalIndexBuilder'"] + E3["entity: 'buildIncrementalIndex'"] + CAP -->|"name 精确匹配
❌ 找不到"| MISS["误报: 能力缺失"] + CAP -->|"name 模糊匹配
✅ 找到"| FOUND["正确: 能力已实现"] +``` + +修复方案：**从精确匹配改为包含匹配**。`countImplementingEntities()` 改为 `LIKE '%cap_name%'` 的模糊查询，并去掉下划线和空格后再对比。 + +这个 bug 暴露了一个更深层次的问题：**能力名和实体名之间的映射不是 1:1 的。** 一个能力可能由多个实体共同实现，一个实体也可能实现多个能力。完美的映射需要语义理解，模糊匹配是务实但不够完美的折中。 + +## 三者的关系 + +```mermaid +flowchart TB + subgraph "Drift 金字塔" + DOC["Documentation Drift
severity=1
README 说支持但代码没有"] + CAP["Capability Drift
severity=2
文档声称但代码没实现"] + ARCH["Architecture Drift
severity=1
调用关系违反分层约定"] + end + + DOC --> CAP + CAP --> ARCH + + subgraph "影响范围递增" + D1["单个文件
（README 不准确）"] + D2["单个功能
（能力未实现）"] + D3["整个系统
（架构腐化）"] + end + + DOC --> D1 + CAP --> D2 + ARCH --> D3 +``` + +- **Documentation Drift** 是表层问题，影响单个文件 +- **Capability Drift** 是功能问题，影响单个能力 +- **Architecture Drift** 是结构问题，影响整个系统 + +三者又是递进关系：文档不准确 → 能力声明不可信 → 架构约定被忽视。 + +## 总结 + +漂移检测是 CodeScope 区别于传统代码搜索工具的核心能力： + +- **Documentation Drift**：README 语义 vs 代码实体，severity=1，confidence=0.8 +- **Capability Drift**：声明能力 vs 实现代码，severity=2，confidence=0.9 +- **Architecture Drift**：分层约定 vs 实际调用，severity=1，confidence=0.7 + +三个检测器覆盖了从文档到代码、从功能到架构的所有层面。它们不依赖用户输入，索引完成后自动运行，持续追踪代码库的健康状态。 + +--- + +## 后记 + +至此，CodeScope 拆解系列八篇文章全部完成： + +| # | 文章 | 核心模块 | +|---|------|----------| +| 1 | 双语言架构 | Rust MCP Server + C++ Engine FFI | +| 2 | Worker 子进程隔离 | fork+exec + DB 合并 | +| 3 | 并行调度器 | 三种调度模式 | +| 4 | 两阶段索引 | Fast Scan + Background Enhance | +| 5 | tree-sitter 解析器与统一 IR | 8 种语言，一个模型 | +| 6 | SQLite 图谱存储 | Schema + FTS + 批处理 | +| 7 | 验证管线 | Claim → Evidence → Verdict | +| 8 | 漂移检测 | 文档、能力、架构三位一体 | + +CodeScope 从"代码搜索引擎"到"项目真相引擎"的演进，核心在于它不再只是**索引代码**，而是**理解代码**——从符号级别到语义级别，从静态分析到验证推理。每一步都面临了工程上的取舍和妥协，这些取舍和妥协正是这个系列试图记录的内容。 \ No newline at end of file diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index ccd891d..7d2b197 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -385,6 +385,32 @@ target_link_libraries(astgraph_engine PUBLIC # Set -DLADYBUG_REQUIRED=ON to make the build fail when LadybugDB # is not found (useful for production builds that require Cypher). option(LADYBUG_REQUIRED "Fail build if LadybugDB is not found" OFF) + +# ── Windows: LadybugDB intentionally disabled ─────────────────── +# The vendored engine/third_party/ladybug/lib/windows/ ships only +# lbug_shared.lib (a static `ar` archive) with no lbug_shared.dll, and its +# MinGW ABI/runtime is unverified against the build toolchain. CMake's +# find_library(lbug) cannot match lbug_shared.lib by name anyway. Rather than +# risk an ABI-mismatch access violation at initLadybugDB() time, Windows +# builds compile with HAS_LADYBUG undefined and use SQLite as the sole graph +# store (the pre-existing effective behavior, now made explicit). +if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + if(LADYBUG_REQUIRED) + message(FATAL_ERROR + "LADYBUG_REQUIRED=ON is not supported on Windows. " + "LadybugDB is intentionally disabled on Windows (SQLite-only).") + endif() + # Clear stale LadybugDB cache entries from a previous native (macOS/Linux) + # cmake run in the same build directory. Without this, find_library() in the + # else() branch below never re-runs (the cache persists), and the stale value + # remains — typically pointing at the host platform's LadybugDB path, which is + # incompatible with cross-compilation (e.g. macOS .dylib → MinGW linker). + unset(LADYBUG_LIBRARY CACHE) + unset(LADYBUG_INCLUDE_DIR CACHE) + message(STATUS + "LadybugDB: skipped on Windows (SQLite-only graph storage; " + "HAS_LADYBUG undefined by design).") +else() set(LADYBUG_VENDOR_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/ladybug") if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(LADYBUG_VENDOR_LIB_DIR "${LADYBUG_VENDOR_DIR}/lib/macos") @@ -537,6 +563,7 @@ else() "The engine will use SQLite for all graph storage.\n" "To enable: install LadybugDB or set -DLADYBUG_REQUIRED=ON to enforce.") endif() +endif() # Windows LadybugDB-skip guard # ── Tests ──────────────────────────────────────────────────────── # Automated tests live in tests/. Each must run with no command-line diff --git a/engine/src/engine_ffi.cpp b/engine/src/engine_ffi.cpp index 2bd4a7d..d2c36e3 100644 --- a/engine/src/engine_ffi.cpp +++ b/engine/src/engine_ffi.cpp @@ -1505,6 +1505,6 @@ const char *engine_version(void) // No try/catch required — only a static string literal is returned, // so no exceptions are possible. // Keep in sync with RELEASE.md and Cargo.toml version. - static const char kVersion[] = "0.2.3"; + static const char kVersion[] = "0.2.4"; return kVersion; } diff --git a/server/Cargo.toml b/server/Cargo.toml index 396e898..977334d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codescope" -version = "0.2.3" +version = "0.2.4" edition = "2024" [[bin]] diff --git a/server/build.rs b/server/build.rs index 1ff3b99..cc760de 100644 --- a/server/build.rs +++ b/server/build.rs @@ -86,8 +86,13 @@ fn main() { // On native Windows, CMake detects the system correctly and should NOT be // overridden — setting CMAKE_SYSTEM_NAME on Windows would break detection. if target_os == "windows" { - let host_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); - let is_cross = host_os != "windows"; + // NOTE: CARGO_CFG_TARGET_OS gives the CROSS target, not the host. + // Use std::env::consts::OS to get the actual build host: + // "macos" on macOS, "linux" on Linux, "windows" on Windows. + // Previously this line read CARGO_CFG_TARGET_OS again, which during a + // cross-compile returns "windows" and made is_cross always false. + let build_host = std::env::consts::OS; + let is_cross = build_host != "windows"; if is_cross { cmake_args.push("-DCMAKE_SYSTEM_NAME=Windows".to_string()); // MinGW cross-compiler needs the RC compiler for Windows resources @@ -140,76 +145,67 @@ fn main() { println!("cargo:rustc-link-search=native={}", build_dir); println!("cargo:rustc-link-lib=static=astgraph_engine"); - // LadybugDB (optional, for embedded graph storage via Cypher). + // ── LadybugDB (optional, for embedded graph storage via Cypher) ── // Read the CMake cache to determine whether CMake's find_library() // succeeded — this is the single source of truth, ensuring build.rs // and CMakeLists.txt agree on whether HAS_LADYBUG is defined. If // CMake found the library, build.rs links it too; otherwise neither // side references lbug symbols and the C++ engine uses SQLite only. - let cmake_cache = format!("{}/CMakeCache.txt", build_dir); - let lbug_lib = std::fs::read_to_string(&cmake_cache) - .ok() - .and_then(|content| { - for line in content.lines() { - if line.starts_with("LADYBUG_LIBRARY:FILEPATH=") { - let val = line.trim_start_matches("LADYBUG_LIBRARY:FILEPATH="); - if !val.is_empty() && val != "LADYBUG_LIBRARY-NOTFOUND" && val != "NOTFOUND" { - return Some(val.to_string()); + // + // Windows: always None. engine/CMakeLists.txt unconditionally skips + // LadybugDB on Windows (stale entries from a previous native cmake + // run in the same build-release/ directory are proactively purged by + // unset(LADYBUG_LIBRARY CACHE) in the Windows branch, but the defense + // also lives here — no cache-reading on Windows at all). + let lbug_lib: Option = if target_os == "windows" { + None + } else { + let cmake_cache = format!("{}/CMakeCache.txt", build_dir); + std::fs::read_to_string(&cmake_cache) + .ok() + .and_then(|content| { + for line in content.lines() { + if line.starts_with("LADYBUG_LIBRARY:FILEPATH=") { + let val = line.trim_start_matches("LADYBUG_LIBRARY:FILEPATH="); + if !val.is_empty() && val != "LADYBUG_LIBRARY-NOTFOUND" && val != "NOTFOUND" + { + return Some(val.to_string()); + } + return None; } - return None; } - } - None - }); + None + }) + }; if let Some(lib_path) = &lbug_lib { // CMake found liblbug. Derive the directory and link mode from the path. - let is_windows = target_os == "windows"; - let (lib_dir, lib_name, link_mode, is_static) = if is_windows { - // Windows cross-compile: use vendored Windows library directly. - // CMake cache stores the macOS path (from the host build), so - // we ignore it and use the Windows-specific path. - let win_lib_dir = format!("{}/third_party/ladybug/lib/windows", engine_dir); - ( - win_lib_dir, - "lbug_shared".to_string(), - "dylib".to_string(), - false, - ) + // + // NOTE: Windows never reaches this branch. engine/CMakeLists.txt skips + // the LadybugDB find_library() on Windows (the vendored lbug_shared.lib + // is a static archive of unverified MinGW ABI, and no lbug_shared.dll + // exists), so `lbug_lib` stays None on Windows and the `else if` warning + // branch below handles it instead. Graph storage uses SQLite only on + // Windows — this is by design, not a regression. + let lib_dir = std::path::Path::new(lib_path) + .parent() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| ".".to_string()); + let is_static = lib_path.ends_with(".a"); + let link_mode = if is_static { + "static".to_string() } else { - let dir = std::path::Path::new(lib_path) - .parent() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|| ".".to_string()); - let is_static = lib_path.ends_with(".a"); - let mode = if is_static { - "static".to_string() - } else { - "dylib".to_string() - }; - (dir, "lbug".to_string(), mode, is_static) + "dylib".to_string() }; println!("cargo:rustc-link-search=native={}", lib_dir); - println!("cargo:rustc-link-lib={}={}", link_mode, lib_name); + println!("cargo:rustc-link-lib={}=lbug", link_mode); // Embed the library directory in the binary's rpath so the // dynamic linker can find liblbug at runtime without requiring - // DYLD_LIBRARY_PATH (macOS), ldconfig (Linux), or PATH (Windows). - if !is_static && !is_windows { + // DYLD_LIBRARY_PATH (macOS) or ldconfig (Linux). Windows never + // reaches here (see NOTE above), so no PATH-copy logic is needed. + if !is_static { println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir); } - // Windows: copy lbug_shared.dll next to the executable - if is_windows { - let dll_name = format!("{}/lbug_shared.dll", lib_dir); - let out_dir = env::var("OUT_DIR").unwrap(); - let out_path = std::path::Path::new(&out_dir); - // cargo's OUT_DIR is in the build tree; copy to target/release/ - let target_dir = out_path.ancestors().nth(3).unwrap(); - let dll_dest = format!("{}/lbug_shared.dll", target_dir.display()); - if std::path::Path::new(&dll_name).exists() { - let _ = std::fs::copy(&dll_name, &dll_dest); - eprintln!("build.rs: Windows DLL copied to {}", dll_dest); - } - } eprintln!( "build.rs: LadybugDB {} lib found via CMake cache at {}", if is_static { "static" } else { "dynamic" }, @@ -223,9 +219,18 @@ fn main() { eprintln!(" Install LadybugDB: https://ladybugdb.com/docs/getting-started"); } - // C++ standard library: libc++ on macOS, libstdc++ on Linux/Windows + // C++ standard library: libc++ on macOS, libstdc++ on Linux/Windows. + // On Windows (MinGW GNU ABI) link libstdc++ STATICALLY. This is intentionally + // redundant with the `-static` rustflag in .cargo/config.toml + // [target.x86_64-pc-windows-gnu] — the explicit `static=` mode here provides + // defense in depth (works even if config.toml is absent). Together they bake + // the C++ runtime into codescope.exe so it no longer depends on + // libstdc++-6.dll / libgcc_s_seh-1.dll / libwinpthread-1.dll at runtime, + // eliminating the MinGW 14.0.0 vs 16.1.0 runtime-DLL mismatch that caused + // access-violation crashes after SQLite init and PowerShell DLL pollution. match target_os.as_str() { "macos" => println!("cargo:rustc-link-lib=dylib=c++"), + "windows" => println!("cargo:rustc-link-lib=static=stdc++"), _ => println!("cargo:rustc-link-lib=dylib=stdc++"), } } @@ -263,9 +268,24 @@ fn platform_default_compiler(target_os: &str) -> (String, String) { ("gcc".to_string(), "g++".to_string()) } "windows" => { - // Windows: MinGW gcc/g++ (installed via choco) - eprintln!("build.rs: Windows → gcc/g++ (MinGW)"); - ("gcc".to_string(), "g++".to_string()) + // Native Windows: MinGW gcc/g++ (installed via choco). + // Cross-compilation (macOS/Linux → Windows): use the MinGW + // cross-compiler (x86_64-w64-mingw32-gcc) from mingw-w64 Homebrew + // formula, which is on PATH when installed. + let build_host = std::env::consts::OS; + if build_host == "windows" { + eprintln!("build.rs: Windows native → gcc/g++ (MinGW)"); + ("gcc".to_string(), "g++".to_string()) + } else { + eprintln!( + "build.rs: cross-compile {}→windows → x86_64-w64-mingw32-gcc/g++", + build_host + ); + ( + "x86_64-w64-mingw32-gcc".to_string(), + "x86_64-w64-mingw32-g++".to_string(), + ) + } } _ => { eprintln!("build.rs: unknown OS {} → clang fallback", target_os);