From 179e2b899ee9c7db5edc815282becfaad9cb99fe Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 30 Apr 2026 20:30:54 +0200 Subject: [PATCH 01/10] chore: setup develop branch and git flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add develop branch as default for new feature work - master remains release branch (no rename to main) - Document release process: develop → master + tag - Add development workflow section to README - Remove AGENTS.md (plan doc fulfilled) --- AGENTS.md | 228 ----------------------------------------------------- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 19 +++++ 4 files changed, 21 insertions(+), 230 deletions(-) delete mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 6378e59..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,228 +0,0 @@ -# Setup git flow: feature → develop → master - -**Branch:** `chore/setup-develop-branch` -**Status:** Wachten tot `feature/serve-tui` gemerged is -**Eigenaar:** OpenCode (of handmatig) - ---- - -## 1. Doel - -Overstappen van een single-branch flow (alle PRs → `master`) naar een -develop-based flow: - -``` -feature/xxx ─PR→ develop ─PR→ master (release) - │ - └─► CI runs hier - -master ─tag v1.x.x→ GitHub Release -``` - -`master` blijft de release branch (geen rename naar `main`). `develop` wordt de -actieve dev branch waar alle CI op draait. Releases gebeuren via PR -`develop → master` + tag. - ---- - -## 2. Voorwaarde (plan A — clean cut) - -**Voordat deze branch wordt uitgevoerd:** alle open feature branches die nog -bezig zijn moeten eerst gemerged of gesloten worden. Concreet: - -- `feature/serve-tui` — wachten tot OpenCode klaar is en gemerged -- Eventueel andere actieve branches verifiëren met `git branch -r` - -Stale branches die niet meer relevant zijn worden verwijderd voor de -overstap (zie sectie 6). - ---- - -## 3. Stappen - -### 3.1 Maak `develop` branch vanuit master - -```bash -git checkout master -git pull origin master -git checkout -b develop -git push -u origin develop -``` - -### 3.2 Update GitHub default branch naar develop - -Via REST API met PAT (vermijd gh CLI vanwege bedrijfsnetwerk traagheid): - -```powershell -$t = (Get-Content "$env:APPDATA\Claude\claude_desktop_config.json" | ConvertFrom-Json).mcpServers.github.env.GITHUB_PERSONAL_ACCESS_TOKEN -Invoke-RestMethod -Uri "https://api.github.com/repos/flupkede/codesearch" -Method PATCH ` - -Headers @{ Authorization = "Bearer $t"; "Content-Type" = "application/json" } ` - -Body (@{ default_branch = "develop" } | ConvertTo-Json) -``` - -Verifieer: -```powershell -(Invoke-RestMethod -Uri "https://api.github.com/repos/flupkede/codesearch" -Headers @{ Authorization = "Bearer $t" }).default_branch -# → "develop" -``` - -### 3.3 Branch protection rules - -Voor `master` (release branch — strikter): -- Required: PR before merge -- Required: status checks (build, test) als die bestaan -- Allowed source: alleen `develop` -- Geen direct push - -Voor `develop` (active dev — minder strikt): -- Required: PR before merge -- Status checks aanbevolen, niet verplicht in v1 -- Direct push uit voorzichtigheid disabled - -API call (master): -```powershell -$rules = @{ - required_status_checks = $null - enforce_admins = $false - required_pull_request_reviews = @{ - required_approving_review_count = 0 - dismiss_stale_reviews = $false - } - restrictions = $null - allow_deletions = $false - allow_force_pushes = $false -} | ConvertTo-Json -Depth 5 - -Invoke-RestMethod -Uri "https://api.github.com/repos/flupkede/codesearch/branches/master/protection" ` - -Method PUT -Headers @{ Authorization = "Bearer $t"; "Content-Type" = "application/json" } -Body $rules -``` - -Hetzelfde voor `develop` met aangepaste regels. - -### 3.4 CI workflow update - -Check `.github/workflows/`: -- Als er een `ci.yml` of `build.yml` bestaat met trigger op `master`, - vervang door `develop` (of beide) in de `on:` sectie: - -```yaml -on: - push: - branches: [develop, master] - pull_request: - branches: [develop] -``` - -Geen wijziging als er nog geen workflows bestaan — dan slaan we deze stap over. - -### 3.5 Release proces documenteren - -Voeg toe aan `README.md` of een nieuwe `RELEASE.md`: - -```markdown -## Release Process - -1. PR `develop → master` aanmaken -2. Review en merge -3. Tag op master: - git checkout master && git pull - git tag -a v1.x.x -m "Release v1.x.x" - git push origin v1.x.x -4. GitHub Release aanmaken op de tag -``` - -### 3.6 Update CONTRIBUTING / docs - -In `README.md` (of nieuwe `CONTRIBUTING.md`) een sectie toevoegen: - -```markdown -## Development workflow - -- Maak feature branches vanuit `develop`: `git checkout -b feature/xxx develop` -- PR naar `develop` -- `master` is de release branch — alleen via PR `develop → master` -- Branch naming: `feature/xxx`, `fix/xxx`, `chore/xxx`, `docs/xxx` -``` - ---- - -## 4. Verifieer na uitvoer - -- [ ] `develop` branch bestaat lokaal en op GitHub -- [ ] GitHub repo settings tonen `develop` als default branch -- [ ] Nieuwe PR via web UI defaultent naar `develop` als target -- [ ] `master` branch protection actief — direct push faalt -- [ ] CI draait op push naar `develop` (als CI bestaat) -- [ ] README documenteert de nieuwe flow - ---- - -## 5. Stale branches opruimen - -Voor de overstap: identificeer en verwijder branches die niet meer relevant zijn. -Lijst van branches die mogelijk stale zijn (op basis van naam en eerdere PR's): - -``` -feat/mcp-literal-search-tool (vervangen door eerdere fixes) -feat/mcp-rebrand-hybrid-search (gemerged via #15) -feature/LMDBResilience_GitAware_IndexCompact.md -feature/auto-regex-confidence -feature/branch_switch_failing_index -feature/cleanup -feature/fix-get-chunk-collision (gemerged via #28) -feature/fix-serve-multi-repo (gepruned) -feature/fix-serve-multi-repo-2 (gemerged via #25) -feature/fix-serve-shutdown -feature/improve_search_results -feature/mcp-navigation-extras -feature/post-pr8-fixes -feature/resolve_git_worktree_correction -feature/strict-scope-and-schema-version -feature/update_readme -feature/upgrade_tree_sitter -features/5-quiet-actually-quiet -``` - -Aanpak: -1. Voor elke branch: check of laatste commit in master zit (`git log master --grep=""` - of `git branch --merged master`) -2. Als gemerged: `git branch -d ` lokaal + `git push origin --delete ` -3. Als niet gemerged en niet meer relevant: bevestig met user voor verwijderen - -Dit is **handmatig werk**, niet automatiseren — risico om recent werk te verliezen. - ---- - -## 6. Niet in scope - -- Conventional Commits enforcement (commitlint) — apart traject -- Semantic versioning automation (release-please) — apart traject -- Changelog generation — apart traject -- Rename `master` → `main` — bewust niet, conform user voorkeur -- Verplichte CI status checks — wachten tot CI workflow zelf gestabiliseerd is - ---- - -## 7. Risico's - -| Risico | Mitigatie | -|--------|-----------| -| Bestaande feature branches gerebaset op verouderde master | Plan A: alle open branches eerst mergen naar master, dan develop opzetten | -| Open PRs richten naar master ipv develop na switch | Bestaande PRs handmatig her-targeten via GitHub UI of API (`PATCH /repos/.../pulls/N` met `base: develop`) | -| Branch protection blokkeert legitieme master commits | Admin override blijft mogelijk; protection geldt voor PR flow | -| Lokale clones bij andere users hebben oude default | `git remote set-head origin -a` om het bij te werken | - ---- - -## 8. Commit message voorstel - -``` -chore: setup develop branch and git flow - -- Add develop branch as default for new feature work -- master remains release branch (no rename to main) -- Branch protection on master: PR only, source must be develop -- Branch protection on develop: PR required -- Update CI triggers (if applicable) to develop -- Document release process: develop → master + tag -``` diff --git a/Cargo.lock b/Cargo.lock index dd98c92..ca55729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.61" +version = "1.0.62" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 40b6589..ca75327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.61" +version = "1.0.62" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/README.md b/README.md index b2a96e3..76ed55d 100644 --- a/README.md +++ b/README.md @@ -698,6 +698,25 @@ cargo fmt # Format cargo clippy # Lint ``` +### Workflow + +- Create feature branches from `develop`: `git checkout -b feature/xxx develop` +- Open PRs against `develop` +- `master` is the release branch — only updated via PR `develop → master` +- Branch naming: `feature/xxx`, `fix/xxx`, `chore/xxx`, `docs/xxx` + +### Release Process + +1. Create PR `develop → master` +2. Review and merge +3. Tag on master: + ```bash + git checkout master && git pull + git tag -a v1.x.x -m "Release v1.x.x" + git push origin v1.x.x + ``` +4. Create GitHub Release on the tag + --- ## License From aec1b2e14fec57befce1a3952e75b5a127bd68d6 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 30 Apr 2026 22:43:08 +0200 Subject: [PATCH 02/10] docs: rewrite README to reflect current capabilities - Focus on value proposition: multi-repo semantic search for AI agents - Architecture diagram (mermaid) showing search + serve flow - MCP Configuration section with per-agent configs - Local/stdio vs Serve/multi-repo modes documented - 5 MCP tools reference (search, find, explore, get_chunk, status) - Serve mode with groups, lazy FSW, idle eviction, TUI - CLI reference with index add/rm/list, groups management --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 821 ++++++++++++++++------------------------------------- 3 files changed, 250 insertions(+), 575 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca55729..e2867a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.62" +version = "1.0.63" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index ca75327..4c283b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.62" +version = "1.0.63" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/README.md b/README.md index 76ed55d..f264d65 100644 --- a/README.md +++ b/README.md @@ -1,160 +1,111 @@ # codesearch -**Token-efficient MCP server for AI agents — local semantic code search powered by Rust.** +**Cross-repo semantic code search for AI agents — a Rust MCP server with vector + BM25 hybrid search, symbol navigation, and multi-repository orchestration.** -codesearch is designed as the primary bridge between AI agents and your codebase. It provides a Model Context Protocol (MCP) server that enables OpenCode, Claude Code, and other AI assistants to perform intelligent, semantic code searches with minimal token usage — all running locally with no API calls. +codesearch gives AI agents (OpenCode, Claude Code, Cursor, etc.) deep codebase understanding through 5 unified MCP tools. It runs entirely locally — no API calls, no cloud dependencies. Index once, search semantically across multiple repositories simultaneously. -**Use AI to understand your code:** Query your codebase with natural language like *"where do we handle authentication?"* or *"show me all API endpoints"* and get instant, accurate results. +## Why codesearch? -> **Fork notice:** This project is a fork of [demongrep](https://github.com/yxanul/demongrep) by [yxanul](https://github.com/yxanul). Huge thanks to yxanul for creating the original project — it's an excellent piece of work and the foundation everything here builds on. Some features (like global database support) were contributed back to demongrep via PR. codesearch extends it further with incremental indexing, MCP token optimizations, AI agent integration, and more. +- **Multi-repo search**: Fan-out queries across repository groups +- **Hybrid retrieval**: Vector embeddings + BM25 full-text search fused with Reciprocal Rank Fusion +- **Symbol navigation**: Jump to definitions, find usages, trace imports and dependents +- **AST-aware chunking**: Tree-sitter parsing for 9 languages — chunks align to functions/classes, not arbitrary line ranges +- **Token-efficient**: Returns metadata by default; agents fetch full code only when needed via `get_chunk` +- **Zero config for single repos**: `codesearch index && codesearch mcp` — done ---- +## Architecture -## Features +```mermaid +graph TB + Agent[AI Agent / MCP Client] -->|MCP stdio or HTTP| Router{MCP Router} -### 🤖 MCP Server (Primary Use Case) + Router --> Search[search tool] + Router --> Find[find tool] + Router --> Explore[explore tool] + Router --> GetChunk[get_chunk tool] + Router --> Status[status tool] -- **Token-Efficient AI Integration** — Compact responses minimize token usage in AI conversations -- **OpenCode Compatible** — Seamless integration with OpenCode and other MCP-compatible agents -- **Automatic Index Discovery** — Finds your codebase index automatically from any directory -- **Real-Time Updates** — File watcher and git branch detection keep index current during AI sessions -- **Privacy-First** — All processing local, no code leaves your machine, no external API calls + Search -->|mode=semantic| Semantic[Vector ANN + BM25 + RRF Fusion] + Search -->|mode=literal| Literal[Tantivy FTS / Regex] -### 🔍 Core Search Capabilities + Find -->|definition/usages| SymbolIndex[Symbol Index] + Find -->|imports/dependents| DepGraph[Dependency Graph] -- **Semantic Search** — Natural language queries that understand code meaning -- **Hybrid Search** — Vector similarity + BM25 full-text search with RRF fusion -- **Neural Reranking** — Optional cross-encoder reranking for higher accuracy -- **Smart Chunking** — Tree-sitter AST-aware chunking that preserves functions, classes, methods -- **Incremental Indexing** — Only re-indexes changed files (10–100× faster updates) -- **Embedding Cache** — Three-layer caching system for dramatically faster subsequent indexes -- **Git-Aware Index Placement** — Automatically places indexes at git repository roots -- **Automatic Branch Detection** — Detects git branch changes and refreshes the index -- **Global & Local Indexes** — Per-project local indexes or a shared global index -- **Fast** — Sub-second search after initial model load + Explore -->|outline| TreeSitter[Tree-sitter AST] + Explore -->|similar| Semantic ---- + Semantic --> Arroy[arroy ANN vectors] + Semantic --> Tantivy[Tantivy BM25] + Arroy --> LMDB[(LMDB)] + Tantivy --> TantivyIdx[(Tantivy Index)] -## Table of Contents + GetChunk --> LMDB -- [Installation](#installation) -- [Quick Start for MCP](#quick-start-for-mcp) -- [Indexing](#indexing) -- [Git Integration](#git-integration) -- [Embedding Cache](#embedding-cache) -- [Searching](#searching) -- [MCP Server Configuration](#mcp-server-configuration) -- [Other Commands](#other-commands) -- [Search Modes](#search-modes) -- [Global vs Local Indexes](#global-vs-local-indexes) -- [Supported Languages](#supported-languages) -- [Embedding Models](#embedding-models) -- [Configuration](#configuration) -- [How It Works](#how-it-works) -- [Troubleshooting](#troubleshooting) + subgraph "Serve Mode (multi-repo)" + ServeRouter[HTTP Router] -->|project/group routing| Repo1[Repo A] + ServeRouter --> Repo2[Repo B] + ServeRouter --> RepoN[Repo N] + end ---- - -## Installation - -### 📥 Download Pre-built Binary (Recommended) - -The fastest way to get started - download a single executable ready to use. No dependencies, no build process, just extract and run. - -Download the latest release for your platform from [Releases](https://github.com/flupkede/codesearch/releases): - -| Platform | Download | -|---|---| -| **Windows x86_64** | `codesearch-windows-x86_64.zip` | -| **Linux x86_64** | `codesearch-linux-x86_64.tar.gz` | -| **macOS (Apple Silicon)** | `codesearch-macos-arm64.tar.gz` | - -Extract and place the binary somewhere on your `PATH`: - -**Windows (PowerShell):** -```powershell -# Extract zip -Expand-Archive codesearch-windows-x86_64.zip -# Add to PATH or move to directory on PATH -$env:Path += ";$PWD" + Router -->|client mode| ServeRouter ``` -**Linux/macOS:** -```bash -# Extract tar.gz -tar -xzf codesearch-linux-x86_64.tar.gz # or codesearch-macos-arm64.tar.gz -# Move to PATH -sudo mv codesearch /usr/local/bin/ -# Verify installation -codesearch --version -``` - -### 🔨 Building from Source +## Quick Start -If you prefer to build from source or need a custom build, you'll need Rust and a few dependencies. +### Install -#### Prerequisites +Download pre-built binaries from [Releases](https://github.com/flupkede/codesearch/releases): -| Platform | Command | -|---|---| -| **Rust** | `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh` | -| **Ubuntu/Debian** | `sudo apt-get install -y build-essential protobuf-compiler libssl-dev pkg-config` | -| **Fedora/RHEL** | `sudo dnf install -y gcc protobuf-compiler openssl-devel pkg-config` | -| **macOS** | `brew install protobuf openssl pkg-config` | -| **Windows** | `winget install -e --id Google.Protobuf` or `choco install protoc` | +| Platform | Download | +|----------|----------| +| Windows x86_64 | `codesearch-windows-x86_64.zip` | +| Linux x86_64 | `codesearch-linux-x86_64.tar.gz` | +| macOS ARM64 | `codesearch-macos-arm64.tar.gz` | -#### Build Steps +Or build from source: ```bash git clone https://github.com/flupkede/codesearch.git cd codesearch - -# Build release binary cargo build --release - -# Binary location: -# Linux/macOS: target/release/codesearch -# Windows: target\release\codesearch.exe - -# Optionally add to PATH: -# Linux/macOS: -sudo cp target/release/codesearch /usr/local/bin/ -# Windows (PowerShell, as admin): -Copy-Item target\release\codesearch.exe "$env:LOCALAPPDATA\Microsoft\WindowsApps\" ``` - -### Verify Installation +### Index a repository ```bash -codesearch --version -codesearch doctor -``` +# Register and index a repo (adds to ~/.codesearch/repos.json) +codesearch index add /path/to/my-project --alias my-project + +# Incremental update (only changed files) +codesearch index /path/to/my-project ---- +# Full rebuild +codesearch index /path/to/my-project --force -## Quick Start for MCP +# Remove a repo +codesearch index rm /path/to/my-project -Get up and running with AI agents in under 2 minutes. +# List registered repos +codesearch index list +``` -### 1️⃣ Install codesearch +First-time indexing takes 2–5 minutes. Subsequent runs are incremental (10–30s). Branch switches trigger automatic re-indexing. -Download the pre-built binary for your platform from [Releases](https://github.com/flupkede/codesearch/releases) and extract it to your PATH, or build from source (see [Installation](#installation)). +## MCP Configuration -### 2️⃣ Index your codebase +codesearch connects to AI agents via MCP. Two modes: -```bash -cd /path/to/your/project +| Mode | How | Best for | +|------|-----|----------| +| **Local (stdio)** | `codesearch mcp` — single repo, auto-index + file watching | Working on one project | +| **Serve (HTTP)** | `codesearch serve` — multi-repo, TUI dashboard, lazy FSW | Multiple repos, cross-repo search | -# First time: creates index at git root (~2-5 min, depends on codebase size) -codesearch index -``` +### Local / Single Repo -The index is automatically placed at the git repository root, so it works from any subdirectory. +The agent spawns `codesearch mcp` as a subprocess. It auto-detects the nearest index and starts a file watcher. -### 3️⃣ Configure your AI agent +**OpenCode** — `~/.config/opencode/config.json`: -**For OpenCode:** ```json { "mcp": { @@ -167,8 +118,8 @@ The index is automatically placed at the git repository root, so it works from a } ``` -**For Claude Code Desktop:** -Add to `claude_desktop_config.json` (Windows) or `claude_desktop_config.json` (macOS/Linux): +**Claude Code** — `~/.config/claude-code/config.json`: + ```json { "mcpServers": { @@ -180,544 +131,266 @@ Add to `claude_desktop_config.json` (Windows) or `claude_desktop_config.json` (m } ``` -### 4️⃣ Start using AI to understand your code - -Restart your AI agent and start asking questions: -- *"Where is the authentication logic?"* -- *"Show me all API endpoints"* -- *"How do we handle errors in this project?"* - -The AI agent will use codesearch to find relevant code and provide accurate answers with minimal token usage. - ---- - -## Quick Start for CLI - -```bash -# 1. Navigate to your project -cd /path/to/your/project - -# 2. Index the codebase (first time ~30–60s, incremental afterwards) -codesearch index - -# 3. Search with natural language -codesearch search "where do we handle authentication?" -``` - ---- - -## Indexing - -Indexing is the core operation — it parses your code into semantic chunks, generates embeddings, and stores them for fast retrieval. - -```bash -codesearch index [PATH] [OPTIONS] -``` - -| Option | Short | Description | -|---|---|---| -| `--force` | `-f` | Delete existing index and rebuild from scratch (alias: `--full`) | -| `--dry-run` | | Preview what would be indexed | -| `--add` | | Create a new index (combine with `-g` for global) | -| `--global` | `-g` | Target the global index (with `--add`) | -| `--rm` | | Remove the index (alias: `--remove`) | -| `--list` | | Show index status | -| `--model` | | Override embedding model | - -### Incremental Indexing - -When an index already exists, `codesearch index` only processes changed, added, and deleted files — typically 10–100× faster than a full rebuild. - -```bash -codesearch index # Incremental (default) -codesearch index --force # Full rebuild -codesearch index list # Show index status -``` - -### What Gets Indexed - -All text files are included, respecting `.gitignore` and `.codesearchignore`. Binary files, `node_modules/`, `.git/`, etc. are skipped automatically. - -See [Global vs Local Indexes](#global-vs-local-indexes) for where the index is stored. - ---- - -## Git Integration - -codesearch is deeply integrated with git for intelligent index management and automatic updates. - -### Automatic Git Root Detection - -When you run `codesearch index`, the index is automatically placed at the **git repository root** (where `.git/` is located), regardless of your current working directory within the project. - -```bash -cd /projects/myapp/src/api/ -codesearch index # Creates .codesearch.db/ at /projects/myapp/ -``` - -**How it works:** -- Searches upward from the current directory to find `.git/` or `.git` (worktree) file -- Places `.codesearch.db/` at the same level as the git repository -- Detects nested git worktrees and errors on multiple child `.git` directories -- Falls back to current directory if no git repository is found - -This ensures a **single, authoritative index per git repository**, avoiding confusion from multiple indexes in subdirectories. - ---- - -### Automatic Branch Change Detection - -codesearch monitors `.git/HEAD` in real-time and automatically refreshes the index when you switch branches. - -```bash -# Currently on main branch -codesearch index - -# Switch branches -git checkout feature/new-auth - -# Index is automatically refreshed to reflect the new branch files -``` - -**Behavior:** -- The MCP server (and `codesearch serve`) polls `.git/HEAD` every 100ms -- Detects HEAD changes (branch switches) and triggers an incremental re-index -- Updates happen automatically in the background — no manual intervention needed - -This is especially useful when working with different branches in AI coding sessions — the search results always reflect your current branch state. - -### Database Bloat Monitoring - -`codesearch stats` now shows a **bloat ratio** that indicates how much free space exists in the LMDB database: - -```bash -$ codesearch stats -Database: .codesearch.db/ -Files: 1,234 -Chunks: 45,678 -Bloat ratio: 1.2 # 1.2x size indicates 20% free space available -``` - -- **Bloat ratio < 1.5**: Healthy, no action needed -- **Bloat ratio > 2.0**: Consider compacting (future feature) - -The bloat ratio is calculated from LMDB's internal statistics and helps monitor database health over time. - ---- - -## Embedding Cache - -codesearch uses a sophisticated caching system to dramatically speed up subsequent indexing after the initial index is created. - -### How Caching Works - -When you index your codebase, codesearch computes **embeddings** (vector representations) for each code chunk. This is the most time-consuming part of indexing. The cache system stores these embeddings so they don't need to be recomputed. - -```bash -# First time: slow (all embeddings computed) -codesearch index -# Takes ~2-5 minutes for 10k files (depends on CPU) - -# Second time: fast (embeddings loaded from cache) -codesearch index -# Takes ~10-30 seconds (only changed files processed) - -# Switching branches: very fast (embeddings reused from cache) -git checkout feature-branch -# Index auto-refreshes in ~5-10 seconds (only new/changed files) -``` - -### Cache Types +**Claude Desktop** — `claude_desktop_config.json`: -codesearch uses **three cache layers** for optimal performance: - -#### 1. In-Memory Cache (Moka LRU Cache) -- **Location**: RAM during indexing process -- **Size**: 100MB (configurable via `CODESEARCH_CACHE_MAX_MEMORY`) -- **Purpose**: Cache embeddings during a single indexing session -- **Benefit**: Avoids recomputing embeddings for duplicate chunks within the same index run - -#### 2. Persistent Cache (Disk-Based) -- **Location**: `~/.codesearch/embedding_cache//` -- **Size**: Up to 200,000 entries (~300MB) -- **Purpose**: Long-term storage keyed by content hash (SHA256) -- **Benefit**: Embeddings survive MCP restarts and branch switches -- **Key Benefit**: Files with identical content across different branches share the same embedding - -#### 3. Query Cache (Optional) -- **Location**: In-memory during search operations -- **Purpose**: Cache query embeddings for repeated searches -- **Benefit**: Repeated searches with the same query are nearly instant - -### Cache Benefits - -| Scenario | Without Cache | With Cache | -|---|---|---| -| First index (10k files) | ~2-5 min | ~2-5 min (cache empty) | -| Incremental index (1% changed) | ~30 sec | ~10 sec | -| Branch switch (50% overlap) | ~1-2 min | ~10 sec | -| Repeated queries | ~500ms | ~50ms | - -### Cache Management - -```bash -# Show cache statistics (all models) -codesearch cache stats - -# Show cache statistics for specific model -codesearch cache stats bge-small - -# Clear persistent cache for specific model -codesearch cache clear bge-small - -# Clear cache without confirmation -codesearch cache clear bge-small --yes +```json +{ + "mcpServers": { + "codesearch": { + "command": "codesearch", + "args": ["mcp"] + } + } +} ``` -### Cache Size Monitoring - -The persistent cache automatically manages disk usage: -- Default limit: 200,000 entries (~300MB) -- Older entries are evicted when limit is reached (LRU policy) -- Per-model isolation: Each embedding model has its own cache +### Serve / Multi-Repo -**Note:** The persistent cache is separate from the index database (`.codesearch.db/`). Clearing the cache does NOT delete your search index — it only deletes cached embeddings, which will be recomputed on the next index. - ---- - -## Searching - -```bash -codesearch search [OPTIONS] -``` - -| Option | Short | Default | Description | -|---|---|---|---| -| `--max-results` | `-m` | 25 | Maximum results | -| `--per-file` | | 1 | Max matches per file | -| `--content` | `-c` | | Show full chunk content | -| `--scores` | | | Show relevance scores and timing | -| `--compact` | | | File paths only (like `grep -l`) | -| `--sync` | `-s` | | Re-index changed files before searching | -| `--json` | | | JSON output for scripting | -| `--filter-path` | | | Restrict to path (e.g., `src/api/`) | -| `--vector-only` | | | Disable hybrid, vector similarity only | -| `--rerank` | | | Enable neural reranking (~1.7s extra) | -| `--rerank-top` | | 50 | Candidates to rerank | -| `--rrf-k` | | 20 | RRF fusion parameter | +Start the server first, then connect your agent. The server manages all registered repos with a TUI dashboard, lazy filesystem watchers, and idle eviction. ```bash -codesearch search "database connection pooling" -codesearch search "error handling" --content --rerank -codesearch search "validation" --filter-path src/api --json -m 10 -codesearch search "new feature" --sync +# Start the server (default port 39725) +codesearch serve ``` ---- - -## MCP Server Configuration - -The MCP server is codesearch's primary integration point for AI coding agents. It exposes token-efficient tools for semantic code search. The MCP server **auto-detects** the nearest database (local or global) — no project path argument is needed. If no database is found, the server will **not start**. This is intentional: codesearch never creates a database automatically to avoid polluting your projects. - -> **Important:** Always `codesearch index` your project first before using the MCP server. - -### OpenCode (recommended) - -OpenCode is the primary target for codesearch's MCP integration. Add the following to your OpenCode config at `~/.config/opencode/opencode.json`: +**OpenCode** — connect via HTTP: ```json { "mcp": { "codesearch": { - "type": "local", - "command": [ - "codesearch", - "mcp" - ], + "type": "remote", + "url": "http://127.0.0.1:39725/mcp", "enabled": true } } } ``` -No project path required — codesearch auto-detects the database for the current working directory. - -> **⚠️ `codesearch` must be on your system `PATH`** for OpenCode to find it. If you built from source, copy the binary to a directory that's in your `PATH` (e.g., `~/.local/bin/` on Linux/macOS or `C:\Users\\.local\bin\` on Windows). Verify with: `codesearch --version` - -### Claude Code - -Add to `~/.config/claude-code/config.json`: +**Claude Code / Claude Desktop** — force serve connection via `--mode client`: ```json { "mcpServers": { "codesearch": { "command": "codesearch", - "args": ["mcp"] + "args": ["mcp", "--mode", "client"] } } } ``` -On Windows, use the full path to `codesearch.exe` if it's not in your `PATH`. Restart Claude Code after editing the config. - -### What Happens on Startup - -When the MCP server starts, it goes through this sequence: - -1. **Database discovery** — Searches for `.codesearch.db/` at the git root (by detecting `.git/` from the current directory), then walks up parent directories (up to 10 levels for non-git projects), and finally checks the global location (`~/.codesearch.dbs/`). The first database found is used. If none is found, the server exits — it will never create a database on its own. -2. **Incremental index** — Automatically runs an incremental re-index against the detected database, so the index is up-to-date before the agent starts working. -3. **File system watcher (FSW)** — Starts watching the project directory for changes. Any file modifications, additions, or deletions are picked up and the index is updated in the background (with debouncing), keeping the database current throughout the session. -4. **Git HEAD watcher** — Monitors `.git/HEAD` for branch changes. When a branch switch is detected, an automatic incremental re-index is triggered to update the database with files from the new branch. - -> **Important:** Databases are discovered at the *git repository root*, not in subdirectories. Do not manually create `.codesearch.db/` directories inside subfolders — this will cause confusion. One database per git repository, at the git root (or global). +> **Note:** In multi-repo mode, agents must specify `project` or `group` in tool calls. `status` always works without scope. `get_chunk` auto-routes when the chunk_id is unique across repos; if ambiguous, it returns candidates and requires `project`. -### MCP Tools +## MCP Tools Reference -| Tool | Parameters | Description | -|---|---|---| -| `semantic_search` | `query`, `limit`, `compact` (default: true), `filter_path` | Semantic code search. Compact mode returns metadata only (~93% fewer tokens). | -| `find_references` | `symbol`, `limit` (default: 50) | Find all usages/call sites of a symbol across the codebase. | -| `get_file_chunks` | `path`, `compact` (default: true) | Get all indexed chunks from a file. | -| `find_databases` | | Discover available codesearch databases. | -| `index_status` | | Check index existence and statistics. | +### `search` — Code Search -### How AI Agents Use the Tools +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | string | Natural language, code snippet, regex, or exact term | +| `mode` | `"semantic"` \| `"literal"` | Search backend (default: semantic) | +| `filter_path` | string | Path prefix filter (semantic mode) | +| `file_glob` | string | Glob filter (literal mode), e.g. `"src/**/*.rs"` | +| `language` | string | Language filter (literal mode) | +| `regex` | bool | Treat query as regex (literal mode) | +| `phrase` | bool | Exact phrase match (literal mode) | +| `compact` | bool | Metadata only, no code (default: true) | +| `limit` | int | Max results (default: 10 semantic, 20 literal) | +| `project` | string | Target specific repo (multi-repo) | +| `group` | string | Search across repo group (multi-repo) | -The MCP tools are designed to work together in a **search → narrow → read** workflow that minimizes token usage: +**Semantic mode** combines vector similarity (fastembed) + BM25 lexical scoring + exact identifier boosting, fused with RRF. Best for conceptual queries and mixed natural-language + symbol searches. -1. **`semantic_search`** — The agent starts here. A natural language query like `"where do we handle authentication?"` returns a ranked list of matches. With `compact=true` (the default), only metadata is returned: file path, line numbers, chunk kind, signature, and score — roughly 40 tokens per result instead of 600. +**Literal mode** uses Tantivy FTS. Use `regex=true` for patterns with punctuation (`foo::bar`, `Vec`). Use `phrase=true` for multi-word exact matches. -2. **`find_references`** — Once the agent identifies a relevant function or symbol, it can ask for all usages and call sites across the codebase. This is much more efficient than grep-based searching and stays within the codesearch ecosystem. Example: `find_references("authenticate")` returns every location that calls or references that symbol. +### `find` — Symbol Navigation -3. **`get_file_chunks`** — To get a broader view of a specific file's structure, the agent can retrieve all indexed chunks. With `compact=true` this gives an outline (functions, classes, methods with signatures); with `compact=false` it includes full source code. +| Parameter | Type | Description | +|-----------|------|-------------| +| `symbol` | string | Symbol name or file path (for imports) | +| `kind` | `"definition"` \| `"usages"` \| `"imports"` \| `"dependents"` | Navigation type | +| `definition_kind` | string | Filter: Function, Class, Method, Struct, Trait, Enum, Interface | +| `project` / `group` | string | Multi-repo routing | -4. **Targeted file reads** — Finally, the agent reads only the specific lines it needs using its built-in file read tools. +### `explore` — File Exploration -**Example session:** -``` -Agent: semantic_search("auth handler", compact=true) - → 20 results, ~800 tokens total (paths, signatures, scores) - -Agent: find_references("authenticate") - → 8 call sites across 5 files, ~100 tokens - -Agent: read("src/auth/handler.rs", lines 45-75) - → Only the code that matters -``` - -This workflow typically saves **90%+ tokens** compared to returning full code content for every search result. +| Parameter | Type | Description | +|-----------|------|-------------| +| `target` | string | File path (outline) or chunk_id (similar) | +| `kind` | `"outline"` \| `"similar"` | Exploration type | +| `limit` | int | Max results for similar mode | +| `project` / `group` | string | Multi-repo routing | ---- +**Outline** returns all top-level symbols in a file (kind, signature, line range). +**Similar** finds semantically related chunks to a given chunk_id. -## Other Commands +### `get_chunk` — Read Code -| Command | Description | -|---|---| -| `codesearch serve [PATH] -p ` | HTTP server with live file watching (default port 4444) | -| `codesearch stats [PATH]` | Show database statistics | -| `codesearch clear [PATH] [-y]` | Delete the index | -| `codesearch list` | List all indexed repositories | -| `codesearch doctor` | Check installation health | -| `codesearch setup [--model ]` | Pre-download embedding models | +| Parameter | Type | Description | +|-----------|------|-------------| +| `chunk_id` | int | Chunk ID from search/explore results | +| `context_lines` | int | Extra lines before/after (0-20, default: 0) | +| `project` | string | Disambiguate if chunk_id exists in multiple repos | -### HTTP Server API +In multi-repo mode: auto-routes when chunk_id is unique; returns candidates list when ambiguous. -| Method | Endpoint | Description | -|---|---|---| -| GET | `/health` | Health check | -| GET | `/status` | Index statistics | -| POST | `/search` | Search (JSON body: `{"query": "...", "limit": 10}`) | +### `status` — Index Info ---- +| Parameter | Type | Description | +|-----------|------|-------------| +| `kind` | `"index"` \| `"projects"` | What to query | +| `project` / `group` | string | Multi-repo routing | -## Search Modes +## Serve Mode (Multi-Repo) -| Mode | Command | Speed | Best For | -|---|---|---|---| -| **Hybrid** (default) | `codesearch search "query"` | ~75ms | Most queries — balances semantic + keyword | -| **Vector-only** | `codesearch search "query" --vector-only` | ~72ms | Conceptual queries without exact keywords | -| **Hybrid + Reranking** | `codesearch search "query" --rerank` | ~1.8s | Maximum accuracy | +For working across multiple repositories simultaneously: ---- - -## Global vs Local Indexes - -codesearch supports two index locations per project. Only one can be active at a time. - -| | Local Index | Global Index | -|---|---|---| -| **Location** | `/.codesearch.db/` | `~/.codesearch.dbs//` | -| **Created with** | `codesearch index` (default) | `codesearch index --add -g` | -| **Visible to** | Only when inside the project tree | From any directory | -| **Use case** | Per-project, self-contained | Shared/central index, searchable from anywhere | - -**How discovery works:** when you run a command, codesearch looks for a database in this order: -1. `.codesearch.db/` at the git root (automatically detected from current directory) -2. `.codesearch.db/` in parent directories (up to 10 levels, for non-git projects) -3. `~/.codesearch.dbs/` (global) +```bash +codesearch serve +``` -This means you can `cd` into any subfolder and codesearch will still find the project index at the git root. +This starts a background HTTP server with: +- **TUI dashboard** (ratatui) showing repo status, CPU usage, active sessions +- **Lazy filesystem watchers** — activated on first query per repo +- **Idle eviction** (30min) — unused repos are unloaded from memory +- **Session tracking** via MCP keep-alive -### Git Worktrees +### Repository Registration -codesearch works naturally with [git worktrees](https://git-scm.com/docs/git-worktree). Each worktree lives in its own directory and points to a different branch of the same git repository, so each worktree can have its own independent database and MCP server instance. This means you can have separate indexes for different branches — when OpenCode or Claude Code starts in a worktree folder, codesearch auto-detects the database for that specific worktree. +Repos are registered via `codesearch index add`: ```bash -# Main repo on main branch -cd /projects/myapp -codesearch index +# Register a repo (creates index + adds to ~/.codesearch/repos.json) +codesearch index add /path/to/my-project --alias my-project -# Worktree for a feature branch -git worktree add /projects/myapp-feature feature/new-auth -cd /projects/myapp-feature -codesearch index +# Remove a repo +codesearch index rm /path/to/my-project -# Each worktree has its own .codesearch.db/ and MCP instance -# Branch switching within a worktree triggers automatic index refresh +# List registered repos +codesearch index list ``` -```bash -codesearch index # Create local index (default) -codesearch index --add -g # Create global index -codesearch index rm # Remove whichever index exists -codesearch index list # Show which index is active -``` +Serve reads `~/.codesearch/repos.json` on startup and manages all registered repos. ---- +### Groups -## Supported Languages +Groups let you search across related repositories: -### Full AST Chunking (Tree-sitter) +```bash +codesearch groups add my-group repo1 repo2 repo3 +codesearch groups list +``` -Rust (`.rs`), Python (`.py`, `.pyw`, `.pyi`), JavaScript (`.js`, `.mjs`, `.cjs`), TypeScript (`.ts`, `.mts`, `.cts`, `.tsx`, `.jsx`), C (`.c`, `.h`), C++ (`.cpp`, `.cc`, `.cxx`, `.hpp`), C# (`.cs`), Go (`.go`), Java (`.java`) +Then in MCP tools: `group="my-group"` fans out the query to all repos in the group. -### Line-based Chunking +### MCP Connection Modes -Ruby, PHP, Swift, Kotlin, Shell, Markdown, JSON, YAML, TOML, SQL, HTML, CSS/SCSS/SASS/LESS +The `codesearch mcp` command supports three modes: ---- +| Mode | Behavior | +|------|----------| +| `auto` (default) | Connects to serve if running, otherwise local stdio | +| `client` | Always connects to serve, fails if not running | +| `local` | Always uses local DB (classic single-repo stdio) | -## Embedding Models +```bash +codesearch mcp --mode client # force serve connection +``` -| Name | ID | Dimensions | Speed | Notes | -|---|---|---|---|---| -| MiniLM-L6 (Q) | `minilm-l6-q` | 384 | Fastest | **Default** | -| MiniLM-L6 | `minilm-l6` | 384 | Fastest | General use | -| MiniLM-L12 (Q) | `minilm-l12-q` | 384 | Fast | Higher quality | -| BGE Small (Q) | `bge-small-q` | 384 | Fast | General use | -| BGE Base | `bge-base` | 768 | Medium | Higher quality | -| BGE Large | `bge-large` | 1024 | Slow | Highest quality | -| **Jina Code** | **`jina-code`** | 768 | Medium | **Code-specific** | -| Nomic v1.5 | `nomic-v1.5` | 768 | Medium | Long context | -| E5 Multilingual | `e5-multilingual` | 384 | Fast | Non-English code | -| MxBai Large | `mxbai-large` | 1024 | Slow | High quality | +The serve endpoint is available at `/mcp` (Streamable HTTP transport). -The model used for indexing is stored in metadata. Always search with the same model you indexed with, or re-index with `--force` when switching. +## CLI Reference ---- +| Command | Description | +|---------|-------------| +| `codesearch index [PATH]` | Index a repo (incremental; `--force` for full rebuild) | +| `codesearch search ` | CLI search (for testing) | +| `codesearch mcp` | Start MCP stdio server | +| `codesearch serve` | Start multi-repo HTTP server with TUI | +| `codesearch stats` | Show database statistics | +| `codesearch clear` | Delete index | +| `codesearch doctor` | Health check (model, index, config) | +| `codesearch setup` | Download embedding models | +| `codesearch cache stats\|clear` | Manage embedding cache | +| `codesearch groups list\|add\|remove` | Manage repository groups | ## Configuration ### Environment Variables -| Variable | Description | Default | -|---|---|---| -| `CODESEARCH_CACHE_MAX_MEMORY` | Max embedding cache in MB | 500 | -| `CODESEARCH_BATCH_SIZE` | Embedding batch size | Auto | -| `RUST_LOG` | Logging level | `codesearch=info` | - -### Ignore Files - -Create `.codesearchignore` in your project root (same syntax as `.gitignore`). Also respects `.gitignore` and `.osgrepignore`. - -### Global Options - -| Option | Short | Description | -|---|---|---| -| `--loglevel` | | Set log level (error, warn, info, debug, trace) | -| `--quiet` | `-q` | Suppress info, only results/errors | -| `--model` | | Override embedding model | -| `--store` | | Override store name | - ---- - -## How It Works - -1. **File Discovery** — Walks the directory respecting ignore files, detects language, skips binaries. -2. **Git Root Detection** — Automatically finds the git repository root and places `.codesearch.db/` there, ensuring a single index per repository. -3. **Semantic Chunking** — Tree-sitter AST parsing extracts functions, classes, methods with metadata. Falls back to line-based chunking for unsupported languages. -4. **Embedding Generation** — fastembed + ONNX Runtime (CPU), batched, with SHA-256 change detection and **caching**. -5. **Vector Storage** — arroy (ANN search) + LMDB (ACID persistence) in a single `.codesearch.db/` directory at git root. -6. **Incremental Updates** — FileMetaStore tracks hash/mtime/size; only changed files are re-processed. -7. **Git Branch Detection** — Monitors `.git/HEAD` for branch switches and automatically refreshes the index. -8. **Search** — Query → embed → vector search → BM25 → RRF fusion → (optional) reranking. - ---- - -## Troubleshooting - -| Problem | Solution | -|---|---| -| "No database found" | Run `codesearch index` first (creates index at git root) | -| Poor search results | Try `--sync` to update, `--rerank` for accuracy, or `--force` to rebuild | -| Model mismatch warning | Re-index: `codesearch index --force --model ` | -| Out of memory | `CODESEARCH_BATCH_SIZE=32 codesearch index` | -| Port in use (serve) | `codesearch serve --port 5555` | -| Wrong database found | Check where `.codesearch.db/` is located with `codesearch list` | -| Index not updating after branch switch | The Git HEAD watcher refreshes automatically; check `codesearch stats` to verify | -| First index very slow | Normal! First time indexes compute all embeddings (2-5 min). Subsequent indexes use cache (10-30 sec) | -| Cache too large | Clear cache: `codesearch cache clear ` | - -### Git-Specific Troubleshooting - -**"Multiple .git directories detected"** -- This error occurs when codesearch finds nested git repositories -- Solution: Remove the nested `.git` directory or index from the outer repository only +| Variable | Description | +|----------|-------------| +| `CODESEARCH_SERVE_PORT` | Serve mode port (default: 39725) | +| `CODESEARCH_MCP_MODE` | MCP mode: auto, client, local | +| `CODESEARCH_REPOS_CONFIG` | Path to repos.json | +| `CODESEARCH_REPO_IDLE_TIMEOUT_SECS` | Idle eviction timeout (default: 1800) | +| `CODESEARCH_CACHE_MAX_MEMORY` | Embedding cache MB (default: 500) | +| `CODESEARCH_BATCH_SIZE` | Embedding batch size | +| `RUST_LOG` | Log level (e.g. `codesearch=debug`) | + +### `.codesearchignore` + +Place in repo root. Gitignore syntax. Excludes paths from indexing: + +```gitignore +# Vendored code +vendor/ +node_modules/ +# Generated files +*.generated.cs +**/migrations/** +``` -**"Database not at git root"** -- Old versions of codesearch created databases in the current directory -- Solution: Delete the old `.codesearch.db/` directory and run `codesearch index` — it will be recreated at the git root +### `repos.json` -### Debug Logging +Located at `~/.codesearch/repos.json`. Managed by `codesearch index add/rm`. Contains repo aliases → paths and group definitions. See [Serve Mode](#serve-mode-multi-repo). -```bash -RUST_LOG=codesearch=debug codesearch search "query" -RUST_LOG=codesearch::embed=trace codesearch index -``` +## Supported Languages ---- +Tree-sitter AST-aware chunking: + +| Language | Extensions | +|----------|-----------| +| Rust | `.rs` | +| Python | `.py` | +| JavaScript | `.js`, `.jsx` | +| TypeScript | `.ts`, `.tsx` | +| C | `.c`, `.h` | +| C++ | `.cpp`, `.hpp` | +| C# | `.cs` | +| Go | `.go` | +| Java | `.java` | + +All other text files use line-based chunking as fallback. + +## Core Technology + +| Component | Technology | +|-----------|-----------| +| Embedding | fastembed + ONNX Runtime (CPU) | +| Vector store | arroy (Approximate Nearest Neighbors) + LMDB | +| Full-text search | Tantivy (BM25, AND mode) | +| Chunking | Tree-sitter AST parsing | +| Incremental sync | SHA-256 content hashing | +| Caching | 3-layer: in-memory (Moka) → persistent disk → query cache | +| Schema | Versioned via `metadata.json` | ## Development ```bash -cargo build # Debug -cargo build --release # Release -cargo test # Tests -cargo fmt # Format -cargo clippy # Lint -``` - -### Workflow - -- Create feature branches from `develop`: `git checkout -b feature/xxx develop` -- Open PRs against `develop` -- `master` is the release branch — only updated via PR `develop → master` -- Branch naming: `feature/xxx`, `fix/xxx`, `chore/xxx`, `docs/xxx` +# Build +cargo build -### Release Process +# Run tests +cargo test -1. Create PR `develop → master` -2. Review and merge -3. Tag on master: - ```bash - git checkout master && git pull - git tag -a v1.x.x -m "Release v1.x.x" - git push origin v1.x.x - ``` -4. Create GitHub Release on the tag +# Check + lint +cargo clippy --all-targets -- -D warnings ---- +# Format +cargo fmt --all +``` ## License @@ -725,4 +398,6 @@ Apache-2.0 ## Acknowledgements -This project is a fork of [demongrep](https://github.com/yxanul/demongrep) by [yxanul](https://github.com/yxanul). A huge thank you for building such a solid and well-designed foundation — without demongrep, codesearch wouldn't exist. +This project is a fork of [demongrep](https://github.com/yxanul/demongrep) by [yxanul](https://github.com/yxanul). Huge thanks for building such a solid foundation. + +Built with: [fastembed-rs](https://github.com/Anush008/fastembed-rs), [arroy](https://github.com/meilisearch/arroy), [tantivy](https://github.com/quickwit-oss/tantivy), [tree-sitter](https://tree-sitter.github.io/), [ratatui](https://github.com/ratatui/ratatui), [LMDB](http://www.lmdb.tech/). From 6b65d21d39894b2229660f1754e65a4f4e36a2c5 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 30 Apr 2026 23:22:51 +0200 Subject: [PATCH 03/10] fix(serve): track file changes + improve reaper visibility - Add changes_count (AtomicU64) to SharedStores for tracking indexed/removed files - Increment changes_count in perform_incremental_refresh_with_stores and process_batch_with_stores - TUI reads changes_count directly from stores instead of unused repo_changes DashMap - Add debug-level reaper logging showing idle ages for all tracked repos - Session keepalive issue is client-side (rmcp returns 404 for stale sessions, clients must re-initialize) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/index/manager.rs | 18 ++++++++++++++++++ src/serve/mod.rs | 29 ++++++++++++++++++++++++++--- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2867a5..4356259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.63" +version = "1.0.64" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 4c283b6..fa9f9a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.63" +version = "1.0.64" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/index/manager.rs b/src/index/manager.rs index a2c26d3..c3f5139 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -133,6 +133,9 @@ pub struct SharedStores { writer_lock: Option, /// Whether this instance is in readonly mode pub readonly: bool, + /// Counter for number of file changes processed (indexed + removed) since serve start. + /// Incremented by FSW batches and incremental refreshes. Read by TUI/dashboard. + pub changes_count: std::sync::atomic::AtomicU64, } impl SharedStores { @@ -159,6 +162,7 @@ impl SharedStores { fts_store: Arc::new(RwLock::new(fts_store)), writer_lock: lock, readonly: false, + changes_count: std::sync::atomic::AtomicU64::new(0), }) } @@ -177,6 +181,7 @@ impl SharedStores { fts_store: Arc::new(RwLock::new(fts_store)), writer_lock: None, readonly: true, + changes_count: std::sync::atomic::AtomicU64::new(0), }) } @@ -601,6 +606,12 @@ impl IndexManager { // Save file metadata file_meta_store.save(db_path)?; + // Track changes for dashboard/TUI + let total_changes = (changed_files.len() + deleted_files.len()) as u64; + if total_changes > 0 { + stores.changes_count.fetch_add(total_changes, std::sync::atomic::Ordering::Relaxed); + } + let elapsed = start.elapsed(); info!( "✅ Incremental refresh completed in {:.2}s", @@ -996,6 +1007,12 @@ impl IndexManager { // Disable quiet mode after batch processing is complete set_quiet(false); + // Track changes for dashboard/TUI + let batch_changes = (files_to_index.len() + files_to_remove.len()) as u64; + if batch_changes > 0 { + stores.changes_count.fetch_add(batch_changes, std::sync::atomic::Ordering::Relaxed); + } + let elapsed = start.elapsed(); info!( "✅ Batch complete: {} indexed, {} removed in {:.2}s", @@ -1507,6 +1524,7 @@ mod tests { fts_store: Arc::new(RwLock::new(FtsStore::new_with_writer(db_path).unwrap())), writer_lock: None, readonly: false, + changes_count: std::sync::atomic::AtomicU64::new(0), } } diff --git a/src/serve/mod.rs b/src/serve/mod.rs index c8d8062..8a65f9a 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -870,9 +870,17 @@ impl ServeState { } }; - let changes = self.repo_changes.get(alias) - .map(|c| c.load(Ordering::Relaxed)) - .unwrap_or(0); + let changes = match self.repos.get(alias) { + Some(entry) => match entry.value() { + RepoState::Write { stores, .. } | RepoState::Warm { stores } | RepoState::Readonly { stores } => { + stores.changes_count.load(Ordering::Relaxed) + } + RepoState::Conflicted => 0, + }, + None => self.repo_changes.get(alias) + .map(|c| c.load(Ordering::Relaxed)) + .unwrap_or(0), + }; let last_tool = self.last_tool_call.get(alias) .map(|e| (e.value().0.clone(), e.value().1.elapsed())) @@ -1024,6 +1032,21 @@ impl ServeState { .map(|entry| entry.key().clone()) .collect(); + // Log reaper status even when nothing to evict (for debugging idle eviction) + if !self.last_access.is_empty() { + let idle_ages: Vec<(String, u64)> = self.last_access + .iter() + .map(|e| (e.key().clone(), now.duration_since(*e.value()).as_secs())) + .collect(); + tracing::debug!( + "🔍 Reaper check: {} repos tracked, {} eligible for eviction (timeout={}m). Ages: {:?}", + self.last_access.len(), + to_evict.len(), + timeout.as_secs() / 60, + idle_ages, + ); + } + if to_evict.is_empty() { return; } From a485d76691c4b78db8eb47751b34ab694a1e0c60 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 10:27:32 +0200 Subject: [PATCH 04/10] feat(serve): auto-reconnect MCP clients with stale sessions When the server restarts or laptop suspends, all in-memory MCP sessions are lost. Clients that still hold a stale session ID get a 404 from rmcp which most MCP client libraries don't handle gracefully. Add middleware that inspects POST /mcp requests with a session ID. If the JSON-RPC method is "initialize", strip the stale session header so rmcp creates a fresh session automatically. This enables seamless reconnection after server restarts without manual client restart. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/serve/mod.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4356259..e806429 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.64" +version = "1.0.66" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index fa9f9a4..b1eda70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.64" +version = "1.0.66" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 8a65f9a..53d4ef2 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -1533,6 +1533,65 @@ async fn log_mcp_requests( response } +/// Middleware that auto-reconnects MCP clients with stale session IDs. +/// +/// When the server restarts (or after laptop suspend), all in-memory MCP sessions are lost. +/// Clients that still hold a stale session ID get a 404 "Session not found" from rmcp, +/// which most MCP client libraries don't handle gracefully — they appear connected but +/// can't send any requests. +/// +/// Strategy: For POST /mcp, buffer the body and inspect the JSON-RPC method. +/// - If it's an `initialize` request with a session ID → strip the header so rmcp +/// creates a fresh session (the client is trying to re-initialize with a stale ID). +/// - For all other requests → pass through unchanged (session should still be valid +/// if the client was properly initialized). +/// - If the pass-through results in 404, we can't retry (axum middleware limitation), +/// but we transform the 404 into a JSON-RPC error that the client can act on. +async fn auto_reconnect_stale_sessions( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + const MCP_SESSION_HEADER: &str = "mcp-session-id"; + + let path = req.uri().path().to_string(); + let is_mcp = path == crate::constants::MCP_ENDPOINT_PATH; + let has_session = req.headers().contains_key(MCP_SESSION_HEADER); + + // Only process POST to MCP endpoint with a session ID + if !is_mcp || !has_session || req.method() != axum::http::Method::POST { + return next.run(req).await; + } + + // Buffer the body to inspect the JSON-RPC method + let (mut parts, body) = req.into_parts(); + let body_bytes = match axum::body::to_bytes(body, 4 * 1024 * 1024).await { + Ok(bytes) => bytes, + Err(e) => { + tracing::warn!("Failed to buffer MCP request body: {}", e); + return axum::response::Response::builder() + .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(axum::body::Body::from("Internal error")) + .unwrap(); + } + }; + + // Check if this is an initialize request + let is_initialize = serde_json::from_slice::(&body_bytes) + .ok() + .and_then(|v| v.get("method").and_then(|m| m.as_str()).map(|s| s == "initialize")) + .unwrap_or(false); + + if is_initialize { + // Client is re-initializing with a stale session ID. + // Strip the header so rmcp treats it as a fresh connection. + tracing::info!("🔄 MCP client re-initializing — stripping stale session ID"); + parts.headers.remove(axum::http::header::HeaderName::from_static(MCP_SESSION_HEADER)); + } + + let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes)); + next.run(req).await +} + /// Run the MCP serve mode. /// /// This is the entry point called from CLI when `codesearch serve` is invoked. @@ -1615,7 +1674,7 @@ pub async fn run_serve( config, ); - // Build axum router with request logging + // Build axum router with request logging + stale session recovery let app = axum::Router::new() .route(HEALTH_PATH, axum::routing::get(health_handler)) .route("/repos", axum::routing::post(add_repo_handler)) @@ -1623,6 +1682,7 @@ pub async fn run_serve( .route("/repos/:alias/reindex", axum::routing::post(reindex_handler)) .nest_service(MCP_ENDPOINT_PATH, mcp_service) .layer(axum::middleware::from_fn(log_mcp_requests)) + .layer(axum::middleware::from_fn(auto_reconnect_stale_sessions)) .with_state(serve_state.clone()); // Bind TCP listener BEFORE spawning background warmup, so we know the port is live. From 40a007a75c90bfdca3b673132746da30f744dee9 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 13:29:17 +0200 Subject: [PATCH 05/10] feat(serve): transparent MCP session reconnect on stale 404 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the server restarts or laptop suspends, in-memory MCP sessions are lost. Previously only initialize requests were handled; tools/call with a stale session still got a 404 that clients couldn't recover from. New approach: after rmcp returns 404 for a stale session, the middleware automatically performs a full reconnect cycle: 1. Sends an internal initialize request to get a fresh session ID 2. Retries the original request with the new session ID 3. Returns the response with the new session ID header so the client updates its stored session ID transparently Uses ReconnectState (reqwest::Client + mcp_url) via from_fn_with_state. The client never sees the 404 — it gets the actual tool response. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/serve/mod.rs | 730 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 527 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e806429..e97bd12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.66" +version = "1.0.67" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index b1eda70..c8eecc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.66" +version = "1.0.67" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 53d4ef2..c8b471c 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -14,24 +14,24 @@ mod tui; use std::net::SocketAddr; use std::path::PathBuf; -use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; use anyhow::{Context, Result}; use axum::response::Json as AxumJson; use colored::Colorize; use dashmap::{DashMap, DashSet}; use rmcp::transport::{ - StreamableHttpServerConfig, StreamableHttpService, - streamable_http_server::session::local::LocalSessionManager, + streamable_http_server::session::local::LocalSessionManager, StreamableHttpServerConfig, + StreamableHttpService, }; use serde_json::json; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; use crate::constants::{ - DEFAULT_SERVE_PORT, HEALTH_PATH, MCP_ENDPOINT_PATH, SERVE_PORT_ENV, DB_DIR_NAME, - REPO_IDLE_TIMEOUT_SECS, REAPER_INTERVAL_SECS, REPO_IDLE_TIMEOUT_ENV, + DB_DIR_NAME, DEFAULT_SERVE_PORT, HEALTH_PATH, MCP_ENDPOINT_PATH, REAPER_INTERVAL_SECS, + REPO_IDLE_TIMEOUT_ENV, REPO_IDLE_TIMEOUT_SECS, SERVE_PORT_ENV, }; use crate::db_discovery::repos::ReposConfig; use crate::index::{IndexManager, SharedStores}; @@ -79,7 +79,12 @@ fn format_tool_call_ago(tool_name: &str, elapsed: std::time::Duration) -> String } else if secs < 3600 { format!("{} ({}m ago)", tool_name, secs / 60) } else { - format!("{} ({}h {}m ago)", tool_name, secs / 3600, (secs % 3600) / 60) + format!( + "{} ({}h {}m ago)", + tool_name, + secs / 3600, + (secs % 3600) / 60 + ) } } @@ -190,9 +195,14 @@ impl ServeState { }, }; - let mtime = std::fs::metadata(&config_path).and_then(|m| m.modified()).ok(); + let mtime = std::fs::metadata(&config_path) + .and_then(|m| m.modified()) + .ok(); - let current_mtime = *self.config_mtime.read().map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))?; + let current_mtime = *self + .config_mtime + .read() + .map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))?; if mtime == current_mtime { return Ok(()); // no change } @@ -201,16 +211,26 @@ impl ServeState { let new_config = match ReposConfig::load_from(&config_path) { Ok(c) => c, Err(e) => { - tracing::warn!("Failed to reload repos config: {}. Keeping current config.", e); - *self.config_mtime.write().map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = mtime; + tracing::warn!( + "Failed to reload repos config: {}. Keeping current config.", + e + ); + *self + .config_mtime + .write() + .map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = mtime; return Ok(()); } }; // Compute removed aliases under read lock (don't hold it long) let removed: Vec = { - let old = self.config.read().map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))?; - old.repos.keys() + let old = self + .config + .read() + .map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))?; + old.repos + .keys() .filter(|k| !new_config.repos.contains_key(*k)) .cloned() .collect() @@ -230,12 +250,19 @@ impl ServeState { // Note: these are two separate writes, so a concurrent reader could observe // the new config with the old mtime (or vice versa). This causes at most a // spurious extra reload on the next call, which is benign. - *self.config.write().map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = new_config; - *self.config_mtime.write().map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = mtime; + *self + .config + .write() + .map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = new_config; + *self + .config_mtime + .write() + .map_err(|e| anyhow::anyhow!("Mutex poisoned: {}", e))? = mtime; #[cfg(test)] { - self.reload_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.reload_count + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); } Ok(()) @@ -248,7 +275,11 @@ impl ServeState { fn stop_fsw(&self, alias: &str) -> Option> { if let Some(mut entry) = self.repos.get_mut(alias) { match entry.value_mut() { - RepoState::Write { cancel_token, stores, .. } => { + RepoState::Write { + cancel_token, + stores, + .. + } => { cancel_token.cancel(); tracing::info!("Stopped FSW for '{}'", alias); return Some(stores.clone()); @@ -275,7 +306,11 @@ impl ServeState { let config = match self.config.read() { Ok(c) => c, Err(e) => { - tracing::error!("Cannot restart FSW for '{}': config lock poisoned: {}", alias, e); + tracing::error!( + "Cannot restart FSW for '{}': config lock poisoned: {}", + alias, + e + ); return; } }; @@ -309,7 +344,9 @@ impl ServeState { &project_path, &db_path_bg, &stores_bg, - ).await { + ) + .await + { tracing::error!("Post-reindex refresh for '{}' failed: {}", alias_bg, e); } @@ -361,8 +398,12 @@ impl ServeState { } let path = { - let config = self.config.read().map_err(|e| format!("Mutex poisoned: {}", e))?; - config.resolve(alias) + let config = self + .config + .read() + .map_err(|e| format!("Mutex poisoned: {}", e))?; + config + .resolve(alias) .ok_or_else(|| format!("Unknown alias '{}'", alias))? }; @@ -386,27 +427,24 @@ impl ServeState { info!("Warmup '{}': opened in write mode", alias); s } - Err(_) => { - match SharedStores::new_readonly(&db_path, dims) { - Ok(s) => { - info!("Warmup '{}': opened in readonly mode", alias); - let stores_arc = Arc::new(s); - self.repos.insert( - alias.to_string(), - RepoState::Readonly { - stores: stores_arc.clone(), - }, - ); - return Ok(()); - } - Err(e) => { - warn!("Warmup '{}': failed to open: {}", alias, e); - self.repos - .insert(alias.to_string(), RepoState::Conflicted); - return Err(Self::conflicted_msg(alias)); - } + Err(_) => match SharedStores::new_readonly(&db_path, dims) { + Ok(s) => { + info!("Warmup '{}': opened in readonly mode", alias); + let stores_arc = Arc::new(s); + self.repos.insert( + alias.to_string(), + RepoState::Readonly { + stores: stores_arc.clone(), + }, + ); + return Ok(()); } - } + Err(e) => { + warn!("Warmup '{}': failed to open: {}", alias, e); + self.repos.insert(alias.to_string(), RepoState::Conflicted); + return Err(Self::conflicted_msg(alias)); + } + }, }; // Build vector index from existing data @@ -446,35 +484,44 @@ impl ServeState { } }); - // Store as Warm — FSW will be started lazily on first query - self.repos.insert( - alias.to_string(), - RepoState::Warm { - stores: stores_arc, - }, - ); - self.touch_access(alias); + // Store as Warm — FSW will be started lazily on first query. + // Do NOT touch_access: warmup is background activity, not a real query. + // The idle timer should only reset when a user/agent actually queries this repo. + self.repos + .insert(alias.to_string(), RepoState::Warm { stores: stores_arc }); Ok(()) } /// Try to open a repo by alias. Returns a clone of the Arc /// if successful, or an error string if conflicted/unknown. + /// + /// `touch`: when true, records the access time for idle-eviction tracking. + /// Pass false for fan-out paths (e.g., multi-repo status, get_chunk candidate + /// scanning) that should NOT reset the idle timer on every repo. pub(crate) async fn get_or_open_stores( &self, alias: &str, + touch: bool, ) -> std::result::Result, String> { let _ = self.reload_if_changed(); // Fast path: already opened if let Some(entry) = self.repos.get(alias) { - self.touch_access(alias); + if touch { + self.touch_access(alias); + } return match entry.value() { RepoState::Write { stores, .. } | RepoState::Readonly { stores } => { Ok(stores.clone()) } RepoState::Warm { stores } => { - // Lazy FSW start: transition Warm → Write on first actual query + // Lazy FSW start: transition Warm → Write only on real query access. + // Fan-out/candidate-detection callers pass touch=false and must not + // trigger Warm → Write or start FSW. let stores = stores.clone(); + if !touch { + return Ok(stores); + } drop(entry); // release DashMap read guard before mutation // Only one caller should do the transition; use a compare-and-swap pattern. @@ -486,8 +533,12 @@ impl ServeState { if let RepoState::Warm { stores } = mut_entry.value() { let stores = stores.clone(); let path = { - let config = self.config.read().map_err(|e| format!("Mutex poisoned: {}", e))?; - config.resolve(alias) + let config = self + .config + .read() + .map_err(|e| format!("Mutex poisoned: {}", e))?; + config + .resolve(alias) .ok_or_else(|| format!("Unknown alias '{}'", alias))? }; @@ -511,8 +562,12 @@ impl ServeState { // Slow path: need to open let path = { - let config = self.config.read().map_err(|e| format!("Mutex poisoned: {}", e))?; - config.resolve(alias) + let config = self + .config + .read() + .map_err(|e| format!("Mutex poisoned: {}", e))?; + config + .resolve(alias) .ok_or_else(|| format!("Unknown alias '{}'", alias))? }; @@ -548,12 +603,14 @@ impl ServeState { stores: stores_arc.clone(), }, ); + if touch { + self.touch_access(alias); + } return Ok(stores_arc); } Err(e) => { warn!("Failed to open repo '{}': {}", alias, e); - self.repos - .insert(alias.to_string(), RepoState::Conflicted); + self.repos.insert(alias.to_string(), RepoState::Conflicted); return Err(Self::conflicted_msg(alias)); } } @@ -600,11 +657,7 @@ impl ServeState { tokio::spawn(async move { // Pre-start FSW so changes during initial refresh aren't lost if let Err(e) = im_for_task.start_watching().await { - tracing::warn!( - "Could not pre-start FSW for '{}': {}", - alias_clone, - e - ); + tracing::warn!("Could not pre-start FSW for '{}': {}", alias_clone, e); } // Initial incremental refresh @@ -615,11 +668,7 @@ impl ServeState { ) .await { - tracing::error!( - "Initial refresh for '{}' failed: {}", - alias_clone, - e - ); + tracing::error!("Initial refresh for '{}' failed: {}", alias_clone, e); } if token_for_task.is_cancelled() { @@ -628,11 +677,7 @@ impl ServeState { // Main file watcher loop — runs until cancel_token fires if let Err(e) = im_for_task.start_file_watcher(token_for_task).await { - tracing::error!( - "File watcher for '{}' stopped: {}", - alias_clone, - e - ); + tracing::error!("File watcher for '{}' stopped: {}", alias_clone, e); } }); @@ -659,6 +704,9 @@ impl ServeState { cancel_token, }, ); + if touch { + self.touch_access(alias); + } Ok(stores_arc) } @@ -697,7 +745,11 @@ impl ServeState { } if let Err(e) = im_for_task.start_watching().await { - tracing::warn!("Lazy FSW start for '{}': pre-start failed: {}", alias_bg, e); + tracing::warn!( + "Lazy FSW start for '{}': pre-start failed: {}", + alias_bg, + e + ); } if token_for_task.is_cancelled() { @@ -711,7 +763,8 @@ impl ServeState { Err(e) => { tracing::warn!( "Lazy FSW for '{}': IndexManager init failed: {} — live updates disabled", - alias_bg, e + alias_bg, + e ); } } @@ -782,33 +835,44 @@ impl ServeState { /// Triggers reload_if_changed first. pub(crate) fn config_snapshot(&self) -> ReposConfig { let _ = self.reload_if_changed(); - self.config.read() + self.config + .read() .map(|guard| guard.clone()) .unwrap_or_default() } /// Resolve a group name to its constituent aliases. /// Returns an error if the group doesn't exist. - pub(crate) fn resolve_group_aliases(&self, group: &str) -> std::result::Result, String> { + pub(crate) fn resolve_group_aliases( + &self, + group: &str, + ) -> std::result::Result, String> { let _ = self.reload_if_changed(); let config = match self.config.read() { Ok(c) => c, Err(e) => return Err(format!("Config lock poisoned: {}", e)), }; - config.groups.get(group) + config + .groups + .get(group) .cloned() .ok_or_else(|| format!("Unknown group '{}'", group)) } /// Record that a repo was just accessed (query or reindex). - /// Called from `get_or_open_stores`, `warmup_repo`, and `reindex_handler`. - fn touch_access(&self, alias: &str) { - self.last_access.insert(alias.to_string(), std::time::Instant::now()); + /// Called from `get_or_open_stores(touch=true)`, and `reindex_handler`. + /// NOT called from `warmup_repo` — background warmup is not a real query. + pub(crate) fn touch_access(&self, alias: &str) { + self.last_access + .insert(alias.to_string(), std::time::Instant::now()); } /// Record a tool call for a specific repo (for dashboard display). pub(crate) fn record_tool_call(&self, alias: &str, tool_name: &str) { - self.last_tool_call.insert(alias.to_string(), (tool_name.to_string(), std::time::Instant::now())); + self.last_tool_call.insert( + alias.to_string(), + (tool_name.to_string(), std::time::Instant::now()), + ); } /// Record that changes were made to a repo (index/reindex). @@ -816,7 +880,9 @@ impl ServeState { pub(crate) fn record_changes(&self, alias: &str, count: u64) { self.repo_changes .entry(alias.to_string()) - .and_modify(|c| { c.fetch_add(count, Ordering::Relaxed); }) + .and_modify(|c| { + c.fetch_add(count, Ordering::Relaxed); + }) .or_insert_with(|| AtomicU64::new(count)); } @@ -872,25 +938,34 @@ impl ServeState { let changes = match self.repos.get(alias) { Some(entry) => match entry.value() { - RepoState::Write { stores, .. } | RepoState::Warm { stores } | RepoState::Readonly { stores } => { + RepoState::Write { stores, .. } + | RepoState::Warm { stores } + | RepoState::Readonly { stores } => { stores.changes_count.load(Ordering::Relaxed) } RepoState::Conflicted => 0, }, - None => self.repo_changes.get(alias) + None => self + .repo_changes + .get(alias) .map(|c| c.load(Ordering::Relaxed)) .unwrap_or(0), }; - let last_tool = self.last_tool_call.get(alias) + let last_tool = self + .last_tool_call + .get(alias) .map(|e| (e.value().0.clone(), e.value().1.elapsed())) .map(|(name, ago)| format_tool_call_ago(&name, ago)); - result.push((alias.clone(), RepoStatusInfo { - status: label, - changes, - last_tool_call: last_tool, - })); + result.push(( + alias.clone(), + RepoStatusInfo { + status: label, + changes, + last_tool_call: last_tool, + }, + )); } result } @@ -933,12 +1008,19 @@ impl ServeState { eprintln!("{}", top.bright_black()); // Header - eprintln!("{} {:7} {} {:<24} {}", - "│".bright_black(), "Project".bold(), "│".bright_black(), - "Status".bold(), "│".bright_black(), - "Changes".bold(), "│".bright_black(), - "Last Tool Call".bold(), "│".bright_black(), - w_alias = alias_w, w_status = status_w, + eprintln!( + "{} {:7} {} {:<24} {}", + "│".bright_black(), + "Project".bold(), + "│".bright_black(), + "Status".bold(), + "│".bright_black(), + "Changes".bold(), + "│".bright_black(), + "Last Tool Call".bold(), + "│".bright_black(), + w_alias = alias_w, + w_status = status_w, ); eprintln!("{}", mid.bright_black()); @@ -961,11 +1043,17 @@ impl ServeState { // Replace the plain text with the colored version let status_display = status_padded.replace(status_plain, &status_colored.to_string()); let tool_str = info.last_tool_call.as_deref().unwrap_or("—"); - eprintln!("{} {:7} {} {:<24} {}", - "│".bright_black(), alias, "│".bright_black(), - status_display, "│".bright_black(), - info.changes, "│".bright_black(), - tool_str, "│".bright_black(), + eprintln!( + "{} {:7} {} {:<24} {}", + "│".bright_black(), + alias, + "│".bright_black(), + status_display, + "│".bright_black(), + info.changes, + "│".bright_black(), + tool_str, + "│".bright_black(), w_alias = alias_w, ); } @@ -973,27 +1061,46 @@ impl ServeState { eprintln!("{}", bot.bright_black()); // Overall status - let has_error = repos.iter().any(|(_, r)| matches!(r.status, RepoStateLabel::Error)); + let has_error = repos + .iter() + .any(|(_, r)| matches!(r.status, RepoStateLabel::Error)); let health = if has_error { "Error".red().bold().to_string() } else { "Healthy".green().bold().to_string() }; - let open_count = repos.iter().filter(|(_, r)| matches!(r.status, RepoStateLabel::Open)).count(); - let warm_count = repos.iter().filter(|(_, r)| matches!(r.status, RepoStateLabel::Warm)).count(); - let closed_count = repos.iter().filter(|(_, r)| matches!(r.status, RepoStateLabel::Closed | RepoStateLabel::NoIndex)).count(); + let open_count = repos + .iter() + .filter(|(_, r)| matches!(r.status, RepoStateLabel::Open)) + .count(); + let warm_count = repos + .iter() + .filter(|(_, r)| matches!(r.status, RepoStateLabel::Warm)) + .count(); + let closed_count = repos + .iter() + .filter(|(_, r)| matches!(r.status, RepoStateLabel::Closed | RepoStateLabel::NoIndex)) + .count(); eprintln!(); - eprintln!(" {} {} {} {} {} {} {} {}", - "Status:".dimmed(), health, - "Open:".dimmed(), format!("{}", open_count).green(), - "Warm:".dimmed(), format!("{}", warm_count).yellow(), - "Closed:".dimmed(), format!("{}", closed_count).dimmed(), + eprintln!( + " {} {} {} {} {} {} {} {}", + "Status:".dimmed(), + health, + "Open:".dimmed(), + format!("{}", open_count).green(), + "Warm:".dimmed(), + format!("{}", warm_count).yellow(), + "Closed:".dimmed(), + format!("{}", closed_count).dimmed(), ); - eprintln!(" {} {} {} {}", - "Active Sessions:".dimmed(), format!("{}", active).cyan(), - "Total Since Start:".dimmed(), format!("{}", total).dimmed(), + eprintln!( + " {} {} {} {}", + "Active Sessions:".dimmed(), + format!("{}", active).cyan(), + "Total Since Start:".dimmed(), + format!("{}", total).dimmed(), ); eprintln!(); } @@ -1019,7 +1126,8 @@ impl ServeState { let now = std::time::Instant::now(); // Collect aliases to evict (can't mutate DashMap while iterating) - let to_evict: Vec = self.last_access + let to_evict: Vec = self + .last_access .iter() .filter(|entry| { let alias = entry.key(); @@ -1034,7 +1142,8 @@ impl ServeState { // Log reaper status even when nothing to evict (for debugging idle eviction) if !self.last_access.is_empty() { - let idle_ages: Vec<(String, u64)> = self.last_access + let idle_ages: Vec<(String, u64)> = self + .last_access .iter() .map(|e| (e.key().clone(), now.duration_since(*e.value()).as_secs())) .collect(); @@ -1101,10 +1210,16 @@ async fn reindex_handler( axum::extract::Path(alias): axum::extract::Path, axum::extract::Query(params): axum::extract::Query>, axum::extract::State(state): axum::extract::State>, -) -> (axum::http::StatusCode, axum::response::Json) { +) -> ( + axum::http::StatusCode, + axum::response::Json, +) { use axum::http::StatusCode; - let force = params.get("force").map(|v| v == "true" || v == "1").unwrap_or(false); + let force = params + .get("force") + .map(|v| v == "true" || v == "1") + .unwrap_or(false); // Resolve the project path for this alias let project_path = { @@ -1163,7 +1278,7 @@ async fn reindex_handler( Some(s) => s, None => { // FSW not running -- try opening normally - match state.get_or_open_stores(&alias).await { + match state.get_or_open_stores(&alias, true).await { Ok(s) => s, Err(e) => { state.active_reindexes.remove(&guard_alias); @@ -1182,16 +1297,13 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); tokio::spawn(async move { - tracing::info!("Force reindex for '{}': clearing stores and reindexing", alias_bg); + tracing::info!( + "Force reindex for '{}': clearing stores and reindexing", + alias_bg + ); // 2. Clear data and reindex - match IndexManager::force_reindex_with_stores( - &project_path, - &db_path, - &stores, - ) - .await - { + match IndexManager::force_reindex_with_stores(&project_path, &db_path, &stores).await { Ok(()) => { tracing::info!("Force reindex complete for '{}'", alias_bg); } @@ -1206,9 +1318,8 @@ async fn reindex_handler( g_state.active_reindexes.remove(&g_alias); }); } else { - // Incremental refresh: ensure the repo is opened, then refresh - let stores = match state.get_or_open_stores(&alias).await { + let stores = match state.get_or_open_stores(&alias, true).await { Ok(s) => s, Err(e) => { state.active_reindexes.remove(&guard_alias); @@ -1225,7 +1336,10 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); tokio::spawn(async move { - tracing::info!("🔄 Incremental reindex triggered for '{}' via HTTP API", alias_bg); + tracing::info!( + "🔄 Incremental reindex triggered for '{}' via HTTP API", + alias_bg + ); match IndexManager::perform_incremental_refresh_with_stores( &project_path, &db_path, @@ -1273,7 +1387,10 @@ struct AddRepoRequest { async fn add_repo_handler( axum::extract::State(state): axum::extract::State>, axum::extract::Json(body): axum::extract::Json, -) -> (axum::http::StatusCode, axum::response::Json) { +) -> ( + axum::http::StatusCode, + axum::response::Json, +) { use axum::http::StatusCode; // Canonicalize the path @@ -1349,13 +1466,8 @@ async fn add_repo_handler( let alias_bg = alias.clone(); let state_bg = state.clone(); - match crate::index::index_quiet( - Some(index_path.clone()), - false, - body.global, - cancel_token, - ) - .await + match crate::index::index_quiet(Some(index_path.clone()), false, body.global, cancel_token) + .await { Ok(()) => { tracing::info!("Index created for '{}' ({})", alias, index_path.display()); @@ -1404,7 +1516,10 @@ async fn add_repo_handler( async fn remove_repo_handler( axum::extract::Path(alias): axum::extract::Path, axum::extract::State(state): axum::extract::State>, -) -> (axum::http::StatusCode, axum::response::Json) { +) -> ( + axum::http::StatusCode, + axum::response::Json, +) { use axum::http::StatusCode; // 1. Resolve project path from config @@ -1463,7 +1578,11 @@ async fn remove_repo_handler( }; config.unregister_alias(&alias); if let Err(e) = config.save() { - tracing::warn!("Failed to save repos config after removing '{}': {}", alias, e); + tracing::warn!( + "Failed to save repos config after removing '{}': {}", + alias, + e + ); } } @@ -1533,21 +1652,36 @@ async fn log_mcp_requests( response } -/// Middleware that auto-reconnects MCP clients with stale session IDs. +/// State for the MCP session reconnect middleware. +/// +/// Holds a reqwest client and the local MCP URL so the middleware can +/// perform an internal `initialize` + retry when it detects a stale session. +#[derive(Clone)] +struct ReconnectState { + client: reqwest::Client, + mcp_url: String, +} + +/// Middleware that transparently reconnects MCP clients with stale session IDs. /// /// When the server restarts (or after laptop suspend), all in-memory MCP sessions are lost. /// Clients that still hold a stale session ID get a 404 "Session not found" from rmcp, /// which most MCP client libraries don't handle gracefully — they appear connected but /// can't send any requests. /// -/// Strategy: For POST /mcp, buffer the body and inspect the JSON-RPC method. -/// - If it's an `initialize` request with a session ID → strip the header so rmcp -/// creates a fresh session (the client is trying to re-initialize with a stale ID). -/// - For all other requests → pass through unchanged (session should still be valid -/// if the client was properly initialized). -/// - If the pass-through results in 404, we can't retry (axum middleware limitation), -/// but we transform the 404 into a JSON-RPC error that the client can act on. +/// Strategy (full transparent retry): +/// 1. Buffer the request body. +/// 2. If it's an `initialize` with a stale session ID → strip the header so rmcp +/// creates a fresh session (client is explicitly re-initializing). +/// 3. For all other requests → pass through to rmcp normally. +/// 4. If rmcp returns 404 (stale session detected): +/// a. Send an internal `initialize` request to ourselves (no session header). +/// b. Extract the new session ID from the initialize response header. +/// c. Retry the original request with the new session ID. +/// d. Forward the retry response to the client with the new session ID header +/// so the client updates its stored session ID automatically. async fn auto_reconnect_stale_sessions( + axum::extract::State(reconnect): axum::extract::State, req: axum::extract::Request, next: axum::middleware::Next, ) -> axum::response::Response { @@ -1557,12 +1691,12 @@ async fn auto_reconnect_stale_sessions( let is_mcp = path == crate::constants::MCP_ENDPOINT_PATH; let has_session = req.headers().contains_key(MCP_SESSION_HEADER); - // Only process POST to MCP endpoint with a session ID + // Only intercept POST to /mcp with a session ID if !is_mcp || !has_session || req.method() != axum::http::Method::POST { return next.run(req).await; } - // Buffer the body to inspect the JSON-RPC method + // Buffer the body so we can replay it on retry let (mut parts, body) = req.into_parts(); let body_bytes = match axum::body::to_bytes(body, 4 * 1024 * 1024).await { Ok(bytes) => bytes, @@ -1575,21 +1709,131 @@ async fn auto_reconnect_stale_sessions( } }; - // Check if this is an initialize request + // If this is an initialize request with a stale session ID, strip the header + // so rmcp treats it as a fresh connection (client is explicitly re-initializing). let is_initialize = serde_json::from_slice::(&body_bytes) .ok() .and_then(|v| v.get("method").and_then(|m| m.as_str()).map(|s| s == "initialize")) .unwrap_or(false); if is_initialize { - // Client is re-initializing with a stale session ID. - // Strip the header so rmcp treats it as a fresh connection. tracing::info!("🔄 MCP client re-initializing — stripping stale session ID"); parts.headers.remove(axum::http::header::HeaderName::from_static(MCP_SESSION_HEADER)); + let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes)); + return next.run(req).await; } - let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes)); - next.run(req).await + // Pass through to rmcp + let stale_session_id = parts + .headers + .get(MCP_SESSION_HEADER) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_owned()); + let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes.clone())); + let response = next.run(req).await; + + // If rmcp returned 404, the session is stale — perform transparent reconnect + if response.status() != axum::http::StatusCode::NOT_FOUND { + return response; + } + + tracing::info!( + "🔄 MCP stale session detected (404) — auto-reconnecting transparently" + ); + + // Step 1: Send an initialize request to ourselves to get a fresh session ID. + // We use a minimal MCP initialize payload (the server doesn't validate client info). + let init_payload = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": { "name": "codesearch-reconnect", "version": "1.0" } + } + }); + + let init_response = match reconnect + .client + .post(&reconnect.mcp_url) + .header("content-type", "application/json") + .header("accept", "text/event-stream") + .json(&init_payload) + .send() + .await + { + Ok(r) => r, + Err(e) => { + tracing::warn!("Auto-reconnect: initialize request failed: {}", e); + return axum::response::Response::builder() + .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) + .body(axum::body::Body::from("MCP reconnect failed")) + .unwrap(); + } + }; + + // Extract the new session ID from the initialize response header + let new_session_id = init_response + .headers() + .get("mcp-session-id") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_owned()); + + let Some(new_session_id) = new_session_id else { + tracing::warn!("Auto-reconnect: initialize succeeded but no session ID in response"); + return axum::response::Response::builder() + .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) + .body(axum::body::Body::from("MCP reconnect: no session ID")) + .unwrap(); + }; + + tracing::info!( + "🔄 Auto-reconnect: new session {} (replaced stale {})", + &new_session_id[..8.min(new_session_id.len())], + stale_session_id.as_deref().map(|s| &s[..8.min(s.len())]).unwrap_or("?") + ); + + // Step 2: Retry the original request with the new session ID + let retry_response = match reconnect + .client + .post(&reconnect.mcp_url) + .header("content-type", "application/json") + .header("accept", "text/event-stream") + .header("mcp-session-id", &new_session_id) + .body(body_bytes.to_vec()) + .send() + .await + { + Ok(r) => r, + Err(e) => { + tracing::warn!("Auto-reconnect: retry request failed: {}", e); + return axum::response::Response::builder() + .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) + .body(axum::body::Body::from("MCP reconnect retry failed")) + .unwrap(); + } + }; + + // Convert reqwest response back to axum response, injecting the new session ID header + // so the client updates its stored session ID automatically. + let status = retry_response.status(); + let mut builder = axum::response::Response::builder().status(status); + for (name, value) in retry_response.headers() { + builder = builder.header(name, value); + } + // Always include the new session ID so the client can update its stored value + builder = builder.header("mcp-session-id", &new_session_id); + + let body_bytes = retry_response.bytes().await.unwrap_or_default(); + builder + .body(axum::body::Body::from(body_bytes)) + .unwrap_or_else(|_| { + axum::response::Response::builder() + .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(axum::body::Body::from("Response build error")) + .unwrap() + }) } /// Run the MCP serve mode. @@ -1652,14 +1896,15 @@ pub async fn run_serve( // Create the MCP service factory — each session gets a fresh CodesearchService // that uses serve_state for repo routing. let state_for_factory = serve_state.clone(); - let service_factory = move || -> std::result::Result { - let session_id = state_for_factory.session_connected(); - info!("🔌 MCP client connected (session #{})", session_id); - // We create a minimal service; actual repo routing is handled inside - // the tool handlers via serve_state. - crate::mcp::CodesearchService::new_for_serve(state_for_factory.clone()) - .map_err(std::io::Error::other) - }; + let service_factory = + move || -> std::result::Result { + let session_id = state_for_factory.session_connected(); + info!("🔌 MCP client connected (session #{})", session_id); + // We create a minimal service; actual repo routing is handled inside + // the tool handlers via serve_state. + crate::mcp::CodesearchService::new_for_serve(state_for_factory.clone()) + .map_err(std::io::Error::other) + }; // Build session manager with extended keep_alive (default is 5 min which kills // idle MCP sessions too aggressively). 30 minutes matches our repo idle eviction. @@ -1668,21 +1913,29 @@ pub async fn run_serve( let session_manager = Arc::new(session_manager); let config = StreamableHttpServerConfig::default(); - let mcp_service = StreamableHttpService::new( - service_factory, - session_manager, - config, - ); + let mcp_service = StreamableHttpService::new(service_factory, session_manager, config); + + // Build reconnect middleware state (reqwest client + local MCP URL for internal retries) + let reconnect_state = ReconnectState { + client: reqwest::Client::new(), + mcp_url: format!("http://{}{}", addr, MCP_ENDPOINT_PATH), + }; // Build axum router with request logging + stale session recovery let app = axum::Router::new() .route(HEALTH_PATH, axum::routing::get(health_handler)) .route("/repos", axum::routing::post(add_repo_handler)) .route("/repos/:alias", axum::routing::delete(remove_repo_handler)) - .route("/repos/:alias/reindex", axum::routing::post(reindex_handler)) + .route( + "/repos/:alias/reindex", + axum::routing::post(reindex_handler), + ) .nest_service(MCP_ENDPOINT_PATH, mcp_service) .layer(axum::middleware::from_fn(log_mcp_requests)) - .layer(axum::middleware::from_fn(auto_reconnect_stale_sessions)) + .layer(axum::middleware::from_fn_with_state( + reconnect_state, + auto_reconnect_stale_sessions, + )) .with_state(serve_state.clone()); // Bind TCP listener BEFORE spawning background warmup, so we know the port is live. @@ -1759,11 +2012,10 @@ pub async fn run_serve( // 3 seconds after the cancel_token is cancelled. This gives in-flight HTTP // requests time to complete while preventing a permanent hang on open sessions. let cancel_for_deadline = cancel_token.clone(); - let server = axum::serve(listener, app) - .with_graceful_shutdown(async move { - cancel_token.cancelled().await; - info!("🛑 codesearch serve shutting down..."); - }); + let server = axum::serve(listener, app).with_graceful_shutdown(async move { + cancel_token.cancelled().await; + info!("🛑 codesearch serve shutting down..."); + }); tokio::select! { result = server => { @@ -1810,16 +2062,22 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("testalias".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("testalias".to_string())) + .unwrap(); let state = state_with_config(config); // First call: DB missing → error, NOT cached as Conflicted - let err = match state.get_or_open_stores("testalias").await { + let err = match state.get_or_open_stores("testalias", true).await { Err(e) => e, Ok(_) => panic!("expected error for missing DB"), }; - assert!(err.contains("Database not found"), "expected 'not found', got: {}", err); + assert!( + err.contains("Database not found"), + "expected 'not found', got: {}", + err + ); assert!(!state.repos.contains_key("testalias")); // Create a minimal DB so next call succeeds @@ -1835,7 +2093,7 @@ mod tests { drop(_stores); // Second call: should succeed without restart - let res = state.get_or_open_stores("testalias").await; + let res = state.get_or_open_stores("testalias", true).await; assert!(res.is_ok(), "expected ok after recreating DB, got: Err"); } @@ -1846,15 +2104,25 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("testalias".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("testalias".to_string())) + .unwrap(); let state = state_with_config(config); - let err = match state.get_or_open_stores("testalias").await { + let err = match state.get_or_open_stores("testalias", true).await { Err(e) => e, Ok(_) => panic!("expected error for missing DB"), }; - assert!(err.contains("codesearch index add"), "error should mention 'index add': {}", err); - assert!(err.contains("codesearch index rm"), "error should mention 'index rm': {}", err); + assert!( + err.contains("codesearch index add"), + "error should mention 'index add': {}", + err + ); + assert!( + err.contains("codesearch index rm"), + "error should mention 'index rm': {}", + err + ); } #[tokio::test] @@ -1873,15 +2141,21 @@ mod tests { let _lock = SharedStores::new(&db_path, 384).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("testalias".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("testalias".to_string())) + .unwrap(); let state = state_with_config(config); - let err = match state.get_or_open_stores("testalias").await { + let err = match state.get_or_open_stores("testalias", true).await { Err(e) => e, Ok(_) => panic!("expected conflict error"), }; assert!(err.contains("Stop"), "error should mention 'Stop': {}", err); - assert!(err.contains("retry"), "error should mention 'retry': {}", err); + assert!( + err.contains("retry"), + "error should mention 'retry': {}", + err + ); } #[test] @@ -1893,7 +2167,9 @@ mod tests { std::fs::create_dir(&repo_a).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_a.clone(), Some("a".to_string())).unwrap(); + config + .register_with_alias(repo_a.clone(), Some("a".to_string())) + .unwrap(); config.save_to(&config_file).unwrap(); let state = ServeState::new(config, Some(config_file.clone())); @@ -1903,7 +2179,9 @@ mod tests { let repo_b = tmp.path().join("repo-b"); std::fs::create_dir(&repo_b).unwrap(); let mut config2 = ReposConfig::load_from(&config_file).unwrap(); - config2.register_with_alias(repo_b, Some("b".to_string())).unwrap(); + config2 + .register_with_alias(repo_b, Some("b".to_string())) + .unwrap(); // Small sleep to ensure mtime changes on Windows std::thread::sleep(std::time::Duration::from_millis(150)); @@ -1932,12 +2210,14 @@ mod tests { drop(_stores); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("x".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("x".to_string())) + .unwrap(); config.save_to(&config_file).unwrap(); let state = ServeState::new(config, Some(config_file.clone())); // Open alias x so it lands in DashMap - let _ = state.get_or_open_stores("x").await.unwrap(); + let _ = state.get_or_open_stores("x", true).await.unwrap(); assert!(state.repos.contains_key("x")); // Rewrite config without x @@ -1948,11 +2228,15 @@ mod tests { config2.save_to(&config_file).unwrap(); // Next query for x should fail as unknown - let err = match state.get_or_open_stores("x").await { + let err = match state.get_or_open_stores("x", true).await { Err(e) => e, Ok(_) => panic!("expected unknown alias after removal"), }; - assert!(err.contains("Unknown alias"), "expected unknown alias, got: {}", err); + assert!( + err.contains("Unknown alias"), + "expected unknown alias, got: {}", + err + ); assert!(!state.repos.contains_key("x")); } @@ -1965,7 +2249,9 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path, Some("a".to_string())).unwrap(); + config + .register_with_alias(repo_path, Some("a".to_string())) + .unwrap(); config.save_to(&config_file).unwrap(); let state = ServeState::new(config, Some(config_file.clone())); @@ -1991,7 +2277,9 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("testalias".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("testalias".to_string())) + .unwrap(); let config_file = tmp.path().join("repos.json"); config.save_to(&config_file).unwrap(); @@ -1999,8 +2287,14 @@ mod tests { let state = Arc::new(ServeState::new(config, Some(config_file))); let app = axum::Router::new() - .route(crate::constants::HEALTH_PATH, axum::routing::get(health_handler)) - .route("/repos/:alias/reindex", axum::routing::post(reindex_handler)) + .route( + crate::constants::HEALTH_PATH, + axum::routing::get(health_handler), + ) + .route( + "/repos/:alias/reindex", + axum::routing::post(reindex_handler), + ) .with_state(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -2021,9 +2315,20 @@ mod tests { .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND, "expected 404 from our handler"); - let body: serde_json::Value = resp.json().await.expect("handler should return JSON body for 404"); - assert!(body.get("error").is_some(), "expected JSON error body, got: {}", body); + assert_eq!( + resp.status(), + reqwest::StatusCode::NOT_FOUND, + "expected 404 from our handler" + ); + let body: serde_json::Value = resp + .json() + .await + .expect("handler should return JSON body for 404"); + assert!( + body.get("error").is_some(), + "expected JSON error body, got: {}", + body + ); // POST to known alias → 202 Accepted or 500 (DB missing), but NOT axum's built-in 404 // The key assertion is that the route IS registered (we get our handler's response, not axum's empty 404) @@ -2035,11 +2340,17 @@ mod tests { let status = resp.status(); let body: serde_json::Value = resp.json().await.expect("handler should return JSON body"); assert!( - status == reqwest::StatusCode::ACCEPTED || status == reqwest::StatusCode::INTERNAL_SERVER_ERROR, + status == reqwest::StatusCode::ACCEPTED + || status == reqwest::StatusCode::INTERNAL_SERVER_ERROR, "expected 202 or 500 from our handler (not axum's 404), got {}: {}", - status, body + status, + body + ); + assert!( + body.get("status").is_some(), + "expected JSON with 'status' field, got: {}", + body ); - assert!(body.get("status").is_some(), "expected JSON with 'status' field, got: {}", body); } #[test] @@ -2051,7 +2362,9 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("a".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("a".to_string())) + .unwrap(); config.save_to(&config_file).unwrap(); let state = ServeState::new(config, Some(config_file.clone())); @@ -2073,7 +2386,9 @@ mod tests { std::fs::create_dir(&repo_path).unwrap(); let mut config = ReposConfig::default(); - config.register_with_alias(repo_path.clone(), Some("testalias".to_string())).unwrap(); + config + .register_with_alias(repo_path.clone(), Some("testalias".to_string())) + .unwrap(); let config_file = tmp.path().join("repos.json"); config.save_to(&config_file).unwrap(); @@ -2081,7 +2396,10 @@ mod tests { let state = Arc::new(ServeState::new(config, Some(config_file))); let app = axum::Router::new() - .route("/repos/:alias/reindex", axum::routing::post(reindex_handler)) + .route( + "/repos/:alias/reindex", + axum::routing::post(reindex_handler), + ) .with_state(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -2103,8 +2421,10 @@ mod tests { .unwrap(); let status1 = resp1.status(); assert!( - status1 == reqwest::StatusCode::ACCEPTED || status1 == reqwest::StatusCode::INTERNAL_SERVER_ERROR, - "first request should be 202 or 500, got {}", status1 + status1 == reqwest::StatusCode::ACCEPTED + || status1 == reqwest::StatusCode::INTERNAL_SERVER_ERROR, + "first request should be 202 or 500, got {}", + status1 ); // If the first request was accepted (202), the reindex is running in background. From 4b51cce521a278acf0695b5601075bd4e0b63d65 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 13:36:54 +0200 Subject: [PATCH 06/10] fix(tui): normalize CPU% by core count (was showing 474% instead of ~32%) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/serve/tui.rs | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e97bd12..218448e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.67" +version = "1.0.68" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index c8eecc0..4065a6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.67" +version = "1.0.68" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/serve/tui.rs b/src/serve/tui.rs index 1ca53d6..4a92f23 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -443,13 +443,25 @@ fn cpu_usage_str(sys_system: &mut Option) -> String { }; // Create System instance on first call, reuse on subsequent calls. - let sys = sys_system.get_or_insert_with(System::new); + // Refresh CPUs once on creation so sys.cpus().len() is populated. + let sys = sys_system.get_or_insert_with(|| { + let mut s = System::new(); + s.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing()); + s + }); // Refresh only our process (cpu) sys.refresh_processes(ProcessesToUpdate::Some(&[pid]), true); match sys.process(pid) { - Some(proc) => format!("{:.0}%", proc.cpu_usage()), + Some(proc) => { + // sysinfo reports cpu_usage() as total across all logical CPUs + // (e.g. 474% on a 15-core machine). Divide by CPU count to get + // a 0-100% value that makes sense to humans. + let num_cpus = sys.cpus().len().max(1) as f32; + let pct = proc.cpu_usage() / num_cpus; + format!("{:.0}%", pct) + } None => "—".into(), } } From be6994bd3f09be94c0b17f4fc24cd85d8a532c32 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 14:02:13 +0200 Subject: [PATCH 07/10] =?UTF-8?q?fix(serve):=20idle=20eviction=20=E2=80=94?= =?UTF-8?q?=20touch=20only=20on=20direct=20query,=20not=20fan-out?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_or_open_stores now takes touch:bool. Fan-out paths (unscoped get_chunk, group routing) pass false so they don't reset the idle timer on every repo. Only direct single-repo queries pass true. After get_chunk candidate detection resolves to a single repo, touch_access is called explicitly for just that repo. Warmup no longer resets idle timers (touch: false). --- AGENTS.md | 163 ++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/mcp/mod.rs | 1280 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 1025 insertions(+), 422 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..debc17c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,163 @@ +# AGENTS.md — features/fixes branch plan + +## Branch: `features/fixes` +**Base:** `develop` +**Goal:** Fix idle eviction bug + improve search quality to reduce agent grep fallback + +--- + +## Fix 1: Idle eviction — `get_or_open_stores` touches ALL repos on fan-out + +### Problem + +`get_or_open_stores()` calls `touch_access()` unconditionally (lines 456, 470 in `src/serve/mod.rs`). +When `get_chunk` is called without `project`/`group` (`allow_unscoped=true`), `resolve_repo_stores_multi` +fans out to ALL repos via `get_or_open_stores()` — resetting the idle timer on every repo. + +Result: repos are never idle, reaper never evicts. The 30-minute timeout is effectively disabled +whenever any agent uses `get_chunk` without explicit project scope. + +Same issue affects `status` tool with explicit project/group (goes through `get_or_open_stores`), +though the unscoped `status` path uses `repo_statuses_lightweight()` which is safe. + +### Fix: Add `touch: bool` parameter to `get_or_open_stores` + +**File:** `src/serve/mod.rs` + +1. Change signature: `pub(crate) async fn get_or_open_stores(&self, alias: &str, touch: bool)` +2. Only call `self.touch_access(alias)` when `touch == true` +3. Update all call sites: + - `warmup_repo` (line 456) → `touch: false` (warmup should NOT reset idle timer) + - `get_or_open_stores` fast path (line 470) → keep `touch: true` (direct query access) + - `resolve_repo_stores_multi` fan-out (line 3113 in `src/mcp/mod.rs`) → `touch: false` + - `resolve_repo_stores_multi` single project (line 3130) → `touch: true` + - `resolve_repo_stores_multi` group members (line 3141) → `touch: false` + - Lazy FSW transition (line ~487) → `touch: true` (first real query) + - `spawn_fsw_for_warm` (line ~722) → `touch: false` + - `reindex_handler` (lines 1166, 1211) → `touch: true` + - Test call sites → `touch: true` + +4. After `get_chunk` candidate detection resolves to a single repo, explicitly call + `serve_state.touch_access(&resolved_alias)` for just that repo. + +5. After group fan-out search completes, touch only repos that contributed results + (or touch all group members — acceptable since the agent explicitly requested the group). + +### Validation +- `cargo check && cargo clippy --all-targets -- -D warnings` +- `cargo test --lib` +- Manual: start serve with 3+ repos, call `get_chunk` without project, verify reaper log + shows idle ages increasing (not resetting) for untouched repos + +--- + +## Fix 2: Search quality — reduce agent grep fallback + +### Problem + +Agents fall back to `grep` when `codesearch_search` returns poor or zero results. +Root causes: + +1. **Top-N cutoff too aggressive** — retrieval pool is `limit * 3`, fusion drops relevant results +2. **Exact identifier boost too weak** — `EXACT_MATCH_RRF_K = 5.0` doesn't sufficiently + prioritize exact code matches over semantic similarity +3. **No auto-fallback** — when semantic search returns few results, no automatic literal retry +4. **minilm-l6 weak on code** — embedding model is NL-trained, code identifiers get poor vectors. + Not fixable without model change, but compensated by stronger FTS fusion. + +### Fix 2a: Increase retrieval pool + +**File:** `src/mcp/mod.rs` + +Change all `limit * 3` to `limit * 5` in the semantic search pipeline. +This gives the RRF fusion more candidates to work with, reducing the chance +that a relevant result falls outside the retrieval window. + +Affected locations (all in `src/mcp/mod.rs`): +- Line 3698: `store.search(&query_embedding, limit * 3)` → `limit * 5` +- Line 3753: `fts_store.search(&request.query, limit * 3, ...)` → `limit * 5` +- Line 3864: `fts_store.search(&request.query, limit * 3, ...)` → `limit * 5` +- Line 3925: `store.search(&query_embedding, limit * 3)` → `limit * 5` +- Line 3955: `fts_store.search(&request.query, limit * 3, ...)` → `limit * 5` +- Line 4108: `fts_store.search(&request.query, limit * 3, ...)` → `limit * 5` + +Also in `src/search/mod.rs` (CLI search path) — same pattern. + +Leave `search_exact` at `limit * 2` (exact matches are already high-precision). +Leave `search_phrase` at `limit * 3` (phrase search is already precise). + +### Fix 2b: Stronger exact identifier boost + +**File:** `src/rerank/mod.rs` + +Change `EXACT_MATCH_RRF_K` from `5.0` to `2.0`. + +Lower K = steeper rank curve = exact matches get proportionally higher RRF scores. +At K=5, an exact match at rank 1 gets score `1/(5+1) = 0.167`. +At K=2, an exact match at rank 1 gets score `1/(2+1) = 0.333` — 2x stronger signal. + +This ensures that when an agent searches for `"evict_idle_repos"`, the chunk containing +that exact identifier dominates the fusion result even if the embedding similarity is low. + +### Fix 2c: Auto-fallback to literal search + +**File:** `src/mcp/mod.rs`, in `semantic_search()` (line ~3620) + +After the hybrid search completes and results are built: + +```rust +// If semantic/hybrid returned fewer than 3 results and query looks like code, +// auto-fallback to literal search and merge results. +if results.len() < 3 && has_identifiers { + // Try literal FTS search as fallback + let literal_results = fts_store.search(&request.query, limit, None)?; + // Deduplicate by chunk_id and append + for lr in literal_results { + if !results.iter().any(|r| r.id == lr.chunk_id) { + // Convert FtsResult to SearchResult and append + } + } +} +``` + +Implementation details: +- Only trigger when `results.len() < 3` AND `has_identifiers` (code-like query) +- Use `with_fts_store_read_for` to run the fallback FTS search +- Deduplicate by `chunk_id` before merging +- Cap total results at `limit` +- Log when fallback triggers: `tracing::debug!("Auto-fallback: semantic returned {} results, trying literal", results.len())` + +### Fix 2d: Increase `search_exact` retrieval for identifiers + +**File:** `src/mcp/mod.rs` + +Change `search_exact(ident, limit * 2, ...)` to `search_exact(ident, limit * 3, ...)` +in the identifier boost paths (lines 3762, 3876, 3968, 4120). + +More exact candidates = better chance the right chunk survives RRF fusion. + +### Validation +- `cargo check && cargo clippy --all-targets -- -D warnings` +- `cargo test --lib` +- Manual test queries that previously required grep fallback: + - `codesearch search "evict_idle_repos"` — should find the function + - `codesearch search "touch_access"` — should find the method + - `codesearch search "Database cleared"` — should find the log message (already fixed by AND mode) + - `codesearch search "EXACT_MATCH_RRF_K"` — should find the constant + +--- + +## Execution order + +1. **Fix 1** — idle eviction (`touch` parameter) +2. **Fix 2a** — retrieval pool `limit * 5` +3. **Fix 2b** — `EXACT_MATCH_RRF_K` = 2.0 +4. **Fix 2c** — auto-fallback to literal +5. **Fix 2d** — `search_exact` retrieval `limit * 3` +6. Validate all together +7. Commit + +## Commits + +One commit per fix, or group 2a-2d into a single "search quality" commit. +Prefer: 2 commits total (Fix 1 + Fix 2). diff --git a/Cargo.lock b/Cargo.lock index 218448e..6041bb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.68" +version = "1.0.69" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 4065a6c..ac1e23c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.68" +version = "1.0.69" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 631e1b9..8ef19eb 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -296,21 +296,15 @@ mod tests { #[test] fn test_path_prefix_windows_backslashes() { - let result = super::prefix_path_with_alias( - r"C:\repo\src\main.rs", - Some("myrepo"), - r"C:\repo", - ); + let result = + super::prefix_path_with_alias(r"C:\repo\src\main.rs", Some("myrepo"), r"C:\repo"); assert_eq!(result, "myrepo/src/main.rs"); } #[test] fn test_path_prefix_unc_prefix() { - let result = super::prefix_path_with_alias( - r"\\?\C:\repo\src\main.rs", - Some("myrepo"), - r"C:\repo", - ); + let result = + super::prefix_path_with_alias(r"\\?\C:\repo\src\main.rs", Some("myrepo"), r"C:\repo"); // After normalization, UNC prefix is stripped by normalize_path_str assert!( result.starts_with("myrepo/"), @@ -326,41 +320,27 @@ mod tests { #[test] fn test_path_prefix_mixed_separators() { - let result = super::prefix_path_with_alias( - r"C:\repo/src\main.rs", - Some("myrepo"), - r"C:\repo", - ); + let result = + super::prefix_path_with_alias(r"C:\repo/src\main.rs", Some("myrepo"), r"C:\repo"); assert_eq!(result, "myrepo/src/main.rs"); } #[test] fn test_path_prefix_no_alias() { - let result = super::prefix_path_with_alias( - "C:/repo/src/main.rs", - None, - "C:/repo", - ); + let result = super::prefix_path_with_alias("C:/repo/src/main.rs", None, "C:/repo"); assert_eq!(result, "src/main.rs"); } #[test] fn test_path_prefix_empty_alias() { - let result = super::prefix_path_with_alias( - "C:/repo/src/main.rs", - Some(""), - "C:/repo", - ); + let result = super::prefix_path_with_alias("C:/repo/src/main.rs", Some(""), "C:/repo"); assert_eq!(result, "src/main.rs"); } #[test] fn test_path_prefix_preserves_path_outside_root() { - let result = super::prefix_path_with_alias( - "C:/other/src/main.rs", - Some("myrepo"), - "C:/repo", - ); + let result = + super::prefix_path_with_alias("C:/other/src/main.rs", Some("myrepo"), "C:/repo"); // Path doesn't start with root — returned normalized, no alias prefix assert_eq!(result, "C:/other/src/main.rs"); } @@ -370,16 +350,10 @@ mod tests { // Simulate two stores for aliases "a" and "b", each returning a result // with absolute path = "/abs/root/src/main.rs". After applying prefix_path_with_alias, // assert results have path = "a/src/main.rs" and "b/src/main.rs". - let result_a = super::prefix_path_with_alias( - "/abs/root/src/main.rs", - Some("a"), - "/abs/root", - ); - let result_b = super::prefix_path_with_alias( - "/abs/root/src/main.rs", - Some("b"), - "/abs/root", - ); + let result_a = + super::prefix_path_with_alias("/abs/root/src/main.rs", Some("a"), "/abs/root"); + let result_b = + super::prefix_path_with_alias("/abs/root/src/main.rs", Some("b"), "/abs/root"); assert_eq!(result_a, "a/src/main.rs"); assert_eq!(result_b, "b/src/main.rs"); } @@ -388,22 +362,15 @@ mod tests { fn test_single_project_result_is_alias_prefixed() { // Single store for alias "myrepo", result with path = "/abs/root/src/lib.rs", // project root "/abs/root" → assert path becomes "myrepo/src/lib.rs". - let result = super::prefix_path_with_alias( - "/abs/root/src/lib.rs", - Some("myrepo"), - "/abs/root", - ); + let result = + super::prefix_path_with_alias("/abs/root/src/lib.rs", Some("myrepo"), "/abs/root"); assert_eq!(result, "myrepo/src/lib.rs"); } #[test] fn test_stdio_mode_paths_not_prefixed() { // alias None → path normalized, no prefix added. - let result = super::prefix_path_with_alias( - "C:/repo/src/main.rs", - None, - "C:/repo", - ); + let result = super::prefix_path_with_alias("C:/repo/src/main.rs", None, "C:/repo"); assert_eq!(result, "src/main.rs"); } @@ -1081,7 +1048,10 @@ mod tests { #[test] fn test_has_chunk_id_and_score_fts_result() { - let result = crate::fts::FtsResult { chunk_id: 42, score: 0.85 }; + let result = crate::fts::FtsResult { + chunk_id: 42, + score: 0.85, + }; assert_eq!(super::HasChunkId::chunk_id(&result), 42); assert!((super::HasScore::score(&result) - 0.85).abs() < f32::EPSILON); } @@ -1118,29 +1088,59 @@ mod tests { // Simulate results from 3 stores with overlapping chunk_ids across repos let store1_results = vec![ - crate::fts::FtsResult { chunk_id: 1, score: 0.5 }, - crate::fts::FtsResult { chunk_id: 2, score: 0.8 }, - crate::fts::FtsResult { chunk_id: 3, score: 0.3 }, + crate::fts::FtsResult { + chunk_id: 1, + score: 0.5, + }, + crate::fts::FtsResult { + chunk_id: 2, + score: 0.8, + }, + crate::fts::FtsResult { + chunk_id: 3, + score: 0.3, + }, ]; let store2_results = vec![ - crate::fts::FtsResult { chunk_id: 1, score: 0.9 }, // same chunk_id, different alias — NOT a dup - crate::fts::FtsResult { chunk_id: 4, score: 0.7 }, - crate::fts::FtsResult { chunk_id: 2, score: 0.4 }, // same chunk_id, different alias — NOT a dup + crate::fts::FtsResult { + chunk_id: 1, + score: 0.9, + }, // same chunk_id, different alias — NOT a dup + crate::fts::FtsResult { + chunk_id: 4, + score: 0.7, + }, + crate::fts::FtsResult { + chunk_id: 2, + score: 0.4, + }, // same chunk_id, different alias — NOT a dup ]; let store3_results = vec![ - crate::fts::FtsResult { chunk_id: 3, score: 0.6 }, // same chunk_id, different alias — NOT a dup - crate::fts::FtsResult { chunk_id: 5, score: 0.2 }, + crate::fts::FtsResult { + chunk_id: 3, + score: 0.6, + }, // same chunk_id, different alias — NOT a dup + crate::fts::FtsResult { + chunk_id: 5, + score: 0.2, + }, ]; // Apply the same dedup logic as with_fts_store_read_multi: key is (alias, chunk_id) let mut all_results: Vec = Vec::new(); let mut seen_ids: HashMap<(String, u32), usize> = HashMap::new(); - for (alias, results) in aliases.iter().zip([&store1_results, &store2_results, &store3_results]) { + for (alias, results) in + aliases + .iter() + .zip([&store1_results, &store2_results, &store3_results]) + { for r in results { let key = (alias.to_string(), super::HasChunkId::chunk_id(r)); if let Some(&existing_idx) = seen_ids.get(&key) { - if super::HasScore::score(r) > super::HasScore::score(&all_results[existing_idx]) { + if super::HasScore::score(r) + > super::HasScore::score(&all_results[existing_idx]) + { all_results[existing_idx] = r.clone(); } } else { @@ -1158,25 +1158,46 @@ mod tests { }); // Verify: 8 unique (alias, chunk_id) pairs — NO cross-alias dedup - assert_eq!(all_results.len(), 8, "Should have 8 unique (alias, chunk_id) pairs across 3 repos"); + assert_eq!( + all_results.len(), + 8, + "Should have 8 unique (alias, chunk_id) pairs across 3 repos" + ); // Check sort: first result should be highest score - assert!((all_results[0].score - 0.9).abs() < f32::EPSILON, "First result should have highest score"); + assert!( + (all_results[0].score - 0.9).abs() < f32::EPSILON, + "First result should have highest score" + ); // Check sort: scores should be descending for i in 1..all_results.len() { - assert!(all_results[i].score <= all_results[i - 1].score, + assert!( + all_results[i].score <= all_results[i - 1].score, "Results should be sorted by score descending, but [{}]={} > [{}]={}", - i - 1, all_results[i - 1].score, i, all_results[i].score); + i - 1, + all_results[i - 1].score, + i, + all_results[i].score + ); } } #[test] fn test_multi_store_dedup_no_overlap() { // Non-overlapping results — all should be kept - let store1 = vec![crate::fts::FtsResult { chunk_id: 1, score: 0.5 }]; - let store2 = vec![crate::fts::FtsResult { chunk_id: 2, score: 0.8 }]; - let store3 = vec![crate::fts::FtsResult { chunk_id: 3, score: 0.3 }]; + let store1 = vec![crate::fts::FtsResult { + chunk_id: 1, + score: 0.5, + }]; + let store2 = vec![crate::fts::FtsResult { + chunk_id: 2, + score: 0.8, + }]; + let store3 = vec![crate::fts::FtsResult { + chunk_id: 3, + score: 0.3, + }]; let mut all_results: Vec = Vec::new(); let mut seen_ids: std::collections::HashMap = std::collections::HashMap::new(); @@ -1185,7 +1206,9 @@ mod tests { for r in results { let id = super::HasChunkId::chunk_id(r); if let Some(&existing_idx) = seen_ids.get(&id) { - if super::HasScore::score(r) > super::HasScore::score(&all_results[existing_idx]) { + if super::HasScore::score(r) + > super::HasScore::score(&all_results[existing_idx]) + { all_results[existing_idx] = r.clone(); } } else { @@ -1195,15 +1218,28 @@ mod tests { } } - assert_eq!(all_results.len(), 3, "All 3 non-overlapping results should be kept"); + assert_eq!( + all_results.len(), + 3, + "All 3 non-overlapping results should be kept" + ); } #[test] fn test_multi_store_dedup_all_same_ids() { // All stores return same chunk_ids — only keep each once with max score - let store1 = vec![crate::fts::FtsResult { chunk_id: 1, score: 0.3 }]; - let store2 = vec![crate::fts::FtsResult { chunk_id: 1, score: 0.9 }]; - let store3 = vec![crate::fts::FtsResult { chunk_id: 1, score: 0.6 }]; + let store1 = vec![crate::fts::FtsResult { + chunk_id: 1, + score: 0.3, + }]; + let store2 = vec![crate::fts::FtsResult { + chunk_id: 1, + score: 0.9, + }]; + let store3 = vec![crate::fts::FtsResult { + chunk_id: 1, + score: 0.6, + }]; let mut all_results: Vec = Vec::new(); let mut seen_ids: std::collections::HashMap = std::collections::HashMap::new(); @@ -1212,7 +1248,9 @@ mod tests { for r in results { let id = super::HasChunkId::chunk_id(r); if let Some(&existing_idx) = seen_ids.get(&id) { - if super::HasScore::score(r) > super::HasScore::score(&all_results[existing_idx]) { + if super::HasScore::score(r) + > super::HasScore::score(&all_results[existing_idx]) + { all_results[existing_idx] = r.clone(); } } else { @@ -1223,8 +1261,11 @@ mod tests { } assert_eq!(all_results.len(), 1, "Should deduplicate to 1 result"); - assert!((all_results[0].score - 0.9).abs() < f32::EPSILON, - "Should keep highest score 0.9, got {}", all_results[0].score); + assert!( + (all_results[0].score - 0.9).abs() < f32::EPSILON, + "Should keep highest score 0.9, got {}", + all_results[0].score + ); } // === Serde roundtrip tests for group field === @@ -1388,12 +1429,14 @@ mod tests { #[test] fn test_routing_decomposition_none_input() { // No routing params → all None/false, needs_local_db = true - let (stores, stores_vec, is_multi, needs_local_db) = - decompose_routing_ctx::(None); + let (stores, stores_vec, is_multi, needs_local_db) = decompose_routing_ctx::(None); assert!(stores.is_none(), "stores should be None"); assert!(stores_vec.is_none(), "stores_vec should be None"); assert!(!is_multi, "is_multi should be false"); - assert!(needs_local_db, "needs_local_db should be true — no serve-state stores"); + assert!( + needs_local_db, + "needs_local_db should be true — no serve-state stores" + ); } #[test] @@ -1402,9 +1445,15 @@ mod tests { let (stores, stores_vec, is_multi, needs_local_db) = decompose_routing_ctx(Some(vec![arc_val(1)])); assert!(stores.is_some(), "stores should be Some for single repo"); - assert!(stores_vec.is_none(), "stores_vec should be None for single repo"); + assert!( + stores_vec.is_none(), + "stores_vec should be None for single repo" + ); assert!(!is_multi, "is_multi should be false for single repo"); - assert!(!needs_local_db, "needs_local_db should be false — we have a store"); + assert!( + !needs_local_db, + "needs_local_db should be false — we have a store" + ); assert_eq!(*stores.unwrap(), 1); } @@ -1414,9 +1463,15 @@ mod tests { let (stores, stores_vec, is_multi, needs_local_db) = decompose_routing_ctx(Some(vec![arc_val(1), arc_val(2)])); assert!(stores.is_none(), "stores should be None for multi-store"); - assert!(stores_vec.is_some(), "stores_vec should be Some for multi-store"); + assert!( + stores_vec.is_some(), + "stores_vec should be Some for multi-store" + ); assert!(is_multi, "is_multi should be true for 2+ stores"); - assert!(!needs_local_db, "needs_local_db should be false — we have stores"); + assert!( + !needs_local_db, + "needs_local_db should be false — we have stores" + ); let sv = stores_vec.unwrap(); assert_eq!(sv.len(), 2); } @@ -1440,7 +1495,10 @@ mod tests { decompose_routing_ctx::(Some(vec![])); // Empty vec: is_multi=false (len=0 not > 1), stores=None (len=0 not 1) assert!(stores.is_none(), "empty vec → stores None"); - assert!(stores_vec.is_none(), "empty vec → stores_vec None (is_multi=false)"); + assert!( + stores_vec.is_none(), + "empty vec → stores_vec None (is_multi=false)" + ); assert!(!is_multi, "empty vec → is_multi false"); assert!(needs_local_db, "empty vec → needs_local_db true"); } @@ -1457,8 +1515,7 @@ mod tests { fn test_routing_single_project_maps_to_single_store() { // A single project alias → vec of length 1 → single-store path let multi = Some(vec![arc_val(42)]); - let (stores, stores_vec, is_multi, needs_local_db) = - decompose_routing_ctx(multi); + let (stores, stores_vec, is_multi, needs_local_db) = decompose_routing_ctx(multi); assert!(!is_multi); assert!(stores.is_some()); assert_eq!(*stores.unwrap(), 42); @@ -1470,8 +1527,7 @@ mod tests { fn test_routing_group_maps_to_multi_store() { // A group with 3 aliases → vec of length 3 → multi-store path let multi = Some(vec![arc_val(1), arc_val(2), arc_val(3)]); - let (stores, stores_vec, is_multi, needs_local_db) = - decompose_routing_ctx(multi); + let (stores, stores_vec, is_multi, needs_local_db) = decompose_routing_ctx(multi); assert!(is_multi); assert!(stores.is_none(), "multi-store → no single override"); assert_eq!(stores_vec.unwrap().len(), 3); @@ -1485,12 +1541,24 @@ mod tests { // Simulate merging FTS results from multiple stores with overlapping chunk_ids // This is the pattern used by with_fts_store_read_multi let mut base: Vec = vec![ - crate::fts::FtsResult { chunk_id: 1, score: 0.5 }, - crate::fts::FtsResult { chunk_id: 2, score: 0.8 }, + crate::fts::FtsResult { + chunk_id: 1, + score: 0.5, + }, + crate::fts::FtsResult { + chunk_id: 2, + score: 0.8, + }, ]; let exact = vec![ - crate::fts::FtsResult { chunk_id: 1, score: 0.9 }, // higher score - crate::fts::FtsResult { chunk_id: 3, score: 0.7 }, // new chunk + crate::fts::FtsResult { + chunk_id: 1, + score: 0.9, + }, // higher score + crate::fts::FtsResult { + chunk_id: 3, + score: 0.7, + }, // new chunk ]; super::merge_exact_into_fts(&mut base, exact); @@ -1678,12 +1746,22 @@ mod tests { // This content does NOT match the regex let regex2 = regex::Regex::new(r"zzz_definitely_not_in_code").unwrap(); let content2 = "fn foo() {}\nfn bar() {}"; - assert!(super::match_line_for_literal(content2, "zzz_definitely_not_in_code", Some(®ex2)).is_none()); + assert!(super::match_line_for_literal( + content2, + "zzz_definitely_not_in_code", + Some(®ex2) + ) + .is_none()); // Non-anchorable regex with no matches → empty (scan path would skip) let regex3 = regex::Regex::new(r"\bimpl\s+\w+\s+for\s+\w+").unwrap(); let content3 = "fn simple() {}\nstruct Foo;"; - assert!(super::match_line_for_literal(content3, r"\bimpl\s+\w+\s+for\s+\w+", Some(®ex3)).is_none()); + assert!(super::match_line_for_literal( + content3, + r"\bimpl\s+\w+\s+for\s+\w+", + Some(®ex3) + ) + .is_none()); } // ─── looks_like_code_pattern detector tests ─────────────────────── @@ -1720,7 +1798,9 @@ mod tests { #[test] fn test_looks_like_code_pattern_plain_identifier_false() { - assert!(!super::looks_like_code_pattern("ActivitiesListModelResponse")); + assert!(!super::looks_like_code_pattern( + "ActivitiesListModelResponse" + )); assert!(!super::looks_like_code_pattern("foo_bar")); } @@ -1762,10 +1842,8 @@ mod tests { fn test_literal_lc_natural_language_weak_score() { // Use a score demonstrably less than f32::MAX let weak_score = super::LITERAL_LOW_CONFIDENCE_BM25 / 2.0; - let (lc, hint) = super::compute_literal_low_confidence( - Some(weak_score), - "how do we handle auth", - ); + let (lc, hint) = + super::compute_literal_low_confidence(Some(weak_score), "how do we handle auth"); assert_eq!(lc, Some(true)); assert!(hint.unwrap().contains("semantic")); } @@ -1776,11 +1854,12 @@ mod tests { // BM25 IDF artefacts (e.g. `or` in a snake_case name) must not // cause false low_confidence signals when results exist. let weak_score = super::LITERAL_LOW_CONFIDENCE_BM25 / 2.0; - let (lc, hint) = super::compute_literal_low_confidence( - Some(weak_score), - "CodesearchService", + let (lc, hint) = + super::compute_literal_low_confidence(Some(weak_score), "CodesearchService"); + assert_eq!( + lc, None, + "single identifier with results must not be flagged low_confidence" ); - assert_eq!(lc, None, "single identifier with results must not be flagged low_confidence"); assert_eq!(hint, None); } @@ -1788,7 +1867,10 @@ mod tests { fn test_literal_lc_does_not_fire_on_strong_results() { // Strong BM25 score (well above floor) must NOT be flagged low_confidence. let (lc, hint) = super::compute_literal_low_confidence(Some(41.5), "anything"); - assert_eq!(lc, None, "strong BM25 results must not be flagged low_confidence"); + assert_eq!( + lc, None, + "strong BM25 results must not be flagged low_confidence" + ); assert_eq!(hint, None); } @@ -1798,7 +1880,7 @@ mod tests { // when the BM25 score is below the floor. let (lc, hint) = super::compute_literal_low_confidence( Some(super::LITERAL_LOW_CONFIDENCE_BM25 - 0.5), - "how do we handle authentication", // multi-word natural language + "how do we handle authentication", // multi-word natural language ); assert_eq!(lc, Some(true)); assert!(hint.is_some()); @@ -1865,26 +1947,52 @@ mod tests { Some("ignored".to_string()) } else if low_confidence == Some(true) { suggested_tool.as_ref().map(|tool| { - format!("Top result has weak BM25 score; consider using `{}` for better matches.", tool) + format!( + "Top result has weak BM25 score; consider using `{}` for better matches.", + tool + ) }) } else { None }; let n = note.expect("note must be present when low_confidence is true"); - assert!(n.starts_with("Top result"), "note must read as a sentence, got: {}", n); - assert!(n.contains("find with kind='definition'"), "note must reference the suggested tool: {}", n); + assert!( + n.starts_with("Top result"), + "note must read as a sentence, got: {}", + n + ); + assert!( + n.contains("find with kind='definition'"), + "note must reference the suggested tool: {}", + n + ); } // ─── MCP mode selection tests ──────────────────────────────────── #[test] fn test_mcp_mode_from_str() { - assert_eq!("auto".parse::().unwrap(), super::McpMode::Auto); - assert_eq!("client".parse::().unwrap(), super::McpMode::Client); - assert_eq!("local".parse::().unwrap(), super::McpMode::Local); - assert_eq!("AUTO".parse::().unwrap(), super::McpMode::Auto); - assert_eq!("Client".parse::().unwrap(), super::McpMode::Client); + assert_eq!( + "auto".parse::().unwrap(), + super::McpMode::Auto + ); + assert_eq!( + "client".parse::().unwrap(), + super::McpMode::Client + ); + assert_eq!( + "local".parse::().unwrap(), + super::McpMode::Local + ); + assert_eq!( + "AUTO".parse::().unwrap(), + super::McpMode::Auto + ); + assert_eq!( + "Client".parse::().unwrap(), + super::McpMode::Client + ); assert!("invalid".parse::().is_err()); } @@ -1911,7 +2019,11 @@ mod tests { fn test_mcp_mode_from_str_covers_all() { // Verify all valid modes parse correctly for mode in &["auto", "client", "local", "AUTO", "Client", "LOCAL"] { - assert!(mode.parse::().is_ok(), "failed to parse: {}", mode); + assert!( + mode.parse::().is_ok(), + "failed to parse: {}", + mode + ); } assert!("invalid".parse::().is_err()); } @@ -1931,7 +2043,8 @@ mod tests { fn test_auto_promoted_skipped_when_user_sets_regex() { let user_set_regex = true; let user_set_phrase = false; - let auto_promoted = !user_set_regex && !user_set_phrase && super::looks_like_code_pattern("foo = null"); + let auto_promoted = + !user_set_regex && !user_set_phrase && super::looks_like_code_pattern("foo = null"); assert!(!auto_promoted); } @@ -1939,7 +2052,8 @@ mod tests { fn test_auto_promoted_skipped_when_user_sets_phrase() { let user_set_regex = false; let user_set_phrase = true; - let auto_promoted = !user_set_regex && !user_set_phrase && super::looks_like_code_pattern("foo = null"); + let auto_promoted = + !user_set_regex && !user_set_phrase && super::looks_like_code_pattern("foo = null"); assert!(!auto_promoted); } @@ -2013,10 +2127,15 @@ mod tests { }; let mut lines: Vec = Vec::new(); if response.auto_promoted_to_regex == Some(true) { - lines.push("# auto-promoted to regex mode (query contained code-like punctuation)".to_string()); + lines.push( + "# auto-promoted to regex mode (query contained code-like punctuation)".to_string(), + ); } for item in &response.results { - lines.push(format!("{}:{}:{}", item.path, item.start_line, item.snippet)); + lines.push(format!( + "{}:{}:{}", + item.path, item.start_line, item.snippet + )); } let output = lines.join("\n"); assert!(output.starts_with("# auto-promoted")); @@ -2041,10 +2160,15 @@ mod tests { }; let mut lines: Vec = Vec::new(); if response.auto_promoted_to_regex == Some(true) { - lines.push("# auto-promoted to regex mode (query contained code-like punctuation)".to_string()); + lines.push( + "# auto-promoted to regex mode (query contained code-like punctuation)".to_string(), + ); } for item in &response.results { - lines.push(format!("{}:{}:{}", item.path, item.start_line, item.snippet)); + lines.push(format!( + "{}:{}:{}", + item.path, item.start_line, item.snippet + )); } let output = lines.join("\n"); assert!(!output.starts_with('#')); @@ -2076,8 +2200,8 @@ use rmcp::{ handler::server::router::tool::ToolRouter, handler::server::wrapper::Parameters, model::{ - CallToolRequestParams, CallToolResult, Content, Implementation, ListToolsResult, PaginatedRequestParams, - ServerCapabilities, ServerInfo, + CallToolRequestParams, CallToolResult, Content, Implementation, ListToolsResult, + PaginatedRequestParams, ServerCapabilities, ServerInfo, }, service::RequestContext, tool, tool_handler, tool_router, ErrorData as McpError, RoleClient, RoleServer, ServerHandler, @@ -2139,10 +2263,12 @@ mod reconnect { impl ServerHandler for McpProxyService { fn get_info(&self) -> ServerInfo { ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) - .with_server_info(Implementation::new("codesearch", env!("CARGO_PKG_VERSION")) - .with_title("codesearch (serve proxy)")) + .with_server_info( + Implementation::new("codesearch", env!("CARGO_PKG_VERSION")) + .with_title("codesearch (serve proxy)"), + ) .with_instructions( - "Proxy to a running codesearch serve hub. All tool calls are forwarded to the hub." + "Proxy to a running codesearch serve hub. All tool calls are forwarded to the hub.", ) } @@ -2194,10 +2320,7 @@ fn read_model_metadata(db_path: &Path) -> (String, usize) { .and_then(|v| v.as_str()) .unwrap_or("unknown") .to_string(); - let dims = json - .get("dimensions") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; + let dims = json.get("dimensions").and_then(|v| v.as_u64()).unwrap_or(0) as usize; // If metadata has explicit dimensions, use those; otherwise infer from model name. let dims = if dims > 0 { dims @@ -2209,7 +2332,10 @@ fn read_model_metadata(db_path: &Path) -> (String, usize) { return (model_name, dims); } } - ("unknown".to_string(), crate::constants::DEFAULT_EMBEDDING_DIMENSIONS) + ( + "unknown".to_string(), + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS, + ) } /// RRF score threshold below which results are considered low-confidence. @@ -2280,19 +2406,27 @@ trait HasScore { } impl HasChunkId for crate::vectordb::SearchResult { - fn chunk_id(&self) -> u32 { self.id } + fn chunk_id(&self) -> u32 { + self.id + } } impl HasScore for crate::vectordb::SearchResult { - fn score(&self) -> f32 { self.score } + fn score(&self) -> f32 { + self.score + } } impl HasChunkId for crate::fts::FtsResult { - fn chunk_id(&self) -> u32 { self.chunk_id } + fn chunk_id(&self) -> u32 { + self.chunk_id + } } impl HasScore for crate::fts::FtsResult { - fn score(&self) -> f32 { self.score } + fn score(&self) -> f32 { + self.score + } } // === Simple Glob Matcher === @@ -2525,7 +2659,11 @@ fn strip_alias_prefix(path: &str, alias: Option<&String>) -> String { /// Prefix a result path with its repo alias for group queries, normalizing /// Windows backslashes to forward slashes in the process. When `alias` is /// None or empty, the path is still normalized (useful for stdio mode). -pub(crate) fn prefix_path_with_alias(path: &str, alias: Option<&str>, project_root: &str) -> String { +pub(crate) fn prefix_path_with_alias( + path: &str, + alias: Option<&str>, + project_root: &str, +) -> String { let normalized = crate::cache::normalize_path_str(path); let normalized_root = crate::cache::normalize_path_str(project_root) .trim_end_matches('/') @@ -2585,7 +2723,11 @@ fn truncate_line_around_match(line: &str, match_start_byte: usize, max_chars: us chars[start..end].iter().collect() } -fn match_line_for_literal(content: &str, query: &str, regex: Option<&Regex>) -> Option<(usize, String)> { +fn match_line_for_literal( + content: &str, + query: &str, + regex: Option<&Regex>, +) -> Option<(usize, String)> { if query.is_empty() { return None; } @@ -2810,8 +2952,8 @@ fn compute_literal_low_confidence( let is_single_identifier = word_count == 1 && !has_code_chars; let suggest_semantic = "search with mode='semantic'"; - let suggest_regex = "search with mode='literal' and regex=true"; - let suggest_find = "find with kind='definition' or kind='usages'"; + let suggest_regex = "search with mode='literal' and regex=true"; + let suggest_find = "find with kind='definition' or kind='usages'"; match top_score { Some(score) if score < LITERAL_LOW_CONFIDENCE_BM25 => { @@ -2820,11 +2962,19 @@ fn compute_literal_low_confidence( // IDF artefact, not a quality signal. Trust the results. return (None, None); } - let hint = if is_natural_language { suggest_semantic } else { suggest_find }; + let hint = if is_natural_language { + suggest_semantic + } else { + suggest_find + }; (Some(true), Some(hint.to_string())) } None => { - let hint = if is_natural_language { suggest_semantic } else { suggest_regex }; + let hint = if is_natural_language { + suggest_semantic + } else { + suggest_regex + }; (Some(true), Some(hint.to_string())) } Some(_) => (None, None), @@ -2848,18 +2998,30 @@ fn parse_import_lines(content: &str, start_line: usize) -> Vec { } let parsed = if let Some(rest) = trimmed.strip_prefix("use ") { - Some(("use".to_string(), rest.trim().trim_end_matches(';').to_string())) + Some(( + "use".to_string(), + rest.trim().trim_end_matches(';').to_string(), + )) } else if let Some(rest) = trimmed.strip_prefix("using ") { // C# using directive — skip `using (...)` statements and `using var` declarations if rest.starts_with('(') || rest.starts_with("var ") { None } else { - Some(("using".to_string(), rest.trim().trim_end_matches(';').to_string())) + Some(( + "using".to_string(), + rest.trim().trim_end_matches(';').to_string(), + )) } } else if let Some(rest) = trimmed.strip_prefix("import ") { - Some(("import".to_string(), rest.trim().trim_end_matches(';').to_string())) + Some(( + "import".to_string(), + rest.trim().trim_end_matches(';').to_string(), + )) } else if let Some(rest) = trimmed.strip_prefix("from ") { - Some(("import".to_string(), rest.trim().trim_end_matches(';').to_string())) + Some(( + "import".to_string(), + rest.trim().trim_end_matches(';').to_string(), + )) } else if trimmed.starts_with("#include") { Some(( "include".to_string(), @@ -3000,11 +3162,15 @@ impl CodesearchService { let dims = json .get("dimensions") .and_then(|v| v.as_u64()) - .unwrap_or(crate::constants::DEFAULT_EMBEDDING_DIMENSIONS as u64) as usize; + .unwrap_or(crate::constants::DEFAULT_EMBEDDING_DIMENSIONS as u64) + as usize; let mt = ModelType::parse(model_name).unwrap_or_default(); (mt, dims) } else { - (ModelType::default(), crate::constants::DEFAULT_EMBEDDING_DIMENSIONS) + ( + ModelType::default(), + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS, + ) }; Ok(Self { @@ -3023,9 +3189,7 @@ impl CodesearchService { /// /// In serve mode, the service does not have a single local DB; instead /// it routes requests to the repo identified by `project`/`group`. - pub(crate) fn new_for_serve( - serve_state: Arc, - ) -> Result { + pub(crate) fn new_for_serve(serve_state: Arc) -> Result { Ok(Self { tool_router: Self::tool_router(), db_path: PathBuf::from("serve://multi-repo"), @@ -3110,7 +3274,7 @@ impl CodesearchService { if !aliases.is_empty() { let mut all_stores = Vec::with_capacity(aliases.len()); for alias in &aliases { - all_stores.push(serve_state.get_or_open_stores(alias).await?); + all_stores.push(serve_state.get_or_open_stores(alias, false).await?); } return Ok(Some((all_stores, aliases))); } @@ -3120,14 +3284,15 @@ impl CodesearchService { } // Must have serve_state to route - let serve_state = self.serve_state.as_ref() - .ok_or_else(|| "project/group routing requires `codesearch serve` to be running.".to_string())?; + let serve_state = self.serve_state.as_ref().ok_or_else(|| { + "project/group routing requires `codesearch serve` to be running.".to_string() + })?; // Validate params types::validate_project_group(project, group, true)?; if let Some(ref alias) = project { - let stores = serve_state.get_or_open_stores(alias).await?; + let stores = serve_state.get_or_open_stores(alias, true).await?; return Ok(Some((vec![stores], vec![alias.clone()]))); } @@ -3138,7 +3303,7 @@ impl CodesearchService { } let mut all_stores = Vec::with_capacity(aliases.len()); for alias in &aliases { - all_stores.push(serve_state.get_or_open_stores(alias).await?); + all_stores.push(serve_state.get_or_open_stores(alias, false).await?); } return Ok(Some((all_stores, aliases))); } @@ -3158,8 +3323,12 @@ impl CodesearchService { allow_unscoped: bool, tool_name: &str, ) -> std::result::Result { - let resolved = self.resolve_repo_stores_multi(project, group, allow_unscoped).await?; - let is_multi = resolved.as_ref().is_some_and(|(stores, _)| stores.len() > 1); + let resolved = self + .resolve_repo_stores_multi(project, group, allow_unscoped) + .await?; + let is_multi = resolved + .as_ref() + .is_some_and(|(stores, _)| stores.len() > 1); let (stores, stores_vec, store_aliases, project_alias) = match &resolved { None => (None, None, None, None), Some((store_vec, aliases)) if store_vec.len() == 1 => { @@ -3205,6 +3374,9 @@ impl CodesearchService { if let Some(ref aliases) = store_aliases { for alias in aliases { serve_state.record_tool_call(alias, tool_name); + // Explicit multi-repo/group query: treat as access. + // (Unscoped multi fan-out is skipped by the outer condition.) + serve_state.touch_access(alias); } } if let Some(ref alias) = project_alias { @@ -3341,7 +3513,8 @@ impl CodesearchService { R: Clone + HasChunkId + HasScore, { let mut all_results: Vec = Vec::new(); - let mut seen_ids: std::collections::HashMap<(String, u32), usize> = std::collections::HashMap::new(); + let mut seen_ids: std::collections::HashMap<(String, u32), usize> = + std::collections::HashMap::new(); for (idx, store_arc) in stores.iter().enumerate() { let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); @@ -3392,7 +3565,8 @@ impl CodesearchService { R: Clone + HasChunkId + HasScore, { let mut all_results: Vec = Vec::new(); - let mut seen_ids: std::collections::HashMap<(String, u32), usize> = std::collections::HashMap::new(); + let mut seen_ids: std::collections::HashMap<(String, u32), usize> = + std::collections::HashMap::new(); for (idx, store_arc) in stores.iter().enumerate() { let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); @@ -3491,7 +3665,11 @@ impl CodesearchService { &self, Parameters(request): Parameters, ) -> Result { - let kind = request.kind.as_deref().unwrap_or("definition").to_lowercase(); + let kind = request + .kind + .as_deref() + .unwrap_or("definition") + .to_lowercase(); tracing::info!( "📥 find(symbol={:?}, kind={}, project={:?}, group={:?})", request.symbol, @@ -3622,7 +3800,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing (multi-store for group fan-out) - let ctx = match self.resolve_routing(&request.project, &request.group, false, "search").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "search") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -3635,7 +3816,11 @@ impl CodesearchService { tracing::debug!( "MCP semantic_search: query='{}', limit={}, compact={}, mode='{}', multi={}", - request.query, limit, compact, mode, ctx.is_multi + request.query, + limit, + compact, + mode, + ctx.is_multi ); // Ensure database exists (skip if serve-mode with routed stores) @@ -3647,19 +3832,32 @@ impl CodesearchService { // === Multi-store group fan-out === if ctx.is_multi { - return self.semantic_search_multi( - &request, &identifiers, limit, compact, - ctx.stores_vec.unwrap(), - ctx.store_aliases.as_ref().unwrap(), - &ctx.alias_roots, - ).await; + return self + .semantic_search_multi( + &request, + &identifiers, + limit, + compact, + ctx.stores_vec.unwrap(), + ctx.store_aliases.as_ref().unwrap(), + &ctx.alias_roots, + ) + .await; } // === Mode: "lexical" — FTS only, no embedding === if mode == "lexical" { tracing::debug!("MCP: mode=lexical — skipping embedding service"); return self - .semantic_search_lexical(&request, &identifiers, limit, compact, ctx.stores, ctx.project_alias.as_deref(), &ctx.alias_roots) + .semantic_search_lexical( + &request, + &identifiers, + limit, + compact, + ctx.stores, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + ) .await; } @@ -3695,7 +3893,7 @@ impl CodesearchService { .with_vector_store_read_for( |store| { store - .search(&query_embedding, limit * 3) + .search(&query_embedding, limit * 5) .context("Error searching vector store") }, ctx.stores.clone(), @@ -3730,7 +3928,14 @@ impl CodesearchService { results.push(r); } } - return self.build_semantic_response(results, &request, compact, has_identifiers, ctx.project_alias.as_deref(), &ctx.alias_roots); + return self.build_semantic_response( + results, + &request, + compact, + has_identifiers, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + ); } // === Modes: "hybrid" | "auto" — full hybrid search === @@ -3749,44 +3954,46 @@ impl CodesearchService { let mut results = match self .with_fts_store_read_for( |fts_store| { - let fts_results = fts_store - .search(&request.query, limit * 3, structural_intent) - .unwrap_or_default(); + let fts_results = fts_store + .search(&request.query, limit * 5, structural_intent) + .unwrap_or_default(); - let fused = if identifiers.is_empty() { - rrf_fusion(&vector_results, &fts_results, vector_k as f32) - } else { - let mut all_exact: Vec = Vec::new(); - for ident in &identifiers { - if let Ok(exact) = - fts_store.search_exact(ident, limit * 2, structural_intent) - { - for r in exact { - if !all_exact.iter().any(|e| e.chunk_id == r.chunk_id) { - all_exact.push(r); + let fused = if identifiers.is_empty() { + rrf_fusion(&vector_results, &fts_results, vector_k as f32) + } else { + let mut all_exact: Vec = Vec::new(); + for ident in &identifiers { + if let Ok(exact) = + fts_store.search_exact(ident, limit * 3, structural_intent) + { + for r in exact { + if !all_exact.iter().any(|e| e.chunk_id == r.chunk_id) { + all_exact.push(r); + } } } } - } - tracing::debug!( - "MCP: FTS found {} results, exact found {} results", - fts_results.len(), - all_exact.len() - ); + tracing::debug!( + "MCP: FTS found {} results, exact found {} results", + fts_results.len(), + all_exact.len() + ); - rrf_fusion_with_exact( - &vector_results, - &fts_results, - &all_exact, - vector_k as f32, - fts_k as f32, - EXACT_MATCH_RRF_K, - ) - }; + rrf_fusion_with_exact( + &vector_results, + &fts_results, + &all_exact, + vector_k as f32, + fts_k as f32, + EXACT_MATCH_RRF_K, + ) + }; - Ok(fused) - }, ctx.stores.clone()) + Ok(fused) + }, + ctx.stores.clone(), + ) .await { Ok(fused) => { @@ -3835,8 +4042,79 @@ impl CodesearchService { boost_kind(&mut results, target_kind); } + // Auto-fallback: if hybrid search returned very few results for a code-like query, + // run literal FTS and merge missing chunks. + if results.len() < 3 && has_identifiers { + tracing::debug!( + "Auto-fallback: semantic returned {} results, trying literal", + results.len() + ); + + let literal_results = self + .with_fts_store_read_for( + |fts_store| fts_store.search(&request.query, limit, None), + ctx.stores.clone(), + ) + .await + .unwrap_or_default(); + + let mut existing_ids: std::collections::HashSet = + results.iter().map(|r| r.id).collect(); + + for fts in literal_results { + if results.len() >= limit { + break; + } + if existing_ids.contains(&fts.chunk_id) { + continue; + } + + let maybe_resolved = self + .with_vector_store_read_for( + |store| { + if let Ok(Some(chunk)) = store.get_chunk(fts.chunk_id) { + Ok(Some(crate::vectordb::SearchResult { + id: fts.chunk_id, + content: chunk.content, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + docstring: chunk.docstring, + context: chunk.context, + hash: chunk.hash, + distance: 0.0, + score: fts.score, + context_prev: chunk.context_prev, + context_next: chunk.context_next, + })) + } else { + Ok(None) + } + }, + ctx.stores.clone(), + ) + .await + .ok() + .flatten(); + + if let Some(resolved) = maybe_resolved { + existing_ids.insert(resolved.id); + results.push(resolved); + } + } + } + tracing::debug!("MCP: Final {} results after hybrid search", results.len()); - self.build_semantic_response(results, &request, compact, has_identifiers, ctx.project_alias.as_deref(), &ctx.alias_roots) + self.build_semantic_response( + results, + &request, + compact, + has_identifiers, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + ) } // === Helper methods (not exposed as tools) === @@ -3861,7 +4139,7 @@ impl CodesearchService { if mode == "lexical" { let fts_results = self .with_fts_store_read_multi( - |fts_store| fts_store.search(&request.query, limit * 3, structural_intent), + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), stores.clone(), aliases, ) @@ -3873,7 +4151,7 @@ impl CodesearchService { for ident in identifiers { let exact = self .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(ident, limit * 2, structural_intent), + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), stores.clone(), aliases, ) @@ -3883,19 +4161,37 @@ impl CodesearchService { } all_fts.sort_by(|a, b| { - b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal) + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) }); - let results = self.resolve_fts_to_search_results_multi(&all_fts, limit, &stores).await; + let results = self + .resolve_fts_to_search_results_multi(&all_fts, limit, &stores) + .await; if let Some(target_kind) = structural_intent { // We need mutable results but we have them as vectordb::SearchResult let mut mutable_results = results; boost_kind(&mut mutable_results, target_kind); - return self.build_semantic_response(mutable_results, request, compact, !identifiers.is_empty(), None, alias_roots); + return self.build_semantic_response( + mutable_results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + ); } - return self.build_semantic_response(results, request, compact, !identifiers.is_empty(), None, alias_roots); + return self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + ); } // === Modes requiring embedding: "semantic", "hybrid", "auto" === @@ -3904,7 +4200,8 @@ impl CodesearchService { Ok(g) => g, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error initializing embedding service: {}", e + "Error initializing embedding service: {}", + e ))])); } }; @@ -3913,7 +4210,8 @@ impl CodesearchService { Ok(e) => e, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error embedding query: {}", e + "Error embedding query: {}", + e ))])); } } @@ -3922,7 +4220,11 @@ impl CodesearchService { // Search vector stores across all repos let vector_results = self .with_vector_store_read_multi( - |store| store.search(&query_embedding, limit * 3).context("Error searching vector store"), + |store| { + store + .search(&query_embedding, limit * 5) + .context("Error searching vector store") + }, stores.clone(), aliases, ) @@ -3943,7 +4245,14 @@ impl CodesearchService { results.push(r); } } - return self.build_semantic_response(results, request, compact, !identifiers.is_empty(), None, alias_roots); + return self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + ); } // === Modes: "hybrid" | "auto" — full hybrid search === @@ -3952,7 +4261,7 @@ impl CodesearchService { // FTS search across all stores let fts_results = self .with_fts_store_read_multi( - |fts_store| fts_store.search(&request.query, limit * 3, structural_intent), + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), stores.clone(), aliases, ) @@ -3965,7 +4274,7 @@ impl CodesearchService { for ident in identifiers { let exact = self .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(ident, limit * 2, structural_intent), + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), stores.clone(), aliases, ) @@ -4008,7 +4317,10 @@ impl CodesearchService { mapped.push(r); } else { // Chunk from FTS but not in vector results — resolve from stores - if let Some(resolved) = self.resolve_chunk_from_stores(f.chunk_id, f.rrf_score, &stores).await { + if let Some(resolved) = self + .resolve_chunk_from_stores(f.chunk_id, f.rrf_score, &stores) + .await + { mapped.push(resolved); } } @@ -4019,7 +4331,14 @@ impl CodesearchService { boost_kind(&mut mapped, target_kind); } - self.build_semantic_response(mapped, request, compact, !identifiers.is_empty(), None, alias_roots) + self.build_semantic_response( + mapped, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + ) } /// Resolve a single chunk from multiple stores (used for FTS-only hits in multi-store fusion). @@ -4104,9 +4423,7 @@ impl CodesearchService { let mut fts_results = self .with_fts_store_read_for( - |fts_store| { - fts_store.search(&request.query, limit * 3, structural_intent) - }, + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), stores.clone(), ) .await @@ -4116,9 +4433,7 @@ impl CodesearchService { for ident in identifiers { let exact = match self .with_fts_store_read_for( - |fts_store| { - fts_store.search_exact(ident, limit * 2, structural_intent) - }, + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), stores.clone(), ) .await @@ -4145,7 +4460,14 @@ impl CodesearchService { boost_kind(&mut results, target_kind); } - self.build_semantic_response(results, request, compact, !identifiers.is_empty(), project_alias, alias_roots) + self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + project_alias, + alias_roots, + ) } /// Build the final SemanticSearchResponse with low-confidence signaling. @@ -4284,7 +4606,10 @@ impl CodesearchService { ); // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "find").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -4307,13 +4632,17 @@ impl CodesearchService { .unwrap_or_default() } else { match self - .with_fts_store_read_for(|fts_store| fts_store.search(&request.symbol, limit * 3, None), ctx.stores.clone()) + .with_fts_store_read_for( + |fts_store| fts_store.search(&request.symbol, limit * 3, None), + ctx.stores.clone(), + ) .await { Ok(r) => r, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {}", e + "Error searching: {}", + e ))])); } } @@ -4399,7 +4728,8 @@ impl CodesearchService { Ok(items) => items, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error opening database: {}", e + "Error opening database: {}", + e ))])); } } @@ -4427,8 +4757,13 @@ impl CodesearchService { &self, Parameters(request): Parameters, ) -> Result { - self.find_usages_impl(request.symbol.clone(), request.limit.unwrap_or(20), request.project, request.group) - .await + self.find_usages_impl( + request.symbol.clone(), + request.limit.unwrap_or(20), + request.project, + request.group, + ) + .await } /// Shared implementation for find_usages (used by `find(kind="usages")`). @@ -4465,13 +4800,17 @@ impl CodesearchService { .unwrap_or_default() } else { match self - .with_fts_store_read_for(|fts_store| fts_store.search(&symbol, limit * 2, None), ctx.stores.clone()) + .with_fts_store_read_for( + |fts_store| fts_store.search(&symbol, limit * 2, None), + ctx.stores.clone(), + ) .await { Ok(r) => r, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {}", e + "Error searching: {}", + e ))])); } } @@ -4543,7 +4882,8 @@ impl CodesearchService { Ok(items) => items, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error opening database: {}", e + "Error opening database: {}", + e ))])); } } @@ -4570,7 +4910,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "explore").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "explore") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -4578,7 +4921,8 @@ impl CodesearchService { // Outline operates on a single repo — reject group fan-out if ctx.is_multi { return Ok(CallToolResult::success(vec![Content::text( - "Tool 'explore' operates on a single repo. Use 'project' instead of 'group'.".to_string(), + "Tool 'explore' operates on a single repo. Use 'project' instead of 'group'." + .to_string(), )])); } @@ -4685,7 +5029,10 @@ impl CodesearchService { request.project, ); // Resolve project/group routing — allow unscoped for smart candidate detection - let ctx = match self.resolve_routing(&request.project, &request.group, true, "get_chunk").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, true, "get_chunk") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -4736,6 +5083,7 @@ impl CodesearchService { // Record tool call for the specific repo that served this chunk if let Some(ref serve_state) = self.serve_state { serve_state.record_tool_call(alias, "get_chunk"); + serve_state.touch_access(alias); } let store = store_arc.vector_store.read().await; store.get_chunk(request.chunk_id).unwrap_or_default() @@ -4761,7 +5109,10 @@ impl CodesearchService { for store_arc in sv { let store = store_arc.vector_store.read().await; match store.get_chunk(request.chunk_id) { - Ok(Some(c)) => { found = Some(c); break; } + Ok(Some(c)) => { + found = Some(c); + break; + } Ok(None) => continue, Err(_) => break, } @@ -4769,13 +5120,12 @@ impl CodesearchService { found } } else { - self - .with_vector_store_read_for( - |store| store.get_chunk(request.chunk_id), - ctx.stores.clone(), - ) - .await - .unwrap_or_default() + self.with_vector_store_read_for( + |store| store.get_chunk(request.chunk_id), + ctx.stores.clone(), + ) + .await + .unwrap_or_default() }; let mut chunk = match chunk { @@ -4850,7 +5200,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "find").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -4888,7 +5241,10 @@ impl CodesearchService { } if seen_ids.insert(meta.id) { if let Ok(Some(chunk)) = store.get_chunk(meta.id) { - all_items.extend(parse_import_lines(&chunk.content, chunk.start_line)); + all_items.extend(parse_import_lines( + &chunk.content, + chunk.start_line, + )); } } } @@ -4942,9 +5298,7 @@ impl CodesearchService { for keyword in IMPORT_FTS_KEYWORDS { let hits = self .with_fts_store_read_multi( - |fts_store| { - fts_store.search_exact(keyword, fallback_limit, None) - }, + |fts_store| fts_store.search_exact(keyword, fallback_limit, None), sv.clone(), ctx.store_aliases.as_ref().unwrap(), ) @@ -4964,7 +5318,8 @@ impl CodesearchService { let store = store_arc.vector_store.read().await; if let Ok(Some(chunk)) = store.get_chunk(*chunk_id) { if crate::cache::normalize_path_str(&chunk.path) == normalized { - resolved.extend(parse_import_lines(&chunk.content, chunk.start_line)); + resolved + .extend(parse_import_lines(&chunk.content, chunk.start_line)); } break; } @@ -4976,9 +5331,7 @@ impl CodesearchService { for keyword in IMPORT_FTS_KEYWORDS { let hits = self .with_fts_store_read_for( - |fts_store| { - fts_store.search_exact(keyword, fallback_limit, None) - }, + |fts_store| fts_store.search_exact(keyword, fallback_limit, None), ctx.stores.clone(), ) .await @@ -4997,7 +5350,10 @@ impl CodesearchService { for (chunk_id, _) in &all_hits { if let Some(chunk) = store.get_chunk(*chunk_id)? { if crate::cache::normalize_path_str(&chunk.path) == normalized { - out.extend(parse_import_lines(&chunk.content, chunk.start_line)); + out.extend(parse_import_lines( + &chunk.content, + chunk.start_line, + )); } } } @@ -5026,7 +5382,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "find").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -5074,9 +5433,7 @@ impl CodesearchService { // Multi-store FTS search let exact_hits = self .with_fts_store_read_multi( - |fts_store| { - fts_store.search_exact(&search_term, high_limit, import_kind) - }, + |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), sv.clone(), sa, ) @@ -5085,9 +5442,7 @@ impl CodesearchService { if exact_hits.is_empty() { self.with_fts_store_read_multi( - |fts_store| { - fts_store.search(&search_term, high_limit, import_kind) - }, + |fts_store| fts_store.search(&search_term, high_limit, import_kind), sv.clone(), sa, ) @@ -5100,9 +5455,7 @@ impl CodesearchService { // Single-store FTS search let exact_hits = self .with_fts_store_read_for( - |fts_store| { - fts_store.search_exact(&search_term, high_limit, import_kind) - }, + |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), ctx.stores.clone(), ) .await @@ -5110,9 +5463,7 @@ impl CodesearchService { if exact_hits.is_empty() { self.with_fts_store_read_for( - |fts_store| { - fts_store.search(&search_term, high_limit, import_kind) - }, + |fts_store| fts_store.search(&search_term, high_limit, import_kind), ctx.stores.clone(), ) .await @@ -5141,26 +5492,19 @@ impl CodesearchService { } let term_lower = search_term.to_lowercase(); - let import_statement = if chunk - .content - .to_lowercase() - .contains(&term_lower) - { - chunk - .content - .lines() - .find(|l| { - l.to_lowercase() - .contains(&term_lower) - }) - .unwrap_or("") - .to_string() - } else { - chunk - .signature - .filter(|s| !s.is_empty()) - .unwrap_or(chunk.content.lines().next().unwrap_or("").to_string()) - }; + let import_statement = + if chunk.content.to_lowercase().contains(&term_lower) { + chunk + .content + .lines() + .find(|l| l.to_lowercase().contains(&term_lower)) + .unwrap_or("") + .to_string() + } else { + chunk.signature.filter(|s| !s.is_empty()).unwrap_or( + chunk.content.lines().next().unwrap_or("").to_string(), + ) + }; out.push(DependentItem { path: chunk.path, @@ -5183,56 +5527,51 @@ impl CodesearchService { match self .with_vector_store_read_for( |store| { - let mut seen_paths = HashSet::new(); - let mut out = Vec::new(); - let term_lower = search_term.to_lowercase(); - for f in &fts_results { - if let Some(chunk) = store.get_chunk(f.chunk_id)? { - if !is_import_kind(&chunk.kind) { - continue; - } - - let norm = crate::cache::normalize_path_str(&chunk.path); - if !seen_paths.insert(norm) { - continue; - } + let mut seen_paths = HashSet::new(); + let mut out = Vec::new(); + let term_lower = search_term.to_lowercase(); + for f in &fts_results { + if let Some(chunk) = store.get_chunk(f.chunk_id)? { + if !is_import_kind(&chunk.kind) { + continue; + } - // Extract the specific import line(s) that mention the - // module name, rather than returning the entire chunk content. - let import_statement = if chunk - .content - .to_lowercase() - .contains(&term_lower) - { - chunk - .content - .lines() - .find(|l| { - l.to_lowercase() - .contains(&term_lower) - }) - .unwrap_or("") - .to_string() - } else { - chunk - .signature - .filter(|s| !s.is_empty()) - .unwrap_or(chunk.content.lines().next().unwrap_or("").to_string()) - }; + let norm = crate::cache::normalize_path_str(&chunk.path); + if !seen_paths.insert(norm) { + continue; + } - out.push(DependentItem { - path: chunk.path, - line: chunk.start_line, - import_statement, - }); + // Extract the specific import line(s) that mention the + // module name, rather than returning the entire chunk content. + let import_statement = + if chunk.content.to_lowercase().contains(&term_lower) { + chunk + .content + .lines() + .find(|l| l.to_lowercase().contains(&term_lower)) + .unwrap_or("") + .to_string() + } else { + chunk.signature.filter(|s| !s.is_empty()).unwrap_or( + chunk.content.lines().next().unwrap_or("").to_string(), + ) + }; + + out.push(DependentItem { + path: chunk.path, + line: chunk.start_line, + import_statement, + }); - if out.len() >= limit { - break; + if out.len() >= limit { + break; + } } } - } - Ok(out) - }, ctx.stores.clone()) + Ok(out) + }, + ctx.stores.clone(), + ) .await { Ok(items) => items, @@ -5268,7 +5607,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "explore").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "explore") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -5345,9 +5687,13 @@ impl CodesearchService { match self .with_vector_store_read_for( |store| { - let embedding = store - .get_embedding(request.chunk_id)? - .ok_or_else(|| anyhow::anyhow!("embedding not found for chunk_id {}", request.chunk_id))?; + let embedding = + store.get_embedding(request.chunk_id)?.ok_or_else(|| { + anyhow::anyhow!( + "embedding not found for chunk_id {}", + request.chunk_id + ) + })?; let mut neighbors = store.search(&embedding, limit + 1)?; neighbors.retain(|r| r.id != request.chunk_id); @@ -5398,7 +5744,10 @@ impl CodesearchService { Parameters(request): Parameters, ) -> Result { // Resolve project/group routing - let ctx = match self.resolve_routing(&request.project, &request.group, false, "search").await { + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "search") + .await + { Ok(c) => c, Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), }; @@ -5407,11 +5756,10 @@ impl CodesearchService { let output_format = request.format.as_deref().unwrap_or("json"); // Auto-regex promotion: detect code patterns that BM25 would destroy - let user_set_regex = request.regex.unwrap_or(false); + let user_set_regex = request.regex.unwrap_or(false); let user_set_phrase = request.phrase.unwrap_or(false); - let auto_promoted = !user_set_regex - && !user_set_phrase - && looks_like_code_pattern(&request.query); + let auto_promoted = + !user_set_regex && !user_set_phrase && looks_like_code_pattern(&request.query); let (effective_query, effective_regex) = if auto_promoted { let escaped = regex::escape(&request.query); @@ -5479,7 +5827,8 @@ impl CodesearchService { } } if let Some(ref glob) = glob_filter { - let relative_path = chunk.path + let relative_path = chunk + .path .strip_prefix(&project_root_normalized) .unwrap_or(&chunk.path) .trim_start_matches('/'); @@ -5488,7 +5837,9 @@ impl CodesearchService { } } if let Some((match_offset, snippet)) = match_line_for_literal( - &chunk.content, &effective_query, snippet_regex.as_ref(), + &chunk.content, + &effective_query, + snippet_regex.as_ref(), ) { let match_line = chunk.start_line + match_offset; items.push(LiteralSearchResultItem { @@ -5497,7 +5848,11 @@ impl CodesearchService { end_line: match_line, snippet, score: 0.0, // No BM25 score — scan-path results are unranked - kind: if chunk.kind.is_empty() { None } else { Some(chunk.kind) }, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, signature: chunk.signature.filter(|s| !s.is_empty()), }); if items.len() >= limit { @@ -5519,7 +5874,8 @@ impl CodesearchService { let mut items: Vec = Vec::new(); for (_, chunk) in all_chunks { if let Some(ref lang) = lang_filter { - let file_lang = Language::from_path(std::path::Path::new(&chunk.path)); + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); if file_lang.name() != lang { continue; } @@ -5535,7 +5891,9 @@ impl CodesearchService { } } if let Some((match_offset, snippet)) = match_line_for_literal( - &chunk.content, &effective_query, snippet_regex.as_ref(), + &chunk.content, + &effective_query, + snippet_regex.as_ref(), ) { let match_line = chunk.start_line + match_offset; items.push(LiteralSearchResultItem { @@ -5544,7 +5902,11 @@ impl CodesearchService { end_line: match_line, snippet, score: 0.0, // No BM25 score — scan-path results are unranked - kind: if chunk.kind.is_empty() { None } else { Some(chunk.kind) }, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, signature: chunk.signature.filter(|s| !s.is_empty()), }); if items.len() >= limit { @@ -5561,7 +5923,8 @@ impl CodesearchService { Ok(items) => items, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error scanning chunks: {}", e + "Error scanning chunks: {}", + e ))])); } } @@ -5603,7 +5966,8 @@ impl CodesearchService { Ok(r) => r, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {}", e + "Error searching: {}", + e ))])); } } @@ -5618,13 +5982,15 @@ impl CodesearchService { let store = store_arc.vector_store.read().await; if let Ok(Some(chunk)) = store.get_chunk(fts_result.chunk_id) { if let Some(ref lang) = lang_filter { - let file_lang = Language::from_path(std::path::Path::new(&chunk.path)); + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); if file_lang.name() != lang { continue; } } if let Some(ref glob) = glob_filter { - let relative_path = chunk.path + let relative_path = chunk + .path .strip_prefix(&project_root_normalized) .unwrap_or(&chunk.path) .trim_start_matches('/'); @@ -5633,13 +5999,16 @@ impl CodesearchService { } } let match_info = match_line_for_literal( - &chunk.content, &effective_query, snippet_regex.as_ref(), + &chunk.content, + &effective_query, + snippet_regex.as_ref(), ); if regex_enabled && match_info.is_none() { continue; } - let (match_offset, snippet) = match_info - .unwrap_or_else(|| (0, chunk.content.lines().next().unwrap_or("").to_string())); + let (match_offset, snippet) = match_info.unwrap_or_else(|| { + (0, chunk.content.lines().next().unwrap_or("").to_string()) + }); let match_line = chunk.start_line + match_offset; items.push(LiteralSearchResultItem { path: chunk.path, @@ -5647,7 +6016,11 @@ impl CodesearchService { end_line: match_line, snippet, score: fts_result.score, - kind: if chunk.kind.is_empty() { None } else { Some(chunk.kind) }, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, signature: chunk.signature.filter(|s| !s.is_empty()), }); if items.len() >= limit { @@ -5662,61 +6035,72 @@ impl CodesearchService { match self .with_vector_store_read_for( |store| { - let items: Vec = fts_results - .iter() - .filter_map(|fts_result| { - let chunk = store.get_chunk(fts_result.chunk_id).ok()??; - Some((chunk, fts_result.score)) - }) - .filter(|(chunk, _)| { - if let Some(ref lang) = lang_filter { - let file_lang = Language::from_path(std::path::Path::new(&chunk.path)); - if file_lang.name() != lang { - return false; + let items: Vec = fts_results + .iter() + .filter_map(|fts_result| { + let chunk = store.get_chunk(fts_result.chunk_id).ok()??; + Some((chunk, fts_result.score)) + }) + .filter(|(chunk, _)| { + if let Some(ref lang) = lang_filter { + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); + if file_lang.name() != lang { + return false; + } } - } - if let Some(ref glob) = glob_filter { - let relative_path = chunk - .path - .strip_prefix(&project_root_normalized) - .unwrap_or(&chunk.path) - .trim_start_matches('/'); - if !simple_glob_match(glob, relative_path) { - return false; + if let Some(ref glob) = glob_filter { + let relative_path = chunk + .path + .strip_prefix(&project_root_normalized) + .unwrap_or(&chunk.path) + .trim_start_matches('/'); + if !simple_glob_match(glob, relative_path) { + return false; + } } - } - true - }) - .take(limit) - .filter_map(|(chunk, score)| { - let match_info = match_line_for_literal( - &chunk.content, &effective_query, snippet_regex.as_ref(), - ); - if regex_enabled && match_info.is_none() { - return None; - } - let (match_offset, snippet) = match_info - .unwrap_or_else(|| (0, chunk.content.lines().next().unwrap_or("").to_string())); - let match_line = chunk.start_line + match_offset; - Some(LiteralSearchResultItem { - path: chunk.path, - start_line: match_line, - end_line: match_line, - snippet, - score, - kind: if chunk.kind.is_empty() { None } else { Some(chunk.kind) }, - signature: chunk.signature.filter(|s| !s.is_empty()), + true }) - }) - .collect(); - Ok(items) - }, ctx.stores.clone()) + .take(limit) + .filter_map(|(chunk, score)| { + let match_info = match_line_for_literal( + &chunk.content, + &effective_query, + snippet_regex.as_ref(), + ); + if regex_enabled && match_info.is_none() { + return None; + } + let (match_offset, snippet) = match_info.unwrap_or_else(|| { + (0, chunk.content.lines().next().unwrap_or("").to_string()) + }); + let match_line = chunk.start_line + match_offset; + Some(LiteralSearchResultItem { + path: chunk.path, + start_line: match_line, + end_line: match_line, + snippet, + score, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, + signature: chunk.signature.filter(|s| !s.is_empty()), + }) + }) + .collect(); + Ok(items) + }, + ctx.stores.clone(), + ) .await { Ok(items) => items, Err(e) => { return Ok(CallToolResult::success(vec![Content::text(format!( - "Error resolving search results: {}", e + "Error resolving search results: {}", + e ))])); } } @@ -5730,7 +6114,8 @@ impl CodesearchService { // Compute low-confidence signal let top_score = items.first().map(|i| i.score); - let (low_confidence, suggested_tool) = compute_literal_low_confidence(top_score, &request.query); + let (low_confidence, suggested_tool) = + compute_literal_low_confidence(top_score, &request.query); // Build note let note = if auto_promoted { @@ -5755,7 +6140,11 @@ impl CodesearchService { auto_promoted_to_regex: if auto_promoted { Some(true) } else { None }, note, low_confidence, - suggested_tool: if low_confidence == Some(true) { suggested_tool } else { None }, + suggested_tool: if low_confidence == Some(true) { + suggested_tool + } else { + None + }, }; // Instrument BM25 score for threshold calibration @@ -5773,7 +6162,10 @@ impl CodesearchService { let output = if output_format == "grep" { let mut lines: Vec = Vec::new(); if response.auto_promoted_to_regex == Some(true) { - lines.push("# auto-promoted to regex mode (query contained code-like punctuation)".to_string()); + lines.push( + "# auto-promoted to regex mode (query contained code-like punctuation)" + .to_string(), + ); } if response.low_confidence == Some(true) { if let Some(ref hint) = response.suggested_tool { @@ -5781,7 +6173,10 @@ impl CodesearchService { } } for item in &response.results { - lines.push(format!("{}:{}:{}", item.path, item.start_line, item.snippet)); + lines.push(format!( + "{}:{}:{}", + item.path, item.start_line, item.snippet + )); } lines.join("\n") } else { @@ -5792,7 +6187,11 @@ impl CodesearchService { } /// Internal implementation for index_status with optional project/group routing. - async fn index_status_impl(&self, project: Option, group: Option) -> Result { + async fn index_status_impl( + &self, + project: Option, + group: Option, + ) -> Result { // When no project/group specified in serve mode, return lightweight aggregated // status WITHOUT opening any databases. Only a specific project/group request // should trigger DB activation. @@ -5802,9 +6201,18 @@ impl CodesearchService { let repo_count = config.repos.len(); let group_count = config.groups.len(); let statuses = serve_state.repo_statuses_lightweight(); - let open_count = statuses.iter().filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Open)).count(); - let warm_count = statuses.iter().filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Warm)).count(); - let closed_count = statuses.iter().filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Closed)).count(); + let open_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Open)) + .count(); + let warm_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Warm)) + .count(); + let closed_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Closed)) + .count(); let status = if open_count + warm_count > 0 { "ready".to_string() @@ -5931,7 +6339,10 @@ impl CodesearchService { // Single-store path let stats = match self - .with_vector_store_read_for(|store| store.stats().context("Error getting index stats"), ctx.stores.clone()) + .with_vector_store_read_for( + |store| store.stats().context("Error getting index stats"), + ctx.stores.clone(), + ) .await { Ok(s) => s, @@ -6015,8 +6426,24 @@ impl CodesearchService { if let Some(stores) = serve_state.get_opened_stores(alias) { let vs = stores.vector_store.read().await; match vs.stats() { - Ok(stats) => (stats.total_chunks, stats.total_files, model_name, serve_state.repo_lock_status(alias).unwrap_or("unknown").to_string()), - Err(_) => (0, 0, model_name, serve_state.repo_lock_status(alias).unwrap_or("unknown").to_string()), + Ok(stats) => ( + stats.total_chunks, + stats.total_files, + model_name, + serve_state + .repo_lock_status(alias) + .unwrap_or("unknown") + .to_string(), + ), + Err(_) => ( + 0, + 0, + model_name, + serve_state + .repo_lock_status(alias) + .unwrap_or("unknown") + .to_string(), + ), } } else { // Repo NOT opened — use metadata only, no DB open @@ -6072,7 +6499,12 @@ impl CodesearchService { if let Ok(store) = VectorStore::new(&db_path, dims) { if let Ok(stats) = store.stats() { - (stats.total_chunks, stats.total_files, model_name, lock.to_string()) + ( + stats.total_chunks, + stats.total_files, + model_name, + lock.to_string(), + ) } else { (0, 0, model_name, lock.to_string()) } @@ -6189,7 +6621,10 @@ fn contains_symbol_as_word(sig: &str, symbol: &str) -> bool { if let Some(rel) = sig[start..].find(symbol) { let abs = start + rel; let before_ok = abs == 0 - || matches!(sig_bytes.get(abs - 1), Some(&b' ') | Some(&b'\t') | Some(&b'\n')); + || matches!( + sig_bytes.get(abs - 1), + Some(&b' ') | Some(&b'\t') | Some(&b'\n') + ); let after_char = sig[abs + sym_len..].chars().next(); let after_ok = matches!(after_char, None | Some('(' | '<' | ':' | ' ' | '\t')); if before_ok && after_ok { @@ -6327,7 +6762,7 @@ async fn probe_serve_health(serve_url: &str) -> bool { /// 4. On success, hot-swaps the peer — tool calls resume immediately /// 5. After 5 minutes of failure, exits cleanly (Claude Desktop detects the disconnect) async fn run_mcp_client(serve_url: &str, cancel_token: CancellationToken) -> Result<()> { - use rmcp::{ServiceExt, transport::stdio}; + use rmcp::{transport::stdio, ServiceExt}; let mcp_url = format!("{}{}", serve_url, crate::constants::MCP_ENDPOINT_PATH); tracing::info!("🔗 Connecting to codesearch serve at {}", mcp_url); @@ -6367,7 +6802,8 @@ async fn run_mcp_client(serve_url: &str, cancel_token: CancellationToken) -> Res serve_down_since = Some(std::time::Instant::now()); tracing::warn!( "codesearch serve not yet available ({}). Proxy is up, will retry every {}s.", - e, reconnect::INTERVAL_SECS + e, + reconnect::INTERVAL_SECS ); // Seed a synthetic disconnect so the main loop starts reconnecting. let tx = disconnect_tx.clone(); @@ -6465,8 +6901,8 @@ async fn connect_to_serve( use rmcp::transport::streamable_http_client::{ StreamableHttpClientTransportConfig, StreamableHttpClientWorker, }; - let config = StreamableHttpClientTransportConfig::with_uri(mcp_url) - .reinit_on_expired_session(true); + let config = + StreamableHttpClientTransportConfig::with_uri(mcp_url).reinit_on_expired_session(true); StreamableHttpClientWorker::new(reqwest::Client::new(), config) }; @@ -6476,7 +6912,8 @@ async fn connect_to_serve( "Failed to connect to codesearch serve at {}.\n\ Error: {}\n\ Is `codesearch serve` running?", - mcp_url, e + mcp_url, + e ) })?; @@ -6558,7 +6995,10 @@ pub async fn run_mcp_server( tracing::warn!("Failed to initialize file logger: {}", e); } if probe_serve_health(&serve_url).await { - tracing::info!("📡 MCP mode: auto — serve detected at {}, connecting as client", serve_url); + tracing::info!( + "📡 MCP mode: auto — serve detected at {}, connecting as client", + serve_url + ); return run_mcp_client(&serve_url, cancel_token).await; } tracing::info!("📡 MCP mode: auto — no serve detected, falling back to local stdio"); From 3eb88444d4020aa0dc7d1b4b487ab7277ed8d3be Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 14:04:43 +0200 Subject: [PATCH 08/10] fix(search): improve search quality to reduce agent grep fallback - Increase retrieval pool: limit*3 -> limit*5 in semantic pipeline - Stronger exact identifier boost: EXACT_MATCH_RRF_K 5.0 -> 2.0 - Increase search_exact pool: limit*2 -> limit*3 - Auto-fallback to literal FTS when semantic returns <3 results and query contains identifiers (has_identifiers check) - Same limit*5 change in CLI search path (src/search/mod.rs) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/rerank/mod.rs | 2 +- src/search/mod.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6041bb7..8faf64c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.69" +version = "1.0.70" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index ac1e23c..6401164 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.69" +version = "1.0.70" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/rerank/mod.rs b/src/rerank/mod.rs index 6165dea..0ba97a5 100644 --- a/src/rerank/mod.rs +++ b/src/rerank/mod.rs @@ -16,7 +16,7 @@ pub use neural::NeuralReranker; pub const DEFAULT_RRF_K: f32 = 20.0; /// RRF k parameter for exact matches (lower = stronger boost) -pub const EXACT_MATCH_RRF_K: f32 = 5.0; +pub const EXACT_MATCH_RRF_K: f32 = 2.0; /// Fused search result combining vector and FTS scores #[derive(Debug, Clone)] diff --git a/src/search/mod.rs b/src/search/mod.rs index b37a3da..ac62edd 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -10,9 +10,9 @@ use crate::chunker::SemanticChunker; use crate::embed::{EmbeddingService, ModelType}; use crate::file::FileWalker; use crate::fts::FtsStore; -use crate::{info_print, warn_print}; use crate::rerank::{rrf_fusion, vector_only, FusedResult, NeuralReranker, DEFAULT_RRF_K}; use crate::vectordb::VectorStore; +use crate::{info_print, warn_print}; /// Configuration options for search operations #[derive(Debug, Clone)] @@ -495,7 +495,7 @@ pub async fn search(query: &str, path: Option, options: SearchOptions) options.max_results } else if has_identifiers { // Identifier queries: fetch fewer results as exact matches are prioritized - std::cmp::max(options.max_results * 3, 100) + std::cmp::max(options.max_results * 5, 100) } else { // Semantic queries: need more candidates for good fusion std::cmp::max(options.max_results * 5, 200) From 4f3d69a9edadb99e2bd7c305559de5a223b7a1a9 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 16:55:07 +0200 Subject: [PATCH 09/10] fix(mcp): client-side proxy reconnect with transport-aware retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace server-side reconnect middleware (which couldn't fix broken TCP) with client-side reconnect in McpProxyService (--mode client). When list_tools or call_tool hits a transport error (broken TCP, stale session 404, keep-alive failure), the proxy: 1. Detects the transport error via is_transport_error_msg() 2. Calls force_reconnect() — clears peer + signals main loop 3. Retries up to PROXY_MAX_RETRY_ATTEMPTS (3) with backoff 4. The main loop receives the disconnect signal and reconnects to serve This handles both server restarts and laptop suspend correctly since the proxy controls the rmcp client connection directly. Removed: ReconnectState + auto_reconnect_stale_sessions middleware from serve/mod.rs (server-side middleware cannot fix dead TCP connections). --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/mcp/mod.rs | 148 +++++++++++++++++++++++++++++----- src/serve/mod.rs | 202 ++--------------------------------------------- 4 files changed, 138 insertions(+), 216 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8faf64c..47957aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.70" +version = "1.0.71" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 6401164..b2d2043 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.70" +version = "1.0.71" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 8ef19eb..76a1377 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -2241,15 +2241,59 @@ struct McpProxyService { /// Shared peer handle — hot-swapped on reconnect. /// `None` means we're reconnecting to serve; tool calls return a retry-able error. peer: std::sync::Arc>>>, + /// Signal to the main loop in `run_mcp_client` that the current peer is dead + /// and a fresh `connect_to_serve` should be attempted. Sent from `call_tool` / + /// `list_tools` when rmcp returns a transport-level error so we can recover + /// from server restarts and TCP keep-alive failures without bubbling the error + /// up to Claude Desktop. + disconnect_tx: tokio::sync::mpsc::Sender<()>, } impl McpProxyService { #[allow(dead_code)] fn new(peer: rmcp::service::Peer) -> Self { + // Direct constructor used by tests / single-shot scenarios. + // No reconnect plumbing — the dummy channel is never read. + let (tx, _rx) = tokio::sync::mpsc::channel(1); Self { peer: std::sync::Arc::new(tokio::sync::RwLock::new(Some(peer))), + disconnect_tx: tx, } } + + /// Force a reconnect: clear the shared peer and signal the main loop in + /// `run_mcp_client` to call `connect_to_serve` again. Brief sleep gives + /// the main loop time to actually reconnect before the caller retries. + async fn force_reconnect(&self) { + *self.peer.write().await = None; + let _ = self.disconnect_tx.send(()).await; + tokio::time::sleep(std::time::Duration::from_millis( + crate::mcp::PROXY_RETRY_BACKOFF_MS, + )) + .await; + } +} + +/// Maximum number of attempts when forwarding a request to serve. +/// Each retry includes a forced reconnect, so this also bounds reconnect attempts +/// per individual tool call. +const PROXY_MAX_RETRY_ATTEMPTS: u32 = 3; + +/// Backoff between proxy retries, also used as the post-reconnect settle delay. +const PROXY_RETRY_BACKOFF_MS: u64 = 500; + +/// Heuristic: does this error message describe a transport-level failure +/// (broken TCP, server gone, stale keep-alive, stale session) that warrants +/// a forced reconnect + retry, as opposed to a real tool-level error that +/// the caller should see? +fn is_transport_error_msg(msg: &str) -> bool { + msg.contains("Transport send error") + || msg.contains("error sending request") + || msg.contains("Transport error") + || msg.contains("connection closed") + || msg.contains("error decoding response body") + || msg.contains("Session not found") + || msg.contains("404") } /// Reconnect-related constants for the MCP proxy. @@ -2277,17 +2321,48 @@ impl ServerHandler for McpProxyService { request: Option, _cx: RequestContext, ) -> Result { - let peer = self.peer.read().await.clone(); - match peer { - Some(p) => p - .list_tools(request) - .await - .map_err(|e| McpError::internal_error(e.to_string(), None)), - None => Err(McpError::internal_error( - "codesearch serve is reconnecting — please retry in a moment".to_string(), - None, - )), + let mut last_err: Option = None; + for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { + let peer = self.peer.read().await.clone(); + match peer { + Some(p) => match p.list_tools(request.clone()).await { + Ok(r) => return Ok(r), + Err(e) => { + let msg = e.to_string(); + if !is_transport_error_msg(&msg) + || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 + { + return Err(McpError::internal_error(msg, None)); + } + tracing::warn!( + "list_tools attempt {}/{} failed (transport): {} — forcing reconnect", + attempt + 1, + PROXY_MAX_RETRY_ATTEMPTS, + msg + ); + last_err = Some(msg); + self.force_reconnect().await; + } + }, + None => { + if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { + tokio::time::sleep(std::time::Duration::from_millis( + PROXY_RETRY_BACKOFF_MS, + )) + .await; + continue; + } + return Err(McpError::internal_error( + "codesearch serve is reconnecting — please retry in a moment".to_string(), + None, + )); + } + } } + Err(McpError::internal_error( + last_err.unwrap_or_else(|| "transport error after retries".to_string()), + None, + )) } async fn call_tool( @@ -2295,17 +2370,49 @@ impl ServerHandler for McpProxyService { request: CallToolRequestParams, _cx: RequestContext, ) -> Result { - let peer = self.peer.read().await.clone(); - match peer { - Some(p) => p - .call_tool(request) - .await - .map_err(|e| McpError::internal_error(e.to_string(), None)), - None => Err(McpError::internal_error( - "codesearch serve is reconnecting — please retry in a moment".to_string(), - None, - )), + let mut last_err: Option = None; + for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { + let peer = self.peer.read().await.clone(); + match peer { + Some(p) => match p.call_tool(request.clone()).await { + Ok(r) => return Ok(r), + Err(e) => { + let msg = e.to_string(); + if !is_transport_error_msg(&msg) + || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 + { + return Err(McpError::internal_error(msg, None)); + } + tracing::warn!( + "call_tool('{}') attempt {}/{} failed (transport): {} — forcing reconnect", + request.name, + attempt + 1, + PROXY_MAX_RETRY_ATTEMPTS, + msg + ); + last_err = Some(msg); + self.force_reconnect().await; + } + }, + None => { + if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { + tokio::time::sleep(std::time::Duration::from_millis( + PROXY_RETRY_BACKOFF_MS, + )) + .await; + continue; + } + return Err(McpError::internal_error( + "codesearch serve is reconnecting — please retry in a moment".to_string(), + None, + )); + } + } } + Err(McpError::internal_error( + last_err.unwrap_or_else(|| "transport error after retries".to_string()), + None, + )) } } @@ -6780,6 +6887,7 @@ async fn run_mcp_client(serve_url: &str, cancel_token: CancellationToken) -> Res // even before the serve connection is established. let proxy = McpProxyService { peer: peer_state.clone(), + disconnect_tx: disconnect_tx.clone(), }; let server = proxy .serve(stdio()) diff --git a/src/serve/mod.rs b/src/serve/mod.rs index c8b471c..8590a4a 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -1652,189 +1652,7 @@ async fn log_mcp_requests( response } -/// State for the MCP session reconnect middleware. -/// -/// Holds a reqwest client and the local MCP URL so the middleware can -/// perform an internal `initialize` + retry when it detects a stale session. -#[derive(Clone)] -struct ReconnectState { - client: reqwest::Client, - mcp_url: String, -} - -/// Middleware that transparently reconnects MCP clients with stale session IDs. -/// -/// When the server restarts (or after laptop suspend), all in-memory MCP sessions are lost. -/// Clients that still hold a stale session ID get a 404 "Session not found" from rmcp, -/// which most MCP client libraries don't handle gracefully — they appear connected but -/// can't send any requests. -/// -/// Strategy (full transparent retry): -/// 1. Buffer the request body. -/// 2. If it's an `initialize` with a stale session ID → strip the header so rmcp -/// creates a fresh session (client is explicitly re-initializing). -/// 3. For all other requests → pass through to rmcp normally. -/// 4. If rmcp returns 404 (stale session detected): -/// a. Send an internal `initialize` request to ourselves (no session header). -/// b. Extract the new session ID from the initialize response header. -/// c. Retry the original request with the new session ID. -/// d. Forward the retry response to the client with the new session ID header -/// so the client updates its stored session ID automatically. -async fn auto_reconnect_stale_sessions( - axum::extract::State(reconnect): axum::extract::State, - req: axum::extract::Request, - next: axum::middleware::Next, -) -> axum::response::Response { - const MCP_SESSION_HEADER: &str = "mcp-session-id"; - - let path = req.uri().path().to_string(); - let is_mcp = path == crate::constants::MCP_ENDPOINT_PATH; - let has_session = req.headers().contains_key(MCP_SESSION_HEADER); - - // Only intercept POST to /mcp with a session ID - if !is_mcp || !has_session || req.method() != axum::http::Method::POST { - return next.run(req).await; - } - - // Buffer the body so we can replay it on retry - let (mut parts, body) = req.into_parts(); - let body_bytes = match axum::body::to_bytes(body, 4 * 1024 * 1024).await { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!("Failed to buffer MCP request body: {}", e); - return axum::response::Response::builder() - .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) - .body(axum::body::Body::from("Internal error")) - .unwrap(); - } - }; - - // If this is an initialize request with a stale session ID, strip the header - // so rmcp treats it as a fresh connection (client is explicitly re-initializing). - let is_initialize = serde_json::from_slice::(&body_bytes) - .ok() - .and_then(|v| v.get("method").and_then(|m| m.as_str()).map(|s| s == "initialize")) - .unwrap_or(false); - - if is_initialize { - tracing::info!("🔄 MCP client re-initializing — stripping stale session ID"); - parts.headers.remove(axum::http::header::HeaderName::from_static(MCP_SESSION_HEADER)); - let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes)); - return next.run(req).await; - } - // Pass through to rmcp - let stale_session_id = parts - .headers - .get(MCP_SESSION_HEADER) - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_owned()); - let req = axum::http::Request::from_parts(parts, axum::body::Body::from(body_bytes.clone())); - let response = next.run(req).await; - - // If rmcp returned 404, the session is stale — perform transparent reconnect - if response.status() != axum::http::StatusCode::NOT_FOUND { - return response; - } - - tracing::info!( - "🔄 MCP stale session detected (404) — auto-reconnecting transparently" - ); - - // Step 1: Send an initialize request to ourselves to get a fresh session ID. - // We use a minimal MCP initialize payload (the server doesn't validate client info). - let init_payload = serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": { - "protocolVersion": "2025-03-26", - "capabilities": {}, - "clientInfo": { "name": "codesearch-reconnect", "version": "1.0" } - } - }); - - let init_response = match reconnect - .client - .post(&reconnect.mcp_url) - .header("content-type", "application/json") - .header("accept", "text/event-stream") - .json(&init_payload) - .send() - .await - { - Ok(r) => r, - Err(e) => { - tracing::warn!("Auto-reconnect: initialize request failed: {}", e); - return axum::response::Response::builder() - .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) - .body(axum::body::Body::from("MCP reconnect failed")) - .unwrap(); - } - }; - - // Extract the new session ID from the initialize response header - let new_session_id = init_response - .headers() - .get("mcp-session-id") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_owned()); - - let Some(new_session_id) = new_session_id else { - tracing::warn!("Auto-reconnect: initialize succeeded but no session ID in response"); - return axum::response::Response::builder() - .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) - .body(axum::body::Body::from("MCP reconnect: no session ID")) - .unwrap(); - }; - - tracing::info!( - "🔄 Auto-reconnect: new session {} (replaced stale {})", - &new_session_id[..8.min(new_session_id.len())], - stale_session_id.as_deref().map(|s| &s[..8.min(s.len())]).unwrap_or("?") - ); - - // Step 2: Retry the original request with the new session ID - let retry_response = match reconnect - .client - .post(&reconnect.mcp_url) - .header("content-type", "application/json") - .header("accept", "text/event-stream") - .header("mcp-session-id", &new_session_id) - .body(body_bytes.to_vec()) - .send() - .await - { - Ok(r) => r, - Err(e) => { - tracing::warn!("Auto-reconnect: retry request failed: {}", e); - return axum::response::Response::builder() - .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) - .body(axum::body::Body::from("MCP reconnect retry failed")) - .unwrap(); - } - }; - - // Convert reqwest response back to axum response, injecting the new session ID header - // so the client updates its stored session ID automatically. - let status = retry_response.status(); - let mut builder = axum::response::Response::builder().status(status); - for (name, value) in retry_response.headers() { - builder = builder.header(name, value); - } - // Always include the new session ID so the client can update its stored value - builder = builder.header("mcp-session-id", &new_session_id); - - let body_bytes = retry_response.bytes().await.unwrap_or_default(); - builder - .body(axum::body::Body::from(body_bytes)) - .unwrap_or_else(|_| { - axum::response::Response::builder() - .status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) - .body(axum::body::Body::from("Response build error")) - .unwrap() - }) -} /// Run the MCP serve mode. /// @@ -1915,13 +1733,13 @@ pub async fn run_serve( let mcp_service = StreamableHttpService::new(service_factory, session_manager, config); - // Build reconnect middleware state (reqwest client + local MCP URL for internal retries) - let reconnect_state = ReconnectState { - client: reqwest::Client::new(), - mcp_url: format!("http://{}{}", addr, MCP_ENDPOINT_PATH), - }; - - // Build axum router with request logging + stale session recovery + // Build axum router with request logging. + // Stale-session recovery is handled client-side by the stdio proxy's retry + // loop in `McpProxyService` (see src/mcp/mod.rs). Remote MCP clients that + // are not spec-compliant must reconnect themselves — we do not attempt a + // server-side transparent reconnect because that path opened a session leak + // and could not actually reach OpenCode (TCP keep-alive failure happens + // before the request hits this middleware). let app = axum::Router::new() .route(HEALTH_PATH, axum::routing::get(health_handler)) .route("/repos", axum::routing::post(add_repo_handler)) @@ -1932,10 +1750,6 @@ pub async fn run_serve( ) .nest_service(MCP_ENDPOINT_PATH, mcp_service) .layer(axum::middleware::from_fn(log_mcp_requests)) - .layer(axum::middleware::from_fn_with_state( - reconnect_state, - auto_reconnect_stale_sessions, - )) .with_state(serve_state.clone()); // Bind TCP listener BEFORE spawning background warmup, so we know the port is live. @@ -2444,4 +2258,4 @@ mod tests { assert_eq!(body["status"], "conflict"); } } -} +} \ No newline at end of file From 9e43b45a29daecc6055413d84130fc8c067a305d Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 1 May 2026 18:28:24 +0200 Subject: [PATCH 10/10] fix(serve): validate config_path from env var (CodeQL path traversal) CODESEARCH_REPOS_CONFIG env var was used directly as a filesystem path without validation or canonicalization. CodeQL flagged this as 'Uncontrolled data used in path expression'. - repos.rs: validate env-var override has .json extension (fail-fast) - serve/mod.rs: canonicalize config_path before fs::metadata/load_from to resolve symlinks and normalize .. components Fixes: CodeQL alert on src/serve/mod.rs reload_if_changed() --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/db_discovery/repos.rs | 13 ++++++++++++- src/serve/mod.rs | 7 +++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47957aa..3ee4cdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -634,7 +634,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.0.71" +version = "1.0.72" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index b2d2043..8867500 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.0.71" +version = "1.0.72" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/src/db_discovery/repos.rs b/src/db_discovery/repos.rs index 522edc9..0a1ef69 100644 --- a/src/db_discovery/repos.rs +++ b/src/db_discovery/repos.rs @@ -235,7 +235,18 @@ pub fn config_dir() -> Result { pub fn config_path() -> Result { if let Ok(override_path) = std::env::var(crate::constants::REPOS_CONFIG_ENV) { - return Ok(PathBuf::from(override_path)); + let path = PathBuf::from(&override_path); + // Validate the env-var override points to a .json file to prevent + // path traversal / arbitrary file read (CodeQL: uncontrolled data in path). + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext.eq_ignore_ascii_case("json") { + return Ok(path); + } + anyhow::bail!( + "{} must point to a .json file, got: {}", + crate::constants::REPOS_CONFIG_ENV, + override_path + ); } Ok(config_dir()?.join(REPOS_CONFIG_FILE)) } diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 8590a4a..d3dd6c4 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -195,6 +195,13 @@ impl ServeState { }, }; + // Canonicalize to resolve symlinks and prevent path traversal. + // CodeQL: path derives from env var (CODESEARCH_REPOS_CONFIG) — validate before use. + let config_path = match std::fs::canonicalize(&config_path) { + Ok(p) => p, + Err(_) => return Ok(()), // file doesn't exist yet — nothing to reload + }; + let mtime = std::fs::metadata(&config_path) .and_then(|m| m.modified()) .ok();