From edb9c095c1d305f902908405b5e1e35f0de55aa9 Mon Sep 17 00:00:00 2001 From: Claude Lin & Lay Date: Fri, 14 Aug 2026 21:29:24 +0900 Subject: [PATCH] feat(mcp): add vector_ids to search for stored-content retrieval [mcp, bridge, docs, tests] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `search` に `vector_ids` パラメータを追加し、索引が既に保持している本文を 返せるようにした。従来は `doc` / `wiki_doc` 以外の型に本文を返す手段が無く、 `search` であたりを付けたあと `gh` や grep で取り直す一往復が必要だった。 設計は既存の導出作法に合わせた。モードは列挙パラメータで選ばせず、 パラメータ集合から導く。優先順位は `vector_ids` -> `query` 空 -> hybrid search。 専用 tool は追加しない。#104 / #105 で 4 tool を `search` へ統合した判断を維持する （本文取得の専用 tool は削除された `get_doc_content` の復活そのものになる）。 - `src/fetch.ts` (new): D1 `search_docs` の保存済み content を 1 回のバッチ クエリで読み出す。GitHub API は呼ばない。`scan.ts` と同じ理由でモジュールを 分けている（MCP server を立てずに実 D1 に対して検証できる）。 - 返す文字列が索引由来かつ 8000 文字 truncate 済みであることを応答が示す: top-level に `content_source: "index"` / `content_max_chars`、各行に `content_chars` / `content_truncated`。フラグは長さから導くため、自然長が ちょうど上限の本文も truncate 済みとして報告する（安全側）。 - 未知・失効 id は `not_found` に載せ、残りは返す。`vector_id` は永続識別子 として約束しない（`legacy-vector-id.ts` の移行履歴があるため）。 - `search` の全 result 行と `same_entity.others` の各要素に `vector_id` を追加。 others に載せるのは、実体集約が comment / review / 旧 diff をそこへ畳むため、 無いと本モードが本来返すべき行だけ到達不能になるからである。 - `include_content` の既存挙動（contents API から完全なファイル、5 件上限）は 変更しない。取得元も上限の根拠も違うため統合しない。 - bridge 側の静的 schema mirror (`mcp-server/server/tools.js`) と manifest も 同期。drift-check が CI gate として効いている。 Closes #239 --- README.ja.md | 53 ++++- README.md | 53 ++++- docs/0-requirements.ja.md | 14 ++ docs/0-requirements.md | 14 ++ mcp-server/README.md | 2 +- mcp-server/manifest.json | 2 +- mcp-server/server/tools.js | 31 ++- mcp-server/test/search-tool-schema.test.js | 19 ++ src/fetch.test.ts | 147 ++++++++++++ src/fetch.ts | 189 +++++++++++++++ src/fetch.workers.test.ts | 261 +++++++++++++++++++++ src/mcp-stateless-contract.test.ts | 31 +++ src/mcp.ts | 107 ++++++++- 13 files changed, 908 insertions(+), 15 deletions(-) create mode 100644 src/fetch.test.ts create mode 100644 src/fetch.ts create mode 100644 src/fetch.workers.test.ts diff --git a/README.ja.md b/README.ja.md index 9d133d2..a3b321c 100644 --- a/README.ja.md +++ b/README.ja.md @@ -95,19 +95,20 @@ GitHub webhooks + GitHub API ## MCP Tools -この MCP サーバーが公開するツールは 1 つに統合されています。意味検索、時系列 activity scan、doc 本文取得のいずれも `search` のパラメータ経由で扱えます。以前の build で分かれていた `get_issue_context` / `get_doc_content` / `list_recent_activity` は削除され、用途は下記パラメータに吸収されました。 +この MCP サーバーが公開するツールは 1 つに統合されています。意味検索、時系列 activity scan、doc 本文取得、`vector_id` 指定の保存済み本文取得のいずれも `search` のパラメータ経由で扱えます。以前の build で分かれていた `get_issue_context` / `get_doc_content` / `list_recent_activity` は削除され、用途は下記パラメータに吸収されました。 ### `search` GitHub の issue / pull request / release / documentation / **GitHub Wiki page** / commit diff / comment 系 (issue と PR の top-level comment、PR review 本文、PR インラインレビューコメント) を対象にした統合検索ツールです。 -`query` と `sort` の組み合わせで、以下の 3 モードを切り替えます。 +パラメータ集合から、以下の 4 モードが導かれます。 1. **ハイブリッド意味検索 (既定)** — dense BGE-M3 (Vectorize) + sparse BM25 (D1 FTS5) を Reciprocal Rank Fusion (RRF, k=60) で合成し、`@cf/baai/bge-reranker-base` cross-encoder で rerank。自然言語 `query` を渡します。 2. **時系列 activity scan** — `query` を省略または空にし、`sort` を `"updated_desc"` / `"created_desc"` に設定します。`since` / `until` を併用して窓を絞れます。従来の `list_recent_activity` を置き換えます。`[since, until)` の窓は索引側で適用されるので、窓に行があればどれだけ古い窓でも返ります。窓が 1 ページに収まらないときは応答に `truncated: true` が付き、「該当なし」と「読み切れていない」を区別できます。返った最古の行の時刻を次の `until` にして遡ってください。 3. **doc 本文取得** — `include_content: true` を指定すると、`type="doc"` 結果の本文が GitHub contents API 経由で取得され、該当行の `content` フィールドに inline されます。API fan-out を抑えるため先頭の数件に絞られます。従来の `get_doc_content` を置き換えます。 +4. **保存済み本文の取得** — `vector_ids`（先行する結果が持つ `vector_id`）を渡します。索引済みの全 type がその行の本文を返します——doc だけでなく issue / PR / comment / review / release / diff も対象です。`search` であたりを付けたあと本文を読むための `gh` / grep の一往復が不要になります。D1 から返すので GitHub API は呼びません。返る文字列が何であって何でないかは下記「保存済み本文の取得」を参照してください。 -structured filter (`repo` / `state` / `labels` / `milestone` / `assignee` / `type`) はすべてのモードで有効です。 +structured filter (`repo` / `state` / `labels` / `milestone` / `assignee` / `type`) は、保存済み本文の取得を除くすべてのモードで有効です。保存済み本文の取得では行をサーバが選ぶのではなく呼び出し側が名指しするため、filter は適用しません。 search モードは「1件もマッチしなかったフィルタ」を `filters_unmatched` に載せます (常に存在し、すべて成立していれば `[]`)。`repo` はフルスラッグ `owner/repo` の完全一致なので、短いリポジトリ名を渡すと母集合が空になり、本当にヒットゼロだった場合と同じ形のレスポンスが返ります。このフィールドがその2つを区別します。効くのは多段のエージェンティック検索で、ゼロが正常な中間結果として読まれてしまい、フィルタ不成立が表に出ないまま終わる場面です。 @@ -131,6 +132,7 @@ bot (`sender.login` が `[bot]` で終わる) と trim 後 10 文字未満の bo | `since` | ISO 8601 文字列 | `updated_at >= since` の結果だけを残します。scan モードの既定は `until` の 7 日前 (`until` 省略時は現在の 7 日前)。 | | `until` | ISO 8601 文字列 | `updated_at < until` の結果だけを残します。 | | `include_content` | boolean | 上位 doc 結果に本文を inline する (既定 `false`)。 | +| `vector_ids` | string[] | 保存済み本文の取得。読み出す行の `vector_id`、1 回あたり最大 50 件。他モードより優先され、指定時は `query` / `sort` と全 filter が無視されます。下記「保存済み本文の取得」参照。 | | `graph_expand` | boolean | opt-in の GraphRAG 拡張（search モードのみ）。`true` のとき fusion 後の上位結果を seed に Decision-Structure の mention グラフ（D1 `doc_edges`）を辿り、関連 wiki ページを `graph_hop` / `graph_from` 付きで別配列 `graph_results` として返す（下記「検索の 2 軸」参照）。既定 `false` は標準ハイブリッド検索とバイト単位で同一（グラフ未参照）。 | | `graph_hops` | number | `graph_expand` のグラフ探索深度（1 または 2、既定 1）。`graph_expand` が `false` のときは無視。 | @@ -155,6 +157,51 @@ bot (`sender.login` が `[bot]` で終わる) と trim 後 10 文字未満の bo 代表になるのはその group で最上位に来た行です。したがって「いつ変わったか」を問う検索では、現在の版ではなく該当する古い commit diff が返ります。他の行を吸収した結果には `same_entity` フィールドが付き（`count` は自身を含む件数、`others[]` は畳んだ各行の type / URL / 時刻 / score）、畳んだ分は捨てられません。完全な規則は [docs/0-requirements.ja.md](docs/0-requirements.ja.md) を参照してください。 +#### 保存済み本文の取得 + +すべての result 行——および `same_entity.others` の各要素——は `vector_id` を持ちます。この id を `vector_ids` として渡し返すと、索引がその行について保持している本文が返ります。 + +返るのは索引が持つ**本文の複製**であって、生きているソースそのものではありません。実体は embedding input であり、取り込み時に 8000 文字で truncate されています。inline された文字列自体はどちらであるかを示さないため、応答が示します——top-level に `content_source: "index"` と `content_max_chars`、各行に `content_chars` と `content_truncated`。`content_truncated: true` の行は断片です。末尾が必要なら GitHub から読み直してください。 + +未知・失効した id は `not_found` に載り、残りの行はそのまま返ります。この部分成功は意図的なものです。`vector_id` は「その結果集合の中で行に到達するための取っ手」であって**永続識別子ではありません**。採番は既に一度移行しているので、保存して後で使うのではなく、そのつど新しい結果から取ってください。 + +`include_content` とは別軸であり、そちらの挙動は変わりません。doc は完全なファイルが必要なので GitHub からファイル全体を読み直しており、API fan-out を抑えるため件数上限があります。本モードは D1 を読み、上限は渡された id の数です。 + +id が載るのは search モードの結果だけです。scan モードの行は structured store 由来なので持ちません。id は不透明（`{type 接頭辞}:{base64url sha256}`）です。結果からコピーしてください。手で組み立てないでください。 + +```json +{ + "vector_ids": [ + "i:d0qhtOi9Lxc4yuMbgbDD1BvcpptqrMWpphGMGw4t79I", + "ic:kPjVFYzpd5y9Y2RWQ1KstYDZYsSDzmxhqQphaHKHHRU" + ] +} +``` + +```json +{ + "count": 2, + "mode": "fetch", + "requested": 2, + "content_source": "index", + "content_max_chars": 8000, + "not_found": [], + "results": [ + { + "vector_id": "i:d0qhtOi9Lxc4yuMbgbDD1BvcpptqrMWpphGMGw4t79I", + "repo": "Liplus-Project/github-rag-mcp", + "type": "issue", + "state": "open", + "number": 239, + "updated_at": "2026-08-14T00:00:00Z", + "content": "feat(mcp): add vector_ids to search ...", + "content_chars": 4213, + "content_truncated": false + } + ] +} +``` + #### 検索の 2 軸 search モードは 2 つの軸を分けて返します。単一順位への融合は行いません。 diff --git a/README.md b/README.md index 8872a6b..36a1157 100644 --- a/README.md +++ b/README.md @@ -95,19 +95,20 @@ See: ## MCP Tools -This MCP server exposes a single consolidated tool. All retrieval modes — semantic search, time-ordered activity scan, and inline doc content fetch — are reached through `search` via its parameter set. Earlier builds split these across `get_issue_context`, `get_doc_content`, and `list_recent_activity`; those tools have been removed and their use cases now fold into the parameters below. +This MCP server exposes a single consolidated tool. All retrieval modes — semantic search, time-ordered activity scan, inline doc content fetch, and stored-content fetch by `vector_id` — are reached through `search` via its parameter set. Earlier builds split these across `get_issue_context`, `get_doc_content`, and `list_recent_activity`; those tools have been removed and their use cases now fold into the parameters below. ### `search` Unified search across GitHub issues, pull requests, releases, repository documentation, GitHub Wiki pages, commit diffs, and comment / review surfaces (top-level comments on issues and PRs, PR review bodies, and PR inline review comments). -Three modes are selected by the combination of `query` and `sort`: +Four modes are selected by the parameter set: 1. **Hybrid semantic search (default)** — dense BGE-M3 over Vectorize + sparse BM25 over D1 FTS5, fused via Reciprocal Rank Fusion (RRF, k=60), then re-scored with the `@cf/baai/bge-reranker-base` cross-encoder. Pass a natural-language `query`. 2. **Time-ordered activity scan** — omit or leave `query` empty and set `sort` to `"updated_desc"` or `"created_desc"`. Optionally narrow with `since` / `until` to list recent activity across every type. This subsumes the previous `list_recent_activity` tool. The `[since, until)` window is applied inside the index, so any window holding rows returns rows however far back it sits; the response carries `truncated: true` when the window holds more than one page, which is what separates "no such rows" from "the read stopped short". Walk backwards by re-issuing the scan with `until` set to the oldest row returned. 3. **Doc / wiki content fetch** — set `include_content: true`. For result rows whose `type` is `"doc"`, the raw file content is fetched from the GitHub contents API; for `type: "wiki_doc"` rows, the raw markup is fetched from `raw.githubusercontent.com/wiki/`. Both are inlined as a `content` field. Capped at the first few rows of each type to bound API fan-out. This subsumes the previous `get_doc_content` tool. +4. **Stored-content fetch** — pass `vector_ids` (the `vector_id` values carried by earlier results). Every indexed type returns the body text the index already holds for that exact row — issues, PRs, comments, reviews, releases and diffs included, not just docs — so locating something with `search` and then reading it no longer costs a round trip through `gh` or grep. Served from D1: no GitHub API call is made. See [Stored-content fetch](#stored-content-fetch) below for what the returned text is and is not. -Structured filters (`repo`, `state`, `labels`, `milestone`, `assignee`, `type`) apply in every mode. +Structured filters (`repo`, `state`, `labels`, `milestone`, `assignee`, `type`) apply in every mode except stored-content fetch, where the rows are named rather than selected. Search mode reports filters that matched nothing at all in `filters_unmatched` (always present, `[]` when every filter matched something). `repo` is an exact match on the full `owner/repo` slug, so a bare repository name selects an empty population and returns a response shaped exactly like a genuine zero-hit search — this field is what separates the two. It matters most in multi-step agentic search, where a zero reads as a normal intermediate result and the mis-specified filter would otherwise never surface. @@ -131,6 +132,7 @@ Bot-authored comments (`sender.login` ending in `[bot]`) and comments shorter th | `since` | ISO 8601 string | Keep only results with `updated_at >= since`. In scan mode, defaults to 7 days before `until` (before now when `until` is omitted). | | `until` | ISO 8601 string | Keep only results with `updated_at < until`. | | `include_content` | boolean | Inline raw content on top doc results (default `false`). | +| `vector_ids` | string[] | Stored-content fetch. The `vector_id` values of the rows to read back, max 50 per call. Takes precedence over the other modes: `query`, `sort` and every filter are ignored when present. See [Stored-content fetch](#stored-content-fetch). | | `graph_expand` | boolean | Opt-in GraphRAG expansion (search mode only). When `true`, after fusion the top results seed a traversal of the Decision-Structure mention graph (D1 `doc_edges`); related wiki pages come back in a separate `graph_results` array tagged `graph_hop` / `graph_from` — see Retrieval axes below. Default `false` = byte-identical to standard hybrid retrieval (no graph read). | | `graph_hops` | number | Graph traversal depth for `graph_expand` (1 or 2, default 1). Ignored when `graph_expand` is `false`. | @@ -155,6 +157,51 @@ One thing is indexed as several rows: a file is a `doc` row plus one `diff` row The representative is the highest-ranked row of the group, so a query about when something changed still returns the relevant old commit diff rather than the current version. A result that absorbed other rows carries a `same_entity` field (`count` including itself, plus `others[]` with the type, URL, timestamp and score of each collapsed row) so nothing is lost. See [docs/0-requirements.md](docs/0-requirements.md) for the full rule. +#### Stored-content fetch + +Every result row — and every `same_entity.others` entry — carries a `vector_id`. Passing those ids back as `vector_ids` returns the body text the index holds for exactly those rows. + +What comes back is the **index's copy** of the body, not the live source: it is the embedding input, truncated by the ingest pipeline at 8000 characters. Inlined text carries no mark of which it is, so the response says so — `content_source: "index"` and `content_max_chars` at the top level, `content_chars` and `content_truncated` on each row. A row flagged `content_truncated: true` is a prefix; read the rest from GitHub if the tail matters. + +Unknown or stale ids come back in `not_found` and the remaining rows still return. That partial success is deliberate: `vector_id` is a handle for reaching a row in the result set it arrived in, **not a durable identifier**. The id scheme has been migrated once already, so do not store one for later use — take it from a fresh result. + +This is a different axis from `include_content`, which is unchanged: that flag re-reads whole files from GitHub because a doc needs its full text, and it is capped to bound API fan-out. Fetch mode reads D1 and is bounded by the ids you listed. + +Ids come from search-mode results only; scan-mode rows are read from the structured store and carry none. They are opaque (`{type prefix}:{base64url sha256}`) — copy them from a result, never build one by hand: + +```json +{ + "vector_ids": [ + "i:d0qhtOi9Lxc4yuMbgbDD1BvcpptqrMWpphGMGw4t79I", + "ic:kPjVFYzpd5y9Y2RWQ1KstYDZYsSDzmxhqQphaHKHHRU" + ] +} +``` + +```json +{ + "count": 2, + "mode": "fetch", + "requested": 2, + "content_source": "index", + "content_max_chars": 8000, + "not_found": [], + "results": [ + { + "vector_id": "i:d0qhtOi9Lxc4yuMbgbDD1BvcpptqrMWpphGMGw4t79I", + "repo": "Liplus-Project/github-rag-mcp", + "type": "issue", + "state": "open", + "number": 239, + "updated_at": "2026-08-14T00:00:00Z", + "content": "feat(mcp): add vector_ids to search ...", + "content_chars": 4213, + "content_truncated": false + } + ] +} +``` + #### Retrieval axes Search mode reports two axes separately and never fuses them into one ranking. diff --git a/docs/0-requirements.ja.md b/docs/0-requirements.ja.md index b8a224b..7014d5c 100644 --- a/docs/0-requirements.ja.md +++ b/docs/0-requirements.ja.md @@ -507,10 +507,12 @@ Parameters: - `fusion` optional — `rrf` (default) / `dense_only` / `sparse_only` - `rerank` optional — `true` (default) / `false` - `since` / `until` optional — 半開区間 `[since, until)` の時間窓 +- `vector_ids` optional — 保存済み本文の取得。下記参照 Returns: - repository、type、state、labels、milestone、assignees、URL、RRF fused score を含む ranked match +- 全 result（および `same_entity.others` の各要素）に `vector_id` — その行の索引キーであり、`vector_ids` が取る取っ手 - 追加 debug フィールド: `dense_score`、`sparse_score`、`dense_rank`、`sparse_rank`、`rerank_score`（rerank 無効時または fallback 時は null） - 同一実体の他の行を吸収した結果には `same_entity`（Entity Aggregation 参照）。`top_k` は行数ではなく実体数で数える - top-level metadata: `fusion`、`dense_candidates`、`sparse_candidates`、`rerank_requested`、`rerank_applied`、`filters_unmatched` @@ -525,6 +527,18 @@ Returns: scan mode は top-level に `truncated` を追加する。窓が応答に載せた以上の行を持つとき true になる（endpoint が cap 一杯まで返した、または merge 後の件数が `top_k` を超えた）。これが「該当なし」と「読み切れていない」を呼び出し側に区別させる: 返った最古の行の時刻を次の `until` にして遡ればよい。両者を区別できない欠損調査ツールは、存在しない欠損を報告し実在する取り込みを見落とす——#178 の再検証で 1 日に 2 度踏んだ誤りがこれである。 +**保存済み本文の取得（`vector_ids`）.** 先行する結果が持つ `vector_id` を渡すと、索引がその行について既に保持している本文を返す。対象は索引済みの全 type——`issue` / `pull_request` / `issue_comment` / `pr_review` / `pr_review_comment` / `release` / `diff` / `doc` / `wiki_doc`。従来は `doc` と `wiki_doc` にしか本文を返す経路が無く、`search` であたりを付けたあと本文を読むには `gh` や grep での一往復が必要だった。本文は D1 から読む。GitHub API を新たに叩かず、subrequest も増えない。 + +これは tool ではなく mode である。#104 / #105 で 4 tool が `search` 1 本へ統合されており、本文取得の専用 tool は削除された `get_doc_content` を別名で復活させることに等しい。統合時の設計意図が「mode はパラメータ集合で表現する」であったため、本 mode もその作法に乗る。優先順位は `vector_ids` → `query` 空 → hybrid search。fetch mode では metadata filter は適用しない——行はサーバが選ぶのではなく呼び出し側が名指しするからである。 + +返るのは索引が持つ本文の複製——embedding input であり、取り込み時に 8000 文字で truncate されている——であって、生きているソースそのものではない。inline された文字列自体はどちらであるかを示さないので、形が示す: 応答は `content_source: "index"` と `content_max_chars` を持ち、各行は `content_chars` と `content_truncated` を持つ。このフラグは取り込み時に記録した値ではなく長さから導いているため、本文の自然長がちょうど上限だった行も truncate 済みとして報告される。この向きの誤りは不要な読み直し 1 回で済むが、逆向きの誤りは断片を完全な本文として通してしまう。 + +id が載るのは search mode の結果だけである——scan mode は structured store を読んでおり、その行は索引キーを持たない。未知・失効した id は `not_found` に載せ、残りの行はそのまま返す。部分成功は利便ではなく契約である。`vector_id` は「その結果集合の中で行に到達するための取っ手」であって永続識別子ではない——採番は既に一度移行している（`src/pipeline/legacy-vector-id.ts`）ため、古い id を再送した呼び出し側にも生きている行は返さなければならない。 + +`include_content` は別軸であり、既存挙動は変更しない。doc は完全なファイルが必要なので GitHub contents API（および `raw.githubusercontent.com/wiki`）からファイル全体を読み直しており、5 件上限はその API fan-out を抑えるためにある。fetch mode は D1 を読み、上限は呼び出し側が挙げた id の数（1 回あたり最大 50、`top_k` の上限に合わせた）である。同じ「content」でも取得元も上限の根拠も異なる。1 つのフラグに両方を載せることは、1 つの保証で 2 つの挙動を覆うことになる。 + +非スコープ: 保存されている文字列が embedding input そのものであるため、8000 文字の上限が embedding と保存の双方を兼ねている状態にある。FTS 側に truncate しない本文を持たせるのが根本の形だが、既存行の再 index が必要になる。fetch mode は「既に持っているものを返す」に限定する。 + ### `get_issue_context` Purpose: diff --git a/docs/0-requirements.md b/docs/0-requirements.md index e76d0ec..760d864 100644 --- a/docs/0-requirements.md +++ b/docs/0-requirements.md @@ -513,10 +513,12 @@ Parameters: - `fusion` optional — `rrf` (default) / `dense_only` / `sparse_only` - `rerank` optional — `true` (default) / `false` - `since` / `until` optional — half-open time window `[since, until)` +- `vector_ids` optional — stored-content fetch; see below Returns: - ranked matches with repository, type, state, labels, milestone, assignees, URL, and RRF fused `score` +- `vector_id` on every result (and on every `same_entity.others` entry) — the row's index key and the handle `vector_ids` takes - additional debug fields per result: `dense_score`, `sparse_score`, `dense_rank`, `sparse_rank`, `rerank_score` (null when rerank disabled or when graceful fallback engaged) - `same_entity` on results that absorbed other rows of the same entity (see Entity Aggregation); `top_k` counts entities, not rows - top-level metadata: `fusion`, `dense_candidates`, `sparse_candidates`, `rerank_requested`, `rerank_applied`, `filters_unmatched` @@ -531,6 +533,18 @@ The check is an existence probe (`SELECT 1 FROM search_docs WHERE repo = ? LIMIT Scan mode adds one top-level field, `truncated`, which is true when the window holds more rows than the response carries — either an endpoint filled its row cap, or the merged set was longer than `top_k`. This is what tells a caller that zero results means "no such rows" rather than "the read stopped short": walk backwards by re-issuing the scan with `until` set to the oldest row returned. A gap-hunting tool that cannot separate those two answers reports absent rows that exist and misses rows that do not, which is how #178 was mis-diagnosed twice in one day. +**Stored-content fetch (`vector_ids`).** Pass the `vector_id` values carried by earlier results and the tool returns the body text the index already holds for those exact rows. Every indexed type is covered — `issue`, `pull_request`, `issue_comment`, `pr_review`, `pr_review_comment`, `release`, `diff`, `doc`, `wiki_doc` — where before only `doc` and `wiki_doc` had any path to a body at all, so locating something with `search` and then reading it cost a second round trip through `gh` or grep. The text is read from D1; no GitHub API call is made, and the subrequest budget is untouched. + +This is a mode, not a tool. Four tools were consolidated into `search` in #104 / #105, and a dedicated body-fetch tool would be the removed `get_doc_content` returning under a new name; the consolidation's own design intent was that modes are expressed by the parameter set, so this one is too. Precedence is `vector_ids` → empty `query` → hybrid search. Metadata filters do not apply in fetch mode: the rows are named by the caller, not selected by the server. + +What comes back is the index's copy of the body — the embedding input, truncated by the ingest pipeline at 8000 characters — and not the live source. Inlined text carries no mark of which it is, so the shape states it: the response carries `content_source: "index"` and `content_max_chars`, and each row carries `content_chars` plus `content_truncated`. The flag is read off the length rather than recorded at ingest, so a body whose natural length is exactly the ceiling reports as truncated; that error direction costs one needless re-read, while the other direction would pass a prefix off as a whole document. + +Ids are carried by search-mode results only — scan mode reads the structured store, whose rows have no index key. Unknown or stale ids are listed in `not_found` and the remaining rows still return. Partial success is the contract rather than a convenience, because `vector_id` is explicitly a handle for reaching a row in the result set it arrived in — not a durable identifier. The id scheme has been migrated once already (`src/pipeline/legacy-vector-id.ts`), so a caller replaying an old id must still get the rows that are live. + +`include_content` is a separate axis and stays unchanged: it re-reads whole files from the GitHub contents API (and `raw.githubusercontent.com/wiki`) because a doc needs its full text, and its 5-row cap exists to bound that API fan-out. Fetch mode reads D1 and is bounded by the ids the caller listed (max 50 per call, mirroring the `top_k` ceiling). Same word, different source and different bound — one flag over both would be one guarantee over two behaviors. + +Out of scope: the stored text is the embedding input, so one 8000-character limit currently serves both embedding and retention. Giving the FTS side an untruncated body is the root-level shape, but it requires re-indexing existing rows. Fetch mode returns what the index already holds. + ### `get_issue_context` Purpose: diff --git a/mcp-server/README.md b/mcp-server/README.md index 45c42d4..65f03b2 100644 --- a/mcp-server/README.md +++ b/mcp-server/README.md @@ -107,7 +107,7 @@ A single consolidated tool, `search`, covers every retrieval mode. Earlier build | Tool | Description | |---|---| -| `search` | Unified search across GitHub issues, pull requests, releases, repository documentation, **GitHub Wiki pages**, commit diffs, and comment / review surfaces. Three modes are selected by the combination of `query` and `sort`: (1) **hybrid semantic search** — dense BGE-M3 over Vectorize + sparse BM25 over D1 FTS5, fused via RRF, then re-scored with the `bge-reranker-base` cross-encoder; (2) **time-ordered activity scan** — leave `query` empty and set `sort` to `updated_desc` / `created_desc`, optionally narrowed with `since` / `until`; (3) **doc / wiki content fetch** — set `include_content: true` to inline raw markup on top `doc` and `wiki_doc` rows. Structured filters (`repo`, `state`, `labels`, `milestone`, `assignee`, `type`, `top_k`, `fusion`, `rerank`) apply in every mode. Search mode also accepts an opt-in `graph_expand` (with `graph_hops`, 1 or 2) that performs a GraphRAG expansion over the Decision-Structure mention graph and appends related wiki entries; it is off by default. | +| `search` | Unified search across GitHub issues, pull requests, releases, repository documentation, **GitHub Wiki pages**, commit diffs, and comment / review surfaces. Four modes are selected by the parameter set: (1) **hybrid semantic search** — dense BGE-M3 over Vectorize + sparse BM25 over D1 FTS5, fused via RRF, then re-scored with the `bge-reranker-base` cross-encoder; (2) **time-ordered activity scan** — leave `query` empty and set `sort` to `updated_desc` / `created_desc`, optionally narrowed with `since` / `until`; (3) **doc / wiki content fetch** — set `include_content: true` to inline raw markup on top `doc` and `wiki_doc` rows; (4) **stored-content fetch** — pass `vector_ids` (the `vector_id` values carried by earlier results) to read back the body text the index holds for those exact rows, for every type and with no GitHub API call. That text is the indexed copy, truncated at the 8000-character ingest ceiling, so each row carries `content_truncated`; unknown ids land in `not_found` and the rest still return. Structured filters (`repo`, `state`, `labels`, `milestone`, `assignee`, `type`, `top_k`, `fusion`, `rerank`) apply in every mode except stored-content fetch, where the rows are named rather than selected. Search mode also accepts an opt-in `graph_expand` (with `graph_hops`, 1 or 2) that performs a GraphRAG expansion over the Decision-Structure mention graph and appends related wiki entries; it is off by default. | The `type` filter accepts: `issue`, `pull_request`, `release`, `doc`, `wiki_doc`, `diff`, `issue_comment`, `pr_review`, `pr_review_comment`, or `all` (default). diff --git a/mcp-server/manifest.json b/mcp-server/manifest.json index d7e7bca..89f2b4f 100644 --- a/mcp-server/manifest.json +++ b/mcp-server/manifest.json @@ -42,7 +42,7 @@ "tools": [ { "name": "search", - "description": "Unified search across GitHub issues, PRs, releases, docs, GitHub Wiki pages, commit diffs, issue/PR top-level comments, PR reviews, and PR inline review comments. Three modes selected via query/sort axes: (1) hybrid semantic search (dense BGE-M3 + sparse BM25 fused via RRF + cross-encoder rerank), (2) time-ordered activity scan (empty query + sort), (3) doc/wiki content fetch (include_content=true inlines raw markup for top doc and wiki_doc results). Structured filters: repo, state, labels, milestone, assignee, type (issue / pull_request / release / doc / wiki_doc / diff / issue_comment / pr_review / pr_review_comment)." + "description": "Unified search across GitHub issues, PRs, releases, docs, GitHub Wiki pages, commit diffs, issue/PR top-level comments, PR reviews, and PR inline review comments. Four modes derived from the parameter set: (1) hybrid semantic search (dense BGE-M3 + sparse BM25 fused via RRF + cross-encoder rerank), (2) time-ordered activity scan (empty query + sort), (3) doc/wiki content fetch (include_content=true inlines raw markup for top doc and wiki_doc results), (4) stored-content fetch (vector_ids returns the body text the index holds for the named rows, any type, no GitHub API call, truncated at the 8000-character ingest ceiling). Structured filters: repo, state, labels, milestone, assignee, type (issue / pull_request / release / doc / wiki_doc / diff / issue_comment / pr_review / pr_review_comment)." } ], "compatibility": { diff --git a/mcp-server/server/tools.js b/mcp-server/server/tools.js index c5aaef4..e95b554 100644 --- a/mcp-server/server/tools.js +++ b/mcp-server/server/tools.js @@ -17,13 +17,16 @@ export const TOOLS = [ description: "Unified search across GitHub issues, PRs, releases, repository documentation, " + "GitHub Wiki pages, commit diffs, issue/PR top-level comments, PR reviews, and " + - "PR inline review comments. Three modes: " + + "PR inline review comments. Four modes, all derived from the parameter set: " + "(1) hybrid semantic search — dense BGE-M3 + sparse BM25 over D1 FTS5 fused via RRF, then re-scored " + "by @cf/baai/bge-reranker-base (toggle with rerank: false); " + "(2) time-ordered activity scan — omit or empty query with sort=\"updated_desc\" / \"created_desc\", " + "optionally narrow via since / until; " + - "(3) doc content fetch — include_content: true inlines raw content on top doc and wiki_doc results. " + - "Structured filters (repo, state, labels, milestone, assignee, type) apply across all modes; " + + "(3) doc content fetch — include_content: true inlines raw content on top doc and wiki_doc results; " + + "(4) stored-content fetch — vector_ids reads back the body text the index holds for the named rows, " + + "for every type and with no GitHub API call, truncated at the 8000-character ingest ceiling. " + + "Structured filters (repo, state, labels, milestone, assignee, type) apply across modes 1-3 " + + "(mode 4 names its rows, so nothing is filtered there); " + "type: \"wiki_doc\" narrows to GitHub Wiki pages only; repo takes the full slug (owner/repo) and matches " + "exactly, so a bare repository name selects nothing. In search mode the response carries " + "filters_unmatched: any filter listed there matched no row in the index at all, which separates a " + @@ -31,6 +34,9 @@ export const TOOLS = [ "Results are aggregated per underlying entity: a file's doc row and its commit diffs are one result, " + "an issue or PR and its comments / reviews are one result. top_k therefore counts distinct entities, " + "and a result that absorbed others carries same_entity { count, others[] } with links to them. " + + "Every result row — and every same_entity.others entry — carries vector_id, the handle mode 4 takes. " + + "It is a handle for reaching a row you just found, not a durable identifier: the id scheme has been " + + "migrated before and may be again, so do not store one for later use. " + "Two retrieval axes are reported separately, never fused into one ranking. results is the keyword axis " + "(dense + sparse, scored and ranked; count counts these). graph_results is the relationship axis, " + "present only with graph_expand: true — candidates reached through the Decision-Structure mention graph, " + @@ -145,6 +151,25 @@ export const TOOLS = [ "the first few rows of each surface to bound API fan-out. Other rows are unaffected. " + "Default: false.", }, + vector_ids: { + type: "array", + items: { type: "string" }, + maxItems: 50, + description: + "Stored-content fetch. Pass the vector_id values carried by earlier search-mode results " + + "(scan-mode rows come from the structured store and carry none) to read back the " + + "body text the index holds for those exact rows, for every type — issue, pull_request, " + + "issue_comment, pr_review, pr_review_comment, release, diff, doc, wiki_doc. " + + "Served from D1: no GitHub API call is made. Takes precedence over the other modes — query, sort, " + + "and every metadata filter are ignored when this is present, because the rows are named rather " + + "than selected. " + + "The text is the INDEXED copy of the body (the embedding input), truncated at 8000 characters — " + + "not the live source. Each row carries content_truncated so a prefix is never mistaken for a whole " + + "body, and the response carries content_source: \"index\". Unknown or stale ids are listed in " + + "not_found and the remaining rows still return. Max 50 ids per call. " + + "Treat vector_id as a handle for a row you just found, not a durable identifier: the id scheme has " + + "been migrated before and may be again, so do not store one for later use.", + }, graph_expand: { type: "boolean", description: diff --git a/mcp-server/test/search-tool-schema.test.js b/mcp-server/test/search-tool-schema.test.js index c4f1a3d..becbff9 100644 --- a/mcp-server/test/search-tool-schema.test.js +++ b/mcp-server/test/search-tool-schema.test.js @@ -52,6 +52,25 @@ test("tool and type descriptions document the wiki surface", () => { // exact-match requirement on `repo` has to be stated here — a bare repository // name silently selects nothing, and the caller has no way to see that from the // zero-result response alone. +// gh#239: fetch mode returns the index's copy of a body — the embedding input, +// truncated at the ingest ceiling. A caller reading this schema is the one that +// decides whether to trust the text as whole, so the provenance, the ceiling, +// the per-row flag and the partial-success field all have to be stated here. +test("vector_ids description states provenance, ceiling, and partial success", () => { + const param = search?.inputSchema?.properties?.vector_ids; + assert.ok(param, "vector_ids param is present in the mirrored schema"); + assert.equal(param.type, "array"); + assert.equal(param.items?.type, "string"); + assert.match(param.description, /INDEXED copy/); + assert.match(param.description, /8000/); + assert.match(param.description, /content_truncated/); + assert.match(param.description, /not_found/); + assert.match(param.description, /no GitHub API call/i); + // The id is a handle for the result set it arrived in, not a citation: this + // repository has migrated its vector id scheme once already. + assert.match(param.description, /not a durable identifier/); +}); + test("repo description states the full-slug exact match and the unmatched-filter signal", () => { const repoParam = search?.inputSchema?.properties?.repo; assert.ok(repoParam, "repo param is present in the mirrored schema"); diff --git a/src/fetch.test.ts b/src/fetch.test.ts new file mode 100644 index 0000000..3e5ab21 --- /dev/null +++ b/src/fetch.test.ts @@ -0,0 +1,147 @@ +/** + * Unit tests for the fetch-mode item assembly (issue #239). + * + * `buildFetchItem` is the half that decides what a caller can tell about the + * text it gets back: which columns are meaningful for the row's type, and + * whether the content is a whole body or a prefix. The D1 read around it is + * exercised against a real database in `fetch.workers.test.ts`. + */ + +import { describe, it, expect } from "vitest"; +import { buildFetchItem, FETCH_CONTENT_MAX_CHARS } from "./fetch.js"; +import { MAX_EMBEDDING_INPUT_CHARS } from "./pipeline/embedding.js"; + +/** A `search_docs` row as `getDocsByVectorIds` returns it. */ +function row(overrides: Record): Record { + return { + repo: "o/r", + type: "issue", + state: "open", + number: 7, + updated_at: "2026-01-01T00:00:00Z", + content: "title\n\nbody", + tag_name: "", + doc_path: "", + commit_sha: "", + file_path: "", + file_status: "", + commit_date: "", + commit_author: "", + ...overrides, + }; +} + +describe("buildFetchItem: content provenance", () => { + it("reports the ingest ceiling so the caller can act on it", () => { + // The ceiling is the pipeline's, not a constant this module invented: a + // change to the embedding input limit must move both together or the flag + // starts lying about a limit that no longer applies. + expect(FETCH_CONTENT_MAX_CHARS).toBe(MAX_EMBEDDING_INPUT_CHARS); + }); + + it("marks a body short of the ceiling as whole", () => { + const item = buildFetchItem("i:1", row({ content: "x".repeat(100) })); + expect(item.content_chars).toBe(100); + expect(item.content_truncated).toBe(false); + }); + + it("marks a body at the ceiling as truncated", () => { + const item = buildFetchItem( + "i:2", + row({ content: "x".repeat(MAX_EMBEDDING_INPUT_CHARS) }), + ); + expect(item.content_chars).toBe(MAX_EMBEDDING_INPUT_CHARS); + // Safer side: a natural body of exactly this length reads as truncated, so + // the caller re-reads something whole rather than trusting a prefix. + expect(item.content_truncated).toBe(true); + }); + + it("counts characters, not bytes, so multi-byte bodies are not over-reported", () => { + const item = buildFetchItem("i:3", row({ content: "判断記録" })); + expect(item.content).toBe("判断記録"); + expect(item.content_chars).toBe(4); + expect(item.content_truncated).toBe(false); + }); + + it("returns an empty body rather than throwing when the column is absent", () => { + const item = buildFetchItem("i:4", { type: "issue" }); + expect(item.content).toBe(""); + expect(item.content_chars).toBe(0); + expect(item.content_truncated).toBe(false); + expect(item.repo).toBe(""); + }); +}); + +describe("buildFetchItem: per-type identity fields", () => { + it("carries the vector_id it was asked for", () => { + expect(buildFetchItem("i:5", row({})).vector_id).toBe("i:5"); + }); + + it("attaches nothing type-specific to an issue row", () => { + const item = buildFetchItem("i:6", row({ type: "issue" })); + expect(item.number).toBe(7); + expect(item.doc_path).toBeUndefined(); + expect(item.commit_sha).toBeUndefined(); + expect(item.tag_name).toBeUndefined(); + }); + + it("attaches tag_name to a release row", () => { + const item = buildFetchItem("r:1", row({ type: "release", tag_name: "v1.2.3" })); + expect(item.tag_name).toBe("v1.2.3"); + expect(item.doc_path).toBeUndefined(); + }); + + it("attaches doc_path to a doc row", () => { + const item = buildFetchItem("d:1", row({ type: "doc", doc_path: "docs/0-requirements.md" })); + expect(item.doc_path).toBe("docs/0-requirements.md"); + expect(item.wiki_path).toBeUndefined(); + }); + + it("reads a wiki_doc slug out of the shared doc_path column", () => { + // wiki_doc reuses `doc_path` for the page slug at the schema level; the + // row's `type` is what separates the two surfaces. + const item = buildFetchItem("w:1", row({ type: "wiki_doc", doc_path: "Decision-Structure" })); + expect(item.wiki_path).toBe("Decision-Structure"); + expect(item.doc_path).toBeUndefined(); + }); + + it("attaches the commit locator to a diff row", () => { + const item = buildFetchItem( + "f:1", + row({ + type: "diff", + commit_sha: "6acdbd3", + file_path: "src/mcp.ts", + file_status: "modified", + commit_date: "2026-02-02T00:00:00Z", + commit_author: "smileygames", + }), + ); + expect(item.commit_sha).toBe("6acdbd3"); + expect(item.file_path).toBe("src/mcp.ts"); + expect(item.file_status).toBe("modified"); + expect(item.commit_date).toBe("2026-02-02T00:00:00Z"); + expect(item.commit_author).toBe("smileygames"); + }); + + it("attaches the diff locator an inline review comment was left on", () => { + const item = buildFetchItem( + "rc:1", + row({ type: "pr_review_comment", file_path: "src/fetch.ts", commit_sha: "abc1234" }), + ); + expect(item.file_path).toBe("src/fetch.ts"); + expect(item.commit_sha).toBe("abc1234"); + // No file_status / commit_date: the inline-comment ingest writes neither, + // and an empty string would read as an observed value. + expect(item.file_status).toBeUndefined(); + expect(item.commit_date).toBeUndefined(); + }); + + it("keeps a comment row to its parent number, with no diff fields", () => { + const item = buildFetchItem("c:1", row({ type: "issue_comment", number: 239 })); + expect(item.type).toBe("issue_comment"); + expect(item.number).toBe(239); + expect(item.file_path).toBeUndefined(); + expect(item.commit_sha).toBeUndefined(); + }); +}); diff --git a/src/fetch.ts b/src/fetch.ts new file mode 100644 index 0000000..cb80e8b --- /dev/null +++ b/src/fetch.ts @@ -0,0 +1,189 @@ +/** + * Fetch mode — the `vector_ids` branch of the `search` tool. + * + * Hands back the body text the index already holds for rows the caller names. + * The search path reads that text on every query — the reranker is fed from it, + * and dense-only candidates have it backfilled from D1 (`getDocsByVectorIds`) — + * and then drops it on the way out for every type except `doc` / `wiki_doc`. So + * locating something with `search` and then reading it took a second round trip + * through `gh` or grep. This branch closes that trip without touching GitHub. + * + * What comes back is NOT the source body. It is the index's copy: the embedding + * input, truncated at `MAX_EMBEDDING_INPUT_CHARS` by the ingest pipeline. Once + * inlined the two are indistinguishable, so the shape states which it is — + * `content_source: "index"` and `content_max_chars` on the response, + * `content_truncated` per item. + * + * A separate axis from `include_content`, not a widening of it. That flag reads + * the GitHub contents API because a doc needs the *whole* file, and its cap + * (`INCLUDE_CONTENT_MAX_DOCS`) exists to bound API fan-out. This one reads D1 + * and is bounded by the ids the caller listed. Same word ("content"), different + * source and different bound — one flag over both would be one guarantee over + * two behaviors. + * + * No `url` field: identity columns are echoed so a caller can correlate rows, + * but the URL was already carried by the search result the `vector_id` came + * from, and rebuilding it here would be a second copy of `buildResultUrl` to + * drift against. + * + * Lives outside `mcp.ts` — like `scan.ts` — so the assembly can be exercised + * against a real D1 in the workers pool without standing up the MCP server. + */ + +import { getDocsByVectorIds } from "./graph.js"; +import { MAX_EMBEDDING_INPUT_CHARS } from "./pipeline/embedding.js"; + +/** + * Upper bound on ids accepted per call. Mirrors the `top_k` ceiling (and + * `RERANK_MAX_CANDIDATES`) so one page of search results can be fetched in one + * call, and keeps the `vector_id IN (...)` placeholder list bounded. + * + * Not a content-size cap: the caller names its rows, so nothing here has to + * guess how many bytes are safe to return. + */ +export const FETCH_MAX_VECTOR_IDS = 50; + +/** Where the returned text came from. Constant, and stated in every response. */ +export const FETCH_CONTENT_SOURCE = "index" as const; + +/** Character ceiling the ingest pipeline applied to the stored text. */ +export const FETCH_CONTENT_MAX_CHARS = MAX_EMBEDDING_INPUT_CHARS; + +/** One fetched row: identity columns plus the stored text and its provenance. */ +export interface FetchItem { + vector_id: string; + repo: string; + type: string; + state: string; + number: number; + updated_at: string; + content: string; + content_chars: number; + /** + * True when the stored text sits at the ingest ceiling, i.e. the tail of the + * source body is not in the index and this content is a prefix of it. + * + * Read off the length rather than recorded at ingest, so a body whose natural + * length is exactly the ceiling reports as truncated. That is the safer error: + * the caller re-reads a complete body it did not have to, instead of treating + * a prefix as whole. + */ + content_truncated: boolean; + tag_name?: string; + doc_path?: string; + wiki_path?: string; + commit_sha?: string; + file_path?: string; + file_status?: string; + commit_date?: string; + commit_author?: string; +} + +/** Fetch-mode response payload, serialized as-is by the tool handler. */ +export interface FetchResponse { + count: number; + mode: "fetch"; + /** Distinct ids actually looked up (after de-duplication). */ + requested: number; + content_source: typeof FETCH_CONTENT_SOURCE; + content_max_chars: number; + /** + * Ids that matched no row. Partial success is the contract: one stale id + * never empties the response, because `vector_id` is explicitly a handle + * rather than a durable identifier and a caller replaying an old one must + * still get the rows that are live. + */ + not_found: string[]; + results: FetchItem[]; +} + +/** + * Assemble one item from a `search_docs` row. + * + * Type-conditional fields follow the same rule the search path uses: only the + * columns that carry meaning for that type are attached, so a consumer cannot + * read an empty-string default as a real value. `wiki_doc` reuses the + * `doc_path` column for its page slug (schema-level unification, see + * `resolveRow` in `mcp.ts`), which is why it maps to `wiki_path` here. + */ +export function buildFetchItem( + vectorId: string, + row: Record, +): FetchItem { + const type = String(row.type ?? ""); + const content = String(row.content ?? ""); + const path = String(row.doc_path ?? ""); + const item: FetchItem = { + vector_id: vectorId, + repo: String(row.repo ?? ""), + type, + state: String(row.state ?? ""), + number: Number(row.number ?? 0), + updated_at: String(row.updated_at ?? ""), + content, + content_chars: content.length, + content_truncated: content.length >= MAX_EMBEDDING_INPUT_CHARS, + }; + if (type === "release") item.tag_name = String(row.tag_name ?? ""); + if (type === "doc") item.doc_path = path; + if (type === "wiki_doc") item.wiki_path = path; + if (type === "diff") { + item.commit_sha = String(row.commit_sha ?? ""); + item.file_path = String(row.file_path ?? ""); + item.file_status = String(row.file_status ?? ""); + item.commit_date = String(row.commit_date ?? ""); + item.commit_author = String(row.commit_author ?? ""); + } + if (type === "pr_review_comment") { + // The inline-comment ingest is the other writer of these two columns: they + // locate the comment in the diff it was left on. + item.file_path = String(row.file_path ?? ""); + item.commit_sha = String(row.commit_sha ?? ""); + } + return item; +} + +/** + * Read the stored content of the named rows in ONE batched D1 query. + * + * Ids are de-duplicated and the response preserves the caller's order, so a + * result can be lined up against the list that produced it. Blank ids are + * dropped before the query rather than reported as missing — they name nothing. + * + * A D1 failure is NOT swallowed into an all-missing response: "these rows do + * not exist" is a claim this function must not make on a read it never + * completed. The caller turns a throw into an explicit error. + */ +export async function fetchStoredContent( + db: D1Database, + vectorIds: string[], +): Promise { + const ids = [...new Set(vectorIds.map((v) => v.trim()))].filter( + (v) => v.length > 0, + ); + const rows = + ids.length > 0 + ? await getDocsByVectorIds(db, ids) + : new Map>(); + + const results: FetchItem[] = []; + const notFound: string[] = []; + for (const id of ids) { + const row = rows.get(id); + if (!row) { + notFound.push(id); + continue; + } + results.push(buildFetchItem(id, row)); + } + + return { + count: results.length, + mode: "fetch", + requested: ids.length, + content_source: FETCH_CONTENT_SOURCE, + content_max_chars: FETCH_CONTENT_MAX_CHARS, + not_found: notFound, + results, + }; +} diff --git a/src/fetch.workers.test.ts b/src/fetch.workers.test.ts new file mode 100644 index 0000000..7245fbd --- /dev/null +++ b/src/fetch.workers.test.ts @@ -0,0 +1,261 @@ +/** + * D1-backed tests for fetch mode — the `vector_ids` branch of `search` (#239). + * + * The point of the mode is that every indexed type can hand its body back, not + * just the two that could before, so the coverage here is per type. Rows go in + * through the real ingest writer (`upsertFtsRow`) and come out through the real + * read (`fetchStoredContent`), which is what makes this a check on the schema + * agreeing with the assembly rather than on a hand-built row. + * + * Shared-DB caveat (same as graph.workers.test.ts): isolatedStorage is false, + * so every vector_id here is globally unique and no test cleans up. + */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { env, applyD1Migrations } from "cloudflare:test"; +import { upsertFtsRow, type FtsUpsertRow } from "./fts.js"; +import { fetchStoredContent } from "./fetch.js"; +import { MAX_EMBEDDING_INPUT_CHARS } from "./pipeline/embedding.js"; + +const REPO = "t/fetch-mode"; + +beforeAll(async () => { + await applyD1Migrations(env.DB_FTS, env.TEST_MIGRATIONS); +}); + +function mkRow( + overrides: Partial & + Pick, +): FtsUpsertRow { + return { + repo: REPO, + state: "open", + labels: "", + milestone: "", + assignees: "", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, + }; +} + +/** The seven types the issue names, plus the two that already had a body path. */ +const SURFACES: Array<{ id: string; row: FtsUpsertRow }> = [ + { + id: "fetch:issue", + row: mkRow({ + vectorId: "fetch:issue", + type: "issue", + number: 239, + content: "feat(mcp): add vector_ids\n\n索引が既に保持している本文を取り出せるようにする", + }), + }, + { + id: "fetch:pr", + row: mkRow({ + vectorId: "fetch:pr", + type: "pull_request", + number: 240, + content: "pr title\n\npr body text", + }), + }, + { + id: "fetch:comment", + row: mkRow({ + vectorId: "fetch:comment", + type: "issue_comment", + state: "active", + number: 239, + content: "smileygames\n\ncomment body text", + }), + }, + { + id: "fetch:review", + row: mkRow({ + vectorId: "fetch:review", + type: "pr_review", + state: "APPROVED", + number: 240, + content: "smileygames\n\nreview body text", + }), + }, + { + id: "fetch:review-comment", + row: mkRow({ + vectorId: "fetch:review-comment", + type: "pr_review_comment", + state: "active", + number: 240, + filePath: "src/fetch.ts", + commitSha: "abc1234", + content: "smileygames\n\ninline comment body", + }), + }, + { + id: "fetch:release", + row: mkRow({ + vectorId: "fetch:release", + type: "release", + state: "published", + tagName: "v0.9.0", + content: "v0.9.0\n\nrelease notes body", + }), + }, + { + id: "fetch:diff", + row: mkRow({ + vectorId: "fetch:diff", + type: "diff", + state: "active", + commitSha: "6acdbd3", + filePath: "src/mcp.ts", + fileStatus: "modified", + commitDate: "2026-02-02T00:00:00Z", + commitAuthor: "smileygames", + content: "commit message\n\nsrc/mcp.ts\n\n@@ -1 +1 @@\n-old\n+new", + }), + }, + { + id: "fetch:doc", + row: mkRow({ + vectorId: "fetch:doc", + type: "doc", + state: "active", + docPath: "docs/0-requirements.md", + content: "requirements body", + }), + }, + { + id: "fetch:wiki", + row: mkRow({ + vectorId: "fetch:wiki", + type: "wiki_doc", + state: "active", + docPath: "Decision-Structure", + content: "wiki page body", + }), + }, +]; + +beforeAll(async () => { + for (const s of SURFACES) await upsertFtsRow(env.DB_FTS, s.row); +}); + +describe("fetchStoredContent: every indexed type returns its body", () => { + it("returns content for all nine surfaces in one call", async () => { + const ids = SURFACES.map((s) => s.id); + const res = await fetchStoredContent(env.DB_FTS, ids); + + expect(res.mode).toBe("fetch"); + expect(res.count).toBe(ids.length); + expect(res.not_found).toEqual([]); + // Order follows the request, so a caller can line results up with its list. + expect(res.results.map((r) => r.vector_id)).toEqual(ids); + for (const item of res.results) { + expect(item.content.length).toBeGreaterThan(0); + } + }); + + it("returns the non-doc bodies that had no retrieval path before #239", async () => { + const res = await fetchStoredContent(env.DB_FTS, [ + "fetch:issue", + "fetch:comment", + "fetch:diff", + ]); + const byId = new Map(res.results.map((r) => [r.vector_id, r])); + expect(byId.get("fetch:issue")?.content).toContain("索引が既に保持している本文"); + expect(byId.get("fetch:comment")?.content).toContain("comment body text"); + expect(byId.get("fetch:diff")?.content).toContain("@@ -1 +1 @@"); + }); + + it("carries the per-type identity columns the schema holds", async () => { + const res = await fetchStoredContent(env.DB_FTS, [ + "fetch:diff", + "fetch:release", + "fetch:wiki", + "fetch:review-comment", + ]); + const byId = new Map(res.results.map((r) => [r.vector_id, r])); + + expect(byId.get("fetch:diff")?.commit_sha).toBe("6acdbd3"); + expect(byId.get("fetch:diff")?.file_path).toBe("src/mcp.ts"); + expect(byId.get("fetch:release")?.tag_name).toBe("v0.9.0"); + expect(byId.get("fetch:wiki")?.wiki_path).toBe("Decision-Structure"); + expect(byId.get("fetch:review-comment")?.file_path).toBe("src/fetch.ts"); + }); + + it("states the source and the ceiling on every response", async () => { + const res = await fetchStoredContent(env.DB_FTS, ["fetch:issue"]); + expect(res.content_source).toBe("index"); + expect(res.content_max_chars).toBe(MAX_EMBEDDING_INPUT_CHARS); + expect(res.results[0].content_truncated).toBe(false); + }); +}); + +describe("fetchStoredContent: truncation is visible on the row", () => { + it("flags a row stored at the ingest ceiling", async () => { + // What a long diff patch looks like after the pipeline truncates it: the + // tail is not in the index, and nothing about the text itself says so. + await upsertFtsRow( + env.DB_FTS, + mkRow({ + vectorId: "fetch:truncated", + type: "diff", + state: "active", + content: "d".repeat(MAX_EMBEDDING_INPUT_CHARS), + }), + ); + + const res = await fetchStoredContent(env.DB_FTS, ["fetch:truncated", "fetch:doc"]); + const byId = new Map(res.results.map((r) => [r.vector_id, r])); + expect(byId.get("fetch:truncated")?.content_truncated).toBe(true); + expect(byId.get("fetch:truncated")?.content_chars).toBe(MAX_EMBEDDING_INPUT_CHARS); + // Same call, so the flag is per row and not a property of the response. + expect(byId.get("fetch:doc")?.content_truncated).toBe(false); + }); +}); + +describe("fetchStoredContent: partial success", () => { + it("returns the live rows and lists the unknown ids", async () => { + const res = await fetchStoredContent(env.DB_FTS, [ + "fetch:issue", + "fetch:no-such-id", + "fetch:doc", + ]); + expect(res.results.map((r) => r.vector_id)).toEqual(["fetch:issue", "fetch:doc"]); + expect(res.not_found).toEqual(["fetch:no-such-id"]); + expect(res.count).toBe(2); + expect(res.requested).toBe(3); + }); + + it("reports every id as missing when none resolve, rather than erroring", async () => { + const res = await fetchStoredContent(env.DB_FTS, ["fetch:gone-a", "fetch:gone-b"]); + expect(res.count).toBe(0); + expect(res.results).toEqual([]); + expect(res.not_found).toEqual(["fetch:gone-a", "fetch:gone-b"]); + }); + + it("de-duplicates repeated ids instead of returning the row twice", async () => { + const res = await fetchStoredContent(env.DB_FTS, [ + "fetch:issue", + "fetch:issue", + " fetch:issue ", + ]); + expect(res.requested).toBe(1); + expect(res.results.map((r) => r.vector_id)).toEqual(["fetch:issue"]); + }); + + it("drops blank ids rather than reporting them as missing rows", async () => { + const res = await fetchStoredContent(env.DB_FTS, ["", " ", "fetch:issue"]); + expect(res.requested).toBe(1); + expect(res.not_found).toEqual([]); + expect(res.count).toBe(1); + }); + + it("returns an empty response for an empty id list without issuing a query", async () => { + const res = await fetchStoredContent(env.DB_FTS, []); + expect(res.count).toBe(0); + expect(res.requested).toBe(0); + expect(res.results).toEqual([]); + expect(res.not_found).toEqual([]); + }); +}); diff --git a/src/mcp-stateless-contract.test.ts b/src/mcp-stateless-contract.test.ts index 6210154..9c2a59b 100644 --- a/src/mcp-stateless-contract.test.ts +++ b/src/mcp-stateless-contract.test.ts @@ -139,12 +139,43 @@ describe("worker <-> bridge stateless contract", () => { "top_k", "type", "until", + "vector_ids", ].sort(), ); await remote.reset(); }); + // gh#239: fetch mode returns the index's copy of a body, not the live source. + // A caller that cannot tell those apart reads a prefix as a whole document, + // so the served text — the one surface every client sees — has to say which + // it is, and has to say that the id is a handle rather than a citation. + it("publishes fetch mode as index-derived, truncated, and handle-keyed", async () => { + const remote = createRemoteClient({ + workerUrl: ENDPOINT, + clientVersion: "0.0.0-test", + fetch: fetchInto(workerHandler()), + }); + + const client = await remote.getClient(); + const [search] = (await client.listTools()).tools; + const description = search.description as string; + + expect(description).toContain("vector_ids"); + expect(description).toMatch(/no GitHub API call/i); + + const vectorIds = ( + search.inputSchema as { properties?: Record } + ).properties?.["vector_ids"]; + expect(vectorIds?.description).toMatch(/INDEXED copy/); + expect(vectorIds?.description).toContain("8000"); + expect(vectorIds?.description).toContain("content_truncated"); + expect(vectorIds?.description).toContain("not_found"); + expect(vectorIds?.description).toMatch(/not a durable identifier/); + + await remote.reset(); + }); + // gh#234: the two axes are only usable if the caller is told they exist. // The description is the one place that reaches every client, so the axis // split and its triage rule are asserted on the served text. diff --git a/src/mcp.ts b/src/mcp.ts index 416e5fd..1a1e191 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -3,8 +3,22 @@ * * Tools: * search — hybrid search + time-ordered activity scan + inline doc - * content fetch. Single entry point for GitHub issue / PR / - * release / doc / commit-diff retrieval. + * content fetch + stored-content fetch by vector_id. Single + * entry point for GitHub issue / PR / release / doc / + * commit-diff retrieval. + * + * Modes are derived from the parameter set, never selected by a mode enum, and + * no mode gets a tool of its own (issues #104 / #105 consolidated four tools + * into this one; a body-fetch tool would be the removed `get_doc_content` + * returning under a new name). Precedence, top to bottom: + * + * vector_ids present -> fetch mode (stored content for the named rows) + * query empty -> scan mode (time-ordered metadata) + * otherwise -> search mode (hybrid retrieval) + * + * `include_content` is orthogonal to all three: it inlines doc / wiki_doc files + * read from GitHub, which is a different source and a different bound from + * fetch mode's D1 read (see `fetch.ts`). * * Protocol revision 2026-07-28 (stateless core, issue #224). The server is * built fresh per HTTP request by `createMcpHandler` in `index.ts` — there is @@ -39,6 +53,11 @@ import { import { rerankCandidates, rerankWasApplied, RERANK_MAX_CANDIDATES } from "./rerank.js"; import { queryNeighbors, getDocsByVectorIds, type GraphNeighbor } from "./graph.js"; import { runScan, type ScanRow } from "./scan.js"; +import { + fetchStoredContent, + FETCH_CONTENT_MAX_CHARS, + FETCH_MAX_VECTOR_IDS, +} from "./fetch.js"; import { entityKey, groupByEntity } from "./aggregate.js"; const GITHUB_API = "https://api.github.com"; @@ -283,7 +302,7 @@ export function createRagMcpServer(env: Env): McpServer { description: "Unified search across GitHub issues, PRs, releases, repository documentation, GitHub Wiki pages, " + "commit diffs, issue/PR top-level comments, PR reviews, and PR inline review comments. " + - "Three modes via the query / sort axes:\n" + + "Four modes, all derived from the parameter set:\n" + " 1. Hybrid semantic search (default): dense BGE-M3 over Vectorize + sparse BM25 over D1 FTS5, " + "fused via Reciprocal Rank Fusion (RRF, k=60), then re-scored with a cross-encoder " + "(@cf/baai/bge-reranker-base; set rerank: false to skip).\n" + @@ -291,7 +310,13 @@ export function createRagMcpServer(env: Env): McpServer { "optionally narrow via since / until to list recent activity across every type.\n" + " 3. Doc content fetch: pass include_content: true to inline the raw file content of top doc and wiki_doc results " + "(docs via GitHub Contents API, wiki_docs via raw.githubusercontent.com/wiki; capped at the first few rows of each).\n" + - "Optional metadata filters (repo, state, labels, milestone, assignee, type) apply across all modes; " + + " 4. Stored-content fetch: pass vector_ids (the vector_id values carried by earlier results) to read back the " + + "body text the index holds for those exact rows — the way to read an issue / PR / comment / review / release / " + + "diff body without a second round trip to GitHub. Served from D1, so it makes no GitHub API call; what it " + + "returns is the indexed copy of the body, truncated at " + + `${FETCH_CONTENT_MAX_CHARS} characters, not the live source.\n` + + "Optional metadata filters (repo, state, labels, milestone, assignee, type) apply across modes 1-3 " + + "(mode 4 names its rows, so nothing is filtered there); " + "repo takes the full slug (owner/repo) and matches exactly, so a bare repository name selects nothing. " + "In search mode the response carries filters_unmatched: any filter listed there matched no row in the " + "index at all, which separates a mis-specified filter from a genuine zero-hit result. " + @@ -304,6 +329,9 @@ export function createRagMcpServer(env: Env): McpServer { "Results are aggregated per underlying entity: a file's doc row and its commit diffs are one result, " + "an issue or PR and its comments / reviews are one result. top_k therefore counts distinct entities, " + "and a result that absorbed others carries same_entity { count, others[] } with links to them.\n" + + "Every result row — and every same_entity.others entry — carries vector_id, the handle mode 4 takes. " + + "It is a handle for reaching a row you just found, not a durable identifier: the id scheme has been " + + "migrated before and may be again, so do not store one for later use.\n" + "Two retrieval axes are reported separately, never fused into one ranking. results is the keyword axis " + "(dense + sparse, scored and ranked; count counts these). graph_results is the relationship axis, " + "present only with graph_expand: true — candidates reached through the Decision-Structure mention graph, " + @@ -431,6 +459,26 @@ export function createRagMcpServer(env: Env): McpServer { `${INCLUDE_CONTENT_MAX_DOCS} doc rows in the result set to bound API fan-out. ` + "Non-doc rows are unaffected.", ), + vector_ids: z + .array(z.string()) + .max(FETCH_MAX_VECTOR_IDS) + .optional() + .describe( + "Stored-content fetch. Pass the vector_id values carried by earlier search-mode results " + + "(scan-mode rows come from the structured store and carry none) to read back the " + + "body text the index holds for those exact rows, for every type — issue, pull_request, " + + "issue_comment, pr_review, pr_review_comment, release, diff, doc, wiki_doc. " + + "Served from D1: no GitHub API call is made. Takes precedence over the other modes — query, " + + "sort, and every metadata filter are ignored when this is present, because the rows are named " + + "rather than selected. " + + "The text is the INDEXED copy of the body (the embedding input), truncated at " + + `${FETCH_CONTENT_MAX_CHARS} characters — not the live source. Each row carries content_truncated ` + + "so a prefix is never mistaken for a whole body, and the response carries content_source: \"index\". " + + "Unknown or stale ids are listed in not_found and the remaining rows still return. " + + `Max ${FETCH_MAX_VECTOR_IDS} ids per call. ` + + "Treat vector_id as a handle for a row you just found, not a durable identifier: the id scheme " + + "has been migrated before and may be again, so do not store one for later use.", + ), graph_expand: z .boolean() .optional() @@ -469,9 +517,43 @@ export function createRagMcpServer(env: Env): McpServer { since, until, include_content, + vector_ids, graph_expand, graph_hops, }) => { + // ── Fetch mode (vector_ids): stored content for named rows ─── + // First branch on purpose: a fetch call carries no query, so leaving it + // below the empty-query test would route it into scan mode. The rows are + // named, not selected, which is why no filter applies here. + const fetchIds = vector_ids ?? []; + if (fetchIds.length > 0) { + try { + const payload = await fetchStoredContent(env.DB_FTS, fetchIds); + return { + content: [ + { type: "text" as const, text: JSON.stringify(payload, null, 2) }, + ], + }; + } catch (err) { + // Reported as an error rather than as an all-missing response: an + // empty `results` with every id in `not_found` would assert the rows + // do not exist, which a failed read gives no ground to claim. + console.error( + "search: stored-content fetch failed:", + err instanceof Error ? err.message : String(err), + ); + return { + content: [ + { + type: "text" as const, + text: "Failed to read stored content from the index", + }, + ], + isError: true, + }; + } + } + const requestedTopK = top_k ?? 10; const fusionMode = fusion ?? "rrf"; const rerankEnabled = rerank ?? true; @@ -913,6 +995,14 @@ export function createRagMcpServer(env: Env): McpServer { // ── Format results ─────────────────────────────────────── type ResultItem = { + /** + * The row's index key, and the handle `vector_ids` takes to read this + * row's stored body back (fetch mode). Deliberately NOT promised as a + * durable identifier: this repository has already migrated its id + * scheme once (`pipeline/legacy-vector-id.ts`), so the field is a + * handle for the result set it arrived in, not a citation to store. + */ + vector_id: string; number: number; title: string; state: string; @@ -952,6 +1042,13 @@ export function createRagMcpServer(env: Env): McpServer { same_entity?: { count: number; others: Array<{ + /** + * Carried for the same reason as on the representative: aggregation + * is what puts a comment / review / older diff *here* instead of in + * `results`, so without it the rows whose bodies fetch mode exists + * to return would be the only ones it could not reach. + */ + vector_id: string; type: string; url: string; updated_at: string; @@ -975,6 +1072,7 @@ export function createRagMcpServer(env: Env): McpServer { others: folded.map((o) => { const or = resolveRow(payload.get(o.vectorId)); return { + vector_id: o.vectorId, type: or.type, url: buildResultUrl(or), updated_at: or.updatedAt, @@ -986,6 +1084,7 @@ export function createRagMcpServer(env: Env): McpServer { : undefined; return { + vector_id: f.vectorId, number: r.number, title: "", // Enriched below state: r.state,