diff --git a/docs/site/.vitepress/config.mts b/docs/site/.vitepress/config.mts index 27fe86c..eaebef0 100644 --- a/docs/site/.vitepress/config.mts +++ b/docs/site/.vitepress/config.mts @@ -88,11 +88,19 @@ export default defineConfig({ ], '/ecosystem/': [ { - text: 'Ecosystem', + text: 'Systems', items: [ { text: 'RAG Systems', link: '/ecosystem/rag-systems' }, { text: 'LLM Wiki Systems', link: '/ecosystem/llm-wiki-systems' }, ] + }, + { + text: 'Technologies', + items: [ + { text: 'RAG Technologies', link: '/ecosystem/rag-technologies' }, + { text: 'LLM Wiki Technologies', link: '/ecosystem/llm-wiki-technologies' }, + { text: 'Paradigm Comparison', link: '/ecosystem/paradigm-comparison' }, + ] } ], } @@ -159,11 +167,19 @@ export default defineConfig({ ], '/zh-CN/ecosystem/': [ { - text: '生态', + text: '系统列表', items: [ { text: 'RAG 系统', link: '/zh-CN/ecosystem/rag-systems' }, { text: 'LLM Wiki 系统', link: '/zh-CN/ecosystem/llm-wiki-systems' }, ] + }, + { + text: '关键技术', + items: [ + { text: 'RAG 关键技术', link: '/zh-CN/ecosystem/rag-technologies' }, + { text: 'LLM Wiki 关键技术', link: '/zh-CN/ecosystem/llm-wiki-technologies' }, + { text: '范式对比', link: '/zh-CN/ecosystem/paradigm-comparison' }, + ] } ], } diff --git a/docs/site/ecosystem/llm-wiki-technologies.md b/docs/site/ecosystem/llm-wiki-technologies.md new file mode 100644 index 0000000..dc20f42 --- /dev/null +++ b/docs/site/ecosystem/llm-wiki-technologies.md @@ -0,0 +1,77 @@ +# LLM Wiki Technologies + +Core technologies behind the LLM Wiki (Knowledge Compilation) paradigm — knowledge is pre-compiled at write time, not retrieved at query time. + +``` +Raw docs → LLM distills → wiki pages → [query time] → FTS / graph → read compiled page +``` + +> Coined by Andrej Karpathy: use LLM as a compiler that transforms raw materials into a persistent, structured, auditable knowledge layer. + +## 1. Knowledge Compilation + +| Technology | Notes | +|---|---| +| LLM as compiler | LLM reads raw docs and writes structured wiki pages — pre-compiled, not retrieved at query time. | +| Distillation pipeline | Raw doc → LLM extracts key claims, entities, aliases, related links → source-note page (WikiLoop pattern). | +| Tiered compilation | Compile frequently-used docs first (Tier 0–3). Sage Wiki: auto-promote on 3 hits, auto-demote after 90 days inactive. | +| Incremental update | Only re-process changed documents — avoids full rebuild cost. | +| [STORM](https://github.com/stanford-oval/storm) (Stanford OVAL) | Multi-perspective Wiki generation agent: simulates different viewpoints → retrieves → generates hierarchical outline → writes cited article. | +| Co-STORM | Collaborative variant of STORM — builds dynamic knowledge map during research to guide compilation direction. | +| Agentic compilation loop | Agent loop: retrieve → draft → evaluate → re-retrieve → regenerate until knowledge is stable. | + +## 2. Knowledge Representation + +| Technology | Notes | +|---|---| +| Structured Markdown | All knowledge stored as plain Markdown — human-readable, git-diffable, auditable. | +| Source-note pages | One distilled page per raw document. Contains `key_claims`, entity annotations `【entity\|type】`, `related_to` / `supports` / `contradicts` links. | +| Concept pages | Cross-document synthesis of a concept or methodology. | +| Comparison pages | Side-by-side comparison of approaches, tools, or paradigms. | +| Decision pages | Technical decision records (ADR format): context + options + decision + consequences. | +| Ontology graph | Typed entity-relation graph built during compilation. Sage Wiki: 8 built-in relation types (`implements`, `contradicts`, `trades_off`, …). | +| Schema / Templates | Authoring rules and page templates that guide LLM compilation style, customizable per KB. | + +## 3. Indexing & Search + +| Technology | Notes | +|---|---| +| SQLite FTS5 + BM25 | Core search engine — no vector model needed. Sub-millisecond full-text search. Used by WikiLoop, Sage Wiki, TreeSearch. | +| Alias expansion | Key terms indexed with aliases and cross-language equivalents to maximize FTS recall. | +| Graph traversal | `related_to` links enable multi-hop navigation (similar to wiki page links). BFS expansion on search results. | +| Hybrid (FTS + vector + graph) | Sage Wiki: FTS5 (411µs) + vector (81ms) + ontology graph (1µs) merged via RRF. | +| Chapter-tree indexing | Preserves document H1/H2/H3 hierarchy — structure-preserving alternative to flat chunking. | + +## 4. Knowledge Quality & Maintenance + +| Technology | Notes | +|---|---| +| Git version control | All wiki pages in git — full history, diffs, blame. Knowledge changes are auditable. | +| Lint / health checks | Validate frontmatter, broken source links, missing citations. `wikiloop lint`. | +| Conflict detection | `contradicts` links surface disagreements between sources for human review. | +| Draft staging | Pages with < 2 source references quarantined in `_draft/` — not indexed until verified. | +| Knowledge gap analysis | `wikiloop synthesize --gaps` identifies topics with insufficient coverage. | +| Entity deduplication | Identify different expressions of the same concept and merge into a single node. | + +## 5. Agent Interface (MCP) + +| Technology | Notes | +|---|---| +| [MCP protocol](https://modelcontextprotocol.io) | Model Context Protocol — Anthropic open standard for exposing tools/resources to AI agents. Supports stdio + HTTP transports. | +| `kb_search` | FTS keyword search, returns ranked results with `related` links for graph navigation. | +| `kb_page` | Fetch full page content by ID. Supports batch (up to 5 IDs) or `full=true` for untruncated text. | +| MCP Resources | Read-only resources (URI form): wiki pages, graph schema, raw documents. | +| Iterative search pattern | Agent issues multiple queries from different angles, follows `related` links, synthesizes own answer. | +| Sage Wiki MCP tools | 17 tools: 6 read, 9 write, 2 composite — agents can directly write and compile knowledge. | + +## 6. File Conversion (Input Layer) + +| Technology | Notes | +|---|---| +| markitdown (Microsoft) | Converts PDF, Word, Excel, PPT, HTML to Markdown before distillation. | +| `raw/converted/` pattern | Agent-extracted content written directly here — skips conversion, goes straight to distillation. | +| File watcher | Auto-detects new/changed files in `raw/`, triggers convert → distill → index pipeline. | + +--- + +See also: [RAG Technologies](/ecosystem/rag-technologies) · [Paradigm Comparison](/ecosystem/paradigm-comparison) diff --git a/docs/site/ecosystem/paradigm-comparison.md b/docs/site/ecosystem/paradigm-comparison.md new file mode 100644 index 0000000..d0b9baa --- /dev/null +++ b/docs/site/ecosystem/paradigm-comparison.md @@ -0,0 +1,65 @@ +# Paradigm Comparison: RAG vs LLM Wiki + +Two fundamentally different approaches to building AI knowledge infrastructure. + +## Pipeline Overview + +``` +RAG Pipeline +──────────────────────────────────────────────────────── +Raw docs → chunk → embed → vector store + ↘ [query time] + query → ANN search → LLM → answer + + +LLM Wiki (Knowledge Compilation) +──────────────────────────────────────────────────────── +Raw docs → LLM compiles → structured wiki pages → FTS index + ↘ [query time] + query → FTS / graph → read page → answer +``` + +## Dimension-by-Dimension Comparison + +| Dimension | RAG | LLM Wiki | +|---|---|---| +| **Core operation** | Retrieve at query time | Compile at write time | +| **Storage** | Vector DB + raw chunks | Structured Markdown + SQLite FTS | +| **Search** | ANN similarity search | FTS5 BM25 + graph traversal | +| **Knowledge form** | Implicit (vectors) | Explicit (readable Markdown) | +| **Auditability** | Low | High (git diff, lint) | +| **Multi-hop reasoning** | LLM-dependent | Via `related` graph links | +| **Embedding required** | Yes | No (pure FTS) | +| **Knowledge accumulation** | None (static index) | Compounds over time | +| **Token cost at query time** | High (raw chunks passed to LLM) | Low (pre-compiled page) | +| **Latency** | Higher (ANN + LLM) | Lower (FTS sub-millisecond) | +| **Infrastructure** | Vector DB required | SQLite only | +| **Human readability** | No (vectors opaque) | Yes (plain Markdown) | +| **Best for** | Broad doc Q&A, real-time/live data | Long-term knowledge, agent memory, research | + +## When to Use Each + +**Choose RAG when:** +- You need to query a large, dynamic corpus that changes frequently +- Real-time or near-real-time data ingestion is required +- Semantic similarity across heterogeneous documents matters most +- You don't need to maintain or review the knowledge layer + +**Choose LLM Wiki when:** +- Knowledge needs to accumulate and improve over time +- Auditability and human review of knowledge are important +- Agents need to search and reason across a curated knowledge base +- You want to minimize query-time token costs +- You want offline-capable, zero-infrastructure search + +## They Are Not Mutually Exclusive + +Many production systems combine both: +- **RAG** for broad retrieval across large, changing corpora +- **LLM Wiki** as a curated, high-quality knowledge layer for known domains + +Example hybrid: raw ingestion via RAG pipeline → high-confidence results compiled into wiki pages for long-term reuse. + +--- + +See also: [RAG Technologies](/ecosystem/rag-technologies) · [LLM Wiki Technologies](/ecosystem/llm-wiki-technologies) diff --git a/docs/site/ecosystem/rag-technologies.md b/docs/site/ecosystem/rag-technologies.md new file mode 100644 index 0000000..848a8d3 --- /dev/null +++ b/docs/site/ecosystem/rag-technologies.md @@ -0,0 +1,107 @@ +# RAG Technologies + +Core technologies behind the RAG (Retrieval-Augmented Generation) pipeline — from raw document to generated answer. + +``` +Raw docs → chunk → embed → store → [query time] → retrieve → LLM → answer +``` + +## 1. Document Parsing + +| Technology | Representative Tools | Notes | +|---|---|---| +| Multi-format parsing | [Unstructured.io](https://unstructured.io), [Docling](https://github.com/DS4SD/docling) (IBM) | Layout-aware extraction from PDF/Word/HTML | +| PDF parsing | PyMuPDF, PDFPlumber | Low-level text/table/layout extraction | +| LLM-enhanced parsing | LlamaParse | Cloud API for complex PDFs with tables and figures | +| PDF → Markdown | [Marker](https://github.com/VikParuchuri/marker), [MinerU](https://github.com/opendatalab/MinerU) | High-fidelity conversion, preserves structure | +| OCR cloud services | Azure Document Intelligence, AWS Textract | Enterprise-grade form and table recognition | + +## 2. Chunking Strategies + +| Strategy | Tools | Notes | +|---|---|---| +| Fixed-length chunking | LangChain `CharacterTextSplitter` | Simple, breaks semantic boundaries | +| Recursive splitting | LangChain `RecursiveCharacterTextSplitter` | Respects paragraph → sentence → word hierarchy | +| Semantic chunking | LlamaIndex `SemanticSplitter` | Detects topic boundaries via embedding similarity | +| Chapter-tree indexing | [he-wiki-rag](https://github.com/liuhe37186/he-wiki-rag) | Preserves H1/H2/H3 hierarchy + breadcrumb path | +| Parent-child chunking | LlamaIndex | Retrieve small chunks, return parent as context | +| Proposition chunking | Custom + LLM | Split into atomic facts — highest precision, expensive | +| Late Chunking | Jina AI | Embed full document first, then split embeddings | +| RAPTOR | LlamaIndex | Recursive tree summaries for hierarchical retrieval | +| Document-tree reasoning | [PageIndex](https://github.com/vectifyai/vectify) | LLM traverses summary tree — no vectors needed, 98.7% on FinanceBench | + +## 3. Embedding Models + +| Model | Provider | Notes | +|---|---|---| +| text-embedding-3-large/small | OpenAI | General-purpose, 3072-dim large variant | +| Embed v3 | Cohere | Optimized for retrieval, 1024-dim | +| BGE-M3 | BAAI (open source) | Multilingual, multi-granularity, strong Chinese | +| E5-Mistral-7B | Microsoft (open source) | High-dim, top MTEB ranking | +| NV-Embed-v2 | NVIDIA (open source) | MTEB leader among open models | +| nomic-embed-text | Nomic (open source) | Fully open, local-friendly | +| jina-embeddings-v3 | Jina AI (open source) | Supports Late Chunking | + +## 4. Vector Databases + +| Database | Type | Notes | +|---|---|---| +| [Qdrant](https://github.com/qdrant/qdrant) | Open source | Rust, high-performance, strong filtering. 100k–10M scale. | +| [Milvus](https://github.com/milvus-io/milvus) | Open source | Billion-scale distributed. Commercial: Zilliz Cloud. | +| [Weaviate](https://github.com/weaviate/weaviate) | Open source | Native hybrid search (vector + BM25) | +| [Chroma](https://github.com/chroma-core/chroma) | Open source | Lightweight, embedded, best for prototyping | +| [pgvector](https://github.com/pgvector/pgvector) | Extension | Vector search inside PostgreSQL | +| [LanceDB](https://github.com/lancedb/lancedb) | Open source | Arrow format, embedded, serverless-friendly | +| Pinecone | Managed SaaS | Serverless, zero ops | + +## 5. Retrieval Strategies + +| Strategy | Tools | Notes | +|---|---|---| +| Dense retrieval | All vector DBs (ANN) | Cosine/dot-product semantic similarity | +| Sparse retrieval (BM25) | Elasticsearch, OpenSearch, Tantivy | Term-frequency keyword matching | +| Hybrid retrieval | Weaviate, Qdrant, RRF algorithm | Dense + sparse, merged via Reciprocal Rank Fusion | +| Graph-augmented retrieval | [GraphRAG](https://github.com/microsoft/graphrag), [LightRAG](https://github.com/HKUDS/LightRAG) | Entity/relation graph for multi-hop reasoning | +| Vector Graph RAG | Community | Triples vectorized instead of graph DB — 96.3% on HotpotQA | +| Agentic RAG (A-RAG) | Custom + LLM | Agent autonomously chooses `keyword_search` / `semantic_search` / `chunk_read` tools | + +## 6. Query Optimization + +| Technique | Tools | Notes | +|---|---|---| +| HyDE | LangChain, LlamaIndex | Generate hypothetical answer, use its embedding to retrieve | +| Multi-query / Query rewriting | LangChain `MultiQueryRetriever` | LLM rewrites to multiple sub-questions | +| Step-Back Prompting | LangChain | Abstract specific question to general before retrieval | +| Self-RAG | Research implementation | LLM self-evaluates retrieval quality, decides whether to re-retrieve | +| CRAG | Research implementation | Falls back to web search for low-confidence retrievals | + +## 7. Reranking + +| Tool | Notes | +|---|---| +| Cohere Rerank | Cross-encoder commercial reranker | +| BGE Reranker (BAAI) | Open-source cross-encoder, strong multilingual | +| FlashRank | Lightweight open-source reranker for local deployment | + +## 8. Orchestration Frameworks + +| Framework | Notes | +|---|---| +| [LangChain](https://github.com/langchain-ai/langchain) | Most popular modular RAG pipeline framework | +| [LlamaIndex](https://github.com/run-llama/llama_index) | Data-centric RAG, richest chunking/retrieval strategies | +| [Haystack](https://github.com/deepset-ai/haystack) | Enterprise-grade NLP/RAG pipeline | +| [DSPy](https://github.com/stanfordnlp/dspy) | Programmatic LLM optimization, replaces hand-written prompts | + +## 9. Evaluation + +| Tool | Notes | +|---|---| +| [RAGAS](https://ragas.io) | Most popular RAG eval — faithfulness, context recall, answer relevancy | +| [DeepEval](https://github.com/confident-ai/deepeval) | Unit-test style RAG evaluation | +| [TruLens](https://www.trulens.org) | Production monitoring via RAG Triad | +| LangSmith | LangChain's tracing and eval platform | +| Key metrics | Faithfulness, Context Recall, Answer Relevancy, MRR, NDCG | + +--- + +See also: [LLM Wiki Technologies](/ecosystem/llm-wiki-technologies) · [Paradigm Comparison](/ecosystem/paradigm-comparison) diff --git a/docs/site/zh-CN/ecosystem/llm-wiki-technologies.md b/docs/site/zh-CN/ecosystem/llm-wiki-technologies.md new file mode 100644 index 0000000..3ba4a65 --- /dev/null +++ b/docs/site/zh-CN/ecosystem/llm-wiki-technologies.md @@ -0,0 +1,77 @@ +# LLM Wiki 关键技术 + +LLM Wiki(知识编译范式)背后的核心技术——知识在写入时预编译,查询时直接读取。 + +``` +原始文档 → LLM 提炼 → Wiki 页面 → [查询时] → FTS / 图遍历 → 读取编译页面 → 答案 +``` + +> 由 Andrej Karpathy 提出:将 LLM 用作编译器,把原始材料转化为持久化、结构化、可审计的知识层。 + +## 1. 知识提炼 / 编译 + +| 技术 | 说明 | +|---|---| +| LLM 作为编译器 | LLM 读取原始文档,写入结构化 Wiki 页面——查询时直接读取编译产物。 | +| 提炼管道 | 原始文档 → LLM 提取关键主张、实体、别名、关联链接 → source-note 页面(WikiLoop 模式)。 | +| 分层编译 | 优先编译高频文档(Tier 0–3)。Sage Wiki:命中 3 次自动升级,90 天不活跃自动降级。 | +| 增量更新 | 仅处理变更文档,避免全量重建成本。 | +| [STORM](https://github.com/stanford-oval/storm)(Stanford OVAL) | 多视角 Wiki 生成 Agent:模拟不同视角提问 → 检索 → 生成层级大纲 → 撰写带引用的完整词条。 | +| Co-STORM | STORM 的协作变体,研究过程中构建动态知识图谱指导编译方向。 | +| Agentic 编译循环 | Agent 循环:检索 → 草稿 → 评估 → 补充检索 → 再生成,直到知识稳定。 | + +## 2. 知识表示 + +| 技术 | 说明 | +|---|---| +| 结构化 Markdown | 所有知识以纯文本 Markdown 存储——人类可读,可 git diff,可审计。 | +| Source-note 页面 | 每个原始文档对应一个提炼笔记,包含 `key_claims`、实体标注 `【实体\|类型】`、`related_to` / `supports` / `contradicts` 链接。 | +| Concept 页面 | 对某一概念或方法论的跨文档综合描述。 | +| Comparison 页面 | 方案、工具或范式的横向对比。 | +| Decision 页面 | 技术决策记录(ADR 格式):背景 + 选项 + 决策 + 影响。 | +| 本体图谱(Ontology Graph) | 编译过程中构建的类型化实体-关系图谱。Sage Wiki 内置 8 种关系类型(`implements`、`contradicts`、`trades_off` 等)。 | +| Schema / 模板 | 指导 LLM 编译风格的写作规则和页面模板,每个 KB 可自定义。 | + +## 3. 索引与搜索 + +| 技术 | 说明 | +|---|---| +| SQLite FTS5 + BM25 | 核心搜索引擎——无需向量模型,亚毫秒级全文检索。WikiLoop、Sage Wiki、TreeSearch 均采用。 | +| 别名扩展 | 关键术语索引时内嵌别名和跨语言等价词,最大化 FTS 召回率。 | +| 图遍历 | `related_to` 链接实现多跳导航(类似 Wiki 页面链接)。搜索结果进行 BFS 扩展。 | +| 混合检索(FTS + 向量 + 图) | Sage Wiki:FTS5(411µs)+ 向量(81ms)+ 本体图谱(1µs),通过 RRF 融合。 | +| 章节树索引 | 保留文档 H1/H2/H3 层级——平铺切块的结构保留替代方案。 | + +## 4. 知识质量与维护 + +| 技术 | 说明 | +|---|---| +| Git 版本控制 | 所有 Wiki 页面在 Git 中——完整历史、diff、blame,知识变更完全可审计。 | +| Lint / 健康检查 | 验证 frontmatter、断开的 source 链接、缺失引用。`wikiloop lint`。 | +| 冲突检测 | `contradicts` 链接将来源间的分歧暴露出来,供人工审核。 | +| Draft 暂存 | 来源少于 2 个的页面隔离到 `_draft/`——验证补充后才进入索引。 | +| 知识空白分析 | `wikiloop synthesize --gaps` 识别覆盖不足的主题。 | +| 实体去重 / 解析 | 识别同一概念的不同表述并合并为单一节点。 | + +## 5. Agent 接口(MCP) + +| 技术 | 说明 | +|---|---| +| [MCP 协议](https://modelcontextprotocol.io) | Model Context Protocol——Anthropic 开放标准,向 AI Agent 暴露工具/资源。支持 stdio + HTTP 双传输。 | +| `kb_search` | FTS 关键词搜索,返回带 `related` 链接的排序结果,用于图谱导航。 | +| `kb_page` | 通过 ID 获取完整页面内容,支持批量(最多 5 个 ID)或 `full=true` 获取不截断文本。 | +| MCP Resources | 只读资源(URI 形式):Wiki 页面、图谱 Schema、原始文档。 | +| 迭代搜索模式 | Agent 从不同角度发出多次查询,跟随 `related` 链接,自行综合答案。 | +| Sage Wiki MCP 工具 | 17 个工具:6 读、9 写、2 复合——Agent 可直接写入和编译知识。 | + +## 6. 文件转换(输入层) + +| 技术 | 说明 | +|---|---| +| markitdown(Microsoft) | 将 PDF、Word、Excel、PPT、HTML 转为 Markdown 后再提炼。 | +| `raw/converted/` 模式 | Agent 提取的内容直接写入此目录——跳过转换,直接进入提炼流程。 | +| 文件监听器(Watcher) | 自动检测 `raw/` 下的新文件/变更文件,触发转换 → 提炼 → 索引全流程。 | + +--- + +另见:[RAG 关键技术](/zh-CN/ecosystem/rag-technologies) · [范式对比](/zh-CN/ecosystem/paradigm-comparison) diff --git a/docs/site/zh-CN/ecosystem/paradigm-comparison.md b/docs/site/zh-CN/ecosystem/paradigm-comparison.md new file mode 100644 index 0000000..bc089ea --- /dev/null +++ b/docs/site/zh-CN/ecosystem/paradigm-comparison.md @@ -0,0 +1,65 @@ +# 范式对比:RAG vs LLM Wiki + +构建 AI 知识基础设施的两种根本不同的路径。 + +## 管道对比 + +``` +RAG 管道 +──────────────────────────────────────────────────────── +原始文档 → 切块 → 嵌入 → 向量存储 + ↘ [查询时] + 查询 → ANN 搜索 → LLM → 答案 + + +LLM Wiki(知识编译范式) +──────────────────────────────────────────────────────── +原始文档 → LLM 编译 → 结构化 Wiki 页面 → FTS 索引 + ↘ [查询时] + 查询 → FTS / 图遍历 → 读取页面 → 答案 +``` + +## 多维对比 + +| 维度 | RAG | LLM Wiki | +|---|---|---| +| **核心操作** | 查询时检索 | 写入时编译 | +| **存储** | 向量数据库 + 原始 chunk | 结构化 Markdown + SQLite FTS | +| **搜索方式** | ANN 相似度搜索 | FTS5 BM25 + 图遍历 | +| **知识形态** | 隐式(向量) | 显式(可读 Markdown) | +| **可审计性** | 低 | 高(git diff、lint) | +| **多跳推理** | 依赖 LLM | 通过 `related` 图链接 | +| **是否需要 Embedding** | 是 | 否(纯 FTS) | +| **知识积累** | 无(静态索引) | 随时间复利增长 | +| **查询时 Token 成本** | 高(原始 chunk 传给 LLM) | 低(预编译页面) | +| **延迟** | 较高(ANN + LLM) | 较低(FTS 亚毫秒) | +| **基础设施** | 需要向量数据库 | 仅需 SQLite | +| **人类可读性** | 否(向量不透明) | 是(纯 Markdown) | +| **最适合场景** | 宽泛文档问答、实时/动态数据 | 长期知识管理、Agent 记忆、研究 | + +## 如何选择 + +**选 RAG 的场景:** +- 需要查询频繁变化的大规模语料 +- 需要实时或近实时数据摄入 +- 异构文档间的语义相似度匹配最重要 +- 不需要维护或审查知识层 + +**选 LLM Wiki 的场景:** +- 知识需要随时间积累和改进 +- 知识的可审计性和人工审核很重要 +- Agent 需要在精心策展的知识库上搜索和推理 +- 希望最小化查询时的 Token 成本 +- 希望离线可用、零基础设施依赖 + +## 两者并不互斥 + +许多生产系统将两者结合使用: +- **RAG** 负责对大规模、动态变化语料的宽泛检索 +- **LLM Wiki** 作为精心策展的高质量知识层,覆盖已知领域 + +典型混合模式:RAG 管道宽泛摄入 → 高置信度结果编译为 Wiki 页面供长期复用。 + +--- + +另见:[RAG 关键技术](/zh-CN/ecosystem/rag-technologies) · [LLM Wiki 关键技术](/zh-CN/ecosystem/llm-wiki-technologies) diff --git a/docs/site/zh-CN/ecosystem/rag-technologies.md b/docs/site/zh-CN/ecosystem/rag-technologies.md new file mode 100644 index 0000000..01121a0 --- /dev/null +++ b/docs/site/zh-CN/ecosystem/rag-technologies.md @@ -0,0 +1,107 @@ +# RAG 关键技术 + +RAG(检索增强生成)管道背后的核心技术——从原始文档到生成答案的全链路。 + +``` +原始文档 → 切块 → 嵌入 → 存储 → [查询时] → 检索 → LLM → 答案 +``` + +## 1. 文档解析层 + +| 技术 | 代表工具 | 说明 | +|---|---|---| +| 多格式文档解析 | [Unstructured.io](https://unstructured.io)、[Docling](https://github.com/DS4SD/docling)(IBM) | 版面感知的 PDF/Word/HTML 多格式提取 | +| PDF 底层解析 | PyMuPDF、PDFPlumber | 文本/表格/布局结构提取 | +| LLM 增强解析 | LlamaParse | 处理含复杂表格和图形 PDF 的云端 API | +| PDF 转 Markdown | [Marker](https://github.com/VikParuchuri/marker)、[MinerU](https://github.com/opendatalab/MinerU) | 高保真格式转换,保留文档结构 | +| OCR 云服务 | Azure Document Intelligence、AWS Textract | 企业级表单和表格识别 | + +## 2. 文本切块策略 + +| 策略 | 工具 | 说明 | +|---|---|---| +| 固定长度切块 | LangChain `CharacterTextSplitter` | 简单,但会破坏语义边界 | +| 递归字符切割 | LangChain `RecursiveCharacterTextSplitter` | 按段落→句子→单词层级递归切割 | +| 语义切块 | LlamaIndex `SemanticSplitter` | 通过 Embedding 相似度检测主题边界 | +| 章节树索引 | [he-wiki-rag](https://github.com/liuhe37186/he-wiki-rag) | 保留 H1/H2/H3 层级和面包屑路径 | +| 父子分块 | LlamaIndex | 检索小粒度子块,返回父块作为上下文 | +| 命题切块 | 自定义 + LLM | 拆分为原子事实——精度最高,代价最大 | +| Late Chunking | Jina AI | 先嵌入完整文档再切分 Embedding | +| RAPTOR | LlamaIndex | 递归树状摘要,构建层次化检索结构 | +| 文档树推理 | [PageIndex](https://github.com/vectifyai/vectify) | LLM 遍历摘要树检索,无需向量,FinanceBench 98.7% | + +## 3. Embedding 模型 + +| 模型 | 提供方 | 说明 | +|---|---|---| +| text-embedding-3-large/small | OpenAI | 通用嵌入,large 版 3072 维 | +| Embed v3 | Cohere | 专为检索优化,1024 维 | +| BGE-M3 | BAAI(开源) | 多语言多粒度,中文表现优异 | +| E5-Mistral-7B | Microsoft(开源) | 高维度,MTEB 排名靠前 | +| NV-Embed-v2 | NVIDIA(开源) | 开源模型中 MTEB 领先 | +| nomic-embed-text | Nomic(开源) | 完全开源,本地友好 | +| jina-embeddings-v3 | Jina AI(开源) | 支持 Late Chunking | + +## 4. 向量数据库 + +| 数据库 | 类型 | 说明 | +|---|---|---| +| [Qdrant](https://github.com/qdrant/qdrant) | 开源 | Rust 实现,高性能,强过滤,适合 10 万~千万规模 | +| [Milvus](https://github.com/milvus-io/milvus) | 开源 | 十亿级分布式,商业版:Zilliz Cloud | +| [Weaviate](https://github.com/weaviate/weaviate) | 开源 | 原生混合检索(向量 + BM25) | +| [Chroma](https://github.com/chroma-core/chroma) | 开源 | 轻量嵌入式,原型开发首选 | +| [pgvector](https://github.com/pgvector/pgvector) | 扩展 | PostgreSQL 原生向量搜索扩展 | +| [LanceDB](https://github.com/lancedb/lancedb) | 开源 | Arrow 格式,嵌入式,无服务器友好 | +| Pinecone | 托管 SaaS | Serverless,零运维 | + +## 5. 检索策略 + +| 策略 | 工具 | 说明 | +|---|---|---| +| 密集检索 | 各向量数据库 ANN | 余弦/点积语义相似度近似最近邻搜索 | +| 稀疏检索(BM25) | Elasticsearch、OpenSearch、Tantivy | 基于词频统计的关键词匹配 | +| 混合检索 | Weaviate、Qdrant、RRF 算法 | 密集 + 稀疏,通过 RRF 倒数排名融合 | +| 图增强检索 | [GraphRAG](https://github.com/microsoft/graphrag)、[LightRAG](https://github.com/HKUDS/LightRAG) | 实体/关系图谱支持多跳推理 | +| Vector Graph RAG | 社区 | 三元组向量化替代图数据库,HotpotQA 96.3% | +| Agentic RAG(A-RAG) | 自定义 + LLM | Agent 自主选择 `keyword_search` / `semantic_search` / `chunk_read` 工具 | + +## 6. 查询优化 + +| 技术 | 工具 | 说明 | +|---|---|---| +| HyDE | LangChain、LlamaIndex | 生成假设性答案,用其 Embedding 做检索 | +| Multi-Query 查询改写 | LangChain `MultiQueryRetriever` | LLM 将原问题改写为多个子问题扩大召回 | +| Step-Back Prompting | LangChain | 将具体问题抽象为通用问题再检索 | +| Self-RAG | 论文实现 | LLM 自评估检索质量,决定是否继续检索 | +| CRAG | 论文实现 | 低置信度时回退到网络搜索 | + +## 7. Rerank 精排 + +| 工具 | 说明 | +|---|---| +| Cohere Rerank | 交叉编码器商业精排模型 | +| BGE Reranker(BAAI) | 开源交叉编码器,多语言表现强 | +| FlashRank | 轻量开源精排库,适合本地部署 | + +## 8. 编排框架 + +| 框架 | 说明 | +|---|---| +| [LangChain](https://github.com/langchain-ai/langchain) | 最流行的模块化 RAG 管道框架 | +| [LlamaIndex](https://github.com/run-llama/llama_index) | 数据中心型 RAG,最丰富的切块/检索策略 | +| [Haystack](https://github.com/deepset-ai/haystack) | 企业级 NLP/RAG 管道框架 | +| [DSPy](https://github.com/stanfordnlp/dspy) | 通过编程方式优化 LLM 管道,替代手写 Prompt | + +## 9. 评估框架 + +| 工具 | 说明 | +|---|---| +| [RAGAS](https://ragas.io) | 最流行的 RAG 评估框架——忠实度、上下文召回、答案相关性 | +| [DeepEval](https://github.com/confident-ai/deepeval) | 单元测试风格的 RAG 评估库 | +| [TruLens](https://www.trulens.org) | 基于 RAG Triad 的生产监控工具 | +| LangSmith | LangChain 官方链路追踪与评估平台 | +| 核心指标 | Faithfulness、Context Recall、Answer Relevancy、MRR、NDCG | + +--- + +另见:[LLM Wiki 关键技术](/zh-CN/ecosystem/llm-wiki-technologies) · [范式对比](/zh-CN/ecosystem/paradigm-comparison)