From 716dbb7fe63fb379b275bad4fb8737148aa34092 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 14:57:35 +0800 Subject: [PATCH 01/30] refactor(engine): remove ask functionality and query context in favor of Python strategy layer BREAKING CHANGE: The ask() method and QueryContext have been removed from the Rust engine as retrieval functionality has been migrated to the Python strategy layer. Users should now use Engine.ask() from the Python SDK for retrieval operations. This removes the retrieval-related code from the Rust core including the ask method implementation, QueryContext struct, and RetrieverClient stub. --- .../vectorless-engine/src/engine.rs | 12 -- vectorless-core/vectorless-engine/src/lib.rs | 60 +----- .../vectorless-engine/src/query_context.rs | 179 ------------------ .../vectorless-engine/src/retriever.rs | 25 --- vectorless-core/vectorless-py/src/engine.rs | 33 ---- 5 files changed, 1 insertion(+), 308 deletions(-) delete mode 100644 vectorless-core/vectorless-engine/src/query_context.rs delete mode 100644 vectorless-core/vectorless-engine/src/retriever.rs diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index 359c940..fa81082 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -439,18 +439,6 @@ impl Engine { Ok(doc.info()) } - /// Ask a question — returns a reasoned answer with evidence and trace. - /// - /// Ask a question about the indexed documents. - /// - /// **Note**: Retrieval is now handled by the Python strategy layer. - /// This method returns an error — use Engine.ask() from the Python SDK. - pub async fn ask(&self, _input: &str, _ids: &[String]) -> Result { - Err(Error::Config( - "Retrieval has been migrated to Python. Use Engine.ask() from the Python SDK.".into(), - )) - } - /// Remove a document from the workspace. pub async fn forget(&self, doc_id: &str) -> Result<()> { self.workspace.remove(doc_id).await?; diff --git a/vectorless-core/vectorless-engine/src/lib.rs b/vectorless-core/vectorless-engine/src/lib.rs index d6f656c..526c9fd 100644 --- a/vectorless-core/vectorless-engine/src/lib.rs +++ b/vectorless-core/vectorless-engine/src/lib.rs @@ -7,71 +7,14 @@ //! - [`Engine`] — The main client for indexing and querying documents //! - [`EngineBuilder`] — Builder pattern for client configuration //! - [`IndexContext`] — Unified input for document indexing -//! - [`QueryContext`] — Unified input for document queries //! -//! # Quick Start -//! -//! ```rust,no_run -//! use vectorless::client::{EngineBuilder, IndexContext, QueryContext}; -//! -//! # #[tokio::main] -//! # async fn main() -> Result<(), Box> { -//! // Create a client with default settings -//! let client = EngineBuilder::new() -//! .with_key("sk-...") -//! .with_model("gpt-4o") -//! .build() -//! .await?; -//! -//! // Index a document -//! let result = client.index(IndexContext::from_path("./document.md")).await?; -//! let doc_id = result.doc_id().unwrap(); -//! -//! // Query the document -//! let result = client.query( -//! QueryContext::new("What is this?").with_doc_ids(vec![doc_id.to_string()]) -//! ).await?; -//! if let Some(item) = result.single() { -//! println!("{}", item.content); -//! } -//! -//! // List all documents -//! for doc in client.list().await? { -//! println!("{}: {}", doc.id, doc.name); -//! } -//! # Ok(()) -//! # } -//! ``` -//! -//! # Events and Progress -//! -//! Monitor operation progress with events: -//! -//! ```rust,no_run -//! # use vectorless::client::{EngineBuilder, EventEmitter, IndexEvent}; -//! # #[tokio::main] -//! # async fn main() -> Result<(), Box> { -//! let events = EventEmitter::new() -//! .on_index(|e| match e { -//! IndexEvent::Complete { doc_id } => println!("Indexed: {}", doc_id), -//! _ => {} -//! }); -//! -//! let client = EngineBuilder::new() -//! .with_events(events) -//! .build() -//! .await?; -//! # Ok(()) -//! # } -//! ``` +//! Retrieval (ask) is handled by the Python strategy layer. mod builder; mod engine; mod index_context; mod indexed_document; mod indexer; -mod query_context; -mod retriever; mod types; mod workspace; @@ -87,7 +30,6 @@ pub use engine::Engine; // ============================================================ pub use index_context::IndexContext; -pub use query_context::QueryContext; // ============================================================ // Result & Info Types diff --git a/vectorless-core/vectorless-engine/src/query_context.rs b/vectorless-core/vectorless-engine/src/query_context.rs deleted file mode 100644 index 48d9ad2..0000000 --- a/vectorless-core/vectorless-engine/src/query_context.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Query context for the Engine API. -//! -//! [`QueryContext`] encapsulates all parameters for a query operation, -//! supporting specific documents or entire workspace queries. -//! -//! # Example -//! -//! ```rust -//! use vectorless::client::QueryContext; -//! -//! // Query specific documents -//! let ctx = QueryContext::new("What is the total revenue?") -//! .with_doc_ids(vec!["doc-1".to_string()]); -//! -//! // Query entire workspace -//! let ctx = QueryContext::new("Explain the algorithm"); -//! ``` - -/// Query scope — determines which documents to search. -#[derive(Debug, Clone)] -pub(crate) enum QueryScope { - /// Query specific documents. - Documents(Vec), - /// Query all documents in the workspace. - Workspace, -} - -/// Context for a query operation. -/// -/// Supports two scopes: -/// - **Specific documents** — via `with_doc_ids()` -/// - **Entire workspace** — default when no scope is set -/// -/// # Convenience -/// -/// Implements `From` and `From<&str>` for quick construction: -/// -/// ```rust -/// use vectorless::client::QueryContext; -/// -/// let ctx: QueryContext = "What is this?".into(); -/// ``` -#[derive(Debug, Clone)] -pub struct QueryContext { - /// The query text. - pub(crate) query: String, - /// Target scope. - pub(crate) scope: QueryScope, - /// Per-operation timeout (seconds). `None` means no timeout. - pub(crate) timeout_secs: Option, - /// Force Orchestrator analysis even when documents are specified. - /// - /// When `true`, the Orchestrator analyzes DocCards to select relevant - /// documents instead of dispatching all specified docs directly. - /// Useful when the user wants the system to decide which documents - /// (or sections) are most relevant to the query. - pub(crate) force_analysis: bool, -} - -impl QueryContext { - /// Create a new query context (defaults to workspace scope). - pub fn new(query: impl Into) -> Self { - Self { - query: query.into(), - scope: QueryScope::Workspace, - timeout_secs: None, - force_analysis: false, - } - } - - /// Set scope to specific documents. - /// - /// Pass a single ID or multiple IDs to restrict the query - /// to those documents only. - pub fn with_doc_ids(mut self, doc_ids: Vec) -> Self { - self.scope = QueryScope::Documents(doc_ids); - self - } - - /// Set scope to entire workspace. - pub fn with_workspace(mut self) -> Self { - self.scope = QueryScope::Workspace; - self - } - - /// Set per-operation timeout in seconds. - pub fn with_timeout_secs(mut self, secs: u64) -> Self { - self.timeout_secs = Some(secs); - self - } - - /// Force the Orchestrator to analyze documents before dispatching Workers. - /// - /// By default, when documents are specified via `with_doc_ids()`, the - /// Orchestrator skips its analysis phase and dispatches Workers to all - /// specified documents directly. Setting this to `true` forces the - /// Orchestrator to analyze DocCards and decide which documents are - /// relevant, even when the user specified documents explicitly. - /// - /// This is useful when querying across many documents where only a subset - /// is likely relevant to the specific question. - pub fn with_force_analysis(mut self, force: bool) -> Self { - self.force_analysis = force; - self - } -} - -impl From for QueryContext { - fn from(query: String) -> Self { - Self::new(query) - } -} - -impl From<&str> for QueryContext { - fn from(query: &str) -> Self { - Self::new(query) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_query_context_new() { - let ctx = QueryContext::new("What is this?"); - assert_eq!(ctx.query, "What is this?"); - } - - #[test] - fn test_query_context_from_string() { - let ctx: QueryContext = "Hello".to_string().into(); - assert_eq!(ctx.query, "Hello"); - } - - #[test] - fn test_query_context_from_str() { - let ctx: QueryContext = "Hello".into(); - assert_eq!(ctx.query, "Hello"); - } - - #[test] - fn test_single_doc_scope() { - let ctx = QueryContext::new("test").with_doc_ids(vec!["doc-1".to_string()]); - assert!( - matches!(ctx.scope, QueryScope::Documents(ref ids) if ids == &["doc-1".to_string()]) - ); - } - - #[test] - fn test_multi_doc_scope() { - let ctx = QueryContext::new("test").with_doc_ids(vec!["a".into(), "b".into()]); - assert!(matches!(ctx.scope, QueryScope::Documents(ref ids) if ids.len() == 2)); - } - - #[test] - fn test_workspace_scope() { - let ctx = QueryContext::new("test"); - assert!(matches!(ctx.scope, QueryScope::Workspace)); - } - - #[test] - fn test_builder_options() { - let ctx = QueryContext::new("test") - .with_doc_ids(vec!["doc-1".to_string()]) - .with_timeout_secs(60); - - assert_eq!(ctx.timeout_secs, Some(60)); - } - - #[test] - fn test_query_context_timeout_default() { - let ctx = QueryContext::new("test"); - assert_eq!(ctx.timeout_secs, None); - } -} diff --git a/vectorless-core/vectorless-engine/src/retriever.rs b/vectorless-core/vectorless-engine/src/retriever.rs deleted file mode 100644 index 1c83372..0000000 --- a/vectorless-core/vectorless-engine/src/retriever.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Document retrieval client — STUB. -//! -//! The strategy layer (agent, orchestrator, worker) has been migrated to Python. -//! This module is a stub that returns an error for any query attempt. -//! All retrieval now goes through the Python Engine.ask() path. - -use vectorless_error::{Error, Result}; - -/// Document retrieval client (stub). -/// -/// All retrieval is now handled by the Python strategy layer. -#[allow(dead_code)] -pub(crate) struct RetrieverClient; - -impl RetrieverClient { - /// Not available — retrieval is handled by Python. - pub async fn query(&self, _question: &str) -> Result<()> { - todo!( - "Document retrieval is now handled by the Python strategy layer. This method should not be called." - ) - } -} diff --git a/vectorless-core/vectorless-py/src/engine.rs b/vectorless-core/vectorless-py/src/engine.rs index da6e794..3b297a0 100644 --- a/vectorless-core/vectorless-py/src/engine.rs +++ b/vectorless-core/vectorless-py/src/engine.rs @@ -10,7 +10,6 @@ use tokio::runtime::Runtime; use ::vectorless_engine::{Engine, EngineBuilder, IngestInput}; -use super::answer::PyAnswer; use super::document::{PyDocument, PyDocumentInfo}; use super::error::VectorlessError; use super::error::to_py_err; @@ -26,15 +25,6 @@ async fn run_ingest(engine: Arc, input: IngestInput) -> PyResult, - question: String, - doc_ids: Vec, -) -> PyResult { - let answer = engine.ask(&question, &doc_ids).await.map_err(to_py_err)?; - Ok(PyAnswer { inner: answer }) -} - async fn run_forget(engine: Arc, doc_id: String) -> PyResult<()> { engine.forget(&doc_id).await.map_err(to_py_err) } @@ -194,29 +184,6 @@ impl PyEngine { future_into_py(py, run_ingest(engine, input)) } - /// Ask a question — returns a reasoned answer with evidence and trace. - /// - /// Args: - /// question: The question to ask (required). - /// doc_ids: List of document IDs to search. Empty = search all. - /// - /// Returns: - /// Answer with content, evidence, confidence, and trace. - /// - /// Raises: - /// VectorlessError: If ask fails. - #[pyo3(signature = (question, doc_ids=None))] - fn ask<'py>( - &self, - py: Python<'py>, - question: String, - doc_ids: Option>, - ) -> PyResult> { - let engine = Arc::clone(&self.inner); - let ids = doc_ids.unwrap_or_default(); - future_into_py(py, run_ask(engine, question, ids)) - } - /// Remove a document by ID. /// /// Args: From 80a3e38ff31759eb651fd7849e5dafa22c62906a Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 15:20:48 +0800 Subject: [PATCH 02/30] refactor(agent): remove unused agent module and update workspace configuration - Remove vectorless-agent crate from workspace members in Cargo.toml - Move vectorless-rerank from active member to commented out section - Remove entire vectorless-agent module including command parsing, configuration, and context modules - Update member list to reflect removal of agent-related components --- Cargo.toml | 2 +- vectorless-core/vectorless-agent/Cargo.toml | 29 - .../vectorless-agent/src/command.rs | 629 ---------------- .../vectorless-agent/src/config.rs | 240 ------ .../vectorless-agent/src/context.rs | 120 --- .../vectorless-agent/src/events.rs | 537 ------------- vectorless-core/vectorless-agent/src/lib.rs | 55 -- .../src/orchestrator/analyze.rs | 159 ---- .../src/orchestrator/dispatch.rs | 92 --- .../src/orchestrator/evaluate.rs | 128 ---- .../vectorless-agent/src/orchestrator/mod.rs | 224 ------ .../src/orchestrator/replan.rs | 249 ------ .../src/orchestrator/supervisor.rs | 159 ---- .../vectorless-agent/src/prompts.rs | 569 -------------- vectorless-core/vectorless-agent/src/state.rs | 312 -------- .../vectorless-agent/src/tools/common.rs | 69 -- .../vectorless-agent/src/tools/mod.rs | 101 --- .../src/tools/orchestrator.rs | 203 ----- .../vectorless-agent/src/tools/worker/cat.rs | 115 --- .../vectorless-agent/src/tools/worker/cd.rs | 262 ------- .../vectorless-agent/src/tools/worker/find.rs | 128 ---- .../vectorless-agent/src/tools/worker/grep.rs | 175 ----- .../vectorless-agent/src/tools/worker/head.rs | 119 --- .../vectorless-agent/src/tools/worker/ls.rs | 124 --- .../vectorless-agent/src/tools/worker/mod.rs | 39 - .../vectorless-agent/src/tools/worker/pwd.rs | 58 -- .../vectorless-agent/src/tools/worker/wc.rs | 109 --- .../vectorless-agent/src/worker/execute.rs | 278 ------- .../vectorless-agent/src/worker/format.rs | 20 - .../vectorless-agent/src/worker/mod.rs | 236 ------ .../vectorless-agent/src/worker/navigation.rs | 448 ----------- .../vectorless-agent/src/worker/planning.rs | 708 ------------------ vectorless-core/vectorless-engine/Cargo.toml | 1 - vectorless-core/vectorless-query/Cargo.toml | 22 - vectorless-core/vectorless-query/src/lib.rs | 45 -- vectorless-core/vectorless-query/src/types.rs | 114 --- .../vectorless-query/src/understand.rs | 246 ------ vectorless-core/vectorless-rerank/Cargo.toml | 19 - .../vectorless-rerank/src/dedup.rs | 216 ------ vectorless-core/vectorless-rerank/src/lib.rs | 103 --- .../vectorless-rerank/src/types.rs | 29 - .../vectorless-retrieval/Cargo.toml | 30 - .../vectorless-retrieval/src/cache.rs | 577 -------------- .../vectorless-retrieval/src/dispatcher.rs | 78 -- .../vectorless-retrieval/src/lib.rs | 31 - .../vectorless-retrieval/src/postprocessor.rs | 130 ---- .../vectorless-retrieval/src/stream.rs | 128 ---- .../vectorless-retrieval/src/types.rs | 245 ------ 48 files changed, 1 insertion(+), 8709 deletions(-) delete mode 100644 vectorless-core/vectorless-agent/Cargo.toml delete mode 100644 vectorless-core/vectorless-agent/src/command.rs delete mode 100644 vectorless-core/vectorless-agent/src/config.rs delete mode 100644 vectorless-core/vectorless-agent/src/context.rs delete mode 100644 vectorless-core/vectorless-agent/src/events.rs delete mode 100644 vectorless-core/vectorless-agent/src/lib.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/analyze.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/dispatch.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/evaluate.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/mod.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/replan.rs delete mode 100644 vectorless-core/vectorless-agent/src/orchestrator/supervisor.rs delete mode 100644 vectorless-core/vectorless-agent/src/prompts.rs delete mode 100644 vectorless-core/vectorless-agent/src/state.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/common.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/mod.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/orchestrator.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/cat.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/cd.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/find.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/grep.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/head.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/ls.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/mod.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/pwd.rs delete mode 100644 vectorless-core/vectorless-agent/src/tools/worker/wc.rs delete mode 100644 vectorless-core/vectorless-agent/src/worker/execute.rs delete mode 100644 vectorless-core/vectorless-agent/src/worker/format.rs delete mode 100644 vectorless-core/vectorless-agent/src/worker/mod.rs delete mode 100644 vectorless-core/vectorless-agent/src/worker/navigation.rs delete mode 100644 vectorless-core/vectorless-agent/src/worker/planning.rs delete mode 100644 vectorless-core/vectorless-query/Cargo.toml delete mode 100644 vectorless-core/vectorless-query/src/lib.rs delete mode 100644 vectorless-core/vectorless-query/src/types.rs delete mode 100644 vectorless-core/vectorless-query/src/understand.rs delete mode 100644 vectorless-core/vectorless-rerank/Cargo.toml delete mode 100644 vectorless-core/vectorless-rerank/src/dedup.rs delete mode 100644 vectorless-core/vectorless-rerank/src/lib.rs delete mode 100644 vectorless-core/vectorless-rerank/src/types.rs delete mode 100644 vectorless-core/vectorless-retrieval/Cargo.toml delete mode 100644 vectorless-core/vectorless-retrieval/src/cache.rs delete mode 100644 vectorless-core/vectorless-retrieval/src/dispatcher.rs delete mode 100644 vectorless-core/vectorless-retrieval/src/lib.rs delete mode 100644 vectorless-core/vectorless-retrieval/src/postprocessor.rs delete mode 100644 vectorless-core/vectorless-retrieval/src/stream.rs delete mode 100644 vectorless-core/vectorless-retrieval/src/types.rs diff --git a/Cargo.toml b/Cargo.toml index 86b89fb..2cf9ec8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ members = [ # "vectorless-core/vectorless-query", # "vectorless-core/vectorless-agent", # "vectorless-core/vectorless-retrieval", + # "vectorless-core/vectorless-rerank", "vectorless-core/vectorless-index", - "vectorless-core/vectorless-rerank", "vectorless-core/vectorless-primitives", "vectorless-core/vectorless-engine", "vectorless-core/vectorless-py", diff --git a/vectorless-core/vectorless-agent/Cargo.toml b/vectorless-core/vectorless-agent/Cargo.toml deleted file mode 100644 index 0d7f793..0000000 --- a/vectorless-core/vectorless-agent/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "vectorless-agent" -version.workspace = true -edition.workspace = true -authors.workspace = true -description.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true - -[dependencies] -vectorless-document = { path = "../vectorless-document" } -vectorless-error = { path = "../vectorless-error" } -vectorless-llm = { path = "../vectorless-llm" } -vectorless-query = { path = "../vectorless-query" } -vectorless-rerank = { path = "../vectorless-rerank" } -vectorless-scoring = { path = "../vectorless-scoring" } -tokio = { workspace = true } -async-trait = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -tracing = { workspace = true } -futures = { workspace = true } -chrono = { workspace = true } -thiserror = { workspace = true } -regex = { workspace = true } - -[lints] -workspace = true diff --git a/vectorless-core/vectorless-agent/src/command.rs b/vectorless-core/vectorless-agent/src/command.rs deleted file mode 100644 index 0738542..0000000 --- a/vectorless-core/vectorless-agent/src/command.rs +++ /dev/null @@ -1,629 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Command parsing for the agent navigation loop. -//! -//! LLM output is parsed into `Command` variants. The parser is intentionally -//! simple and forgiving — unknown input falls back to `Ls` so the agent can -//! re-observe its surroundings. - -use vectorless_document::{NavigationIndex, NodeId}; - -/// Parsed command from LLM output. -#[derive(Debug, Clone, PartialEq)] -pub enum Command { - /// List children of the current node. - Ls, - /// Navigate into a child node by name. - Cd { target: String }, - /// Navigate back to parent. - CdUp, - /// Read node content (collects as evidence). - Cat { target: String }, - /// Search for a keyword in the ReasoningIndex. - Find { keyword: String }, - /// Regex search across node content in the current subtree. - Grep { pattern: String }, - /// Preview first N lines of a node without collecting evidence. - Head { target: String, lines: usize }, - /// Search for nodes by title pattern in the tree. - FindTree { pattern: String }, - /// Show node content size (lines, chars). - Wc { target: String }, - /// Show current navigation path. - Pwd, - /// Evaluate evidence sufficiency. - Check, - /// End navigation. - Done, -} - -/// Strip surrounding quotes from a target string. -/// -/// Handles straight quotes (`"`, `'`) and Unicode smart quotes (U+201C/U+201D, U+2018/U+2019). -fn strip_quotes(s: &str) -> String { - let trimmed = s.trim(); - let chars: Vec = trimmed.chars().collect(); - if chars.len() < 2 { - return trimmed.to_string(); - } - let (first, last) = (chars[0], chars[chars.len() - 1]); - let matching = (first == '"' && last == '"') - || (first == '\'' && last == '\'') - || (first == '\u{201c}' && last == '\u{201d}') - || (first == '\u{2018}' && last == '\u{2019}'); - if matching { - trimmed[chars[0].len_utf8()..trimmed.len() - chars[chars.len() - 1].len_utf8()].to_string() - } else { - trimmed.to_string() - } -} - -/// Parse the first non-empty line of LLM output into a Command. -pub fn parse_command(llm_output: &str) -> Command { - let line = llm_output - .lines() - .find(|l| !l.trim().is_empty()) - .unwrap_or("") - .trim(); - - // Remove common wrapping (markdown code blocks, etc.) - let line = line.trim_start_matches('`').trim_end_matches('`').trim(); - - let parts: Vec<&str> = line.split_whitespace().collect(); - - match parts.as_slice() { - ["ls"] => Command::Ls, - ["cat"] => Command::Cat { - target: ".".to_string(), - }, - ["cd", ".."] => Command::CdUp, - ["cd", target] => Command::Cd { - target: strip_quotes(target), - }, - ["cd", _target, ..] => Command::Cd { - // Handle "cd some name" by joining remaining parts - target: strip_quotes(&parts[1..].join(" ")), - }, - ["cat", target] => Command::Cat { - target: strip_quotes(target), - }, - ["cat", _target, ..] => Command::Cat { - target: strip_quotes(&parts[1..].join(" ")), - }, - ["find", keyword] => Command::Find { - keyword: strip_quotes(keyword), - }, - ["find", _keyword, ..] => Command::Find { - keyword: strip_quotes(&parts[1..].join(" ")), - }, - ["grep", pattern] => Command::Grep { - pattern: strip_quotes(pattern), - }, - ["grep", _pattern, ..] => Command::Grep { - pattern: strip_quotes(&parts[1..].join(" ")), - }, - ["head", target] => Command::Head { - target: strip_quotes(target), - lines: 20, // default - }, - ["head", "-n", n, target @ ..] => Command::Head { - target: strip_quotes(&target.join(" ")), - lines: n.parse().unwrap_or(20), - }, - ["head", _target, ..] => Command::Head { - target: strip_quotes(&parts[1..].join(" ")), - lines: 20, - }, - ["findtree", pattern] => Command::FindTree { - pattern: strip_quotes(pattern), - }, - ["findtree", _pattern, ..] => Command::FindTree { - pattern: strip_quotes(&parts[1..].join(" ")), - }, - ["wc", target] => Command::Wc { - target: strip_quotes(target), - }, - ["wc", _target, ..] => Command::Wc { - target: strip_quotes(&parts[1..].join(" ")), - }, - ["pwd"] => Command::Pwd, - ["check"] => Command::Check, - ["done"] => Command::Done, - _ => Command::Ls, // fallback: re-observe - } -} - -/// Resolve a cd/cat target string to a NodeId using multi-level matching. -/// -/// Matching priority: -/// 1. Exact title match -/// 2. Case-insensitive title match -/// 3. Substring (contains) match -/// 4. Numeric index match ("1" → first child, "2" → second, etc.) -pub fn resolve_target( - target: &str, - nav_index: &NavigationIndex, - current_node: NodeId, -) -> Option { - let target = strip_quotes(target); - let routes = nav_index.get_child_routes(current_node)?; - - // 1. Exact match - if let Some(r) = routes.iter().find(|r| r.title == target) { - return Some(r.node_id); - } - - // 2. Case-insensitive match - let target_lower = target.to_lowercase(); - if let Some(r) = routes - .iter() - .find(|r| r.title.to_lowercase() == target_lower) - { - return Some(r.node_id); - } - - // 3. Substring (contains) match - if let Some(r) = routes - .iter() - .find(|r| r.title.to_lowercase().contains(&target_lower)) - { - return Some(r.node_id); - } - - // 4. Numeric index match ("1" → first child) - if let Ok(idx) = target.parse::() { - if idx > 0 && idx <= routes.len() { - return Some(routes[idx - 1].node_id); - } - } - - None -} - -/// Resolve a cd/cat target with additional context from the tree node titles. -/// -/// Matching priority: -/// 1. Direct children via NavigationIndex (exact, case-insensitive, substring, numeric) -/// 2. Direct children via TreeNode titles (case-insensitive contains) -/// 3. Deep descendant search (BFS, up to depth 4) — enables `cd "Research Labs"` from -/// root when "Research Labs" is a grandchild behind an intermediate wrapper node. -pub fn resolve_target_extended( - target: &str, - nav_index: &NavigationIndex, - current_node: NodeId, - tree: &vectorless_document::DocumentTree, -) -> Option { - let target = strip_quotes(target); - // Try the primary resolver first - if let Some(id) = resolve_target(&target, nav_index, current_node) { - return Some(id); - } - - let target_lower = target.to_lowercase(); - - // Extended: check all direct children by their TreeNode titles - let children: Vec = tree.children_iter(current_node).collect(); - for child_id in &children { - if let Some(node) = tree.get(*child_id) { - if node.title.to_lowercase().contains(&target_lower) { - return Some(*child_id); - } - } - } - - // Deep search: BFS through descendants up to depth 4. - // Returns the shallowest match so `cd "Research Labs"` from root finds it - // at depth 1 even if another "Research Labs" exists deeper. - search_descendants(&target_lower, current_node, tree, 4) -} - -/// BFS search through descendants, returning the shallowest matching NodeId. -fn search_descendants( - target_lower: &str, - start: NodeId, - tree: &vectorless_document::DocumentTree, - max_depth: usize, -) -> Option { - let mut queue: Vec<(NodeId, usize)> = vec![(start, 0)]; - - while let Some((node_id, depth)) = queue.pop() { - if depth >= max_depth { - continue; - } - for child_id in tree.children_iter(node_id) { - if let Some(node) = tree.get(child_id) { - if node.title.to_lowercase().contains(target_lower) { - return Some(child_id); - } - } - queue.push((child_id, depth + 1)); - } - } - - None -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_ls() { - assert_eq!(parse_command("ls"), Command::Ls); - assert_eq!(parse_command(" ls "), Command::Ls); - } - - #[test] - fn test_parse_cd() { - assert_eq!(parse_command("cd .."), Command::CdUp); - assert_eq!( - parse_command("cd Getting Started"), - Command::Cd { - target: "Getting Started".to_string() - } - ); - assert_eq!( - parse_command("cd some long name"), - Command::Cd { - target: "some long name".to_string() - } - ); - // Quoted multi-word targets should have quotes stripped - assert_eq!( - parse_command("cd \"Vectorless Architecture Guide\""), - Command::Cd { - target: "Vectorless Architecture Guide".to_string() - } - ); - assert_eq!( - parse_command("cd 'Vectorless Architecture Guide'"), - Command::Cd { - target: "Vectorless Architecture Guide".to_string() - } - ); - // Smart quotes - assert_eq!( - parse_command("\u{201c}Vectorless Architecture Guide\u{201d}"), - Command::Ls // doesn't start with a command keyword - ); - } - - #[test] - fn test_strip_quotes_straight() { - assert_eq!(strip_quotes("\"hello\""), "hello"); - assert_eq!(strip_quotes("'hello'"), "hello"); - assert_eq!(strip_quotes("hello"), "hello"); - assert_eq!(strip_quotes("\"only left"), "\"only left"); - } - - #[test] - fn test_strip_quotes_smart() { - assert_eq!(strip_quotes("\u{201c}hello\u{201d}"), "hello"); - assert_eq!(strip_quotes("\u{2018}hello\u{2019}"), "hello"); - } - - #[test] - fn test_resolve_target_quoted() { - use vectorless_document::{ChildRoute, DocumentTree}; - - let mut tree = DocumentTree::new("Root", ""); - let root = tree.root(); - let c1 = tree.add_child(root, "Vectorless Architecture Guide", "content"); - - let mut nav_index = NavigationIndex::new(); - nav_index.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "Vectorless Architecture Guide".to_string(), - description: "Main guide".to_string(), - leaf_count: 5, - }], - ); - - // Quoted target should still resolve - assert_eq!( - resolve_target("\"Vectorless Architecture Guide\"", &nav_index, root), - Some(c1) - ); - assert_eq!( - resolve_target("'Vectorless Architecture Guide'", &nav_index, root), - Some(c1) - ); - } - - #[test] - fn test_parse_cat() { - assert_eq!( - parse_command("cat Installation"), - Command::Cat { - target: "Installation".to_string() - } - ); - assert_eq!( - parse_command("cat API Reference"), - Command::Cat { - target: "API Reference".to_string() - } - ); - } - - #[test] - fn test_parse_find() { - assert_eq!( - parse_command("find authentication"), - Command::Find { - keyword: "authentication".to_string() - } - ); - } - - #[test] - fn test_parse_misc() { - assert_eq!(parse_command("pwd"), Command::Pwd); - assert_eq!(parse_command("check"), Command::Check); - assert_eq!(parse_command("done"), Command::Done); - } - - #[test] - fn test_parse_fallback() { - assert_eq!(parse_command(""), Command::Ls); - assert_eq!(parse_command("unknown command"), Command::Ls); - assert_eq!(parse_command("blah blah"), Command::Ls); - } - - #[test] - fn test_parse_with_wrapping() { - assert_eq!(parse_command("`ls`"), Command::Ls); - assert_eq!(parse_command("```ls```"), Command::Ls); - } - - #[test] - fn test_parse_multiline() { - // Should parse the first non-empty line - assert_eq!(parse_command("\n\nls\n\n// listing children"), Command::Ls); - } - - #[test] - fn test_resolve_target_numeric() { - use vectorless_document::{ChildRoute, DocumentTree}; - - let mut tree = DocumentTree::new("Root", ""); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "content"); - let c2 = tree.add_child(root, "API Reference", "content"); - - let mut nav_index = NavigationIndex::new(); - nav_index.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup guide".to_string(), - leaf_count: 3, - }, - ChildRoute { - node_id: c2, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }, - ], - ); - - assert_eq!(resolve_target("1", &nav_index, root), Some(c1)); - assert_eq!(resolve_target("2", &nav_index, root), Some(c2)); - assert_eq!(resolve_target("3", &nav_index, root), None); - } - - #[test] - fn test_resolve_target_exact() { - use vectorless_document::{ChildRoute, DocumentTree}; - - let mut tree = DocumentTree::new("Root", ""); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "content"); - - let mut nav_index = NavigationIndex::new(); - nav_index.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup".to_string(), - leaf_count: 3, - }], - ); - - assert_eq!( - resolve_target("Getting Started", &nav_index, root), - Some(c1) - ); - } - - #[test] - fn test_resolve_target_case_insensitive() { - use vectorless_document::{ChildRoute, DocumentTree}; - - let mut tree = DocumentTree::new("Root", ""); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "content"); - - let mut nav_index = NavigationIndex::new(); - nav_index.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup".to_string(), - leaf_count: 3, - }], - ); - - assert_eq!( - resolve_target("getting started", &nav_index, root), - Some(c1) - ); - assert_eq!( - resolve_target("GETTING STARTED", &nav_index, root), - Some(c1) - ); - } - - #[test] - fn test_resolve_target_contains() { - use vectorless_document::{ChildRoute, DocumentTree}; - - let mut tree = DocumentTree::new("Root", ""); - let root = tree.root(); - let c1 = tree.add_child(root, "API Reference", "content"); - - let mut nav_index = NavigationIndex::new(); - nav_index.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }], - ); - - assert_eq!(resolve_target("api", &nav_index, root), Some(c1)); - assert_eq!(resolve_target("reference", &nav_index, root), Some(c1)); - } - - #[test] - fn test_resolve_target_no_routes() { - let nav_index = NavigationIndex::new(); - let tree = vectorless_document::DocumentTree::new("Root", ""); - assert!(resolve_target("anything", &nav_index, tree.root()).is_none()); - } - - #[test] - fn test_resolve_target_extended_deep_search() { - use vectorless_document::{ChildRoute, DocumentTree}; - - // root → "Wrapper" → "Research Labs" → "Lab B" - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let wrapper = tree.add_child(root, "Quantum Computing Division", "wrapper"); - let labs = tree.add_child(wrapper, "Research Labs", "labs content"); - let lab_b = tree.add_child(labs, "Lab B", "lab b content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ChildRoute { - node_id: wrapper, - title: "Quantum Computing Division".to_string(), - description: "Division".to_string(), - leaf_count: 7, - }], - ); - nav.add_child_routes( - wrapper, - vec![ChildRoute { - node_id: labs, - title: "Research Labs".to_string(), - description: "Labs".to_string(), - leaf_count: 4, - }], - ); - nav.add_child_routes( - labs, - vec![ChildRoute { - node_id: lab_b, - title: "Lab B".to_string(), - description: "Topological".to_string(), - leaf_count: 1, - }], - ); - - // "Research Labs" is a grandchild of root — deep search should find it - assert_eq!( - resolve_target_extended("Research Labs", &nav, root, &tree), - Some(labs) - ); - - // "Lab B" is a great-grandchild — deep search should find it - assert_eq!( - resolve_target_extended("Lab B", &nav, root, &tree), - Some(lab_b) - ); - - // Direct children should still work via primary resolver - assert_eq!( - resolve_target_extended("Quantum Computing Division", &nav, root, &tree), - Some(wrapper) - ); - } - - #[test] - fn test_parse_grep() { - assert_eq!( - parse_command("grep EBITDA"), - Command::Grep { - pattern: "EBITDA".to_string() - } - ); - assert_eq!( - parse_command("grep revenue.*2024"), - Command::Grep { - pattern: "revenue.*2024".to_string() - } - ); - } - - #[test] - fn test_parse_head() { - assert_eq!( - parse_command("head Installation"), - Command::Head { - target: "Installation".to_string(), - lines: 20 - } - ); - assert_eq!( - parse_command("head -n 5 API Reference"), - Command::Head { - target: "API Reference".to_string(), - lines: 5 - } - ); - } - - #[test] - fn test_parse_findtree() { - assert_eq!( - parse_command("findtree revenue"), - Command::FindTree { - pattern: "revenue".to_string() - } - ); - assert_eq!( - parse_command("findtree API Reference"), - Command::FindTree { - pattern: "API Reference".to_string() - } - ); - } - - #[test] - fn test_parse_wc() { - assert_eq!( - parse_command("wc Installation"), - Command::Wc { - target: "Installation".to_string() - } - ); - assert_eq!( - parse_command("wc API Reference"), - Command::Wc { - target: "API Reference".to_string() - } - ); - } -} diff --git a/vectorless-core/vectorless-agent/src/config.rs b/vectorless-core/vectorless-agent/src/config.rs deleted file mode 100644 index d3b784f..0000000 --- a/vectorless-core/vectorless-agent/src/config.rs +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Configuration and output types for the retrieval agent. - -use serde::{Deserialize, Serialize}; - -// --------------------------------------------------------------------------- -// Worker configuration -// --------------------------------------------------------------------------- - -/// Worker configuration — navigation budget settings. -#[derive(Debug, Clone)] -pub struct WorkerConfig { - /// Maximum navigation rounds per Worker loop (ls/cd/cat/grep/head/find etc.). - /// `check` does NOT count against this budget. - pub max_rounds: u32, - /// Hard cap on total LLM calls per Worker (planning + nav + check). - /// Prevents runaway costs regardless of max_rounds. 0 = no limit. - pub max_llm_calls: u32, -} - -impl Default for WorkerConfig { - fn default() -> Self { - Self { - max_rounds: 100, - max_llm_calls: 200, - } - } -} - -impl WorkerConfig { - pub fn new() -> Self { - Self::default() - } -} - -// --------------------------------------------------------------------------- -// Answer pipeline configuration -// --------------------------------------------------------------------------- - -/// Answer pipeline configuration — synthesis settings. -#[derive(Debug, Clone)] -pub struct AnswerConfig { - /// Maximum number of evidence items to feed into synthesis. - pub evidence_cap: usize, -} - -impl Default for AnswerConfig { - fn default() -> Self { - Self { evidence_cap: 20 } - } -} - -// --------------------------------------------------------------------------- -// Aggregated agent configuration -// --------------------------------------------------------------------------- - -/// Aggregated configuration for the entire retrieval agent system. -#[derive(Debug, Clone, Default)] -pub struct AgentConfig { - pub worker: WorkerConfig, - pub answer: AnswerConfig, -} - -impl AgentConfig { - pub fn new() -> Self { - Self::default() - } -} - -// --------------------------------------------------------------------------- -// Output types -// --------------------------------------------------------------------------- - -/// Agent output — the final result of a retrieval operation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Output { - /// Final synthesized answer. - pub answer: String, - /// Collected evidence from navigation. - pub evidence: Vec, - /// Agent execution metrics. - pub metrics: Metrics, - /// Confidence score (0.0–1.0) — derived from LLM evaluate() result. - pub confidence: f32, - /// Reasoning trace steps collected during agent navigation. - pub trace_steps: Vec, -} - -impl Output { - /// Create an empty output (no evidence found). - pub fn empty() -> Self { - Self { - answer: String::new(), - evidence: Vec::new(), - metrics: Metrics::default(), - confidence: 0.0, - trace_steps: Vec::new(), - } - } -} - -/// A single piece of evidence collected during navigation. -/// -/// Re-exported from [`vectorless_rerank::types::Evidence`]. -pub use vectorless_rerank::types::Evidence; - -/// Agent execution metrics. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct Metrics { - pub rounds_used: u32, - pub llm_calls: u32, - pub nodes_visited: usize, - pub budget_exhausted: bool, - pub plan_generated: bool, - pub check_count: u32, - pub evidence_chars: usize, -} - -/// Step result from the navigation loop. -#[derive(Debug, Clone, PartialEq)] -pub enum Step { - /// Continue to next round with the given feedback. - Continue, - /// Navigation is done, proceed to synthesis. - Done, - /// Forced done due to budget exhaustion or error. - ForceDone(String), -} - -// --------------------------------------------------------------------------- -// Worker output (evidence only, no answer) -// --------------------------------------------------------------------------- - -/// Output from a single Worker — pure evidence, no answer synthesis. -/// Rerank handles all answer generation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkerOutput { - /// Collected evidence from document navigation. - pub evidence: Vec, - /// Worker execution metrics. - pub metrics: WorkerMetrics, - /// Document name this Worker was assigned to. - pub doc_name: String, - /// Reasoning trace steps from this Worker. - pub trace_steps: Vec, -} - -/// Metrics specific to a single Worker's execution. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct WorkerMetrics { - /// Number of navigation rounds used. - pub rounds_used: u32, - /// Number of LLM calls made. - pub llm_calls: u32, - /// Number of distinct nodes visited. - pub nodes_visited: usize, - /// Whether the LLM call budget was exhausted. - pub budget_exhausted: bool, - /// Whether a navigation plan was generated. - pub plan_generated: bool, - /// Number of times `check` was called. - pub check_count: u32, - /// Total characters of collected evidence. - pub evidence_chars: usize, -} - -impl From for Output { - fn from(wo: WorkerOutput) -> Self { - Output { - answer: String::new(), - evidence: wo.evidence, - metrics: Metrics { - rounds_used: wo.metrics.rounds_used, - llm_calls: wo.metrics.llm_calls, - nodes_visited: wo.metrics.nodes_visited, - budget_exhausted: wo.metrics.budget_exhausted, - plan_generated: wo.metrics.plan_generated, - check_count: wo.metrics.check_count, - evidence_chars: wo.metrics.evidence_chars, - }, - confidence: 0.0, - trace_steps: wo.trace_steps, - } - } -} - -// --------------------------------------------------------------------------- -// Scope types -// --------------------------------------------------------------------------- - -/// Scope context — determines which path the dispatcher takes. -/// -/// Both variants go through the Orchestrator. The difference is: -/// - `Specified`: user chose specific documents → skip Orchestrator analysis phase -/// - `Workspace`: user didn't specify → Orchestrator analyzes DocCards to select docs -pub enum Scope<'a> { - /// User specified one or more documents (by doc_id). - /// Orchestrator skips analysis, spawns Workers directly. - Specified(Vec>), - /// Workspace scope — user didn't specify documents. - /// Orchestrator analyzes DocCards and selects relevant ones. - Workspace(WorkspaceContext<'a>), -} - -/// Read-only access to a single document's compile artifacts. -pub struct DocContext<'a> { - /// Document content tree. - pub tree: &'a vectorless_document::DocumentTree, - /// Navigation index (includes DocCard). - pub nav_index: &'a vectorless_document::NavigationIndex, - /// Reasoning index (keyword/topic lookup). - pub reasoning_index: &'a vectorless_document::ReasoningIndex, - /// Document name (for evidence source attribution). - pub doc_name: &'a str, -} - -/// Read-only access to multiple documents' compile artifacts. -pub struct WorkspaceContext<'a> { - /// All available documents. - pub docs: Vec>, -} - -impl<'a> WorkspaceContext<'a> { - /// Create a workspace from a slice of DocContexts. - pub fn new(docs: Vec>) -> Self { - Self { docs } - } - - /// Number of documents in the workspace. - pub fn doc_count(&self) -> usize { - self.docs.len() - } - - /// Whether the workspace has only one document. - pub fn is_single(&self) -> bool { - self.docs.len() == 1 - } -} diff --git a/vectorless-core/vectorless-agent/src/context.rs b/vectorless-core/vectorless-agent/src/context.rs deleted file mode 100644 index f984bd5..0000000 --- a/vectorless-core/vectorless-agent/src/context.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Read-only data access wrappers over compile artifacts. -//! -//! These types provide the agent with structured access to the document's -//! navigation index, content tree, and reasoning index — all read-only. - -use vectorless_document::{ChildRoute, NodeId, TopicEntry}; - -// Re-export from config for convenience -pub use super::config::{DocContext, WorkspaceContext}; - -/// A single hit from a keyword search. -#[derive(Debug, Clone)] -pub struct FindHit { - /// The matched keyword. - pub keyword: String, - /// Topic entries matching the keyword. - pub entries: Vec, -} - -impl<'a> DocContext<'a> { - /// List child routes for a given node. - pub fn ls(&self, node: NodeId) -> Option<&[ChildRoute]> { - self.nav_index.get_child_routes(node) - } - - /// Read the full content of a node. - pub fn cat(&self, node: NodeId) -> Option<&str> { - self.tree.get(node).map(|n| n.content.as_str()) - } - - /// Get the title of a node. - pub fn node_title(&self, node: NodeId) -> Option<&str> { - self.tree.get(node).map(|n| n.title.as_str()) - } - - /// Search for a keyword in the reasoning index. - pub fn find(&self, keyword: &str) -> Option { - self.reasoning_index - .topic_entries(keyword) - .map(|entries| FindHit { - keyword: keyword.to_string(), - entries: entries.to_vec(), - }) - } - - /// Search for multiple keywords, collecting all hits. - pub fn find_all(&self, keywords: &[String]) -> Vec { - keywords.iter().filter_map(|kw| self.find(kw)).collect() - } - - /// Get the root node ID. - pub fn root(&self) -> NodeId { - self.tree.root() - } - - /// Get the document's DocCard, if available. - pub fn doc_card(&self) -> Option<&vectorless_document::DocCard> { - self.nav_index.doc_card() - } - - /// Get the navigation entry for a node (overview, hints, tags). - pub fn nav_entry(&self, node: NodeId) -> Option<&vectorless_document::NavEntry> { - self.nav_index.get_entry(node) - } - - /// Get the summary shortcut (pre-computed overview), if available. - pub fn summary_shortcut(&self) -> Option<&vectorless_document::SummaryShortcut> { - self.reasoning_index.summary_shortcut() - } - - /// Find a top-level section by its title, returning its NodeId. - pub fn find_section(&self, title: &str) -> Option { - self.reasoning_index.find_section(title) - } - - /// Get the parent of a node (by searching the tree). - pub fn parent(&self, node: NodeId) -> Option { - self.tree.parent(node) - } -} - -impl<'a> WorkspaceContext<'a> { - /// Search for a keyword across all documents. - pub fn find_cross(&self, keyword: &str) -> Vec<(usize, FindHit)> { - self.docs - .iter() - .enumerate() - .filter_map(|(idx, doc)| doc.find(keyword).map(|hit| (idx, hit))) - .collect() - } - - /// Search multiple keywords across all documents. - pub fn find_cross_all(&self, keywords: &[String]) -> Vec<(usize, Vec)> { - let mut results: Vec<(usize, Vec)> = Vec::new(); - for (idx, doc) in self.docs.iter().enumerate() { - let hits = doc.find_all(keywords); - if !hits.is_empty() { - results.push((idx, hits)); - } - } - results - } - - /// Get all DocCards for documents that have them. - pub fn doc_cards(&self) -> Vec<(usize, &vectorless_document::DocCard)> { - self.docs - .iter() - .enumerate() - .filter_map(|(idx, doc)| doc.doc_card().map(|card| (idx, card))) - .collect() - } - - /// Get a specific document context by index. - pub fn doc(&self, idx: usize) -> Option<&DocContext<'a>> { - self.docs.get(idx) - } -} diff --git a/vectorless-core/vectorless-agent/src/events.rs b/vectorless-core/vectorless-agent/src/events.rs deleted file mode 100644 index e4575c9..0000000 --- a/vectorless-core/vectorless-agent/src/events.rs +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Agent events — rich, structured visibility into the entire retrieval pipeline. -//! -//! Events are organized by pipeline stage: -//! 1. **Query Understanding** — intent analysis, keyword extraction -//! 2. **Orchestrator** — document selection, dispatch, evaluation, replan -//! 3. **Worker** — navigation, evidence collection, budget management -//! 4. **Answer** — synthesis and fusion -//! -//! The stream terminates with `Completed` or `Error`. - -use serde::Serialize; - -/// An event emitted during agent-based retrieval. -/// -/// Each variant carries the data a client needs to understand what happened, -/// not just that something happened. All events are `Clone + Serialize` so -/// they can be broadcast or persisted. -#[derive(Debug, Clone, Serialize)] -pub enum AgentEvent { - // ── Query Understanding ────────────────────────────────────────── - /// Query understanding started. - QueryUnderstandingStarted { query: String }, - - /// Query understanding completed (intent, keywords, strategy decided). - QueryUnderstandingCompleted { - query: String, - intent: String, - keywords: Vec, - strategy_hint: String, - complexity: String, - }, - - // ── Orchestrator ───────────────────────────────────────────────── - /// Orchestrator started. - OrchestratorStarted { - query: String, - doc_count: usize, - skip_analysis: bool, - }, - - /// Orchestrator is analyzing documents to select which to dispatch. - OrchestratorAnalyzing { - doc_count: usize, - keywords: Vec, - }, - - /// A Worker was dispatched to a document. - WorkerDispatched { - doc_idx: usize, - doc_name: String, - task: String, - focus_keywords: Vec, - }, - - /// A Worker finished its task. - WorkerCompleted { - doc_idx: usize, - doc_name: String, - evidence_count: usize, - rounds_used: u32, - llm_calls: u32, - success: bool, - }, - - /// Cross-doc sufficiency evaluation result. - OrchestratorEvaluated { - sufficient: bool, - evidence_count: usize, - missing_info: Option, - }, - - /// Orchestrator is replanning after insufficient evidence. - OrchestratorReplanning { - reason: String, - evidence_count: usize, - }, - - /// Orchestrator completed. - OrchestratorCompleted { - evidence_count: usize, - total_llm_calls: u32, - dispatch_rounds: u32, - }, - - // ── Worker (per-document navigation) ───────────────────────────── - /// Worker started on a document. - WorkerStarted { - doc_name: String, - task: Option, - max_rounds: u32, - }, - - /// Worker generated a navigation plan. - WorkerPlanGenerated { doc_name: String, plan_len: usize }, - - /// A navigation round completed. - WorkerRound { - doc_name: String, - round: u32, - command: String, - success: bool, - elapsed_ms: u64, - }, - - /// Evidence was collected from a node. - EvidenceCollected { - doc_name: String, - node_title: String, - source_path: String, - content_len: usize, - total_evidence: usize, - }, - - /// Worker sufficiency check result. - WorkerSufficiencyCheck { - doc_name: String, - sufficient: bool, - evidence_count: usize, - missing_info: Option, - }, - - /// Worker re-planned after insufficient check. - WorkerReplan { - doc_name: String, - missing_info: String, - plan_len: usize, - }, - - /// Worker budget warning (stuck or half-budget). - WorkerBudgetWarning { - doc_name: String, - warning_type: String, - round: u32, - }, - - /// Worker completed. - WorkerDone { - doc_name: String, - evidence_count: usize, - rounds_used: u32, - llm_calls: u32, - budget_exhausted: bool, - plan_generated: bool, - }, - - // ── Answer Pipeline ────────────────────────────────────────────── - /// Answer synthesis started. - AnswerStarted { - evidence_count: usize, - multi_doc: bool, - }, - - /// Answer synthesis completed. - AnswerCompleted { - answer_len: usize, - confidence: String, - }, - - // ── Terminal ───────────────────────────────────────────────────── - /// Entire retrieval pipeline completed. - Completed { - evidence_count: usize, - llm_calls: u32, - answer_len: usize, - }, - - /// An error occurred. - Error { stage: String, message: String }, -} - -// --------------------------------------------------------------------------- -// Channel + EventEmitter -// --------------------------------------------------------------------------- - -/// Sender for agent events. -pub(crate) type AgentEventSender = tokio::sync::mpsc::Sender; - -/// Receiver for agent events. -pub type AgentEventReceiver = tokio::sync::mpsc::Receiver; - -/// Create a bounded channel for agent events. -pub(crate) fn channel(bound: usize) -> (AgentEventSender, AgentEventReceiver) { - tokio::sync::mpsc::channel(bound) -} - -/// Default channel bound for agent events. -pub const DEFAULT_AGENT_EVENT_BOUND: usize = 256; - -/// A handle for emitting agent events. -/// -/// Wraps an `mpsc::Sender` and silently drops events if the receiver -/// is closed (no panic on send failure). Cheaply clonable. -#[derive(Clone)] -pub struct EventEmitter { - tx: Option, -} - -impl EventEmitter { - /// Create a new emitter with the given sender. - pub fn new(tx: AgentEventSender) -> Self { - Self { tx: Some(tx) } - } - - /// Create a noop emitter that discards all events. - pub fn noop() -> Self { - Self { tx: None } - } - - /// Emit an event. Silently drops if the receiver is closed. - pub fn emit(&self, event: AgentEvent) { - if let Some(ref tx) = self.tx { - let _ = tx.try_send(event); - } - } - - // ── Query Understanding ── - - pub fn emit_query_understanding_started(&self, query: &str) { - self.emit(AgentEvent::QueryUnderstandingStarted { - query: query.to_string(), - }); - } - - pub fn emit_query_understanding_completed( - &self, - query: &str, - intent: &str, - keywords: &[String], - strategy_hint: &str, - complexity: &str, - ) { - self.emit(AgentEvent::QueryUnderstandingCompleted { - query: query.to_string(), - intent: intent.to_string(), - keywords: keywords.to_vec(), - strategy_hint: strategy_hint.to_string(), - complexity: complexity.to_string(), - }); - } - - // ── Orchestrator ── - - pub fn emit_orchestrator_started(&self, query: &str, doc_count: usize, skip_analysis: bool) { - self.emit(AgentEvent::OrchestratorStarted { - query: query.to_string(), - doc_count, - skip_analysis, - }); - } - - pub fn emit_orchestrator_analyzing(&self, doc_count: usize, keywords: &[String]) { - self.emit(AgentEvent::OrchestratorAnalyzing { - doc_count, - keywords: keywords.to_vec(), - }); - } - - pub fn emit_worker_dispatched( - &self, - doc_idx: usize, - doc_name: &str, - task: &str, - focus_keywords: &[String], - ) { - self.emit(AgentEvent::WorkerDispatched { - doc_idx, - doc_name: doc_name.to_string(), - task: task.to_string(), - focus_keywords: focus_keywords.to_vec(), - }); - } - - pub fn emit_worker_completed( - &self, - doc_idx: usize, - doc_name: &str, - evidence_count: usize, - rounds_used: u32, - llm_calls: u32, - success: bool, - ) { - self.emit(AgentEvent::WorkerCompleted { - doc_idx, - doc_name: doc_name.to_string(), - evidence_count, - rounds_used, - llm_calls, - success, - }); - } - - pub fn emit_orchestrator_evaluated( - &self, - sufficient: bool, - evidence_count: usize, - missing_info: Option<&str>, - ) { - self.emit(AgentEvent::OrchestratorEvaluated { - sufficient, - evidence_count, - missing_info: missing_info.map(|s| s.to_string()), - }); - } - - pub fn emit_orchestrator_replanning(&self, reason: &str, evidence_count: usize) { - self.emit(AgentEvent::OrchestratorReplanning { - reason: reason.to_string(), - evidence_count, - }); - } - - pub fn emit_orchestrator_completed( - &self, - evidence_count: usize, - total_llm_calls: u32, - dispatch_rounds: u32, - ) { - self.emit(AgentEvent::OrchestratorCompleted { - evidence_count, - total_llm_calls, - dispatch_rounds, - }); - } - - // ── Worker ── - - pub fn emit_worker_started(&self, doc_name: &str, task: Option<&str>, max_rounds: u32) { - self.emit(AgentEvent::WorkerStarted { - doc_name: doc_name.to_string(), - task: task.map(|s| s.to_string()), - max_rounds, - }); - } - - pub fn emit_worker_plan_generated(&self, doc_name: &str, plan_len: usize) { - self.emit(AgentEvent::WorkerPlanGenerated { - doc_name: doc_name.to_string(), - plan_len, - }); - } - - pub fn emit_worker_round( - &self, - doc_name: &str, - round: u32, - command: &str, - success: bool, - elapsed_ms: u64, - ) { - self.emit(AgentEvent::WorkerRound { - doc_name: doc_name.to_string(), - round, - command: command.to_string(), - success, - elapsed_ms, - }); - } - - pub fn emit_evidence( - &self, - doc_name: &str, - node_title: &str, - source_path: &str, - content_len: usize, - total: usize, - ) { - self.emit(AgentEvent::EvidenceCollected { - doc_name: doc_name.to_string(), - node_title: node_title.to_string(), - source_path: source_path.to_string(), - content_len, - total_evidence: total, - }); - } - - pub fn emit_worker_sufficiency_check( - &self, - doc_name: &str, - sufficient: bool, - evidence_count: usize, - missing_info: Option<&str>, - ) { - self.emit(AgentEvent::WorkerSufficiencyCheck { - doc_name: doc_name.to_string(), - sufficient, - evidence_count, - missing_info: missing_info.map(|s| s.to_string()), - }); - } - - pub fn emit_worker_replan(&self, doc_name: &str, missing_info: &str, plan_len: usize) { - self.emit(AgentEvent::WorkerReplan { - doc_name: doc_name.to_string(), - missing_info: missing_info.to_string(), - plan_len, - }); - } - - pub fn emit_worker_budget_warning(&self, doc_name: &str, warning_type: &str, round: u32) { - self.emit(AgentEvent::WorkerBudgetWarning { - doc_name: doc_name.to_string(), - warning_type: warning_type.to_string(), - round, - }); - } - - pub fn emit_worker_done( - &self, - doc_name: &str, - evidence_count: usize, - rounds_used: u32, - llm_calls: u32, - budget_exhausted: bool, - plan_generated: bool, - ) { - self.emit(AgentEvent::WorkerDone { - doc_name: doc_name.to_string(), - evidence_count, - rounds_used, - llm_calls, - budget_exhausted, - plan_generated, - }); - } - - // ── Answer ── - - pub fn emit_answer_started(&self, evidence_count: usize, multi_doc: bool) { - self.emit(AgentEvent::AnswerStarted { - evidence_count, - multi_doc, - }); - } - - pub fn emit_answer_completed(&self, answer_len: usize, confidence: &str) { - self.emit(AgentEvent::AnswerCompleted { - answer_len, - confidence: confidence.to_string(), - }); - } - - // ── Terminal ── - - pub fn emit_completed(&self, evidence_count: usize, llm_calls: u32, answer_len: usize) { - self.emit(AgentEvent::Completed { - evidence_count, - llm_calls, - answer_len, - }); - } - - pub fn emit_error(&self, stage: &str, message: &str) { - self.emit(AgentEvent::Error { - stage: stage.to_string(), - message: message.to_string(), - }); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_noop_emitter() { - let emitter = EventEmitter::noop(); - emitter.emit_orchestrator_started("test", 1, false); - emitter.emit_worker_started("doc.md", None, 8); - emitter.emit_worker_round("doc.md", 1, "ls", true, 50); - emitter.emit_worker_done("doc.md", 0, 1, 1, false, false); - emitter.emit_completed(0, 1, 0); - // No panic — events silently dropped - } - - #[test] - fn test_event_roundtrip() { - let (tx, mut rx) = channel(DEFAULT_AGENT_EVENT_BOUND); - let emitter = EventEmitter::new(tx); - - emitter.emit_orchestrator_started("what is X?", 1, true); - emitter.emit_worker_started("doc.md", None, 8); - emitter.emit_evidence("doc.md", "Intro", "root/Intro", 100, 1); - emitter.emit_worker_sufficiency_check("doc.md", true, 1, None); - emitter.emit_worker_done("doc.md", 1, 3, 5, false, true); - emitter.emit_completed(1, 6, 42); - - let events: Vec = (0..6).map(|_| rx.blocking_recv().unwrap()).collect(); - - assert!( - matches!(&events[0], AgentEvent::OrchestratorStarted { query, .. } if query == "what is X?") - ); - assert!( - matches!(&events[1], AgentEvent::WorkerStarted { doc_name, .. } if doc_name == "doc.md") - ); - assert!( - matches!(&events[2], AgentEvent::EvidenceCollected { node_title, .. } if node_title == "Intro") - ); - assert!(matches!( - &events[3], - AgentEvent::WorkerSufficiencyCheck { - sufficient: true, - .. - } - )); - assert!(matches!( - &events[4], - AgentEvent::WorkerDone { - evidence_count: 1, - plan_generated: true, - .. - } - )); - assert!(matches!( - &events[5], - AgentEvent::Completed { - evidence_count: 1, - answer_len: 42, - .. - } - )); - } - - #[test] - fn test_serialization() { - let event = AgentEvent::OrchestratorStarted { - query: "test".to_string(), - doc_count: 3, - skip_analysis: false, - }; - let json = serde_json::to_string(&event).unwrap(); - assert!(json.contains("OrchestratorStarted")); - assert!(json.contains("test")); - } -} diff --git a/vectorless-core/vectorless-agent/src/lib.rs b/vectorless-core/vectorless-agent/src/lib.rs deleted file mode 100644 index a566ab6..0000000 --- a/vectorless-core/vectorless-agent/src/lib.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Retrieval agent — struct-based document intelligence. -//! -//! # Architecture -//! -//! The retrieval dispatcher always goes through the Orchestrator. -//! Based on [`Scope`]: -//! -//! - **User specified doc_ids** → Orchestrator skips analysis, spawns Workers directly. -//! - **Workspace / unspecified** → Orchestrator analyzes DocCards, selects docs, spawns Workers. -//! -//! Both paths produce the same [`Output`] type and share the same synthesis logic. -//! -//! ```text -//! dispatch(query, scope) -//! └── Orchestrator (always) -//! ├── Scope::Specified(docs) → skip analysis → N × Worker → synthesis -//! └── Scope::Workspace(ws) → analysis → N × Worker → fusion → synthesis -//! ``` -//! -//! # Agent trait -//! -//! All retrieval agents implement [`Agent`] with `async fn run(self)` (Edition 2024). -//! The trait uses native async functions — no `async-trait` crate needed. - -pub mod command; -pub mod config; -pub mod context; -pub mod events; -pub mod state; -pub mod tools; - -pub mod orchestrator; -pub mod prompts; -pub mod worker; - -pub use config::{DocContext, Evidence, Output, Scope, WorkspaceContext}; -pub use events::{AgentEvent, EventEmitter}; - -/// Agent trait — async, consuming-self execution. -/// -/// Each agent struct holds its own configuration and context. -/// Calling `run(self)` consumes the agent and produces output. -/// -/// Uses Edition 2024 native `async fn` in trait — no `async-trait` crate. -pub trait Agent { - /// The output type produced by this agent. - type Output; - /// Agent name for logging and events. - fn name(&self) -> &str; - /// Execute the agent, consuming self. - async fn run(self) -> vectorless_error::Result; -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/analyze.rs b/vectorless-core/vectorless-agent/src/orchestrator/analyze.rs deleted file mode 100644 index 2edd961..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/analyze.rs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Phase 1: Analyze documents and produce a dispatch plan. -//! -//! Uses the [`QueryPlan`] from query understanding to inform document selection. -//! LLM errors propagate — no silent degradation. - -use tracing::{debug, info}; - -use vectorless_error::Error; -use vectorless_llm::LlmClient; -use vectorless_query::QueryPlan; -use vectorless_scoring::bm25::extract_keywords; - -use super::super::config::WorkspaceContext; -use super::super::prompts::{DispatchEntry, orchestrator_analysis, parse_dispatch_plan}; -use super::super::state::OrchestratorState; -use super::super::tools::orchestrator as orch_tools; - -/// Outcome of the analyze phase. -pub enum AnalyzeOutcome { - /// Produce dispatch entries for Phase 2. - Proceed { - dispatches: Vec, - llm_calls: u32, - }, - /// Cross-doc search already answered the query. - AlreadyAnswered { llm_calls: u32 }, - /// No relevant documents found. - NoResults { llm_calls: u32 }, -} - -/// Analyze documents and produce a dispatch plan. -/// -/// Uses the [`QueryPlan`] for intent-aware analysis: -/// - Intent and key concepts inform the LLM about what to look for -/// - Complexity hints at how many documents may be needed -/// - Strategy hint guides the analysis approach -/// -/// LLM failures propagate as [`Error::LlmReasoning`] — no fallback. -pub async fn analyze( - query: &str, - ws: &WorkspaceContext<'_>, - state: &mut OrchestratorState, - emitter: &crate::EventEmitter, - skip_analysis: bool, - query_plan: &QueryPlan, - llm: &LlmClient, -) -> vectorless_error::Result { - if skip_analysis { - debug!("Phase 1: skipping (user-specified documents)"); - let dispatches = (0..ws.doc_count()) - .map(|idx| DispatchEntry { - doc_idx: idx, - reason: "User-specified document".to_string(), - task: query.to_string(), - }) - .collect(); - return Ok(AnalyzeOutcome::Proceed { - dispatches, - llm_calls: 0, - }); - } - - debug!( - intent = %query_plan.intent, - complexity = %query_plan.complexity, - strategy = query_plan.strategy_hint, - "Phase 1: analyzing doc cards with query understanding" - ); - - let doc_cards_text = orch_tools::ls_docs(ws).feedback; - let keywords = extract_keywords(query); - let find_text = if keywords.is_empty() { - "(no keywords extracted)".to_string() - } else { - orch_tools::find_cross(&keywords, ws).feedback - }; - - info!(keywords = ?keywords, "Phase 1: analyzing"); - debug!( - doc_cards_len = doc_cards_text.len(), - find_results_len = find_text.len(), - "Phase 1: analysis input" - ); - - // Build analysis prompt enriched with query understanding - let concepts_text = if query_plan.key_concepts.is_empty() { - String::new() - } else { - format!("\nKey concepts: {}", query_plan.key_concepts.join(", ")) - }; - - let strategy_text = if query_plan.strategy_hint.is_empty() { - String::new() - } else { - format!("\nRetrieval strategy: {}", query_plan.strategy_hint) - }; - - let rewritten_text = if query_plan.rewritten.is_empty() { - String::new() - } else { - format!( - "\nRewritten queries for matching: {}", - query_plan.rewritten.join("; ") - ) - }; - - let intent_context = format!( - "\nQuery intent: {} (complexity: {}){concepts_text}{strategy_text}{rewritten_text}", - query_plan.intent, query_plan.complexity, - ); - - let (system, user) = - orchestrator_analysis(&super::super::prompts::OrchestratorAnalysisParams { - query, - doc_cards: &doc_cards_text, - find_results: &find_text, - intent_context: &intent_context, - }); - - let analysis_output = llm.complete(&system, &user).await.map_err(|e| { - emitter.emit_error("orchestrator/analysis", &e.to_string()); - Error::LlmReasoning { - stage: "orchestrator/analysis".to_string(), - detail: format!("LLM call failed: {e}"), - } - })?; - - info!( - response_len = analysis_output.len(), - response = %if analysis_output.len() > 500 { &analysis_output[..500] } else { &analysis_output }, - "Phase 1: analysis LLM response" - ); - - let dispatches = match parse_dispatch_plan(&analysis_output, ws.doc_count()) { - Some(entries) => entries, - None => { - info!("Orchestrator: analysis indicates already answered"); - return Ok(AnalyzeOutcome::AlreadyAnswered { llm_calls: 1 }); - } - }; - - info!( - dispatches = dispatches.len(), - "Phase 1: parsed dispatch plan" - ); - - if dispatches.is_empty() { - return Ok(AnalyzeOutcome::NoResults { llm_calls: 1 }); - } - - state.analyze_done = true; - Ok(AnalyzeOutcome::Proceed { - dispatches, - llm_calls: 1, - }) -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/dispatch.rs b/vectorless-core/vectorless-agent/src/orchestrator/dispatch.rs deleted file mode 100644 index 0916ec4..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/dispatch.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Phase 2: Dispatch Workers and collect results. - -use tracing::{info, warn}; - -use vectorless_llm::LlmClient; - -use super::super::Agent; -use super::super::config::{AgentConfig, WorkspaceContext}; -use super::super::events::EventEmitter; -use super::super::prompts::DispatchEntry; -use super::super::state::OrchestratorState; -use super::super::worker::Worker; -use vectorless_query::QueryPlan; - -/// Dispatch Workers in parallel and collect results. -pub async fn dispatch_and_collect( - query: &str, - dispatches: &[DispatchEntry], - ws: &WorkspaceContext<'_>, - config: &AgentConfig, - llm: &LlmClient, - state: &mut OrchestratorState, - emitter: &EventEmitter, - query_plan: &QueryPlan, -) { - let futures: Vec<_> = dispatches - .iter() - .filter_map(|dispatch| { - let doc = match ws.doc(dispatch.doc_idx) { - Some(d) => d, - None => { - warn!(doc_idx = dispatch.doc_idx, "Document not found, skipping"); - return None; - } - }; - - let query = query.to_string(); - let task = dispatch.task.clone(); - let worker_config = config.worker.clone(); - let doc_idx = dispatch.doc_idx; - let doc_name = doc.doc_name.to_string(); - let llm = llm.clone(); - let sub_emitter = EventEmitter::noop(); - let worker_plan = query_plan.clone(); - - Some(async move { - emitter.emit_worker_dispatched(doc_idx, &doc_name, &task, &[]); - let worker = Worker::new( - &query, - Some(&task), - doc, - worker_config, - llm, - sub_emitter, - worker_plan, - ); - let result = worker.run().await; - (doc_idx, doc_name, result) - }) - }) - .collect(); - - let results: Vec<_> = futures::future::join_all(futures).await; - - for (doc_idx, doc_name, result) in results { - match result { - Ok(output) => { - info!( - doc_idx, - evidence = output.evidence.len(), - "Worker completed" - ); - emitter.emit_worker_completed( - doc_idx, - &doc_name, - output.evidence.len(), - output.metrics.rounds_used, - output.metrics.llm_calls, - true, - ); - state.collect_result(doc_idx, output); - } - Err(e) => { - warn!(doc_idx, error = %e, "Worker failed"); - emitter.emit_worker_completed(doc_idx, &doc_name, 0, 0, 0, false); - } - } - } -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/evaluate.rs b/vectorless-core/vectorless-agent/src/orchestrator/evaluate.rs deleted file mode 100644 index 9b0898e..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/evaluate.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Evaluate cross-document evidence sufficiency via LLM. -//! -//! Replaces the old `integrate` module's heuristic sufficiency check. -//! LLM errors propagate — no silent "assume sufficient" fallback. - -use tracing::info; - -use vectorless_error::Error; -use vectorless_llm::LlmClient; - -use super::super::config::Evidence; -use super::super::prompts::{check_sufficiency, parse_sufficiency_response}; - -/// Result of the evidence sufficiency evaluation. -pub struct EvalResult { - /// Whether the collected evidence is sufficient to answer the query. - pub sufficient: bool, - /// Description of what information is still missing (empty if sufficient). - pub missing_info: String, -} - -/// Evaluate cross-document evidence sufficiency via LLM. -/// -/// Propagates LLM errors as [`Error::LlmReasoning`]. -/// The caller decides how to handle insufficiency (replan, abort, etc.). -pub async fn evaluate( - query: &str, - evidence: &[Evidence], - llm: &LlmClient, -) -> vectorless_error::Result { - let evidence_summary = format_evidence_summary(evidence); - let (system, user) = check_sufficiency(query, &evidence_summary); - - info!( - evidence = evidence.len(), - "Evaluating evidence sufficiency..." - ); - let response = llm - .complete(&system, &user) - .await - .map_err(|e| Error::LlmReasoning { - stage: "orchestrator/evaluate".to_string(), - detail: format!("Sufficiency check LLM call failed: {e}"), - })?; - - let sufficient = parse_sufficiency_response(&response); - let missing_info = if sufficient { - String::new() - } else { - // Extract the reason from the response (everything after SUFFICIENT/INSUFFICIENT) - let reason = response - .trim() - .strip_prefix("INSUFFICIENT") - .or_else(|| response.trim().strip_prefix("Insufficient")) - .unwrap_or("") - .trim_start_matches(|c: char| c == '-' || c == ' ' || c == ':'); - if reason.is_empty() { - "Evidence does not fully address the query.".to_string() - } else { - reason.to_string() - } - }; - - info!( - sufficient, - evidence = evidence.len(), - missing_info_len = missing_info.len(), - "Cross-doc sufficiency evaluation" - ); - - Ok(EvalResult { - sufficient, - missing_info, - }) -} - -/// Format evidence summary for sufficiency check. -/// Includes actual content so the check LLM can evaluate relevance. -pub fn format_evidence_summary(evidence: &[Evidence]) -> String { - if evidence.is_empty() { - return "(no evidence)".to_string(); - } - evidence - .iter() - .map(|e| { - let doc = e.doc_name.as_deref().unwrap_or("unknown"); - format!("[{}] (from {})\n{}", e.node_title, doc, e.content) - }) - .collect::>() - .join("\n\n") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_format_evidence_summary() { - let evidence = vec![ - Evidence { - source_path: "root/A".to_string(), - node_title: "A".to_string(), - content: "content".to_string(), - doc_name: Some("doc1".to_string()), - }, - Evidence { - source_path: "root/B".to_string(), - node_title: "B".to_string(), - content: "more content".to_string(), - doc_name: Some("doc2".to_string()), - }, - ]; - let summary = format_evidence_summary(&evidence); - assert!(summary.contains("[A]")); - assert!(summary.contains("doc1")); - assert!(summary.contains("[B]")); - assert!(summary.contains("doc2")); - } - - #[test] - fn test_format_evidence_summary_empty() { - let summary = format_evidence_summary(&[]); - assert!(summary.contains("no evidence")); - } -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/mod.rs b/vectorless-core/vectorless-agent/src/orchestrator/mod.rs deleted file mode 100644 index e17d1f3..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/mod.rs +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Orchestrator agent — supervisor loop for multi-document retrieval. -//! -//! The Orchestrator is a consuming-self struct implementing [`Agent`]: -//! 1. Analyze: LLM selects documents + tasks (informed by QueryPlan) -//! 2. Supervisor loop: dispatch → evaluate → replan if insufficient -//! 3. Rerank: dedup → BM25 scoring → synthesis/fusion - -mod analyze; -mod dispatch; -mod evaluate; -mod replan; -mod supervisor; - -use tracing::info; - -use vectorless_llm::LlmClient; -use vectorless_query::QueryPlan; - -use super::Agent; -use super::config::{AgentConfig, Output, WorkspaceContext}; -use super::events::EventEmitter; -use super::state::OrchestratorState; - -use analyze::{AnalyzeOutcome, analyze}; -use supervisor::run_supervisor_loop; - -/// Maximum supervisor loop iterations to prevent infinite loops. -const MAX_SUPERVISOR_ITERATIONS: u32 = 3; - -/// Orchestrator agent — coordinates multi-document retrieval. -/// -/// Holds all execution context. Calling [`run()`](Agent::run) consumes self. -pub struct Orchestrator<'a> { - query: String, - ws: &'a WorkspaceContext<'a>, - config: AgentConfig, - llm: LlmClient, - emitter: EventEmitter, - skip_analysis: bool, - /// Query understanding plan — produced by `QueryPipeline::understand()`. - /// Contains intent, complexity, key concepts, and strategy hints. - query_plan: QueryPlan, -} - -impl<'a> Orchestrator<'a> { - /// Create a new Orchestrator. - pub fn new( - query: &str, - ws: &'a WorkspaceContext<'a>, - config: AgentConfig, - llm: LlmClient, - emitter: EventEmitter, - skip_analysis: bool, - query_plan: QueryPlan, - ) -> Self { - Self { - query: query.to_string(), - ws, - config, - llm, - emitter, - skip_analysis, - query_plan, - } - } -} - -impl<'a> Agent for Orchestrator<'a> { - type Output = Output; - - fn name(&self) -> &str { - "orchestrator" - } - - async fn run(self) -> vectorless_error::Result { - let Orchestrator { - query, - ws, - config, - llm, - emitter, - skip_analysis, - query_plan, - } = self; - - info!( - docs = ws.doc_count(), - skip_analysis, - intent = %query_plan.intent, - complexity = %query_plan.complexity, - "Orchestrator starting" - ); - emitter.emit_orchestrator_started(&query, ws.doc_count(), skip_analysis); - - let mut state = OrchestratorState::new(); - let mut orch_llm_calls: u32 = 0; - - // --- Phase 1: Analyze — LLM selects documents + tasks --- - let initial_dispatches = match analyze( - &query, - ws, - &mut state, - &emitter, - skip_analysis, - &query_plan, - &llm, - ) - .await? - { - AnalyzeOutcome::Proceed { - dispatches, - llm_calls, - } => { - orch_llm_calls += llm_calls; - dispatches - } - AnalyzeOutcome::AlreadyAnswered { llm_calls } => { - let mut output = Output::empty(); - output.answer = "Already answered by cross-document search.".to_string(); - emitter.emit_orchestrator_completed(0, orch_llm_calls + llm_calls, 0); - return Ok(output); - } - AnalyzeOutcome::NoResults { llm_calls } => { - emitter.emit_orchestrator_completed(0, orch_llm_calls + llm_calls, 0); - return Ok(Output::empty()); - } - }; - - // --- Phase 2: Supervisor loop --- - let outcome = run_supervisor_loop( - &query, - initial_dispatches, - ws, - &config, - &llm, - &mut state, - &emitter, - &query_plan, - skip_analysis, - ) - .await?; - orch_llm_calls += outcome.llm_calls; - - let confidence = compute_confidence( - outcome.eval_sufficient, - outcome.iteration, - state.all_evidence.is_empty(), - ); - - // --- Phase 3: Finalize — rerank + synthesize --- - if state.all_evidence.is_empty() { - emitter.emit_orchestrator_completed(0, orch_llm_calls, 0); - return Ok(state.into_output(String::new())); - } - - let multi_doc = ws.doc_count() > 1; - finalize_output( - &query, - &state, - &emitter, - orch_llm_calls, - multi_doc, - query_plan.intent, - confidence, - ) - .await - } -} - -/// Compute confidence from LLM evaluate() outcome. -fn compute_confidence(eval_sufficient: bool, replan_rounds: u32, no_evidence: bool) -> f32 { - if no_evidence { - return 0.0; - } - if eval_sufficient { - // LLM said sufficient: first round = 0.95, each replan round drops 0.15 - (0.95 - replan_rounds as f32 * 0.15).max(0.5) - } else { - // LLM never said sufficient (budget exhausted or no more docs) - (0.4 - replan_rounds as f32 * 0.1).max(0.1) - } -} - -/// Rerank evidence and emit completion events. -pub async fn finalize_output( - query: &str, - state: &OrchestratorState, - emitter: &EventEmitter, - orch_llm_calls: u32, - multi_doc: bool, - intent: vectorless_query::QueryIntent, - confidence: f32, -) -> vectorless_error::Result { - let rerank_result = - vectorless_rerank::process(query, &state.all_evidence, multi_doc, intent, confidence) - .await?; - - let total_llm_calls = orch_llm_calls + rerank_result.llm_calls; - if !rerank_result.answer.is_empty() { - emitter.emit_answer_completed(rerank_result.answer.len(), "medium"); - } - - let mut output = state.clone_results_into_output(rerank_result.answer); - output.metrics.llm_calls += total_llm_calls; - output.confidence = rerank_result.confidence; - - emitter.emit_orchestrator_completed( - output.evidence.len(), - output.metrics.llm_calls, - output.metrics.rounds_used, - ); - - info!( - evidence = output.evidence.len(), - llm_calls = output.metrics.llm_calls, - confidence = output.confidence, - "Orchestrator complete" - ); - - Ok(output) -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/replan.rs b/vectorless-core/vectorless-agent/src/orchestrator/replan.rs deleted file mode 100644 index 5694c37..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/replan.rs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Replan: LLM-driven re-dispatch after insufficient evidence. -//! -//! After evaluate() returns insufficient, the Orchestrator replans: -//! the LLM analyzes what's missing and decides which documents to query next. -//! This replaces the old heuristic supplement logic. - -use tracing::info; - -use vectorless_error::Error; -use vectorless_llm::LlmClient; -use vectorless_scoring::bm25::extract_keywords; - -use super::super::config::Evidence; -use super::super::prompts::DispatchEntry; - -/// Result of the replan phase. -pub struct ReplanResult { - /// New dispatch targets for the next round. - pub dispatches: Vec, - /// The LLM's reasoning about what was missing. - pub reasoning: String, -} - -/// Replan dispatch targets based on missing information. -/// -/// The LLM reviews: -/// - The original query -/// - What evidence has been collected so far -/// - What information is still missing -/// - Available documents that haven't been dispatched yet -/// -/// Returns new dispatch targets. LLM errors propagate. -pub async fn replan( - query: &str, - missing_info: &str, - collected_evidence: &[Evidence], - dispatched_indices: &[usize], - total_docs: usize, - doc_cards_text: &str, - llm: &LlmClient, -) -> vectorless_error::Result { - let evidence_summary = format_evidence_context(collected_evidence); - let keywords = extract_keywords(query); - let find_text = if keywords.is_empty() { - String::new() - } else { - format!("\nExtracted keywords: {}", keywords.join(", ")) - }; - - let (system, user) = replan_prompt( - query, - missing_info, - &evidence_summary, - dispatched_indices, - doc_cards_text, - &find_text, - ); - - info!( - evidence = collected_evidence.len(), - "Replanning dispatch targets..." - ); - let response = llm - .complete(&system, &user) - .await - .map_err(|e| Error::LlmReasoning { - stage: "orchestrator/replan".to_string(), - detail: format!("Replan LLM call failed: {e}"), - })?; - - info!( - response_len = response.len(), - "Replan LLM response received" - ); - - let dispatches = parse_replan_response(&response, total_docs, dispatched_indices); - let reasoning = response.lines().take(3).collect::>().join(" "); - - info!( - new_dispatches = dispatches.len(), - "Replan produced new dispatch targets" - ); - - Ok(ReplanResult { - dispatches, - reasoning, - }) -} - -/// Format collected evidence for the replan prompt. -/// Includes content so the LLM can reason about what's actually been found. -fn format_evidence_context(evidence: &[Evidence]) -> String { - if evidence.is_empty() { - return "(no evidence collected)".to_string(); - } - evidence - .iter() - .map(|e| { - let doc = e.doc_name.as_deref().unwrap_or("unknown"); - format!("[{}] (from {})\n{}", e.node_title, doc, e.content) - }) - .collect::>() - .join("\n\n") -} - -/// Build the replan prompt. -fn replan_prompt( - query: &str, - missing_info: &str, - evidence_summary: &str, - dispatched: &[usize], - doc_cards: &str, - keywords_text: &str, -) -> (String, String) { - let dispatched_set: Vec = dispatched - .iter() - .map(|&i| format!("doc {}", i + 1)) - .collect(); - let dispatched_text = if dispatched_set.is_empty() { - "None".to_string() - } else { - dispatched_set.join(", ") - }; - - let system = "You are a multi-document retrieval coordinator. The first round of evidence \ - collection was insufficient to fully answer the query. Review what was collected, \ - what's missing, and decide which additional documents to query. - -Output format — for each additional document to query, output a block: -- doc: - reason: - task: - -Only include documents not yet dispatched. If no additional documents are likely to help, \ -respond with: NO_ADDITIONAL_DOCS" - .to_string(); - - let user = format!( - "Original question: {query} - -Missing information: {missing_info} - -Collected evidence so far: -{evidence_summary} - -Already dispatched documents: {dispatched_text} - -Available documents (all): -{doc_cards}{keywords_text} - -Additional documents to query:" - ); - - (system, user) -} - -/// Parse the replan response into dispatch entries. -fn parse_replan_response( - response: &str, - total_docs: usize, - dispatched: &[usize], -) -> Vec { - let trimmed = response.trim(); - - if trimmed.starts_with("NO_ADDITIONAL_DOCS") { - return Vec::new(); - } - - let mut entries = Vec::new(); - let mut current_doc_idx: Option = None; - let mut current_reason = String::new(); - let mut current_task = String::new(); - - for line in trimmed.lines() { - let line = line.trim(); - - if let Some(rest) = line.strip_prefix("- doc:") { - // Flush previous - if let Some(idx) = current_doc_idx.take() { - entries.push(DispatchEntry { - doc_idx: idx, - reason: std::mem::take(&mut current_reason), - task: std::mem::take(&mut current_task), - }); - } - - let doc_num: usize = rest.trim().trim_end_matches(',').parse().unwrap_or(0); - if doc_num > 0 && doc_num <= total_docs { - let idx = doc_num - 1; - // Only include if not already dispatched - if !dispatched.contains(&idx) { - current_doc_idx = Some(idx); - } - } - } else if let Some(rest) = line.strip_prefix("reason:") { - current_reason = rest.trim().to_string(); - } else if let Some(rest) = line.strip_prefix("task:") { - current_task = rest.trim().to_string(); - } - } - - // Flush last - if let Some(idx) = current_doc_idx { - entries.push(DispatchEntry { - doc_idx: idx, - reason: current_reason, - task: current_task, - }); - } - - entries -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_replan_response_basic() { - let response = "\ -- doc: 3 - reason: May contain the missing financial data - task: Find Q4 revenue figures"; - let entries = parse_replan_response(response, 5, &[0, 1]); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].doc_idx, 2); - assert_eq!(entries[0].task, "Find Q4 revenue figures"); - } - - #[test] - fn test_parse_replan_response_already_dispatched() { - let response = "\ -- doc: 1 - reason: Already queried - task: test"; - let entries = parse_replan_response(response, 3, &[0]); - assert!(entries.is_empty()); // doc 1 (idx 0) already dispatched - } - - #[test] - fn test_parse_replan_response_no_additional() { - let response = "NO_ADDITIONAL_DOCS"; - let entries = parse_replan_response(response, 3, &[0, 1]); - assert!(entries.is_empty()); - } -} diff --git a/vectorless-core/vectorless-agent/src/orchestrator/supervisor.rs b/vectorless-core/vectorless-agent/src/orchestrator/supervisor.rs deleted file mode 100644 index d98dd1a..0000000 --- a/vectorless-core/vectorless-agent/src/orchestrator/supervisor.rs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Phase 2: Supervisor loop — dispatch → evaluate → replan. - -use tracing::info; - -use vectorless_llm::LlmClient; -use vectorless_query::QueryPlan; - -use super::super::config::{AgentConfig, WorkspaceContext}; -use super::super::events::EventEmitter; -use super::super::prompts::DispatchEntry; -use super::super::state::OrchestratorState; -use super::super::tools::orchestrator as orch_tools; -use super::MAX_SUPERVISOR_ITERATIONS; -use super::dispatch; -use super::evaluate::evaluate; -use super::replan::replan; - -/// Outcome of the supervisor loop. -pub struct SupervisorOutcome { - /// Number of replan iterations performed. - pub iteration: u32, - /// Whether the LLM evaluator judged evidence sufficient. - pub eval_sufficient: bool, - /// LLM calls consumed within the supervisor loop itself. - pub llm_calls: u32, -} - -/// Run the supervisor loop: dispatch → evaluate → replan. -/// -/// Returns a [`SupervisorOutcome`] summarizing what happened. -pub async fn run_supervisor_loop( - query: &str, - initial_dispatches: Vec, - ws: &WorkspaceContext<'_>, - config: &AgentConfig, - llm: &LlmClient, - state: &mut OrchestratorState, - emitter: &EventEmitter, - query_plan: &QueryPlan, - skip_analysis: bool, -) -> vectorless_error::Result { - let mut current_dispatches = initial_dispatches; - let mut iteration: u32 = 0; - let mut eval_sufficient = false; - let mut llm_calls: u32 = 0; - - loop { - if iteration >= MAX_SUPERVISOR_ITERATIONS { - info!(iteration, "Supervisor loop budget exhausted"); - break; - } - - // Dispatch current plan - if !current_dispatches.is_empty() { - info!( - docs = current_dispatches.len(), - docs_list = ?current_dispatches.iter().map(|d| d.doc_idx).collect::>(), - iteration, - "Dispatching Workers" - ); - dispatch::dispatch_and_collect( - query, - ¤t_dispatches, - ws, - config, - llm, - state, - emitter, - query_plan, - ) - .await; - } - - // No evidence at all — nothing to evaluate - if state.all_evidence.is_empty() { - info!("No evidence collected from any Worker"); - break; - } - - // Skip evaluation for user-specified documents (no replan needed) - if skip_analysis { - eval_sufficient = !state.all_evidence.is_empty(); - break; - } - - // Evaluate sufficiency - let eval_result = evaluate(query, &state.all_evidence, llm).await?; - llm_calls += 1; - - if eval_result.sufficient { - eval_sufficient = true; - info!( - evidence = state.all_evidence.len(), - iteration, "Evidence sufficient — exiting supervisor loop" - ); - break; - } - - // Insufficient — replan - info!( - evidence = state.all_evidence.len(), - missing = eval_result.missing_info.len(), - iteration, - "Evidence insufficient — replanning" - ); - - let doc_cards_text = orch_tools::ls_docs(ws).feedback; - let replan_result = replan( - query, - &eval_result.missing_info, - &state.all_evidence, - &state.dispatched, - ws.doc_count(), - &doc_cards_text, - llm, - ) - .await?; - llm_calls += 1; - - if replan_result.dispatches.is_empty() { - info!("Replan produced no new dispatches — exiting supervisor loop"); - break; - } - - current_dispatches = replan_result.dispatches; - iteration += 1; - } - - Ok(SupervisorOutcome { - iteration, - eval_sufficient, - llm_calls, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_supervisor_outcome_fields() { - let outcome = SupervisorOutcome { - iteration: 2, - eval_sufficient: true, - llm_calls: 5, - }; - assert_eq!(outcome.iteration, 2); - assert!(outcome.eval_sufficient); - assert_eq!(outcome.llm_calls, 5); - } - - #[test] - fn test_max_iterations_constant() { - assert_eq!(MAX_SUPERVISOR_ITERATIONS, 3); - } -} diff --git a/vectorless-core/vectorless-agent/src/prompts.rs b/vectorless-core/vectorless-agent/src/prompts.rs deleted file mode 100644 index 11d26fb..0000000 --- a/vectorless-core/vectorless-agent/src/prompts.rs +++ /dev/null @@ -1,569 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Prompt templates for the retrieval agent. -//! -//! Prompts for agent-level operations: -//! 1. `worker_navigation` — Worker nav loop, every round -//! 2. `orchestrator_analysis` — Orchestrator Phase 1 -//! 3. `worker_dispatch` — Worker first round (when dispatched by Orchestrator) -//! 4. `check_sufficiency` — evidence sufficiency evaluation -//! -//! Post-processing prompts (answer synthesis, multi-doc fusion) have been -//! moved to `rerank/synthesis.rs` and `rerank/fusion.rs`. - -// --------------------------------------------------------------------------- -// Prompt 1: Worker Navigation (used every round in the nav loop) -// --------------------------------------------------------------------------- - -/// Parameters for the sub-agent navigation prompt. -pub struct NavigationParams<'a> { - pub query: &'a str, - /// Sub-task description (None when Worker is called directly). - pub task: Option<&'a str>, - /// Current breadcrumb path. - pub breadcrumb: &'a str, - /// Summary of collected evidence. - pub evidence_summary: &'a str, - /// Description of what's still missing (empty string if nothing). - pub missing_info: &'a str, - /// Feedback from the last command execution. - pub last_feedback: &'a str, - /// Remaining rounds. - pub remaining: u32, - /// Maximum rounds. - pub max_rounds: u32, - /// ReAct history of recent rounds. - pub history: &'a str, - /// Titles of already-visited nodes. - pub visited_titles: &'a str, - /// Navigation plan from bird's-eye analysis (empty if no plan). - pub plan: &'a str, - /// Query intent context from QueryPlan (e.g. "factual — find specific answer"). - /// Empty string if not available. - pub intent_context: &'a str, - /// Formatted keyword index matches (empty if none). - pub keyword_hints: &'a str, -} - -pub fn worker_navigation(params: &NavigationParams) -> (String, String) { - let query = params.query; - let breadcrumb = params.breadcrumb; - let evidence_summary = params.evidence_summary; - let remaining = params.remaining; - let max_rounds = params.max_rounds; - - let task_section = match params.task { - Some(task) => format!( - "\nYour specific task: {}\n(This is a sub-task for the original query.)", - task - ), - None => String::new(), - }; - - let missing_section = if params.missing_info.is_empty() { - String::new() - } else { - format!("\nPotentially missing info: {}", params.missing_info) - }; - - let last_feedback_section = if params.last_feedback.is_empty() { - String::new() - } else { - format!("\nLast command result:\n{}\n", params.last_feedback) - }; - - let history_section = if params.history == "(no history yet)" { - String::new() - } else { - format!("\nPrevious rounds:\n{}\n", params.history) - }; - - let visited_section = if params.visited_titles == "(none)" { - String::new() - } else { - format!( - "\nAlready visited (do not re-read these): {}", - params.visited_titles - ) - }; - - let plan_section = if params.plan.is_empty() { - String::new() - } else { - format!( - "\nNavigation plan (follow this as guidance, adapt if needed):\n{}\n", - params.plan - ) - }; - - let keyword_section = if params.keyword_hints.is_empty() { - String::new() - } else { - format!("\n{}", params.keyword_hints) - }; - - let intent_section = if params.intent_context.is_empty() { - String::new() - } else { - format!("\nQuery context: {}", params.intent_context) - }; - - let system = format!( - "You are a document navigation assistant. You navigate inside a document to find \ - information that answers the user's question. - -Available commands: -- ls List children at current position (with summaries and leaf counts) -- cd Enter a child node (supports relative paths like Section/Sub and absolute paths like /root/Section) -- cd .. Go back to parent node -- cat Read a child node's content (automatically collected as evidence) -- cat Read the current node's content (useful at leaf nodes) -- head Preview first 20 lines of a node (does NOT collect evidence) -- find Search for a keyword in the document index (also supports multi-word like 'Lab C') -- findtree Search for nodes by title pattern (case-insensitive) -- grep Regex search across all content in current subtree -- wc Show content size (lines, words, chars) -- pwd Show current navigation path -- check Evaluate if collected evidence is sufficient -- done End navigation - -SEARCH STRATEGY (important — follow this priority order): -- When keyword matches are shown below, navigate directly to the highest-weight matched node. \ -Do NOT explore other branches first — the keyword index has already identified the most relevant location. -- When find results include content snippets that answer the question, cd to that node and cat it immediately. -- Use find with the EXACT keyword from the list (single word, \ -not multi-word phrases). Example: if hint shows keyword 'performance' pointing to Performance section, \ -use find performance, NOT find \"performance guide\". -- Use ls only when you have no keyword hints or need to discover the structure of an unknown section. -- Use findtree when you know a section title pattern but not the exact name. - -NAVIGATION EFFICIENCY (critical — every round counts): -- Prefer cd with absolute paths (/root/Section/Subsection) or relative paths (Section/Sub) \ -to reach target nodes in ONE command instead of multiple cd steps. -- Do NOT ls before cd if keyword hints or find results already tell you which node to enter. -- Do NOT cd into nodes one level at a time when you can use a multi-segment path. - -Rules: -- Output exactly ONE command per response, nothing else. -- Content from cat is automatically saved as evidence — don't re-cat the same node. -- Do not cat or cd into nodes you have already visited. -- If the current branch has nothing relevant, use cd .. to go back. -- If you're at the root and no children seem relevant, use done. - -STOPPING RULES (critical — follow these strictly): -- After cat collects evidence, immediately check: does the collected text contain information \ - that answers or relates to the user's question? If YES, output done. Do NOT continue searching. -- Do NOT run grep after cat — cat already collected the full content. grep is for locating \ - content BEFORE cat, not after. -- If ls shows '(no navigation data)' or no children, you are at a leaf node. Use cat to read it \ - or cd .. to go back. Do NOT ls again. -- When remaining rounds are low (≤2), prefer done over exploring new branches." - ); - - let user = format!( - "{last_feedback_section}\ -User question: {query}{task_section}{intent_section} - -Current position: /{breadcrumb} -Collected evidence: -{evidence_summary}{missing_section}{keyword_section}{visited_section}{plan_section} -{history_section} -Remaining rounds: {remaining}/{max_rounds} - -Command:" - ); - - (system, user) -} - -// --------------------------------------------------------------------------- -// Prompt 2: Orchestrator Analysis (multi-doc Phase 1) -// --------------------------------------------------------------------------- - -/// Parameters for the orchestrator analysis prompt. -pub struct OrchestratorAnalysisParams<'a> { - pub query: &'a str, - /// Formatted DocCard listing from ls_docs. - pub doc_cards: &'a str, - /// Formatted cross-document search results. - pub find_results: &'a str, - /// Query understanding context (intent, concepts, strategy, complexity). - pub intent_context: &'a str, -} - -pub fn orchestrator_analysis(params: &OrchestratorAnalysisParams) -> (String, String) { - let doc_cards = params.doc_cards; - let find_results = params.find_results; - let query = params.query; - let intent_context = params.intent_context; - - let system = - "You are a multi-document retrieval coordinator. Analyze the user's question, \ - review the available documents, and decide which documents to search and what to look for in each. - -Output format — for each relevant document, output a block: -- doc: - reason: - task: - -Only include documents that are likely to contain relevant information. -If the cross-document search results already fully answer the question, respond with just: ALREADY_ANSWERED".to_string(); - - let user = format!( - "Available documents: -{doc_cards} - -Cross-document search results: -{find_results} -{intent_context} - -User question: {query} - -Relevant documents:" - ); - - (system, user) -} - -// --------------------------------------------------------------------------- -// Prompt 3: Worker Dispatch (first-round prompt when Orchestrator dispatches) -// --------------------------------------------------------------------------- - -/// Parameters for the dispatch prompt. -pub struct WorkerDispatchParams<'a> { - pub original_query: &'a str, - pub task: &'a str, - pub doc_name: &'a str, - pub breadcrumb: &'a str, -} - -pub fn worker_dispatch(params: &WorkerDispatchParams) -> (String, String) { - let doc_name = params.doc_name; - let original_query = params.original_query; - let task = params.task; - let breadcrumb = params.breadcrumb; - - let system = format!( - "You are a document navigation assistant. You are searching inside the document \ - \"{doc_name}\" for specific information. - -Available commands: ls, cd (supports Section/Sub paths and /root/Section absolute paths), \ -cd .., cat, cat , head , find , findtree , grep , wc , \ -pwd, check, done - -SEARCH STRATEGY: -- Prefer find to jump directly to relevant sections over manual ls→cd exploration. -- When find results include content snippets that answer your task, cd to that node and cat it immediately. -- Use multi-segment paths (e.g. cd Research Labs/Lab A) to reach targets in ONE command. -- Do NOT ls before cd if find results already tell you which node to enter. -- Use findtree when you know a section title pattern but not the exact name. - -Rules: -- Output exactly ONE command per response. -- Content from cat is automatically saved as evidence. -- After cat collects evidence, if it relates to your task, use done immediately. -- Do NOT grep after cat — cat already collected the full content. -- If ls shows no children, use cat to read the current node or cd .. to go back. -- When evidence is sufficient, use done." - ); - - let user = format!( - "Original question: {original_query} -Your task: {task} -Document: {doc_name} -Current position: /{breadcrumb} - -Command:" - ); - - (system, user) -} - -// --------------------------------------------------------------------------- -// Prompt 4: Check (evidence sufficiency evaluation) -// --------------------------------------------------------------------------- - -/// Build the check prompt for LLM-based sufficiency evaluation. -pub fn check_sufficiency(query: &str, evidence_summary: &str) -> (String, String) { - let system = "You evaluate whether collected evidence contains information that can answer or \ - relate to the user's question. The evidence is raw document text — it does not need to be \ - a complete or perfect answer. If the evidence mentions or addresses the key concepts from \ - the question, it is sufficient. - -Respond with ONLY 'SUFFICIENT' or 'INSUFFICIENT' followed by a one-line reason. - -Guidelines: -- If the evidence text contains any information directly related to the question's key terms, \ -respond SUFFICIENT. -- If the evidence is completely unrelated or empty, respond INSUFFICIENT. -- Default to SUFFICIENT unless the evidence is clearly irrelevant." - .to_string(); - - let user = format!( - "Question: {query}\n\n\ - Collected evidence:\n\ - {evidence_summary}\n\n\ - Is this sufficient?" - ); - - (system, user) -} - -// --------------------------------------------------------------------------- -// Dispatch plan parsing -// --------------------------------------------------------------------------- - -/// A single dispatch entry parsed from orchestrator analysis. -#[derive(Debug, Clone)] -pub struct DispatchEntry { - /// Document index (0-based). - pub doc_idx: usize, - /// Why this document was selected. - pub reason: String, - /// What to search for in this document. - pub task: String, -} - -/// Parse the LLM output from orchestrator analysis into dispatch entries. -/// -/// Returns `None` if the response is "ALREADY_ANSWERED". -/// Returns empty vec if no valid dispatch entries found. -pub fn parse_dispatch_plan(llm_output: &str, total_docs: usize) -> Option> { - let trimmed = llm_output.trim(); - - if trimmed.starts_with("ALREADY_ANSWERED") { - return None; - } - - let mut entries = Vec::new(); - let mut current_doc_idx: Option = None; - let mut current_reason = String::new(); - let mut current_task = String::new(); - - for line in trimmed.lines() { - let line = line.trim(); - - if let Some(rest) = line.strip_prefix("- doc:") { - // Flush previous entry - if let Some(idx) = current_doc_idx.take() { - entries.push(DispatchEntry { - doc_idx: idx, - reason: std::mem::take(&mut current_reason), - task: std::mem::take(&mut current_task), - }); - } - - let doc_num: usize = rest.trim().trim_end_matches(',').parse().unwrap_or(0); - if doc_num > 0 && doc_num <= total_docs { - current_doc_idx = Some(doc_num - 1); // Convert to 0-based - } else if doc_num > 0 { - tracing::warn!( - requested_doc = doc_num, - total_docs, - "Dispatch plan references out-of-range document, skipping" - ); - } - } else if let Some(rest) = line.strip_prefix("reason:") { - current_reason = rest.trim().to_string(); - } else if let Some(rest) = line.strip_prefix("task:") { - current_task = rest.trim().to_string(); - } - } - - // Flush last entry - if let Some(idx) = current_doc_idx { - entries.push(DispatchEntry { - doc_idx: idx, - reason: current_reason, - task: current_task, - }); - } - - Some(entries) -} - -/// Parse the sufficiency check response. -pub fn parse_sufficiency_response(response: &str) -> bool { - let upper = response.trim().to_uppercase(); - upper.starts_with("SUFFICIENT") && !upper.starts_with("INSUFFICIENT") -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_worker_navigation_without_task() { - let params = NavigationParams { - query: "What is the revenue?", - task: None, - breadcrumb: "root/Financial Statements", - evidence_summary: "- [Revenue] 200 chars", - missing_info: "2024 comparison", - last_feedback: "[1] Q1 Report — Q1 data (5 leaves)\n[2] Q2 Report — Q2 data (5 leaves)", - remaining: 5, - max_rounds: 15, - history: "(no history yet)", - visited_titles: "(none)", - plan: "", - intent_context: "", - keyword_hints: "", - }; - - let (system, user) = worker_navigation(¶ms); - assert!(system.contains("document navigation")); - assert!(system.contains("SEARCH STRATEGY")); - assert!(user.contains("What is the revenue?")); - assert!(user.contains("root/Financial Statements")); - assert!(user.contains("200 chars")); - assert!(user.contains("2024 comparison")); - assert!(user.contains("5/15")); - assert!(!user.contains("sub-task")); - } - - #[test] - fn test_worker_navigation_with_keyword_hints() { - let params = NavigationParams { - query: "What is the revenue?", - task: None, - breadcrumb: "root", - evidence_summary: "(none)", - missing_info: "", - last_feedback: "", - remaining: 8, - max_rounds: 15, - history: "(no history yet)", - visited_titles: "(none)", - plan: "", - intent_context: "", - keyword_hints: "Keyword matches (use find to jump directly):\n - 'revenue' → root > Revenue (weight 0.85)\n", - }; - - let (_, user) = worker_navigation(¶ms); - assert!(user.contains("revenue")); - assert!(user.contains("find")); - } - - #[test] - fn test_worker_navigation_with_task() { - let params = NavigationParams { - query: "Compare 2024 and 2023 revenue", - task: Some("Find revenue data in this document"), - breadcrumb: "root", - evidence_summary: "(none)", - missing_info: "", - last_feedback: "", - remaining: 8, - max_rounds: 15, - history: "(no history yet)", - visited_titles: "(none)", - plan: "", - intent_context: "analytical — comparative analysis", - keyword_hints: "", - }; - - let (_, user) = worker_navigation(¶ms); - assert!(user.contains("Find revenue data")); - assert!(user.contains("sub-task")); - } - - #[test] - fn test_orchestrator_analysis() { - let params = OrchestratorAnalysisParams { - query: "Compare 2024 and 2023 revenue", - doc_cards: "[1] 2024 Report\n[2] 2023 Report", - find_results: "doc 1: keyword 'revenue' matched", - intent_context: "\nQuery intent: analytical (complexity: moderate)", - }; - - let (system, user) = orchestrator_analysis(¶ms); - assert!(system.contains("multi-document")); - assert!(user.contains("2024 Report")); - assert!(user.contains("revenue")); - assert!(user.contains("analytical")); - } - - #[test] - fn test_worker_dispatch() { - let params = WorkerDispatchParams { - original_query: "Compare revenue", - task: "Find 2024 revenue figures", - doc_name: "2024 Annual Report", - breadcrumb: "root", - }; - - let (system, user) = worker_dispatch(¶ms); - assert!(system.contains("2024 Annual Report")); - assert!(user.contains("Compare revenue")); - assert!(user.contains("Find 2024 revenue")); - } - - #[test] - fn test_check_sufficiency() { - let (system, user) = check_sufficiency("What is X?", "- [A] some data"); - assert!(system.contains("SUFFICIENT")); - assert!(user.contains("What is X?")); - } - - // --- Dispatch plan parsing --- - - #[test] - fn test_parse_dispatch_plan_basic() { - let output = "\ -- doc: 1 - reason: Contains revenue data - task: Find 2024 revenue figures -- doc: 2 - reason: Contains comparison data - task: Find 2023 revenue figures"; - - let entries = parse_dispatch_plan(output, 3).unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].doc_idx, 0); - assert_eq!(entries[0].task, "Find 2024 revenue figures"); - assert_eq!(entries[1].doc_idx, 1); - assert_eq!(entries[1].reason, "Contains comparison data"); - } - - #[test] - fn test_parse_dispatch_plan_already_answered() { - let output = "ALREADY_ANSWERED"; - assert!(parse_dispatch_plan(output, 3).is_none()); - } - - #[test] - fn test_parse_dispatch_plan_empty() { - let entries = parse_dispatch_plan("no relevant documents", 3).unwrap(); - assert!(entries.is_empty()); - } - - #[test] - fn test_parse_dispatch_plan_out_of_range() { - let output = "\ -- doc: 99 - reason: test - task: test"; - - let entries = parse_dispatch_plan(output, 3).unwrap(); - assert!(entries.is_empty()); // doc 99 is out of range, skipped - } - - // --- Sufficiency parsing --- - - #[test] - fn test_parse_sufficiency_sufficient() { - assert!(parse_sufficiency_response("SUFFICIENT - we have all data")); - assert!(parse_sufficiency_response("Sufficient")); - } - - #[test] - fn test_parse_sufficiency_insufficient() { - assert!(!parse_sufficiency_response("INSUFFICIENT - missing data")); - assert!(!parse_sufficiency_response("Insufficient")); - } -} diff --git a/vectorless-core/vectorless-agent/src/state.rs b/vectorless-core/vectorless-agent/src/state.rs deleted file mode 100644 index d2d3029..0000000 --- a/vectorless-core/vectorless-agent/src/state.rs +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Agent state types — mutable state that lives within a single retrieve() call. - -use std::collections::HashSet; - -use vectorless_document::NodeId; -use vectorless_document::TraceStep; - -use super::config::{Evidence, Output}; - -// --------------------------------------------------------------------------- -// Worker state -// --------------------------------------------------------------------------- - -/// Mutable navigation state for a Worker loop. -/// -/// Created at loop start, destroyed at loop end. Never escapes the call. -pub struct WorkerState { - /// Navigation breadcrumb (path from root to current node). - pub breadcrumb: Vec, - /// Current position in the document tree. - pub current_node: NodeId, - /// Collected evidence so far. - pub evidence: Vec, - /// Nodes already visited (prevents redundant reads). - pub visited: HashSet, - /// Nodes whose content has been collected via cat. Separate from visited - /// because cd-ing through a node ≠ reading its content. - pub collected_nodes: HashSet, - /// Remaining navigation rounds. - pub remaining: u32, - /// Maximum rounds (for display in prompts). - pub max_rounds: u32, - /// Feedback from the last executed command (injected into next prompt). - pub last_feedback: String, - /// Structured description of what information is still missing. - /// Updated after `check` returns "insufficient". - pub missing_info: String, - /// ReAct history: summary of each round's command + result. - /// Keeps last N entries for prompt injection. - pub history: Vec, - /// Navigation plan generated after bird's-eye view (Phase 1.5). - /// Injected into subsequent prompts as guidance (non-binding). - pub plan: String, - /// Number of times `check` has been called. - pub check_count: u32, - /// Whether a navigation plan was generated in Phase 1.5. - pub plan_generated: bool, - /// Reasoning trace steps collected during navigation. - pub trace_steps: Vec, -} - -/// Maximum number of history entries to keep for prompt injection. -const MAX_HISTORY_ENTRIES: usize = 6; - -impl WorkerState { - /// Create a new state starting at the given root node. - pub fn new(root: NodeId, max_rounds: u32) -> Self { - Self { - breadcrumb: vec!["root".to_string()], - current_node: root, - evidence: Vec::new(), - visited: HashSet::new(), - collected_nodes: HashSet::new(), - remaining: max_rounds, - max_rounds, - last_feedback: String::new(), - missing_info: String::new(), - history: Vec::new(), - plan: String::new(), - check_count: 0, - plan_generated: false, - trace_steps: Vec::new(), - } - } - - /// Consume the remaining rounds. - pub fn dec_round(&mut self) { - if self.remaining > 0 { - self.remaining -= 1; - } - } - - /// Set feedback from tool execution. - pub fn set_feedback(&mut self, feedback: String) { - self.last_feedback = feedback; - } - - /// Navigate into a child node. - pub fn cd(&mut self, node: NodeId, title: &str) { - self.breadcrumb.push(title.to_string()); - self.current_node = node; - } - - /// Navigate back to parent. - /// - /// Returns `false` if already at root. - pub fn cd_up(&mut self, parent: NodeId) -> bool { - if self.breadcrumb.len() <= 1 { - return false; - } - self.breadcrumb.pop(); - self.current_node = parent; - true - } - - /// Add a piece of evidence. - pub fn add_evidence(&mut self, evidence: Evidence) { - self.evidence.push(evidence); - } - - /// Check if evidence has already been collected for a specific node. - pub fn has_evidence_for(&self, node_id: vectorless_document::NodeId) -> bool { - self.collected_nodes.contains(&node_id) - } - - /// Push a history entry (command + result summary). - /// Keeps only the last `MAX_HISTORY_ENTRIES` entries. - pub fn push_history(&mut self, entry: String) { - if self.history.len() >= MAX_HISTORY_ENTRIES { - self.history.remove(0); - } - self.history.push(entry); - } - - /// Format history as text for prompt injection. - pub fn history_text(&self) -> String { - if self.history.is_empty() { - return "(no history yet)".to_string(); - } - self.history - .iter() - .enumerate() - .map(|(i, h)| format!("{}. {}", i + 1, h)) - .collect::>() - .join("\n") - } - - /// Format the breadcrumb as a path string (e.g., "root/Chapter 1/Section 1.2"). - pub fn path_str(&self) -> String { - self.breadcrumb.join("/") - } - - /// Summary of collected evidence for prompts. - pub fn evidence_summary(&self) -> String { - if self.evidence.is_empty() { - return "(none)".to_string(); - } - self.evidence - .iter() - .map(|e| format!("- [{}] {} chars", e.node_title, e.content.len())) - .collect::>() - .join("\n") - } - - /// Evidence with actual content for sufficiency evaluation. - pub fn evidence_for_check(&self) -> String { - if self.evidence.is_empty() { - return "(no evidence collected yet)".to_string(); - } - self.evidence - .iter() - .map(|e| format!("[{}]\n{}", e.node_title, e.content)) - .collect::>() - .join("\n\n") - } - - /// Convert this state into a WorkerOutput (consuming the state), with budget flag. - /// Worker returns evidence only — no answer synthesis. - pub fn into_worker_output( - self, - llm_calls: u32, - budget_exhausted: bool, - doc_name: &str, - ) -> super::config::WorkerOutput { - let evidence_chars: usize = self.evidence.iter().map(|e| e.content.len()).sum(); - super::config::WorkerOutput { - evidence: self.evidence, - metrics: super::config::WorkerMetrics { - rounds_used: self.max_rounds.saturating_sub(self.remaining), - llm_calls, - nodes_visited: self.visited.len(), - budget_exhausted, - plan_generated: self.plan_generated, - check_count: self.check_count, - evidence_chars, - }, - doc_name: doc_name.to_string(), - trace_steps: self.trace_steps, - } - } -} - -// --------------------------------------------------------------------------- -// Orchestrator state -// --------------------------------------------------------------------------- - -/// Mutable state for the Orchestrator loop. -/// -/// Tracks which documents have been dispatched and collects Worker results. -pub struct OrchestratorState { - /// Indices of documents that have been dispatched. - pub dispatched: Vec, - /// Results returned by dispatched Workers. - pub sub_results: Vec, - /// All evidence merged from sub-results. - pub all_evidence: Vec, - /// Whether the analysis phase is complete. - pub analyze_done: bool, - /// Total LLM calls across orchestrator + sub-agents. - pub total_llm_calls: u32, -} - -impl OrchestratorState { - /// Create a new orchestrator state. - pub fn new() -> Self { - Self { - dispatched: Vec::new(), - sub_results: Vec::new(), - all_evidence: Vec::new(), - analyze_done: false, - total_llm_calls: 0, - } - } - - /// Record a dispatch to document at the given index. - pub fn record_dispatch(&mut self, doc_idx: usize) { - if !self.dispatched.contains(&doc_idx) { - self.dispatched.push(doc_idx); - } - } - - /// Collect a Worker result, converting WorkerOutput to Output for internal tracking. - pub fn collect_result(&mut self, doc_idx: usize, result: super::config::WorkerOutput) { - self.total_llm_calls += result.metrics.llm_calls; - self.all_evidence.extend(result.evidence.iter().cloned()); - self.sub_results.push(result.into()); - self.record_dispatch(doc_idx); - } - - /// Clone results into an Output without consuming self. - /// - /// Used by `finalize_output` which needs to borrow state for rerank. - pub fn clone_results_into_output(&self, answer: String) -> Output { - Output { - answer, - evidence: self.all_evidence.clone(), - metrics: super::config::Metrics { - llm_calls: self.total_llm_calls, - nodes_visited: self - .sub_results - .iter() - .map(|r| r.metrics.nodes_visited) - .sum(), - plan_generated: self.sub_results.iter().any(|r| r.metrics.plan_generated), - check_count: self.sub_results.iter().map(|r| r.metrics.check_count).sum(), - evidence_chars: self - .sub_results - .iter() - .map(|r| r.metrics.evidence_chars) - .sum(), - ..Default::default() - }, - confidence: 0.0, - trace_steps: self.collect_trace_steps(), - } - } - - /// Merge all sub-results into a single Output (consuming self). - pub fn into_output(self, answer: String) -> Output { - let trace_steps = self.collect_trace_steps(); - Output { - answer, - evidence: self.all_evidence, - metrics: super::config::Metrics { - llm_calls: self.total_llm_calls, - nodes_visited: self - .sub_results - .iter() - .map(|r| r.metrics.nodes_visited) - .sum(), - plan_generated: self.sub_results.iter().any(|r| r.metrics.plan_generated), - check_count: self.sub_results.iter().map(|r| r.metrics.check_count).sum(), - evidence_chars: self - .sub_results - .iter() - .map(|r| r.metrics.evidence_chars) - .sum(), - ..Default::default() - }, - confidence: 0.0, - trace_steps, - } - } - - /// Collect trace steps from all sub-results. - fn collect_trace_steps(&self) -> Vec { - let mut steps = Vec::new(); - for result in &self.sub_results { - steps.extend(result.trace_steps.iter().cloned()); - } - steps - } -} - -impl Default for OrchestratorState { - fn default() -> Self { - Self::new() - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/common.rs b/vectorless-core/vectorless-agent/src/tools/common.rs deleted file mode 100644 index 740510d..0000000 --- a/vectorless-core/vectorless-agent/src/tools/common.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Common tools shared between Orchestrator and Worker (find, check, done). - -use super::ToolResult; - -/// Execute a `find` command — search for a keyword. -/// -/// Returns formatted search results as feedback text. -pub fn format_find_result(keyword: &str, hits: &[super::super::context::FindHit]) -> String { - if hits.is_empty() { - return format!("No results found for '{}'", keyword); - } - - let mut output = format!("Results for '{}':\n", keyword); - for hit in hits { - for entry in &hit.entries { - output.push_str(&format!( - " - node (depth {}, weight {:.2})\n", - entry.depth, entry.weight - )); - } - } - output -} - -/// Execute a `check` command — evaluate evidence sufficiency. -/// -/// Returns a formatted summary of current evidence for the LLM to evaluate. -pub fn format_check_prompt(evidence_summary: &str, query: &str) -> String { - format!( - "Please evaluate whether the collected evidence is sufficient to answer the query.\n\n\ - Query: {}\n\n\ - Evidence:\n{}\n\n\ - Is this sufficient? Answer YES or NO and briefly explain.", - query, evidence_summary - ) -} - -/// Execute a `done` command — signal loop termination. -pub fn format_done() -> ToolResult { - ToolResult::done("Navigation complete.") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_format_find_result_empty() { - let result = format_find_result("nonexistent", &[]); - assert!(result.contains("No results")); - } - - #[test] - fn test_format_check_prompt() { - let prompt = format_check_prompt("- [Intro] 500 chars", "What is X?"); - assert!(prompt.contains("What is X?")); - assert!(prompt.contains("500 chars")); - } - - #[test] - fn test_format_done() { - let result = format_done(); - assert!(result.should_stop); - assert!(result.success); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/mod.rs b/vectorless-core/vectorless-agent/src/tools/mod.rs deleted file mode 100644 index c44c002..0000000 --- a/vectorless-core/vectorless-agent/src/tools/mod.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Tool definitions for the retrieval agent. -//! -//! Tools are organized by role: -//! - `common` — shared between Orchestrator and Worker (find, check, done) -//! - `worker` — Worker-specific (ls, cd, cd_up, cat, pwd) -//! - `orchestrator` — Orchestrator-specific (ls_docs, find_cross, dispatch) - -pub mod common; -pub mod orchestrator; -pub mod worker; - -/// Result of executing a tool command. -#[derive(Debug, Clone)] -pub struct ToolResult { - /// Text feedback to include in the next LLM prompt. - pub feedback: String, - /// Whether the loop should stop. - pub should_stop: bool, - /// Whether the command executed successfully. - pub success: bool, -} - -impl ToolResult { - /// Create a successful result with feedback. - pub fn ok(feedback: impl Into) -> Self { - Self { - feedback: feedback.into(), - should_stop: false, - success: true, - } - } - - /// Create a result that signals loop termination. - pub fn done(feedback: impl Into) -> Self { - Self { - feedback: feedback.into(), - should_stop: true, - success: true, - } - } - - /// Create a failed result (parse error, invalid target, etc.). - pub fn fail(feedback: impl Into) -> Self { - Self { - feedback: feedback.into(), - should_stop: false, - success: false, - } - } -} - -/// Extract a content snippet around the first occurrence of `keyword`. -/// -/// Returns `None` if the content is empty. If the keyword is not found, -/// returns the beginning of the content instead. -pub fn content_snippet(content: &str, keyword: &str, max_len: usize) -> Option { - if content.trim().is_empty() { - return None; - } - - let keyword_lower = keyword.to_lowercase(); - let content_lower = content.to_lowercase(); - - let start = match content_lower.find(&keyword_lower) { - Some(pos) => { - let back = (max_len / 4).min(pos); - pos - back - } - None => 0, - }; - - let start = content - .char_indices() - .find(|(i, _)| *i >= start) - .map(|(i, _)| i) - .unwrap_or(0); - - let end = content - .char_indices() - .take_while(|(i, _)| *i <= start + max_len) - .last() - .map(|(i, c)| i + c.len_utf8()) - .unwrap_or(content.len()); - - let snippet = content[start..end].trim(); - if snippet.is_empty() { - return None; - } - - let mut result = snippet.to_string(); - if end < content.len() { - result.push_str("..."); - } - if start > 0 { - result = format!("...{}", result); - } - Some(result) -} diff --git a/vectorless-core/vectorless-agent/src/tools/orchestrator.rs b/vectorless-core/vectorless-agent/src/tools/orchestrator.rs deleted file mode 100644 index 4b3d72a..0000000 --- a/vectorless-core/vectorless-agent/src/tools/orchestrator.rs +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Orchestrator tools: ls_docs, find_cross, dispatch. - -use super::ToolResult; -use crate::config::WorkspaceContext; - -/// Execute `ls_docs` — list all document cards. -/// -/// Returns a formatted view of all DocCards for the Orchestrator's Bird's-Eye View. -pub fn ls_docs(ctx: &WorkspaceContext) -> ToolResult { - let cards = ctx.doc_cards(); - - if cards.is_empty() { - return ToolResult::ok("No documents with DocCards available."); - } - - let mut output = format!("Available documents ({} total):\n\n", ctx.doc_count()); - - for (idx, card) in &cards { - output.push_str(&format!( - "[{}] {} — {}\n", - idx + 1, - card.title, - card.overview - )); - - for sec in &card.sections { - output.push_str(&format!( - " → {} ({} leaves)\n", - sec.title, sec.leaf_count - )); - } - - if !card.question_hints.is_empty() { - output.push_str(&format!( - " Can answer: {}\n", - card.question_hints.join(", ") - )); - } - - if !card.topic_tags.is_empty() { - output.push_str(&format!(" Topics: {}\n", card.topic_tags.join(", "))); - } - - output.push('\n'); - } - - // Also mention docs without cards - let with_cards: Vec = cards.iter().map(|(idx, _)| *idx).collect(); - let without_cards: Vec = (0..ctx.doc_count()) - .filter(|i| !with_cards.contains(i)) - .collect(); - - if !without_cards.is_empty() { - output.push_str(&format!( - "Documents without DocCards: {:?}\n", - without_cards - .iter() - .map(|i| format!("doc_{}", i)) - .collect::>() - )); - } - - ToolResult::ok(output) -} - -/// Execute `find_cross` — search keywords across all documents. -/// -/// Returns formatted results showing which documents matched, with content snippets. -pub fn find_cross(keywords: &[String], ctx: &WorkspaceContext) -> ToolResult { - let results = ctx.find_cross_all(keywords); - - if results.is_empty() { - return ToolResult::ok(format!( - "No matches found for keywords: {}", - keywords.join(", ") - )); - } - - let mut output = String::new(); - for (doc_idx, hits) in &results { - let doc = ctx.doc(*doc_idx); - let doc_name = doc.map(|d| d.doc_name).unwrap_or("unknown"); - output.push_str(&format!("Document [{}] {}:\n", doc_idx + 1, doc_name)); - - for hit in hits { - for entry in &hit.entries { - let title = doc - .and_then(|d| d.node_title(entry.node_id)) - .unwrap_or("unknown"); - output.push_str(&format!( - " keyword '{}' → {} (depth {}, weight {:.2})", - hit.keyword, title, entry.depth, entry.weight - )); - // Include content snippet for cross-doc relevance judgment - if let Some(content) = doc.and_then(|d| d.cat(entry.node_id)) { - if let Some(snippet) = super::content_snippet(content, &hit.keyword, 300) { - output.push_str(&format!("\n \"{}\"", snippet)); - } - } - output.push('\n'); - } - } - output.push('\n'); - } - - ToolResult::ok(output) -} - -#[cfg(test)] -mod tests { - use super::*; - use vectorless_document::{DocCard, NavigationIndex, ReasoningIndex, SectionCard}; - - fn build_workspace() -> ( - Vec, - Vec, - Vec, - ) { - let tree1 = vectorless_document::DocumentTree::new("2024 Report", "content"); - let mut nav1 = NavigationIndex::new(); - nav1.set_doc_card(DocCard { - title: "2024 Financial Report".to_string(), - overview: "Annual financial statements".to_string(), - question_hints: vec!["Revenue?".to_string()], - topic_tags: vec!["finance".to_string(), "2024".to_string()], - sections: vec![SectionCard { - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 5, - }], - total_leaves: 10, - }); - - let tree2 = vectorless_document::DocumentTree::new("2023 Report", "content"); - let mut nav2 = NavigationIndex::new(); - nav2.set_doc_card(DocCard { - title: "2023 Financial Report".to_string(), - overview: "Previous year financial statements".to_string(), - question_hints: vec!["Sales?".to_string()], - topic_tags: vec!["finance".to_string(), "2023".to_string()], - sections: vec![SectionCard { - title: "Net Sales".to_string(), - description: "Net sales figures".to_string(), - leaf_count: 4, - }], - total_leaves: 8, - }); - - ( - vec![tree1, tree2], - vec![nav1, nav2], - vec![ReasoningIndex::default(), ReasoningIndex::default()], - ) - } - - #[test] - fn test_ls_docs_shows_cards() { - let (trees, navs, ridxs) = build_workspace(); - let docs = vec![ - crate::config::DocContext { - tree: &trees[0], - nav_index: &navs[0], - reasoning_index: &ridxs[0], - doc_name: "2024", - }, - crate::config::DocContext { - tree: &trees[1], - nav_index: &navs[1], - reasoning_index: &ridxs[1], - doc_name: "2023", - }, - ]; - let ctx = WorkspaceContext::new(docs); - - let result = ls_docs(&ctx); - assert!(result.success); - assert!(result.feedback.contains("2024 Financial Report")); - assert!(result.feedback.contains("2023 Financial Report")); - assert!(result.feedback.contains("Revenue")); - assert!(result.feedback.contains("finance")); - } - - #[test] - fn test_ls_docs_empty() { - let tree = vectorless_document::DocumentTree::new("Empty", ""); - let nav = NavigationIndex::new(); - let ridx = ReasoningIndex::default(); - let docs = vec![crate::config::DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &ridx, - doc_name: "empty", - }]; - let ctx = WorkspaceContext::new(docs); - - let result = ls_docs(&ctx); - assert!(result.success); - assert!(result.feedback.contains("No documents with DocCards")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/cat.rs b/vectorless-core/vectorless-agent/src/tools/worker/cat.rs deleted file mode 100644 index e4aeb05..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/cat.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `cat` — read node content and collect as evidence. - -use crate::command; -use crate::config::{DocContext, Evidence}; -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `cat ` — read node content and collect as evidence. -/// -/// Special targets: -/// - `cat .` or `cat` (no arg) reads the current node's content. -/// - Otherwise resolves the target to a child node by name. -pub fn cat(target: &str, ctx: &DocContext, state: &mut WorkerState) -> ToolResult { - let node_id = if target == "." || target.is_empty() { - state.current_node - } else { - match command::resolve_target_extended(target, ctx.nav_index, state.current_node, ctx.tree) - { - Some(id) => id, - None => { - return ToolResult::fail(format!( - "Target '{}' not found. Use 'ls' to see children, or 'cat .' to read current node.", - target - )); - } - } - }; - - if state.has_evidence_for(node_id) { - let title = ctx.node_title(node_id).unwrap_or("unknown"); - return ToolResult::ok(format!( - "[Already collected: {}]. Use a different target or cd to another branch.", - title - )); - } - - match ctx.cat(node_id) { - Some(content) => { - let title = ctx.node_title(node_id).unwrap_or("unknown").to_string(); - let content_string = content.to_string(); - - state.add_evidence(Evidence { - source_path: format!("{}/{}", state.path_str(), title), - node_title: title.clone(), - content: content_string.clone(), - doc_name: Some(ctx.doc_name.to_string()), - }); - - state.collected_nodes.insert(node_id); - state.visited.insert(node_id); - - ToolResult::ok(format!( - "[Evidence collected: {}]\n{}", - title, content_string - )) - } - None => ToolResult::fail(format!("No content available for '{}'.", target)), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_test_tree() -> (DocumentTree, NavigationIndex, NodeId, NodeId, NodeId) { - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "gs content"); - let c2 = tree.add_child(root, "API Reference", "api content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup guide".to_string(), - leaf_count: 3, - }, - ChildRoute { - node_id: c2, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }, - ], - ); - - (tree, nav, root, c1, c2) - } - - #[test] - fn test_cat_collects_evidence() { - let (tree, nav, root, _, _) = build_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - - let result = cat("Getting Started", &ctx, &mut state); - assert!(result.success); - assert!(result.feedback.contains("Evidence collected")); - assert_eq!(state.evidence.len(), 1); - assert_eq!(state.evidence[0].content, "gs content"); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/cd.rs b/vectorless-core/vectorless-agent/src/tools/worker/cd.rs deleted file mode 100644 index 14972ab..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/cd.rs +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `cd`, `cd_absolute`, `cd_up` — navigation commands. - -use crate::command; -use crate::config::DocContext; -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `cd ` — navigate into a child node. -/// -/// Supports: -/// - Relative names (child of current node): `cd "Getting Started"` -/// - Relative paths with `/`: `cd "Research Labs/Lab B"` -/// - Absolute paths starting with `/`: `cd /root/Chapter 1/Section 1.2` -pub fn cd(target: &str, ctx: &DocContext, state: &mut WorkerState) -> ToolResult { - if target.starts_with('/') { - return cd_absolute(target, ctx, state); - } - - // Relative path with segments: "Research Labs/Lab B" - if target.contains('/') { - return cd_relative_path(target, ctx, state); - } - - match command::resolve_target_extended(target, ctx.nav_index, state.current_node, ctx.tree) { - Some(node_id) => { - let title = ctx.node_title(node_id).unwrap_or(target).to_string(); - state.cd(node_id, &title); - ToolResult::ok(format!("Entered: {}", state.path_str())) - } - None => ToolResult::fail(format!( - "Target '{}' not found. Use ls to see available children.", - target - )), - } -} - -/// Navigate using a relative multi-segment path (e.g., `"Research Labs/Lab B"`). -fn cd_relative_path(path: &str, ctx: &DocContext, state: &mut WorkerState) -> ToolResult { - let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); - if segments.is_empty() { - return ToolResult::fail("Empty path.".to_string()); - } - - let mut current = state.current_node; - let mut breadcrumb = state.breadcrumb.clone(); - - for segment in &segments { - match command::resolve_target_extended(segment, ctx.nav_index, current, ctx.tree) { - Some(node_id) => { - let title = ctx.node_title(node_id).unwrap_or(*segment).to_string(); - breadcrumb.push(title); - current = node_id; - } - None => { - return ToolResult::fail(format!( - "Path segment '{}' not found at '/{}'. Use ls to see available children.", - segment, - breadcrumb.join("/") - )); - } - } - } - - state.breadcrumb = breadcrumb; - state.current_node = current; - state.visited.insert(current); - - ToolResult::ok(format!("Entered: {}", state.path_str())) -} - -/// Navigate using an absolute path (e.g., `/root/Chapter 1/Section 1.2`). -fn cd_absolute(path: &str, ctx: &DocContext, state: &mut WorkerState) -> ToolResult { - let segments: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); - - if segments.is_empty() { - return ToolResult::fail("Empty absolute path.".to_string()); - } - - let root = ctx.root(); - let mut current = root; - - let start_idx = if !segments.is_empty() && segments[0].eq_ignore_ascii_case("root") { - 1 - } else { - 0 - }; - - let mut breadcrumb = vec!["root".to_string()]; - - for segment in &segments[start_idx..] { - match command::resolve_target_extended(segment, ctx.nav_index, current, ctx.tree) { - Some(node_id) => { - let title = ctx.node_title(node_id).unwrap_or(*segment).to_string(); - breadcrumb.push(title); - current = node_id; - } - None => { - return ToolResult::fail(format!( - "Path segment '{}' not found. Stopped at: /{}", - segment, - breadcrumb.join("/") - )); - } - } - } - - state.breadcrumb = breadcrumb; - state.current_node = current; - state.visited.insert(current); - - ToolResult::ok(format!("Entered: {}", state.path_str())) -} - -/// Execute `cd ..` — navigate back to parent. -pub fn cd_up(ctx: &DocContext, state: &mut WorkerState) -> ToolResult { - match ctx.parent(state.current_node) { - Some(parent) => { - if state.cd_up(parent) { - ToolResult::ok(format!("Back to: {}", state.path_str())) - } else { - ToolResult::ok("Already at root.".to_string()) - } - } - None => ToolResult::ok("Already at root (no parent).".to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_test_tree() -> (DocumentTree, NavigationIndex, NodeId, NodeId, NodeId) { - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "gs content"); - let c2 = tree.add_child(root, "API Reference", "api content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup guide".to_string(), - leaf_count: 3, - }, - ChildRoute { - node_id: c2, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }, - ], - ); - - (tree, nav, root, c1, c2) - } - - #[test] - fn test_cd_navigates() { - let (tree, nav, root, c1, _) = build_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - - let result = cd("Getting Started", &ctx, &mut state); - assert!(result.success); - assert_eq!(state.current_node, c1); - assert!(state.path_str().contains("Getting Started")); - } - - #[test] - fn test_cd_up_goes_back() { - let (tree, nav, root, _c1, _) = build_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - - cd("Getting Started", &ctx, &mut state); - let result = cd_up(&ctx, &mut state); - assert!(result.success); - assert_eq!(state.current_node, root); - } - - fn build_deep_tree() -> (DocumentTree, NavigationIndex, NodeId, NodeId, NodeId) { - // Root → "Research Labs" → "Lab B" - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let section = tree.add_child(root, "Research Labs", "section content"); - let lab_b = tree.add_child(section, "Lab B", "lab b content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ChildRoute { - node_id: section, - title: "Research Labs".to_string(), - description: "Lab sections".to_string(), - leaf_count: 4, - }], - ); - nav.add_child_routes( - section, - vec![ChildRoute { - node_id: lab_b, - title: "Lab B".to_string(), - description: "Topological qubits".to_string(), - leaf_count: 1, - }], - ); - - (tree, nav, root, section, lab_b) - } - - #[test] - fn test_cd_relative_path() { - let (tree, nav, root, _, lab_b) = build_deep_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - - let result = cd("Research Labs/Lab B", &ctx, &mut state); - assert!(result.success); - assert_eq!(state.current_node, lab_b); - assert!(state.path_str().contains("Research Labs")); - assert!(state.path_str().contains("Lab B")); - } - - #[test] - fn test_cd_relative_path_partial_fail() { - let (tree, nav, root, _, _) = build_deep_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - - let result = cd("Research Labs/Nonexistent", &ctx, &mut state); - assert!(!result.success); - assert!(result.feedback.contains("Nonexistent")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/find.rs b/vectorless-core/vectorless-agent/src/tools/worker/find.rs deleted file mode 100644 index b48b418..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/find.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `find_tree` — search for nodes by title pattern across the entire tree. - -use crate::config::DocContext; - -use super::super::ToolResult; - -/// Execute `findtree ` — search for nodes by title pattern across the entire tree. -/// -/// Returns all nodes whose title contains the pattern (case-insensitive). -pub fn find_tree(pattern: &str, ctx: &DocContext) -> ToolResult { - let pattern_lower = pattern.to_lowercase(); - let all_nodes = ctx.tree.traverse(); - - let mut results = Vec::new(); - for node_id in &all_nodes { - if let Some(node) = ctx.tree.get(*node_id) { - if node.title.to_lowercase().contains(&pattern_lower) { - let depth = ctx.tree.depth(*node_id); - let leaf_count = ctx.nav_entry(*node_id).map(|e| e.leaf_count).unwrap_or(0); - results.push((node.title.clone(), depth, leaf_count)); - } - } - } - - if results.is_empty() { - return ToolResult::ok(format!("No nodes matching '{}'.", pattern)); - } - - let mut output = format!("Nodes matching '{}' ({} found):\n", pattern, results.len()); - for (title, depth, leaves) in &results { - output.push_str(&format!( - " - {} (depth {}, {} leaves)\n", - title, depth, leaves - )); - } - - ToolResult::ok(output) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_rich_tree() -> (DocumentTree, NavigationIndex, NodeId) { - let mut tree = DocumentTree::new( - "Root", - "Welcome to the financial report.\nThis document covers 2024 and 2023 figures.", - ); - let root = tree.root(); - let c1 = tree.add_child( - root, - "Revenue", - "Total revenue in 2024 was $10.2M.\nQ1 revenue: $2.5M\nQ2 revenue: $2.8M\nEBITDA margin: 32%", - ); - let c2 = tree.add_child( - root, - "Expenses", - "Operating expenses totaled $6.8M.\nR&D spending: $3.1M\nMarketing: $1.2M", - ); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 2, - }, - ChildRoute { - node_id: c2, - title: "Expenses".to_string(), - description: "Cost analysis".to_string(), - leaf_count: 2, - }, - ], - ); - - (tree, nav, root) - } - - macro_rules! rich_ctx { - ($tree:expr, $nav:expr) => { - DocContext { - tree: &$tree, - nav_index: &$nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - } - }; - } - - #[test] - fn test_find_tree() { - let (tree, nav, _root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - - let result = find_tree("revenue", &ctx); - assert!(result.success); - assert!(result.feedback.contains("Revenue")); - } - - #[test] - fn test_find_tree_case_insensitive() { - let (tree, nav, _root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - - let result = find_tree("EXPENSE", &ctx); - assert!(result.success); - assert!(result.feedback.contains("Expenses")); - } - - #[test] - fn test_find_tree_no_match() { - let (tree, nav, _root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - - let result = find_tree("nonexistent_xyz", &ctx); - assert!(result.success); - assert!(result.feedback.contains("No nodes matching")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/grep.rs b/vectorless-core/vectorless-agent/src/tools/worker/grep.rs deleted file mode 100644 index b1555fd..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/grep.rs +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `grep` — regex search across all node content in the current subtree. - -use crate::config::DocContext; -use crate::state::WorkerState; - -use super::super::ToolResult; -use super::collect_subtree; - -/// Execute `grep ` — regex search across all node content in the current subtree. -/// -/// Searches content of the current node and all descendants. Returns matching lines -/// with their node titles, capped at 30 matches to avoid overwhelming feedback. -pub fn grep(pattern: &str, ctx: &DocContext, state: &WorkerState) -> ToolResult { - let re = match regex::Regex::new(pattern) { - Ok(re) => re, - Err(e) => return ToolResult::fail(format!("Invalid regex '{}': {}", pattern, e)), - }; - - let subtree = collect_subtree(state.current_node, ctx.tree); - let mut matches_found = 0; - let mut output = String::new(); - let max_matches = 30; - - for node_id in &subtree { - if matches_found >= max_matches { - output.push_str("\n... (truncated, more matches available)"); - break; - } - - let content = match ctx.cat(*node_id) { - Some(c) if !c.is_empty() => c, - _ => continue, - }; - - let title = ctx.node_title(*node_id).unwrap_or("?"); - - for line in content.lines() { - if matches_found >= max_matches { - break; - } - if re.is_match(line) { - output.push_str(&format!("[{}] {}\n", title, line)); - matches_found += 1; - } - } - } - - if matches_found == 0 { - ToolResult::ok(format!("No matches for /{}/ in subtree.", pattern)) - } else { - ToolResult::ok(format!( - "Found {} match(es) for /{}/:\n{}", - matches_found, pattern, output - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::state::WorkerState; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_rich_tree() -> (DocumentTree, NavigationIndex, NodeId) { - let mut tree = DocumentTree::new( - "Root", - "Welcome to the financial report.\nThis document covers 2024 and 2023 figures.", - ); - let root = tree.root(); - let c1 = tree.add_child( - root, - "Revenue", - "Total revenue in 2024 was $10.2M.\nQ1 revenue: $2.5M\nQ2 revenue: $2.8M\nEBITDA margin: 32%", - ); - let c2 = tree.add_child( - root, - "Expenses", - "Operating expenses totaled $6.8M.\nR&D spending: $3.1M\nMarketing: $1.2M", - ); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 2, - }, - ChildRoute { - node_id: c2, - title: "Expenses".to_string(), - description: "Cost analysis".to_string(), - leaf_count: 2, - }, - ], - ); - - (tree, nav, root) - } - - macro_rules! rich_ctx { - ($tree:expr, $nav:expr) => { - DocContext { - tree: &$tree, - nav_index: &$nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - } - }; - } - - #[test] - fn test_grep_finds_matches() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = grep("revenue", &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("revenue")); - assert!(result.feedback.contains("[Revenue]")); - } - - #[test] - fn test_grep_regex() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = grep("EBITDA|\\$\\d+", &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("EBITDA")); - assert!(result.feedback.contains("$10")); - } - - #[test] - fn test_grep_no_matches() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = grep("nonexistent_term_xyz", &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("No matches")); - } - - #[test] - fn test_grep_invalid_regex() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = grep("[invalid", &ctx, &state); - assert!(!result.success); - assert!(result.feedback.contains("Invalid regex")); - } - - #[test] - fn test_grep_subtree_only() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let mut state = WorkerState::new(root, 15); - - crate::tools::worker::cd::cd("Expenses", &ctx, &mut state); - let result = grep("revenue", &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("No matches")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/head.rs b/vectorless-core/vectorless-agent/src/tools/worker/head.rs deleted file mode 100644 index 764cba7..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/head.rs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `head` — preview first N lines of a node without collecting evidence. - -use crate::command; -use crate::config::DocContext; -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `head ` — preview first N lines of a node without collecting evidence. -pub fn head(target: &str, lines: usize, ctx: &DocContext, state: &WorkerState) -> ToolResult { - let node_id = - match command::resolve_target_extended(target, ctx.nav_index, state.current_node, ctx.tree) - { - Some(id) => id, - None => { - return ToolResult::fail(format!( - "Target '{}' not found. Use ls to see available children.", - target - )); - } - }; - - let content = match ctx.cat(node_id) { - Some(c) => c, - None => return ToolResult::fail(format!("No content for '{}'.", target)), - }; - - let title = ctx.node_title(node_id).unwrap_or("unknown"); - let total_lines = content.lines().count(); - let preview: Vec<&str> = content.lines().take(lines).collect(); - - let mut output = format!( - "[Preview: {} — showing {}/{} lines]\n", - title, - preview.len().min(lines), - total_lines - ); - output.push_str(&preview.join("\n")); - - if total_lines > lines { - output.push_str(&format!( - "\n... ({} more lines, use cat to read all)", - total_lines - lines - )); - } - - ToolResult::ok(output) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::state::WorkerState; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_rich_tree() -> (DocumentTree, NavigationIndex, NodeId) { - let mut tree = DocumentTree::new( - "Root", - "Welcome to the financial report.\nThis document covers 2024 and 2023 figures.", - ); - let root = tree.root(); - let c1 = tree.add_child( - root, - "Revenue", - "Total revenue in 2024 was $10.2M.\nQ1 revenue: $2.5M\nQ2 revenue: $2.8M\nEBITDA margin: 32%", - ); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 2, - }], - ); - - (tree, nav, root) - } - - macro_rules! rich_ctx { - ($tree:expr, $nav:expr) => { - DocContext { - tree: &$tree, - nav_index: &$nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - } - }; - } - - #[test] - fn test_head_preview() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = head("Revenue", 2, &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("Preview")); - assert!(result.feedback.contains("$10.2M")); - assert!(result.feedback.contains("2/4 lines")); - } - - #[test] - fn test_head_not_found() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = head("NonExistent", 10, &ctx, &state); - assert!(!result.success); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/ls.rs b/vectorless-core/vectorless-agent/src/tools/worker/ls.rs deleted file mode 100644 index 3c85bc1..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/ls.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `ls` — list children of the current node. - -use crate::config::DocContext; -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `ls` — list children of the current node. -pub fn ls(ctx: &DocContext, state: &WorkerState) -> ToolResult { - let mut output = String::new(); - - if let Some(entry) = ctx.nav_entry(state.current_node) { - output.push_str(&format!("Current section: {}\n", entry.overview)); - if !entry.question_hints.is_empty() { - output.push_str(&format!( - "Can answer: {}\n", - entry.question_hints.join(", ") - )); - } - output.push('\n'); - } - - match ctx.ls(state.current_node) { - Some(routes) => { - if routes.is_empty() { - output - .push_str("(leaf node — no children)\nUse cd .. to go back or done to finish."); - return ToolResult::ok(output); - } - - for (i, route) in routes.iter().enumerate() { - if route.title == route.description { - output.push_str(&format!( - "[{}] {} ({} leaves)", - i + 1, - route.title, - route.leaf_count - )); - } else { - output.push_str(&format!( - "[{}] {} — {} ({} leaves)", - i + 1, - route.title, - route.description, - route.leaf_count - )); - } - if let Some(nav) = ctx.nav_entry(route.node_id) { - if !nav.question_hints.is_empty() { - output.push_str(&format!( - "\n Can answer: {}", - nav.question_hints.join(", ") - )); - } - if !nav.topic_tags.is_empty() { - output.push_str(&format!("\n Topics: {}", nav.topic_tags.join(", "))); - } - } - output.push('\n'); - } - ToolResult::ok(output) - } - None => { - output.push_str( - "(no navigation data for this node)\nUse cat to read content or cd .. to go back.", - ); - ToolResult::ok(output) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_test_tree() -> (DocumentTree, NavigationIndex, NodeId, NodeId, NodeId) { - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let c1 = tree.add_child(root, "Getting Started", "gs content"); - let c2 = tree.add_child(root, "API Reference", "api content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: c1, - title: "Getting Started".to_string(), - description: "Setup guide".to_string(), - leaf_count: 3, - }, - ChildRoute { - node_id: c2, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }, - ], - ); - - (tree, nav, root, c1, c2) - } - - #[test] - fn test_ls_shows_children() { - let (tree, nav, root, _, _) = build_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let state = WorkerState::new(root, 15); - - let result = ls(&ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("Getting Started")); - assert!(result.feedback.contains("API Reference")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/mod.rs b/vectorless-core/vectorless-agent/src/tools/worker/mod.rs deleted file mode 100644 index 4a9fc6e..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/mod.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Worker tools: ls, cd, cd_up, cat, pwd, grep, head, find_tree, wc. - -mod cat; -mod cd; -mod find; -mod grep; -mod head; -mod ls; -mod pwd; -mod wc; - -pub use cat::cat; -pub use cd::{cd, cd_up}; -pub use find::find_tree; -pub use grep::grep; -pub use head::head; -pub use ls::ls; -pub use pwd::pwd; -pub use wc::wc; - -use vectorless_document::{DocumentTree, NodeId}; - -/// Collect all NodeIds in the subtree rooted at `node` (inclusive). -pub(super) fn collect_subtree(node: NodeId, tree: &DocumentTree) -> Vec { - let mut result = vec![node]; - let mut stack = vec![node]; - - while let Some(current) = stack.pop() { - for child in tree.children_iter(current) { - result.push(child); - stack.push(child); - } - } - - result -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/pwd.rs b/vectorless-core/vectorless-agent/src/tools/worker/pwd.rs deleted file mode 100644 index c5ff06b..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/pwd.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `pwd` — show current navigation path. - -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `pwd` — show current navigation path. -pub fn pwd(state: &WorkerState) -> ToolResult { - ToolResult::ok(format!("Current path: {}", state.path_str())) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::tools::worker::cd::cd; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex}; - - fn build_test_tree() -> (DocumentTree, NavigationIndex) { - let mut tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - let c1 = tree.add_child(root, "API Reference", "api content"); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "API Reference".to_string(), - description: "API docs".to_string(), - leaf_count: 7, - }], - ); - - (tree, nav) - } - - #[test] - fn test_pwd() { - let (tree, nav) = build_test_tree(); - let root = tree.root(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 15); - cd("API Reference", &ctx, &mut state); - - let result = pwd(&state); - assert!(result.success); - assert!(result.feedback.contains("API Reference")); - } -} diff --git a/vectorless-core/vectorless-agent/src/tools/worker/wc.rs b/vectorless-core/vectorless-agent/src/tools/worker/wc.rs deleted file mode 100644 index adc05cf..0000000 --- a/vectorless-core/vectorless-agent/src/tools/worker/wc.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! `wc` — show node content statistics. - -use crate::command; -use crate::config::DocContext; -use crate::state::WorkerState; - -use super::super::ToolResult; - -/// Execute `wc ` — show node content statistics. -pub fn wc(target: &str, ctx: &DocContext, state: &WorkerState) -> ToolResult { - let node_id = - match command::resolve_target_extended(target, ctx.nav_index, state.current_node, ctx.tree) - { - Some(id) => id, - None => { - return ToolResult::fail(format!( - "Target '{}' not found. Use ls to see available children.", - target - )); - } - }; - - let content = match ctx.cat(node_id) { - Some(c) => c, - None => return ToolResult::fail(format!("No content for '{}'.", target)), - }; - - let title = ctx.node_title(node_id).unwrap_or("unknown"); - let lines = content.lines().count(); - let words = content.split_whitespace().count(); - let chars = content.len(); - - ToolResult::ok(format!( - "[{}] {} lines, {} words, {} chars", - title, lines, words, chars - )) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::state::WorkerState; - use vectorless_document::{ChildRoute, DocumentTree, NavigationIndex, NodeId}; - - fn build_rich_tree() -> (DocumentTree, NavigationIndex, NodeId) { - let mut tree = DocumentTree::new( - "Root", - "Welcome to the financial report.\nThis document covers 2024 and 2023 figures.", - ); - let root = tree.root(); - let c1 = tree.add_child( - root, - "Revenue", - "Total revenue in 2024 was $10.2M.\nQ1 revenue: $2.5M\nQ2 revenue: $2.8M\nEBITDA margin: 32%", - ); - - let mut nav = NavigationIndex::new(); - nav.add_child_routes( - root, - vec![ChildRoute { - node_id: c1, - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 2, - }], - ); - - (tree, nav, root) - } - - macro_rules! rich_ctx { - ($tree:expr, $nav:expr) => { - DocContext { - tree: &$tree, - nav_index: &$nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - } - }; - } - - #[test] - fn test_wc_stats() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = wc("Revenue", &ctx, &state); - assert!(result.success); - assert!(result.feedback.contains("Revenue")); - assert!(result.feedback.contains("lines")); - assert!(result.feedback.contains("words")); - assert!(result.feedback.contains("chars")); - } - - #[test] - fn test_wc_not_found() { - let (tree, nav, root) = build_rich_tree(); - let ctx = rich_ctx!(tree, nav); - let state = WorkerState::new(root, 15); - - let result = wc("NonExistent", &ctx, &state); - assert!(!result.success); - } -} diff --git a/vectorless-core/vectorless-agent/src/worker/execute.rs b/vectorless-core/vectorless-agent/src/worker/execute.rs deleted file mode 100644 index 12ac88e..0000000 --- a/vectorless-core/vectorless-agent/src/worker/execute.rs +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Command execution — dispatch parsed Command to tool functions. - -use tracing::{info, warn}; - -use vectorless_llm::LlmClient; - -use super::super::command::{Command, parse_command}; -use super::super::config::{DocContext, Step}; -use super::super::events::EventEmitter; -use super::super::prompts::{check_sufficiency, parse_sufficiency_response}; -use super::super::state::WorkerState; -use super::super::tools::worker as tools; - -/// Execute a single parsed command, mutating state. -/// -/// Returns a `Step` indicating whether to continue or stop. -pub async fn execute_command( - command: &Command, - ctx: &DocContext<'_>, - state: &mut WorkerState, - query: &str, - llm: &LlmClient, - llm_calls: &mut u32, - emitter: &EventEmitter, -) -> Step { - info!( - doc = ctx.doc_name, - command = ?command, - "Executing tool" - ); - match command { - Command::Ls => { - let result = tools::ls(ctx, state); - info!(doc = ctx.doc_name, feedback = %truncate_log(&result.feedback), "ls result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::Cd { target } => { - let result = tools::cd(target, ctx, state); - info!(doc = ctx.doc_name, target, feedback = %truncate_log(&result.feedback), "cd result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::CdUp => { - let result = tools::cd_up(ctx, state); - info!(doc = ctx.doc_name, feedback = %truncate_log(&result.feedback), "cd_up result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::Cat { target } => { - let evidence_before = state.evidence.len(); - let result = tools::cat(target, ctx, state); - info!(doc = ctx.doc_name, target, feedback = %truncate_log(&result.feedback), "cat result"); - state.set_feedback(result.feedback); - if state.evidence.len() > evidence_before { - if let Some(ev) = state.evidence.last() { - info!( - doc = ctx.doc_name, - node = %ev.node_title, - path = %ev.source_path, - len = ev.content.len(), - total = state.evidence.len(), - "Evidence collected" - ); - emitter.emit_evidence( - ctx.doc_name, - &ev.node_title, - &ev.source_path, - ev.content.len(), - state.evidence.len(), - ); - } - } - Step::Continue - } - - Command::Find { keyword } => { - let feedback = match ctx.find(keyword) { - Some(hit) => { - let mut entries = hit.entries.clone(); - entries.sort_by(|a, b| { - b.weight - .partial_cmp(&a.weight) - .unwrap_or(std::cmp::Ordering::Equal) - }); - let mut seen_nodes = std::collections::HashSet::new(); - let mut output = format!("Results for '{}':\n", keyword); - for entry in &entries { - if !seen_nodes.insert(entry.node_id) { - continue; - } - let title = ctx.node_title(entry.node_id).unwrap_or("unknown"); - output.push_str(&format!( - " - {} (depth {}, weight {:.2})", - title, entry.depth, entry.weight - )); - if let Some(content) = ctx.cat(entry.node_id) { - if let Some(snippet) = - super::super::tools::content_snippet(content, keyword, 300) - { - output.push_str(&format!("\n \"{}\"", snippet)); - } - } - output.push('\n'); - } - output - } - None => { - // Fallback: search node titles (like findtree) with content snippets - let pattern_lower = keyword.to_lowercase(); - let all_nodes = ctx.tree.traverse(); - let mut results = Vec::new(); - for node_id in &all_nodes { - if let Some(node) = ctx.tree.get(*node_id) { - if node.title.to_lowercase().contains(&pattern_lower) { - let depth = ctx.tree.depth(*node_id); - results.push((node.title.clone(), *node_id, depth)); - } - } - } - if results.is_empty() { - format!("No results for '{}' in index or titles.", keyword) - } else { - let mut output = format!( - "Results for '{}' (title match, {} found):\n", - keyword, - results.len() - ); - for (title, node_id, depth) in &results { - output.push_str(&format!(" - {} (depth {})", title, depth)); - if let Some(content) = ctx.cat(*node_id) { - if let Some(snippet) = - super::super::tools::content_snippet(content, keyword, 300) - { - output.push_str(&format!("\n \"{}\"", snippet)); - } - } - output.push('\n'); - } - output - } - } - }; - info!(doc = ctx.doc_name, keyword, feedback = %truncate_log(&feedback), "find result"); - state.set_feedback(feedback); - Step::Continue - } - - Command::Pwd => { - let result = tools::pwd(state); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::Check => { - let evidence_text = state.evidence_for_check(); - - let (system, user) = check_sufficiency(query, &evidence_text); - - info!( - doc = ctx.doc_name, - system = %system, - user = %user, - "Check prompt" - ); - - match llm.complete(&system, &user).await { - Ok(response) => { - *llm_calls += 1; - state.check_count += 1; - let sufficient = parse_sufficiency_response(&response); - info!( - doc = ctx.doc_name, - sufficient, - evidence = state.evidence.len(), - response = %response, - "Sufficiency check" - ); - emitter.emit_worker_sufficiency_check( - ctx.doc_name, - sufficient, - state.evidence.len(), - None, - ); - if sufficient { - state.last_feedback = - "Evidence is sufficient. Use done to finish.".to_string(); - Step::Done - } else { - let reason = response - .trim() - .strip_prefix("INSUFFICIENT") - .unwrap_or(response.trim()) - .trim() - .trim_start_matches(|c: char| c == '-' || c == ' '); - if !reason.is_empty() { - state.missing_info = reason.to_string(); - } - state.set_feedback(format!( - "Evidence not yet sufficient: {}", - response.trim() - )); - Step::Continue - } - } - Err(e) => { - warn!(error = %e, "Check LLM call failed"); - state.last_feedback = "Could not evaluate sufficiency.".to_string(); - Step::Continue - } - } - } - - Command::Done => { - state.last_feedback = "Navigation complete.".to_string(); - Step::Done - } - - Command::Grep { pattern } => { - let result = tools::grep(pattern, ctx, state); - info!(doc = ctx.doc_name, pattern, feedback = %truncate_log(&result.feedback), "grep result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::Head { target, lines } => { - let result = tools::head(target, *lines, ctx, state); - info!(doc = ctx.doc_name, target, lines, feedback = %truncate_log(&result.feedback), "head result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::FindTree { pattern } => { - let result = tools::find_tree(pattern, ctx); - info!(doc = ctx.doc_name, pattern, feedback = %truncate_log(&result.feedback), "find_tree result"); - state.set_feedback(result.feedback); - Step::Continue - } - - Command::Wc { target } => { - let result = tools::wc(target, ctx, state); - info!(doc = ctx.doc_name, target, feedback = %truncate_log(&result.feedback), "wc result"); - state.set_feedback(result.feedback); - Step::Continue - } - } -} - -/// Truncate feedback for log output — keep first 300 chars to avoid noisy logs. -fn truncate_log(s: &str) -> std::borrow::Cow<'_, str> { - const MAX: usize = 300; - if s.len() <= MAX { - std::borrow::Cow::Borrowed(s) - } else { - std::borrow::Cow::Owned(format!( - "{}...(truncated, {} chars total)", - &s[..MAX], - s.len() - )) - } -} - -/// Parse the LLM output and detect parse failures. -/// -/// Returns `(command, is_parse_failure)`. -pub fn parse_and_detect_failure(llm_output: &str) -> (Command, bool) { - let command = parse_command(llm_output); - let trimmed = llm_output.trim(); - let is_parse_failure = - matches!(command, Command::Ls) && !trimmed.starts_with("ls") && !trimmed.is_empty(); - (command, is_parse_failure) -} diff --git a/vectorless-core/vectorless-agent/src/worker/format.rs b/vectorless-core/vectorless-agent/src/worker/format.rs deleted file mode 100644 index be9e029..0000000 --- a/vectorless-core/vectorless-agent/src/worker/format.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Formatting helpers for Worker prompts. - -use super::super::config::DocContext; -use super::super::state::WorkerState; - -/// Resolve visited NodeIds to their titles for prompt injection. -pub fn format_visited_titles(state: &WorkerState, ctx: &DocContext<'_>) -> String { - if state.visited.is_empty() { - return "(none)".to_string(); - } - state - .visited - .iter() - .filter_map(|&node_id| ctx.node_title(node_id).map(|t| t.to_string())) - .collect::>() - .join(", ") -} diff --git a/vectorless-core/vectorless-agent/src/worker/mod.rs b/vectorless-core/vectorless-agent/src/worker/mod.rs deleted file mode 100644 index d4ac545..0000000 --- a/vectorless-core/vectorless-agent/src/worker/mod.rs +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Worker agent — document navigation and evidence collection. -//! -//! The Worker is a consuming-self struct implementing [`Agent`]: -//! 1. Bird's-eye: ls(root) for initial overview -//! 2. Navigation planning: LLM generates a plan (keyword hits as context) -//! 3. Navigation loop: LLM → parse → execute → repeat (max N rounds) -//! -//! Dispatched by the Orchestrator, one per document. -//! Returns raw evidence — no answer synthesis. Rerank owns all answer generation. - -mod execute; -mod format; -mod navigation; -mod planning; - -use tracing::info; - -use super::Agent; -use super::config::{DocContext, WorkerConfig, WorkerOutput}; -use super::context::FindHit; -use super::events::EventEmitter; -use super::state::WorkerState; -use super::tools::worker as tools; -use vectorless_error::Error; -use vectorless_llm::LlmClient; -use vectorless_query::QueryPlan; -use vectorless_scoring::bm25::extract_keywords; - -use navigation::run_navigation_loop; -use planning::build_plan_prompt; - -/// Worker agent — navigates a single document to collect evidence. -/// -/// Holds all execution context. Calling [`run()`](Agent::run) consumes self. -pub struct Worker<'a> { - query: String, - task: Option, - ctx: &'a DocContext<'a>, - config: WorkerConfig, - llm: LlmClient, - emitter: EventEmitter, - query_plan: QueryPlan, -} - -impl<'a> Worker<'a> { - /// Create a new Worker. - pub fn new( - query: &str, - task: Option<&str>, - ctx: &'a DocContext<'a>, - config: WorkerConfig, - llm: LlmClient, - emitter: EventEmitter, - query_plan: QueryPlan, - ) -> Self { - Self { - query: query.to_string(), - task: task.map(|s| s.to_string()), - ctx, - config, - llm, - emitter, - query_plan, - } - } -} - -impl<'a> Agent for Worker<'a> { - type Output = WorkerOutput; - - fn name(&self) -> &str { - "worker" - } - - async fn run(self) -> vectorless_error::Result { - let Worker { - query, - task, - ctx, - config, - llm, - emitter, - query_plan, - } = self; - let task_ref = task.as_deref(); - - let intent_context = format!("{} — {}", query_plan.intent, query_plan.strategy_hint); - - emitter.emit_worker_started(ctx.doc_name, task_ref, config.max_rounds); - - info!( - doc = ctx.doc_name, - task = task_ref.unwrap_or("(full query)"), - max_rounds = config.max_rounds, - max_llm_calls = config.max_llm_calls, - "Worker starting" - ); - - let mut llm_calls: u32 = 0; - - // Gather keyword hits as context for LLM planning (not routing rules) - let keywords = extract_keywords(&query); - let index_hits: Vec = ctx.find_all(&keywords); - if !index_hits.is_empty() { - tracing::debug!( - doc = ctx.doc_name, - hit_count = index_hits.len(), - "ReasoningIndex keyword hits available for planning" - ); - } - - // --- Phase 1: Bird's-eye view --- - let mut state = WorkerState::new(ctx.root(), config.max_rounds); - let ls_result = tools::ls(ctx, &state); - state.set_feedback(ls_result.feedback); - - // --- Phase 1.5: Navigation planning --- - if state.remaining > 0 && (config.max_llm_calls == 0 || llm_calls < config.max_llm_calls) { - info!(doc = ctx.doc_name, "Generating navigation plan..."); - let plan_prompt = build_plan_prompt( - &query, - task_ref, - &state.last_feedback, - ctx.doc_name, - &index_hits, - ctx, - query_plan.intent, - ); - let plan_output = llm - .complete(&plan_prompt.0, &plan_prompt.1) - .await - .map_err(|e| Error::LlmReasoning { - stage: "worker/plan".to_string(), - detail: format!("Navigation plan LLM call failed: {e}"), - })?; - llm_calls += 1; - let plan_text = plan_output.trim().to_string(); - if !plan_text.is_empty() { - info!( - doc = ctx.doc_name, - plan = %plan_text, - "Navigation plan generated" - ); - emitter.emit_worker_plan_generated(ctx.doc_name, plan_text.len()); - state.plan = plan_text; - state.plan_generated = true; - } - } - - // --- Phase 2: Navigation loop --- - run_navigation_loop( - &query, - task_ref, - ctx, - &config, - &llm, - &mut state, - &emitter, - &index_hits, - &intent_context, - &mut llm_calls, - ) - .await?; - - let budget_exhausted = - state.remaining == 0 || (config.max_llm_calls > 0 && llm_calls >= config.max_llm_calls); - - let output = state.into_worker_output(llm_calls, budget_exhausted, ctx.doc_name); - - emitter.emit_worker_done( - ctx.doc_name, - output.evidence.len(), - output.metrics.rounds_used, - output.metrics.llm_calls, - output.metrics.budget_exhausted, - output.metrics.plan_generated, - ); - - info!( - doc = ctx.doc_name, - evidence = output.evidence.len(), - rounds = output.metrics.rounds_used, - llm_calls = output.metrics.llm_calls, - "Worker complete" - ); - - Ok(output) - } -} - -#[cfg(test)] -mod truncation_tests { - /// Verify that truncating feedback with multi-byte UTF-8 characters - /// never panics. This mirrors the truncation logic in the navigation loop. - #[test] - fn test_utf8_safe_truncation_ascii() { - let feedback = "a".repeat(200); - let boundary = feedback.ceil_char_boundary(120); - let truncated = &feedback[..boundary]; - assert!(truncated.len() <= 123); // 120 + "..." fits - assert!(truncated.is_char_boundary(truncated.len())); - } - - #[test] - fn test_utf8_safe_truncation_multibyte() { - // Each '中' is 3 bytes in UTF-8 - let feedback = "中文反馈内容测试截断安全".repeat(20); - assert!(feedback.len() > 120); - let boundary = feedback.ceil_char_boundary(120); - let truncated = &feedback[..boundary]; - assert!(truncated.len() <= 120); - assert!(truncated.is_char_boundary(truncated.len())); - } - - #[test] - fn test_utf8_safe_truncation_emoji() { - // Emojis are 4 bytes each - let feedback = "🦀🎉🚀".repeat(50); - assert!(feedback.len() > 120); - let boundary = feedback.ceil_char_boundary(120); - let truncated = &feedback[..boundary]; - assert!(truncated.len() <= 120); - assert!(truncated.is_char_boundary(truncated.len())); - } - - #[test] - fn test_utf8_safe_truncation_short_string() { - // String shorter than limit — no truncation needed - let feedback = "short feedback".to_string(); - let boundary = feedback.ceil_char_boundary(120); - assert_eq!(boundary, feedback.len()); - } -} diff --git a/vectorless-core/vectorless-agent/src/worker/navigation.rs b/vectorless-core/vectorless-agent/src/worker/navigation.rs deleted file mode 100644 index cb6b06b..0000000 --- a/vectorless-core/vectorless-agent/src/worker/navigation.rs +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Phase 2: Navigation loop — LLM-driven command loop until done or budget exhausted. - -use tracing::{debug, info}; - -use super::super::command::Command; -use super::super::config::{DocContext, Step, WorkerConfig}; -use super::super::context::FindHit; -use super::super::events::EventEmitter; -use super::super::prompts::{NavigationParams, worker_dispatch, worker_navigation}; -use super::super::state::WorkerState; -use super::execute::{execute_command, parse_and_detect_failure}; -use super::format::format_visited_titles; -use super::planning::{build_replan_prompt, format_keyword_hints}; -use vectorless_error::Error; -use vectorless_llm::LlmClient; - -/// Run the Phase 2 navigation loop. -/// -/// Loops until budget exhausted, `done`/`force_done`, or error. -/// Mutates `state` and `llm_calls` in place. -pub async fn run_navigation_loop( - query: &str, - task: Option<&str>, - ctx: &DocContext<'_>, - config: &WorkerConfig, - llm: &LlmClient, - state: &mut WorkerState, - emitter: &EventEmitter, - index_hits: &[FindHit], - intent_context: &str, - llm_calls: &mut u32, -) -> vectorless_error::Result<()> { - let use_dispatch_prompt = task.is_some(); - let keyword_hints = format_keyword_hints(index_hits, ctx); - let max_llm = config.max_llm_calls; - - loop { - if state.remaining == 0 { - info!(doc = ctx.doc_name, "Navigation budget exhausted"); - break; - } - if max_llm > 0 && *llm_calls >= max_llm { - info!( - doc = ctx.doc_name, - llm_calls, max_llm, "LLM call budget exhausted" - ); - break; - } - - // Build prompt - let (system, user) = build_round_prompt( - query, - task, - ctx, - state, - intent_context, - &keyword_hints, - use_dispatch_prompt, - config.max_rounds, - ); - - // LLM decision - let round_num = config.max_rounds - state.remaining + 1; - let round_start = std::time::Instant::now(); - info!( - doc = ctx.doc_name, - round = round_num, - max_rounds = config.max_rounds, - "Navigation round: calling LLM..." - ); - let llm_output = llm - .complete(&system, &user) - .await - .map_err(|e| Error::LlmReasoning { - stage: "worker/navigation".to_string(), - detail: format!("Nav loop LLM call failed (round {round_num}): {e}"), - })?; - *llm_calls += 1; - - // Parse command - let (command, is_parse_failure) = handle_parse_failure(&llm_output, ctx.doc_name, state); - if is_parse_failure { - continue; - } - - debug!(doc = ctx.doc_name, ?command, "Parsed command"); - - let is_check = matches!(command, Command::Check); - - // Execute - let step = execute_command(&command, ctx, state, query, llm, llm_calls, emitter).await; - - // Dynamic re-planning after insufficient check - handle_replan( - is_check, query, task, ctx, llm, state, emitter, llm_calls, max_llm, - ) - .await?; - - // Emit round event - let cmd_str = format!("{:?}", command); - let success = !matches!(step, Step::ForceDone(_)); - let round_elapsed = round_start.elapsed().as_millis() as u64; - emitter.emit_worker_round(ctx.doc_name, round_num, &cmd_str, success, round_elapsed); - - push_round_history(state, &cmd_str); - - // Check termination - match step { - Step::Done => { - info!( - doc = ctx.doc_name, - evidence = state.evidence.len(), - "Navigation done" - ); - break; - } - Step::ForceDone(reason) => { - info!(doc = ctx.doc_name, reason = %reason, "Forced done"); - break; - } - Step::Continue => { - if !is_check { - state.dec_round(); - } - } - } - } - - Ok(()) -} - -/// Build the (system, user) prompt pair for a single navigation round. -fn build_round_prompt( - query: &str, - task: Option<&str>, - ctx: &DocContext<'_>, - state: &WorkerState, - intent_context: &str, - keyword_hints: &str, - use_dispatch_prompt: bool, - max_rounds: u32, -) -> (String, String) { - if use_dispatch_prompt && state.remaining == max_rounds { - worker_dispatch(&super::super::prompts::WorkerDispatchParams { - original_query: query, - task: task.unwrap_or(query), - doc_name: ctx.doc_name, - breadcrumb: &state.path_str(), - }) - } else { - let visited_titles = format_visited_titles(state, ctx); - worker_navigation(&NavigationParams { - query, - task, - breadcrumb: &state.path_str(), - evidence_summary: &state.evidence_summary(), - missing_info: &state.missing_info, - last_feedback: &state.last_feedback, - remaining: state.remaining, - max_rounds: state.max_rounds, - history: &state.history_text(), - visited_titles: &visited_titles, - plan: &state.plan, - intent_context, - keyword_hints, - }) - } -} - -/// Parse LLM output and handle parse failures. -/// -/// Returns `(command, is_parse_failure)`. On parse failure, updates state -/// with feedback and pushes a history entry. -fn handle_parse_failure( - llm_output: &str, - doc_name: &str, - state: &mut WorkerState, -) -> (Command, bool) { - if llm_output.trim().len() < 2 { - tracing::warn!( - doc = doc_name, - response = llm_output.trim(), - "LLM response unusually short" - ); - } - let (command, is_parse_failure) = parse_and_detect_failure(llm_output); - if is_parse_failure { - let raw_preview = if llm_output.trim().len() > 200 { - format!("{}...", &llm_output.trim()[..200]) - } else { - llm_output.trim().to_string() - }; - state.last_feedback = format!( - "Your output was not recognized as a valid command:\n\"{}\"\n\n\ - Please output exactly one command (ls, cd, cat, head, find, findtree, grep, wc, pwd, check, or done).", - raw_preview - ); - state.push_history("(unrecognized) → parse failure".to_string()); - } - (command, is_parse_failure) -} - -/// Push a round's command + feedback preview into history and trace. -fn push_round_history(state: &mut WorkerState, cmd_str: &str) { - let feedback_preview = if state.last_feedback.len() > 120 { - let boundary = state.last_feedback.ceil_char_boundary(120); - format!("{}...", &state.last_feedback[..boundary]) - } else { - state.last_feedback.clone() - }; - state.push_history(format!("{} → {}", cmd_str, feedback_preview)); - - let round = state.max_rounds.saturating_sub(state.remaining); - state.trace_steps.push(vectorless_document::TraceStep { - action: cmd_str.to_string(), - observation: state.last_feedback.chars().take(200).collect(), - round, - }); -} - -/// Dynamic re-planning after an insufficient check. -/// -/// If check returned INSUFFICIENT with enough remaining rounds and LLM budget, -/// generates a new navigation plan. Otherwise clears stale replan state. -async fn handle_replan( - is_check: bool, - query: &str, - task: Option<&str>, - ctx: &DocContext<'_>, - llm: &LlmClient, - state: &mut WorkerState, - emitter: &EventEmitter, - llm_calls: &mut u32, - max_llm: u32, -) -> vectorless_error::Result<()> { - if !is_check { - return Ok(()); - } - - if !state.missing_info.is_empty() - && state.remaining >= 3 - && (max_llm == 0 || *llm_calls < max_llm) - { - let missing = state.missing_info.clone(); - info!(doc = ctx.doc_name, missing = %missing, "Re-planning navigation..."); - let replan = build_replan_prompt(query, task, state, ctx); - let new_plan = - llm.complete(&replan.0, &replan.1) - .await - .map_err(|e| Error::LlmReasoning { - stage: "worker/replan".to_string(), - detail: format!("Re-plan LLM call failed: {e}"), - })?; - *llm_calls += 1; - let plan_text = new_plan.trim().to_string(); - if !plan_text.is_empty() { - info!( - doc = ctx.doc_name, - plan = %plan_text, - "Re-plan generated" - ); - emitter.emit_worker_replan(ctx.doc_name, &missing, plan_text.len()); - state.plan = plan_text; - } - state.missing_info.clear(); - } else if !state.missing_info.is_empty() { - state.plan.clear(); - state.missing_info.clear(); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::state::WorkerState; - use vectorless_document::{DocumentTree, NodeId}; - - fn test_ctx() -> (DocumentTree, NodeId) { - let tree = DocumentTree::new("Root", "root content"); - let root = tree.root(); - (tree, root) - } - - #[test] - fn test_handle_parse_failure_valid_command() { - let (tree, root) = test_ctx(); - let nav = vectorless_document::NavigationIndex::new(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 10); - - let (cmd, is_failure) = handle_parse_failure("ls", ctx.doc_name, &mut state); - assert!(!is_failure); - assert!(matches!(cmd, Command::Ls)); - } - - #[test] - fn test_handle_parse_failure_unrecognized() { - let (tree, root) = test_ctx(); - let nav = vectorless_document::NavigationIndex::new(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 10); - - let (_cmd, is_failure) = - handle_parse_failure("random garbage text", ctx.doc_name, &mut state); - assert!(is_failure); - assert!(state.last_feedback.contains("not recognized")); - assert!(state.history.last().unwrap().contains("unrecognized")); - } - - #[test] - fn test_handle_parse_failure_short_response() { - let (tree, root) = test_ctx(); - let nav = vectorless_document::NavigationIndex::new(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let mut state = WorkerState::new(root, 10); - - // Single character response — short but not a parse failure if it's "ls" - let (cmd, is_failure) = handle_parse_failure("ls", ctx.doc_name, &mut state); - assert!(!is_failure); - assert!(matches!(cmd, Command::Ls)); - } - - #[test] - fn test_push_round_history_short_feedback() { - let (_, root) = test_ctx(); - let mut state = WorkerState::new(root, 10); - state.last_feedback = "short feedback".to_string(); - - push_round_history(&mut state, "ls"); - assert_eq!(state.history.len(), 1); - assert!(state.history[0].contains("ls → short feedback")); - } - - #[test] - fn test_push_round_history_long_feedback() { - let (_, root) = test_ctx(); - let mut state = WorkerState::new(root, 10); - state.last_feedback = "a".repeat(200); - - push_round_history(&mut state, "cat"); - assert_eq!(state.history.len(), 1); - assert!(state.history[0].contains("cat → ")); - // Should be truncated with ... - assert!(state.history[0].contains("...")); - } - - #[test] - fn test_push_round_history_respects_max_entries() { - let (_, root) = test_ctx(); - let mut state = WorkerState::new(root, 10); - state.last_feedback = "ok".to_string(); - - for i in 0..8 { - push_round_history(&mut state, &format!("cmd_{i}")); - } - // MAX_HISTORY_ENTRIES is 6, so only last 6 should remain - assert_eq!(state.history.len(), 6); - } - - #[test] - fn test_build_round_prompt_dispatch_first_round() { - let (tree, root) = test_ctx(); - let nav = vectorless_document::NavigationIndex::new(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test_doc", - }; - let mut state = WorkerState::new(root, 10); - // remaining == max_rounds means first round - assert_eq!(state.remaining, 10); - - let (system, user) = build_round_prompt( - "test query", - Some("sub-task"), - &ctx, - &state, - "factual — find answer", - "", - true, // use_dispatch_prompt - 10, - ); - assert!(system.contains("dispatch") || !system.is_empty()); - assert!(user.contains("test query") || user.contains("sub-task")); - } - - #[test] - fn test_build_round_prompt_navigation_subsequent_round() { - let (tree, root) = test_ctx(); - let nav = vectorless_document::NavigationIndex::new(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test_doc", - }; - let mut state = WorkerState::new(root, 10); - state.remaining = 8; // not first round - - let (system, _user) = build_round_prompt( - "test query", - None, - &ctx, - &state, - "factual", - "keyword hints here", - false, // use_dispatch_prompt - 10, - ); - assert!(!system.is_empty()); - } - - #[test] - fn test_utf8_safe_truncation_in_history() { - let (_, root) = test_ctx(); - let mut state = WorkerState::new(root, 10); - // Each '中' is 3 bytes in UTF-8 - state.last_feedback = "中文反馈内容测试截断安全".repeat(20); - - push_round_history(&mut state, "cat"); - let entry = &state.history[0]; - // Should be truncated without panicking - assert!(entry.contains("cat → ")); - assert!(entry.len() < state.last_feedback.len() + 20); - } -} diff --git a/vectorless-core/vectorless-agent/src/worker/planning.rs b/vectorless-core/vectorless-agent/src/worker/planning.rs deleted file mode 100644 index 80149e7..0000000 --- a/vectorless-core/vectorless-agent/src/worker/planning.rs +++ /dev/null @@ -1,708 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Navigation planning prompts — initial plan, re-plan, semantic hints, deep expansion. - -use std::collections::HashSet; - -use vectorless_query::QueryIntent; -use vectorless_scoring::bm25::{Bm25Engine, FieldDocument, extract_keywords}; - -use super::super::config::DocContext; -use super::super::context::FindHit; -use super::super::state::WorkerState; -use super::format::format_visited_titles; - -/// Maximum keyword/semantic hit entries in plan prompt. -const MAX_PLAN_ENTRIES: usize = 15; -/// Maximum section summaries in plan prompt. -const MAX_SECTION_SUMMARIES: usize = 10; -/// Maximum deep expansion entries. -const MAX_EXPANSION_ENTRIES: usize = 8; - -/// Build the navigation planning prompt (Phase 1.5). -pub fn build_plan_prompt( - query: &str, - task: Option<&str>, - ls_output: &str, - doc_name: &str, - keyword_hits: &[FindHit], - ctx: &DocContext<'_>, - intent: QueryIntent, -) -> (String, String) { - let task_section = match task { - Some(t) => format!("\nYour specific task: {}", t), - None => String::new(), - }; - - let query_keywords = extract_keywords(query); - let query_lower = query.to_lowercase(); - - let mut keyword_section = if keyword_hits.is_empty() { - String::new() - } else { - let mut section = - String::from("\nKeyword index matches (use these to prioritize navigation):\n"); - let mut entry_count = 0; - for hit in keyword_hits { - let mut entries = hit.entries.clone(); - entries.sort_by(|a, b| { - b.weight - .partial_cmp(&a.weight) - .unwrap_or(std::cmp::Ordering::Equal) - }); - let mut seen = HashSet::new(); - for entry in &entries { - if !seen.insert(entry.node_id) { - continue; - } - let ancestor_path = build_ancestor_path(entry.node_id, ctx); - section.push_str(&format!( - " - keyword '{}' → {} (depth {}, weight {:.2})\n", - hit.keyword, ancestor_path, entry.depth, entry.weight - )); - if let Some(content) = ctx.cat(entry.node_id) { - if let Some(snippet) = - super::super::tools::content_snippet(content, &hit.keyword, 300) - { - section.push_str(&format!(" \"{}\"\n", snippet)); - } - } - entry_count += 1; - if entry_count >= MAX_PLAN_ENTRIES { - section.push_str(" ... (more hits omitted)\n"); - break; - } - } - if entry_count >= MAX_PLAN_ENTRIES { - break; - } - } - section - }; - - let deep_expansion = build_deep_expansion(keyword_hits, ctx); - if !deep_expansion.is_empty() { - keyword_section.push_str(&deep_expansion); - } - - let semantic_section = build_semantic_hints(&query_keywords, &query_lower, ctx); - - let intent_section = build_intent_signals(intent, ctx); - - let system = "You are a document navigation planner. Given a user question, the top-level \ - document structure, keyword index matches, and semantic hints, output a brief navigation \ - plan: which sections to visit and in what order. Prioritize sections that matched keywords \ - or semantic hints. The plan should be 2-5 steps. Each step should be a specific action \ - like \"cd to X, then cat Y\" or \"grep for Z in current subtree\". \ - Pay attention to 'Can answer' and 'Topics' annotations in the structure listing — \ - they indicate what questions each section addresses. \ - Output only the plan, nothing else.\n\n\ - Example plan for \"What is the Q1 revenue?\":\n\ - 1. cd to Revenue (matched keyword 'revenue')\n\ - 2. ls to see sub-sections\n\ - 3. cat Q1 Report\n\ - 4. check\n\ - 5. done".to_string(); - - let user = format!( - "Document: {doc_name}\n\ - Top-level structure:\n{ls_output}{keyword_section}{semantic_section}{intent_section}\ - User question: {query}{task_section}\n\n\ - Navigation plan:" - ); - - (system, user) -} - -/// Build a focused re-planning prompt when check returns INSUFFICIENT. -pub fn build_replan_prompt( - query: &str, - task: Option<&str>, - state: &WorkerState, - ctx: &DocContext<'_>, -) -> (String, String) { - let task_section = match task { - Some(t) => format!("\nOriginal sub-task: {}", t), - None => String::new(), - }; - - let visited = format_visited_titles(state, ctx); - let evidence_summary = state.evidence_summary(); - - let current_children = match ctx.ls(state.current_node) { - Some(routes) if !routes.is_empty() => { - let items: Vec = routes - .iter() - .map(|r| format!(" - {} ({} leaves)", r.title, r.leaf_count)) - .collect(); - format!("Children at current position:\n{}\n", items.join("\n")) - } - _ => "Current position is a leaf node — consider cd .. to go back.\n".to_string(), - }; - - let sibling_hints = build_sibling_hints(state, ctx); - - let system = "You are re-planning a document navigation strategy. The previous plan did not \ - find sufficient evidence. Given what's been found and what's still missing, generate a \ - focused 2-3 step plan. Each step should be a specific action like \ - \"cd to X, then cat Y\" or \"grep for Z in current subtree\". \ - Prefer exploring unvisited branches. If current branch is exhausted, cd .. and try \ - a different path. Output only the plan, nothing else." - .to_string(); - - let user = format!( - "Original question: {query}{task_section}\n\ - Current position: /{}\n\ - Evidence collected so far:\n{evidence_summary}\n\ - What's missing: {}\n\ - Already visited: {visited}\n\ - {current_children}\ - {sibling_hints}\ - Remaining rounds: {}/{}\n\n\ - Revised navigation plan:", - state.path_str(), - state.missing_info, - state.remaining, - state.max_rounds, - ); - - (system, user) -} - -/// Format keyword index hits into a compact string for LLM context. -/// -/// Returns a string like: -/// ```text -/// Keyword matches (use find to jump directly): -/// - 'complex' → Performance (weight 0.85) -/// "...complexity analysis shows..." -/// - 'latency' → Performance (weight 0.72) -/// "...latency benchmarks indicate..." -/// ``` -pub fn format_keyword_hints(keyword_hits: &[FindHit], ctx: &DocContext<'_>) -> String { - if keyword_hits.is_empty() { - return String::new(); - } - - let mut section = String::from("Keyword matches (use find to jump directly):\n"); - let mut entry_count = 0; - for hit in keyword_hits { - let mut entries = hit.entries.clone(); - entries.sort_by(|a, b| { - b.weight - .partial_cmp(&a.weight) - .unwrap_or(std::cmp::Ordering::Equal) - }); - let mut seen = HashSet::new(); - for entry in &entries { - if !seen.insert(entry.node_id) { - continue; - } - let title = ctx.node_title(entry.node_id).unwrap_or("unknown"); - section.push_str(&format!( - " - '{}' → {} (weight {:.2})\n", - hit.keyword, title, entry.weight - )); - if let Some(content) = ctx.cat(entry.node_id) { - if let Some(snippet) = - super::super::tools::content_snippet(content, &hit.keyword, 300) - { - section.push_str(&format!(" \"{}\"\n", snippet)); - } - } - entry_count += 1; - if entry_count >= MAX_PLAN_ENTRIES { - section.push_str(" ... (more omitted)\n"); - return section; - } - } - } - section -} - -/// Build the ancestor path string for a node (e.g., "root/Chapter 1/Section 1.2"). -pub fn build_ancestor_path(node_id: vectorless_document::NodeId, ctx: &DocContext<'_>) -> String { - let mut path: Vec = ctx.tree.ancestors_iter(node_id).collect(); - path.reverse(); - path.iter() - .filter_map(|&id| ctx.node_title(id)) - .collect::>() - .join("/") -} - -/// Build intent-specific index signals for the planning prompt. -/// -/// Injects pre-computed ReasoningIndex data as context for the LLM: -/// - Summary intent → summary_shortcut (document overview + section summaries) -/// - Navigational intent → section_map matches from query keywords -/// - Factual/Analytical → no additional signals (keyword hits already injected) -fn build_intent_signals(intent: QueryIntent, ctx: &DocContext<'_>) -> String { - match intent { - QueryIntent::Summary => { - let shortcut = match ctx.summary_shortcut() { - Some(s) => s, - None => return String::new(), - }; - let mut section = String::from( - "\nPre-computed document overview (use this to plan breadth-first scan):\n", - ); - if !shortcut.document_summary.is_empty() { - section.push_str(&format!( - "Document summary: {}\n", - shortcut.document_summary - )); - } - let mut summary_count = 0; - for ss in &shortcut.section_summaries { - section.push_str(&format!( - " - Section '{}' (depth {}): {}\n", - ss.title, ss.depth, ss.summary - )); - summary_count += 1; - if summary_count >= MAX_SECTION_SUMMARIES { - section.push_str(" ... (more sections omitted)\n"); - break; - } - } - section - } - QueryIntent::Navigational => { - let root = ctx.root(); - let routes = match ctx.ls(root) { - Some(r) => r, - None => return String::new(), - }; - let mut section = - String::from("\nSection map (known top-level sections for direct navigation):\n"); - for route in routes { - section.push_str(&format!( - " - {} ({} leaves)\n", - route.title, route.leaf_count - )); - } - section - } - _ => String::new(), - } -} - -/// Build semantic hints section using BM25 scoring over child routes. -fn build_semantic_hints( - query_keywords: &[String], - query_lower: &str, - ctx: &DocContext<'_>, -) -> String { - let root = ctx.root(); - let routes = match ctx.ls(root) { - Some(r) => r, - None => return String::new(), - }; - - if routes.is_empty() { - return String::new(); - } - - let field_docs: Vec> = routes - .iter() - .map(|route| { - let nav = ctx.nav_entry(route.node_id); - let overview = nav.map(|n| n.overview.as_str()).unwrap_or(""); - let hints_text = nav.map(|n| n.question_hints.join(" ")).unwrap_or_default(); - let tags_text = nav.map(|n| n.topic_tags.join(" ")).unwrap_or_default(); - let content = if overview.is_empty() && hints_text.is_empty() && tags_text.is_empty() { - String::new() - } else { - format!("{} {} {}", overview, hints_text, tags_text) - }; - FieldDocument::new( - route.title.clone(), - route.title.clone(), - route.description.clone(), - content, - ) - }) - .collect(); - - let engine = Bm25Engine::fit_to_corpus(&field_docs); - let bm25_results: std::collections::HashMap = engine - .search_weighted(query_lower, routes.len()) - .into_iter() - .collect(); - - let mut section = String::new(); - let mut entry_count = 0; - - for route in routes { - let nav = match ctx.nav_entry(route.node_id) { - Some(n) => n, - None => continue, - }; - - let bm25_score = bm25_results.get(&route.title).copied().unwrap_or(0.0); - if bm25_score <= 0.0 { - continue; - } - - let mut annotations = Vec::new(); - - for hint in &nav.question_hints { - let hint_lower = hint.to_lowercase(); - for kw in query_keywords { - if hint_lower.contains(&kw.to_lowercase()) { - annotations.push(format!("question \"{}\"", hint)); - break; - } - } - if !annotations.iter().any(|a| a.contains(&hint.clone())) { - for word in hint_lower.split_whitespace() { - if word.len() > 3 && query_lower.contains(word) { - annotations.push(format!("question \"{}\"", hint)); - break; - } - } - } - } - - for tag in &nav.topic_tags { - let tag_lower = tag.to_lowercase(); - for kw in query_keywords { - if tag_lower.contains(&kw.to_lowercase()) || kw.to_lowercase().contains(&tag_lower) - { - annotations.push(format!("topic \"{}\"", tag)); - break; - } - } - if !annotations - .iter() - .any(|a| a.contains(&format!("topic \"{}\"", tag))) - { - if query_lower.contains(&tag_lower) && tag.len() > 2 { - annotations.push(format!("topic \"{}\"", tag)); - } - } - } - - let annotation_str = if annotations.is_empty() { - String::new() - } else { - format!(", {}", annotations.join(", ")) - }; - - let line = format!( - " - Section '{}' — BM25: {:.2}{}\n", - route.title, bm25_score, annotation_str - ); - section.push_str(&line); - entry_count += 1; - if entry_count >= MAX_PLAN_ENTRIES { - break; - } - } - - if section.is_empty() { - String::new() - } else { - format!( - "\nSemantic hints (BM25-scored sections, higher = more relevant):\n{}", - section - ) - } -} - -/// For keyword hits that land in deep nodes (depth >= 2), expand the parent node's children. -fn build_deep_expansion(keyword_hits: &[FindHit], ctx: &DocContext<'_>) -> String { - if keyword_hits.is_empty() { - return String::new(); - } - - let mut seen_parents = HashSet::new(); - let mut expansion = String::new(); - let mut expansion_count = 0; - - for hit in keyword_hits { - for entry in &hit.entries { - if entry.depth < 2 { - continue; - } - let parent = match ctx.parent(entry.node_id) { - Some(p) => p, - None => continue, - }; - if !seen_parents.insert(parent) { - continue; - } - let routes = match ctx.ls(parent) { - Some(r) => r, - None => continue, - }; - let parent_title = ctx.node_title(parent).unwrap_or("unknown"); - expansion.push_str(&format!( - "Siblings near keyword hit '{}' (under {}):\n", - hit.keyword, parent_title - )); - for route in routes { - let marker = if ctx.node_title(entry.node_id) == Some(&route.title) { - " ← keyword hit" - } else { - "" - }; - expansion.push_str(&format!( - " - {} ({} leaves){}\n", - route.title, route.leaf_count, marker - )); - } - expansion.push('\n'); - expansion_count += 1; - if expansion_count >= MAX_EXPANSION_ENTRIES { - expansion.push_str(" ... (more expansions omitted)\n"); - break; - } - } - if expansion_count >= MAX_EXPANSION_ENTRIES { - break; - } - } - - expansion -} - -/// Build unvisited sibling branch hints for structured backtracking. -fn build_sibling_hints(state: &WorkerState, ctx: &DocContext<'_>) -> String { - let mut hints = String::new(); - - if let Some(parent) = ctx.parent(state.current_node) { - if let Some(routes) = ctx.ls(parent) { - let unvisited: Vec<&vectorless_document::ChildRoute> = routes - .iter() - .filter(|r| !state.visited.contains(&r.node_id)) - .collect(); - if !unvisited.is_empty() { - hints.push_str("Unvisited sibling branches at current level:\n"); - for route in &unvisited { - hints.push_str(&format!( - " - {} ({} leaves)\n", - route.title, route.leaf_count - )); - } - } - } - - if let Some(grandparent) = ctx.parent(parent) { - if let Some(routes) = ctx.ls(grandparent) { - let unvisited_parent_siblings: Vec<&vectorless_document::ChildRoute> = routes - .iter() - .filter(|r| !state.visited.contains(&r.node_id) && r.node_id != parent) - .collect(); - if !unvisited_parent_siblings.is_empty() { - hints.push_str("Unvisited branches at parent level (cd .. then explore):\n"); - for route in &unvisited_parent_siblings { - hints.push_str(&format!( - " - {} ({} leaves)\n", - route.title, route.leaf_count - )); - } - } - } - } - } - - if hints.is_empty() { - String::new() - } else { - format!("\n{}", hints) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DocContext; - use crate::config::Evidence; - use crate::state::WorkerState; - use vectorless_document::{ChildRoute, NavEntry, NodeId}; - use vectorless_scoring::bm25::extract_keywords; - - fn build_semantic_test_tree() -> ( - vectorless_document::DocumentTree, - vectorless_document::NavigationIndex, - NodeId, - NodeId, - NodeId, - ) { - let mut tree = vectorless_document::DocumentTree::new("Root", "root content"); - let root = tree.root(); - let revenue = tree.add_child(root, "Revenue", "revenue content"); - let expenses = tree.add_child(root, "Expenses", "expense content"); - - let mut nav = vectorless_document::NavigationIndex::new(); - nav.add_entry( - root, - NavEntry { - overview: "Annual financial report".to_string(), - question_hints: vec!["What is the financial overview?".to_string()], - topic_tags: vec!["finance".to_string()], - leaf_count: 4, - level: 0, - }, - ); - nav.add_child_routes( - root, - vec![ - ChildRoute { - node_id: revenue, - title: "Revenue".to_string(), - description: "Revenue breakdown".to_string(), - leaf_count: 2, - }, - ChildRoute { - node_id: expenses, - title: "Expenses".to_string(), - description: "Cost analysis".to_string(), - leaf_count: 2, - }, - ], - ); - nav.add_entry( - revenue, - NavEntry { - overview: "Revenue figures for 2024".to_string(), - question_hints: vec![ - "What is the total revenue?".to_string(), - "What was the Q1 revenue?".to_string(), - ], - topic_tags: vec![ - "revenue".to_string(), - "sales".to_string(), - "income".to_string(), - ], - leaf_count: 2, - level: 1, - }, - ); - nav.add_entry( - expenses, - NavEntry { - overview: "Operating expenses".to_string(), - question_hints: vec!["What are the operating costs?".to_string()], - topic_tags: vec!["expenses".to_string(), "costs".to_string()], - leaf_count: 2, - level: 1, - }, - ); - - (tree, nav, root, revenue, expenses) - } - - #[test] - fn test_build_ancestor_path() { - let (tree, nav, root, revenue, _) = build_semantic_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - assert_eq!(build_ancestor_path(revenue, &ctx), "Root/Revenue"); - assert_eq!(build_ancestor_path(root, &ctx), "Root"); - } - - #[test] - fn test_semantic_hints_keyword_match() { - let (tree, nav, _, _, _) = build_semantic_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let keywords = extract_keywords("What is the revenue?"); - let hints = build_semantic_hints(&keywords, &"what is the revenue".to_lowercase(), &ctx); - assert!( - hints.contains("Revenue"), - "Should match Revenue section, got: {}", - hints - ); - assert!(hints.contains("BM25")); - } - - #[test] - fn test_semantic_hints_topic_match() { - let (tree, nav, _, _, _) = build_semantic_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let keywords = extract_keywords("operating costs analysis"); - let hints = - build_semantic_hints(&keywords, &"operating costs analysis".to_lowercase(), &ctx); - assert!( - hints.contains("Expenses"), - "Should match Expenses via topic 'costs', got: {}", - hints - ); - } - - #[test] - fn test_semantic_hints_no_match() { - let (tree, nav, _, _, _) = build_semantic_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let keywords = extract_keywords("xyzzy foobar"); - let hints = build_semantic_hints(&keywords, &"xyzzy foobar".to_lowercase(), &ctx); - assert!(hints.is_empty(), "Should not match, got: {}", hints); - } - - #[test] - fn test_build_replan_prompt() { - let (tree, nav, root, _, _) = build_semantic_test_tree(); - let mut state = WorkerState::new(root, 15); - state.missing_info = "Need Q2 revenue figures".to_string(); - state.add_evidence(Evidence { - source_path: "root/Revenue".to_string(), - node_title: "Revenue".to_string(), - content: "Q1 revenue was $2.5M".to_string(), - doc_name: None, - }); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "test", - }; - let (system, user) = build_replan_prompt("What is total revenue?", None, &state, &ctx); - assert!(system.contains("re-planning")); - assert!(user.contains("What is total revenue?")); - assert!(user.contains("Q2 revenue")); - } - - #[test] - fn test_build_plan_prompt_with_semantic_hints() { - let (tree, nav, _, _, _) = build_semantic_test_tree(); - let ctx = DocContext { - tree: &tree, - nav_index: &nav, - reasoning_index: &vectorless_document::ReasoningIndex::default(), - doc_name: "Financial Report", - }; - let ls_output = - "[1] Revenue — Revenue breakdown (2 leaves)\n[2] Expenses — Cost analysis (2 leaves)\n"; - let (system, user) = build_plan_prompt( - "What is the revenue?", - None, - ls_output, - "Financial Report", - &[], - &ctx, - QueryIntent::Factual, - ); - assert!(system.contains("semantic hints")); - assert!(user.contains("What is the revenue?")); - } -} diff --git a/vectorless-core/vectorless-engine/Cargo.toml b/vectorless-core/vectorless-engine/Cargo.toml index 71533ee..5b57292 100644 --- a/vectorless-core/vectorless-engine/Cargo.toml +++ b/vectorless-core/vectorless-engine/Cargo.toml @@ -17,7 +17,6 @@ vectorless-graph = { path = "../vectorless-graph" } vectorless-index = { path = "../vectorless-index" } vectorless-llm = { path = "../vectorless-llm" } vectorless-metrics = { path = "../vectorless-metrics" } -vectorless-rerank = { path = "../vectorless-rerank" } vectorless-storage = { path = "../vectorless-storage" } vectorless-utils = { path = "../vectorless-utils" } tokio = { workspace = true } diff --git a/vectorless-core/vectorless-query/Cargo.toml b/vectorless-core/vectorless-query/Cargo.toml deleted file mode 100644 index 353e94d..0000000 --- a/vectorless-core/vectorless-query/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "vectorless-query" -version.workspace = true -edition.workspace = true -authors.workspace = true -description.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true - -[dependencies] -vectorless-error = { path = "../vectorless-error" } -vectorless-llm = { path = "../vectorless-llm" } -vectorless-scoring = { path = "../vectorless-scoring" } -serde = { workspace = true } -serde_json = { workspace = true } -tracing = { workspace = true } -tokio = { workspace = true } -chrono = { workspace = true } - -[lints] -workspace = true diff --git a/vectorless-core/vectorless-query/src/lib.rs b/vectorless-core/vectorless-query/src/lib.rs deleted file mode 100644 index 38ecb11..0000000 --- a/vectorless-core/vectorless-query/src/lib.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Query understanding and planning. -//! -//! Analyzes a user's raw query and produces a structured [`QueryPlan`] -//! for downstream modules (Orchestrator, Worker). -//! -//! # Pipeline -//! -//! ```text -//! raw query string -//! → extract keywords (from scoring/bm25) -//! → LLM query understanding (intent, concepts, complexity) -//! → QueryPlan -//! ``` -//! -//! LLM understanding is required — this is a pure reasoning engine. -//! Errors are propagated, not silently degraded. - -mod types; -mod understand; - -pub use types::{QueryIntent, QueryPlan}; - -use vectorless_llm::LlmClient; -use vectorless_scoring::bm25::extract_keywords; - -/// Query understanding pipeline. -/// -/// Produces a [`QueryPlan`] from a raw query string via LLM analysis. -pub struct QueryPipeline; - -impl QueryPipeline { - /// Analyze a query and produce a structured plan. - /// - /// 1. Extract keywords (zero-cost, no LLM) - /// 2. LLM deep understanding (intent, concepts, complexity, strategy) - /// - /// Errors propagate — the caller handles retries or failure. - pub async fn understand(query: &str, llm: &LlmClient) -> vectorless_error::Result { - let keywords = extract_keywords(query); - understand::understand(query, &keywords, llm).await - } -} diff --git a/vectorless-core/vectorless-query/src/types.rs b/vectorless-core/vectorless-query/src/types.rs deleted file mode 100644 index f8e025e..0000000 --- a/vectorless-core/vectorless-query/src/types.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2026 vectorless devices -// SPDX-License-Identifier: Apache-2.0 - -//! Core types for query understanding. - -use serde::{Deserialize, Serialize}; - -/// Query intent classification. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum QueryIntent { - /// Factoid: "What is the Q3 2024 revenue?" - Factual, - /// Analytical: "Compare market risk vs operational risk" - Analytical, - /// Navigation: "Find the section on compliance policy" - Navigational, - /// Summary: "Summarize the main points of this document" - Summary, -} - -impl Default for QueryIntent { - fn default() -> Self { - Self::Factual - } -} - -impl std::fmt::Display for QueryIntent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - QueryIntent::Factual => write!(f, "factual"), - QueryIntent::Analytical => write!(f, "analytical"), - QueryIntent::Navigational => write!(f, "navigational"), - QueryIntent::Summary => write!(f, "summary"), - } - } -} - -/// Query complexity estimation. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum Complexity { - /// Single keyword, simple factoid. - Simple, - /// Multi-concept, requires synthesis. - Moderate, - /// Cross-document, comparative, or multi-faceted. - Complex, -} - -impl Default for Complexity { - fn default() -> Self { - Self::Simple - } -} - -impl std::fmt::Display for Complexity { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Complexity::Simple => write!(f, "simple"), - Complexity::Moderate => write!(f, "moderate"), - Complexity::Complex => write!(f, "complex"), - } - } -} - -/// A sub-query produced by decomposition. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SubQuery { - /// The sub-query text. - pub query: String, - /// Intent of this sub-query. - pub intent: QueryIntent, - /// Pre-identified target documents (if any). - pub target_docs: Option>, -} - -/// A structured query plan — the output of the query understanding pipeline. -/// -/// Produced by `QueryPipeline::understand()`. Consumed by the Orchestrator -/// and Worker agents for strategy selection. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct QueryPlan { - /// The original raw query string. - pub original: String, - /// Detected intent. - pub intent: QueryIntent, - /// Extracted keywords. - pub keywords: Vec, - /// Key concepts identified by LLM (distinct from keywords). - pub key_concepts: Vec, - /// Strategy hint for navigation agents. - pub strategy_hint: String, - /// Estimated complexity. - pub complexity: Complexity, - /// Rewritten queries (produced by LLM for better matching). - pub rewritten: Vec, - /// Decomposed sub-queries (for complex/multi-faceted queries). - pub sub_queries: Vec, -} - -impl QueryPlan { - /// LLM understanding failed — produce a minimal default plan. - pub fn default_for(query: &str, keywords: Vec) -> Self { - Self { - original: query.to_string(), - intent: QueryIntent::Factual, - keywords, - key_concepts: Vec::new(), - strategy_hint: "focused".to_string(), - complexity: Complexity::Simple, - rewritten: Vec::new(), - sub_queries: Vec::new(), - } - } -} diff --git a/vectorless-core/vectorless-query/src/understand.rs b/vectorless-core/vectorless-query/src/understand.rs deleted file mode 100644 index c124395..0000000 --- a/vectorless-core/vectorless-query/src/understand.rs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! LLM-driven query understanding. -//! -//! Uses an LLM call to analyze the query and produce a structured [`QueryPlan`]. -//! Falls back to keyword-only analysis on LLM failure. - -use serde::Deserialize; -use tracing::{info, warn}; - -use vectorless_llm::LlmClient; - -use super::types::{Complexity, QueryIntent, QueryPlan, SubQuery}; - -/// Structured analysis returned by the LLM. -#[derive(Deserialize)] -struct QueryAnalysis { - intent: String, - key_concepts: Vec, - strategy_hint: String, - complexity: String, - rewritten: Option, - sub_queries: Vec, -} - -/// Use LLM to understand the query and produce a QueryPlan. -/// -/// Propagates LLM errors — no silent degradation. The caller decides -/// how to handle failure (retry, abort, etc.). -pub async fn understand( - query: &str, - keywords: &[String], - llm: &LlmClient, -) -> vectorless_error::Result { - let (system, user) = understand_prompt(query, keywords); - info!("Query understanding: calling LLM..."); - let response = llm.complete(&system, &user).await?; - - if response.trim().is_empty() { - warn!("Query understanding: LLM returned empty response"); - return Err(vectorless_error::Error::Config( - "Query understanding failed: LLM returned an empty response. \ - Check your API key, model, and endpoint configuration." - .to_string(), - )); - } - - let analysis = parse_analysis(&response).ok_or_else(|| { - let preview = &response[..response.len().min(300)]; - vectorless_error::Error::Config(format!( - "Query understanding returned unparseable response ({} chars): {}", - response.len(), - preview - )) - })?; - - info!( - intent = %analysis.intent, - complexity = %analysis.complexity, - concepts = ?analysis.key_concepts, - strategy = %analysis.strategy_hint, - rewritten = ?analysis.rewritten, - "Query understanding complete" - ); - Ok(analysis.into_plan(query, keywords)) -} - -/// Parse the LLM's JSON response into a QueryAnalysis. -fn parse_analysis(response: &str) -> Option { - let trimmed = response.trim(); - - // Try to extract JSON from the response (LLM may wrap it in markdown) - let json_str = if trimmed.starts_with("```") { - // Find the first newline after the opening fence (skips language tag) - let after_fence = if let Some(nl) = trimmed.find('\n') { - &trimmed[nl + 1..] - } else { - trimmed - }; - // Strip the closing fence - let without_end = if let Some(end) = after_fence.rfind("```") { - &after_fence[..end] - } else { - after_fence - }; - without_end.trim() - } else { - trimmed - }; - - match serde_json::from_str(json_str) { - Ok(analysis) => Some(analysis), - Err(e) => { - warn!( - error = %e, - json_len = json_str.len(), - "Query understanding: JSON parse failed" - ); - None - } - } -} - -impl QueryAnalysis { - fn into_plan(self, query: &str, keywords: &[String]) -> QueryPlan { - QueryPlan { - original: query.to_string(), - intent: parse_intent(&self.intent), - keywords: keywords.to_vec(), - key_concepts: self.key_concepts, - strategy_hint: self.strategy_hint, - complexity: parse_complexity(&self.complexity), - rewritten: self.rewritten.into_iter().collect(), - sub_queries: self - .sub_queries - .into_iter() - .map(|sq| SubQuery { - query: sq, - intent: QueryIntent::Factual, - target_docs: None, - }) - .collect(), - } - } -} - -fn parse_intent(s: &str) -> QueryIntent { - match s.to_lowercase().as_str() { - "analytical" | "analysis" | "compare" | "comparison" => QueryIntent::Analytical, - "navigational" | "navigation" | "find" | "locate" => QueryIntent::Navigational, - "summary" | "summarize" | "overview" => QueryIntent::Summary, - _ => QueryIntent::Factual, - } -} - -fn parse_complexity(s: &str) -> Complexity { - match s.to_lowercase().as_str() { - "complex" | "high" => Complexity::Complex, - "moderate" | "medium" => Complexity::Moderate, - _ => Complexity::Simple, - } -} - -/// Build the LLM prompt for query understanding. -fn understand_prompt(query: &str, keywords: &[String]) -> (String, String) { - let system = r#"You are a query analysis engine. Analyze the user's query and respond with a JSON object containing: - -- "intent": one of "factual", "analytical", "navigational", "summary" -- "key_concepts": array of the main concepts/entities in the query (distinct from keywords) -- "strategy_hint": one of "focused" (single-topic), "exploratory" (broad scan), "comparative" (cross-reference), or "summary" (aggregate) -- "complexity": one of "simple", "moderate", "complex" -- "rewritten": optional rewritten version of the query for better retrieval (null if not needed) -- "sub_queries": array of sub-query strings if the query can be decomposed (empty array if not) - -Respond with ONLY the JSON object, no additional text."#; - - let user = format!( - "Query: {}\nExtracted keywords: [{}]", - query, - keywords.join(", ") - ); - - (system.to_string(), user) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_intent() { - assert_eq!(parse_intent("factual"), QueryIntent::Factual); - assert_eq!(parse_intent("analytical"), QueryIntent::Analytical); - assert_eq!(parse_intent("analysis"), QueryIntent::Analytical); - assert_eq!(parse_intent("navigational"), QueryIntent::Navigational); - assert_eq!(parse_intent("summary"), QueryIntent::Summary); - assert_eq!(parse_intent("unknown"), QueryIntent::Factual); - } - - #[test] - fn test_parse_complexity() { - assert_eq!(parse_complexity("simple"), Complexity::Simple); - assert_eq!(parse_complexity("moderate"), Complexity::Moderate); - assert_eq!(parse_complexity("complex"), Complexity::Complex); - assert_eq!(parse_complexity("high"), Complexity::Complex); - assert_eq!(parse_complexity("unknown"), Complexity::Simple); - } - - #[test] - fn test_parse_analysis_json() { - let response = r#"{"intent":"factual","key_concepts":["revenue","Q3"],"strategy_hint":"focused","complexity":"simple","rewritten":null,"sub_queries":[]}"#; - let analysis = parse_analysis(response).unwrap(); - assert_eq!(analysis.intent, "factual"); - assert_eq!(analysis.key_concepts.len(), 2); - assert!(analysis.rewritten.is_none()); - } - - #[test] - fn test_parse_analysis_markdown_wrapped() { - let response = "```json\n{\"intent\":\"analytical\",\"key_concepts\":[\"risk\"],\"strategy_hint\":\"comparative\",\"complexity\":\"moderate\",\"rewritten\":\"compare risks\",\"sub_queries\":[]}\n```"; - let analysis = parse_analysis(response).unwrap(); - assert_eq!(analysis.intent, "analytical"); - } - - #[test] - fn test_parse_analysis_invalid() { - assert!(parse_analysis("not json").is_none()); - } - - #[test] - fn test_parse_analysis_code_fence_no_newline() { - // Edge case: ```json{"intent":...}``` with no newline after language tag - let response = "```json\n{\"intent\":\"factual\",\"key_concepts\":[\"test\"],\"strategy_hint\":\"focused\",\"complexity\":\"simple\",\"rewritten\":null,\"sub_queries\":[]}\n```"; - let analysis = parse_analysis(response).unwrap(); - assert_eq!(analysis.intent, "factual"); - } - - #[test] - fn test_parse_analysis_code_fence_no_closing() { - // LLM sometimes omits the closing fence - let response = "```json\n{\"intent\":\"summary\",\"key_concepts\":[\"overview\"],\"strategy_hint\":\"summary\",\"complexity\":\"simple\",\"rewritten\":null,\"sub_queries\":[]}"; - let analysis = parse_analysis(response).unwrap(); - assert_eq!(analysis.intent, "summary"); - } - - #[test] - fn test_parse_analysis_keys_starting_with_fence_letters() { - // The old trim_start_matches(|c| 'j' | 's' | 'o' | 'n') would eat - // JSON keys starting with those letters. Verify this works correctly. - let response = r#"{"intent":"navigational","key_concepts":["journal","offset","node"],"strategy_hint":"focused","complexity":"moderate","rewritten":null,"sub_queries":[]}"#; - let analysis = parse_analysis(response).unwrap(); - assert_eq!(analysis.intent, "navigational"); - assert_eq!(analysis.key_concepts, vec!["journal", "offset", "node"]); - } - - #[test] - fn test_default_plan() { - let plan = QueryPlan::default_for("test query", vec!["test".to_string()]); - assert_eq!(plan.original, "test query"); - assert_eq!(plan.intent, QueryIntent::Factual); - assert_eq!(plan.keywords.len(), 1); - assert!(plan.key_concepts.is_empty()); - assert!(plan.sub_queries.is_empty()); - } -} diff --git a/vectorless-core/vectorless-rerank/Cargo.toml b/vectorless-core/vectorless-rerank/Cargo.toml deleted file mode 100644 index f9a9112..0000000 --- a/vectorless-core/vectorless-rerank/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "vectorless-rerank" -version.workspace = true -edition.workspace = true -authors.workspace = true -description.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true - -[dependencies] -serde = { workspace = true } -serde_json = { workspace = true } -tracing = { workspace = true } -vectorless-error = { path = "../vectorless-error" } -vectorless-query = { path = "../vectorless-query" } - -[lints] -workspace = true diff --git a/vectorless-core/vectorless-rerank/src/dedup.rs b/vectorless-core/vectorless-rerank/src/dedup.rs deleted file mode 100644 index 713c608..0000000 --- a/vectorless-core/vectorless-rerank/src/dedup.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Evidence deduplication and quality filtering. - -use std::collections::HashSet; - -use crate::types::Evidence; - -/// Minimum characters for an evidence item to be considered meaningful. -const MIN_EVIDENCE_CHARS: usize = 50; - -/// Jaccard similarity threshold for content dedup. -const SIMILARITY_THRESHOLD: f64 = 0.8; - -/// Filter low-quality and duplicate evidence. -/// -/// Steps: -/// 1. Drop evidence with no meaningful content (< MIN_EVIDENCE_CHARS) -/// 2. Deduplicate by source overlap (same path in same doc) -/// 3. Deduplicate by content similarity (Jaccard on token sets) -pub fn dedup(evidence: &[Evidence]) -> Vec { - // Step 1: Quality filter - let quality: Vec<&Evidence> = evidence - .iter() - .filter(|e| e.content.len() >= MIN_EVIDENCE_CHARS) - .collect(); - - // Step 2: Deduplicate by source overlap - let mut seen_sources: HashSet = HashSet::new(); - let source_deduped: Vec<&Evidence> = quality - .into_iter() - .filter(|e| { - let doc_key = e.doc_name.as_deref().unwrap_or("_unknown"); - let key = format!("{}:{}", doc_key, e.source_path); - seen_sources.insert(key) - }) - .collect(); - - // Step 3: Deduplicate by content similarity - let mut deduped: Vec = Vec::new(); - for ev in source_deduped { - let tokens = tokenize(&ev.content); - let dominated = deduped - .iter() - .any(|existing| jaccard(&tokens, &tokenize(&existing.content)) >= SIMILARITY_THRESHOLD); - if !dominated { - deduped.push(ev.clone()); - } - } - - deduped -} - -/// Tokenize text into a set of lowercase words. -fn tokenize(text: &str) -> HashSet { - text.to_lowercase() - .split_whitespace() - .map(|s| s.to_string()) - .collect() -} - -/// Compute Jaccard similarity between two sets. -fn jaccard(a: &HashSet, b: &HashSet) -> f64 { - if a.is_empty() && b.is_empty() { - return 1.0; - } - let intersection = a.intersection(b).count() as f64; - let union = a.union(b).count() as f64; - intersection / union -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_evidence(title: &str, content: &str) -> Evidence { - Evidence { - source_path: format!("root/{}", title), - node_title: title.to_string(), - content: content.to_string(), - doc_name: Some("doc".to_string()), - } - } - - #[test] - fn test_quality_filter() { - let evidence = vec![ - make_evidence("A", "short"), // < 50 chars, filtered - make_evidence("B", &"x".repeat(60)), // kept - ]; - let result = dedup(&evidence); - assert_eq!(result.len(), 1); - assert_eq!(result[0].node_title, "B"); - } - - #[test] - fn test_source_dedup() { - let evidence = vec![ - make_evidence( - "A", - &"content A with enough text to pass the quality filter threshold".to_string(), - ), - make_evidence( - "A", - &"different content A but same source path that is long enough".to_string(), - ), - ]; - let result = dedup(&evidence); - assert_eq!(result.len(), 1); - } - - #[test] - fn test_content_similarity_dedup() { - let base = "This is a piece of evidence about machine learning algorithms and their applications in real world scenarios".to_string(); - let similar = "This is a piece of evidence about machine learning algorithms and their applications in real world".to_string(); - let different = - "Completely unrelated content about quantum physics and particle accelerators at CERN" - .to_string(); - let evidence = vec![ - make_evidence("A", &base), - make_evidence("B", &similar), // high similarity, should be deduped - make_evidence("C", &different), // different, kept - ]; - let result = dedup(&evidence); - assert!(result.len() >= 2); // at least A and C - } - - #[test] - fn test_empty_input() { - let result = dedup(&[]); - assert!(result.is_empty()); - } - - #[test] - fn test_jaccard_identical() { - let a = tokenize("hello world foo"); - let b = tokenize("hello world foo"); - assert!((jaccard(&a, &b) - 1.0).abs() < 0.001); - } - - #[test] - fn test_jaccard_disjoint() { - let a = tokenize("aaa bbb"); - let b = tokenize("ccc ddd"); - assert!((jaccard(&a, &b)).abs() < 0.001); - } - - #[test] - fn test_source_dedup_none_doc_name() { - // Evidence with doc_name: None should use "_unknown" as doc key, - // so same source_path with None doc_name still deduplicates correctly. - let evidence = vec![ - Evidence { - source_path: "root/section_a".to_string(), - node_title: "A".to_string(), - content: "content A with enough text to pass the quality filter threshold" - .to_string(), - doc_name: None, - }, - Evidence { - source_path: "root/section_a".to_string(), - node_title: "A2".to_string(), - content: "different content but same source path that should be deduped" - .to_string(), - doc_name: None, - }, - ]; - let result = dedup(&evidence); - assert_eq!(result.len(), 1); - } - - #[test] - fn test_source_dedup_mixed_doc_name() { - // Same source_path but different doc_name should produce different dedup keys, - // so both survive source dedup. Content must be sufficiently different too. - let evidence = vec![ - Evidence { - source_path: "root/section".to_string(), - node_title: "A".to_string(), - content: "Revenue for Q4 was twelve million dollars driven by SaaS growth in the enterprise segment".to_string(), - doc_name: Some("doc_a".to_string()), - }, - Evidence { - source_path: "root/section".to_string(), - node_title: "B".to_string(), - content: "The encryption module uses AES-256 for data at rest and TLS 1.3 for all network communication".to_string(), - doc_name: Some("doc_b".to_string()), - }, - ]; - let result = dedup(&evidence); - assert_eq!(result.len(), 2); - } - - #[test] - fn test_source_dedup_none_vs_some_doc_name() { - // None doc_name ("_unknown") and Some doc_name produce different keys, - // so both survive source dedup. Content must be sufficiently different too. - let evidence = vec![ - Evidence { - source_path: "root/section".to_string(), - node_title: "A".to_string(), - content: "The database uses a log-structured merge tree with write-ahead logging for durability".to_string(), - doc_name: None, - }, - Evidence { - source_path: "root/section".to_string(), - node_title: "B".to_string(), - content: "Authentication requires Bearer tokens with automatic refresh after twenty-four hours".to_string(), - doc_name: Some("doc_x".to_string()), - }, - ]; - let result = dedup(&evidence); - assert_eq!(result.len(), 2); - } -} diff --git a/vectorless-core/vectorless-rerank/src/lib.rs b/vectorless-core/vectorless-rerank/src/lib.rs deleted file mode 100644 index 875c102..0000000 --- a/vectorless-core/vectorless-rerank/src/lib.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Result reranking — dedup + format. -//! -//! Post-processing pipeline that runs after the agent collects raw evidence: -//! -//! ```text -//! agent (collect evidence) -//! → rerank::process() -//! → dedup (quality filter + dedup) -//! → format as answer (no LLM — return original text) -//! → Output with final answer -//! ``` -//! -//! This is a document retrieval engine. The answer IS the evidence. -//! No LLM synthesis, no rewriting. Find what you find, return what you find. - -pub mod dedup; -pub mod types; - -use tracing::info; - -use types::{Evidence, RerankOutput}; -use vectorless_query::QueryIntent; - -/// Process agent output through the rerank pipeline. -/// -/// Deduplicates evidence, then returns the original text as the answer. -/// No LLM calls — the Worker already retrieved the exact passages. -pub async fn process( - _query: &str, - evidence: &[Evidence], - _multi_doc: bool, - intent: QueryIntent, - confidence: f32, -) -> vectorless_error::Result { - let deduped = dedup::dedup(evidence); - if deduped.is_empty() { - info!("No evidence after dedup"); - return Ok(RerankOutput { - answer: String::new(), - llm_calls: 0, - confidence: 0.0, - }); - } - - info!( - evidence = deduped.len(), - intent = %intent, - "Evidence after dedup" - ); - - let answer = match intent { - QueryIntent::Navigational => format_locations(&deduped), - _ => format_evidence_as_answer(&deduped), - }; - - info!( - evidence = deduped.len(), - answer_len = answer.len(), - confidence, - "Rerank complete" - ); - - Ok(RerankOutput { - answer, - llm_calls: 0, - confidence, - }) -} - -/// Format evidence as a location listing for navigational queries. -fn format_locations(evidence: &[Evidence]) -> String { - if evidence.is_empty() { - return "No matching locations found.".to_string(); - } - let mut result = "Found at:\n".to_string(); - for e in evidence { - let doc = e.doc_name.as_deref().unwrap_or("unknown"); - result.push_str(&format!( - "- **{}** in {} at {}\n", - e.node_title, doc, e.source_path - )); - } - result -} - -/// Format collected evidence directly as the answer. -fn format_evidence_as_answer(evidence: &[Evidence]) -> String { - evidence - .iter() - .map(|e| { - let doc = e.doc_name.as_deref().unwrap_or(""); - if doc.is_empty() { - format!("[{}]\n{}", e.node_title, e.content) - } else { - format!("[{} — {}]\n{}", e.node_title, doc, e.content) - } - }) - .collect::>() - .join("\n\n") -} diff --git a/vectorless-core/vectorless-rerank/src/types.rs b/vectorless-core/vectorless-rerank/src/types.rs deleted file mode 100644 index 73d19ce..0000000 --- a/vectorless-core/vectorless-rerank/src/types.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Rerank result types. - -use serde::{Deserialize, Serialize}; - -/// A single piece of evidence collected during navigation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Evidence { - /// Navigation path where this evidence was found (e.g., "Root/API Reference/Auth"). - pub source_path: String, - /// Title of the node. - pub node_title: String, - /// Content of the node. - pub content: String, - /// Source document name (set by Orchestrator in multi-doc scenarios). - pub doc_name: Option, -} - -/// Output from the rerank pipeline. -pub struct RerankOutput { - /// Synthesized answer. - pub answer: String, - /// Number of LLM calls used during synthesis/fusion. - pub llm_calls: u32, - /// Confidence score (0.0–1.0) — derived from LLM evaluate() result. - pub confidence: f32, -} diff --git a/vectorless-core/vectorless-retrieval/Cargo.toml b/vectorless-core/vectorless-retrieval/Cargo.toml deleted file mode 100644 index b364f76..0000000 --- a/vectorless-core/vectorless-retrieval/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "vectorless-retrieval" -version.workspace = true -edition.workspace = true -authors.workspace = true -description.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true - -[dependencies] -vectorless-agent = { path = "../vectorless-agent" } -vectorless-document = { path = "../vectorless-document" } -vectorless-error = { path = "../vectorless-error" } -vectorless-llm = { path = "../vectorless-llm" } -vectorless-query = { path = "../vectorless-query" } -vectorless-storage = { path = "../vectorless-storage" } -vectorless-utils = { path = "../vectorless-utils" } -tokio = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -tracing = { workspace = true } -futures = { workspace = true } -parking_lot = { workspace = true } - -[dev-dependencies] -indextree = { workspace = true } - -[lints] -workspace = true diff --git a/vectorless-core/vectorless-retrieval/src/cache.rs b/vectorless-core/vectorless-retrieval/src/cache.rs deleted file mode 100644 index c924732..0000000 --- a/vectorless-core/vectorless-retrieval/src/cache.rs +++ /dev/null @@ -1,577 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Tiered reasoning cache for the retrieval pipeline. -//! -//! Provides three levels of caching to avoid redundant computation: -//! -//! - **L1 (Exact)**: Cache full retrieval results keyed by exact query fingerprint. -//! Identical queries return instantly. -//! -//! - **L2 (Path Pattern)**: Cache navigation decisions for tree paths. If a previous -//! query navigated through Section 3.2, a new query about the same section can -//! reuse those path cues even when the full query differs. -//! -//! - **L3 (Strategy Score)**: Cache node scores from keyword/BM25 strategies. -//! Node scores are independent of the query, so they can be shared across -//! different queries on the same document. - -use std::collections::{HashMap, VecDeque}; -use std::sync::RwLock; -use std::time::Instant; - -use tracing::warn; - -use vectorless_document::NodeId; -use vectorless_utils::fingerprint::Fingerprint; - -/// A tiered reasoning cache for the retrieval pipeline. -/// -/// Thread-safe via `RwLock`. Each tier has independent size limits -/// and TTL-based expiration. -pub struct ReasoningCache { - /// L1: Exact query → cached candidate list. - l1: RwLock, - /// L2: Node path pattern → cached navigation cue score. - l2: RwLock, - /// L3: Node content fingerprint → cached strategy score. - l3: RwLock, - /// Configuration. - config: ReasoningCacheConfig, -} - -/// Configuration for the reasoning cache. -#[derive(Debug, Clone)] -pub struct ReasoningCacheConfig { - /// Maximum L1 entries (exact query results). - pub l1_max: usize, - /// Maximum L2 entries (path patterns). - pub l2_max: usize, - /// Maximum L3 entries (strategy scores). - pub l3_max: usize, -} - -impl Default for ReasoningCacheConfig { - fn default() -> Self { - Self { - l1_max: 200, - l2_max: 1000, - l3_max: 5000, - } - } -} - -// ---- L1: Exact Query Cache ---- - -#[derive(Debug, Clone)] -struct L1Entry { - /// Fingerprint of the workspace + document set used for this query. - scope_fp: Fingerprint, - /// Cached candidate nodes (pre-sorted by score). - candidates: Vec, - /// Strategy used. - strategy: String, - /// When cached. - created_at: Instant, -} - -/// A cached candidate from a previous retrieval. -#[derive(Debug, Clone)] -pub struct CachedCandidate { - /// Node ID. - pub node_id: NodeId, - /// Relevance score. - pub score: f32, - /// Depth in tree. - pub depth: usize, -} - -struct L1Store { - entries: HashMap, - order: VecDeque, // For LRU eviction — O(1) pop_front -} - -// ---- L2: Path Pattern Cache ---- - -#[derive(Debug, Clone)] -struct L2Entry { - /// Score for this navigation cue. - confidence: f32, - /// How many times this path was relevant. - hit_count: usize, - created_at: Instant, -} - -struct L2Store { - entries: HashMap, // Key: "doc_fp:node_path" - order: VecDeque, -} - -// ---- L3: Strategy Score Cache ---- - -#[derive(Debug, Clone)] -struct L3Entry { - /// BM25/Keyword score. - score: f32, - /// Which strategy produced this score. - strategy: String, - created_at: Instant, -} - -struct L3Store { - entries: HashMap, // Key: node content fingerprint - order: VecDeque, -} - -// ---- Public API ---- - -impl ReasoningCache { - /// Create a new reasoning cache with default configuration. - pub fn new() -> Self { - Self::with_config(ReasoningCacheConfig::default()) - } - - /// Create with custom configuration. - pub fn with_config(config: ReasoningCacheConfig) -> Self { - Self { - l1: RwLock::new(L1Store { - entries: HashMap::new(), - order: VecDeque::new(), - }), - l2: RwLock::new(L2Store { - entries: HashMap::new(), - order: VecDeque::new(), - }), - l3: RwLock::new(L3Store { - entries: HashMap::new(), - order: VecDeque::new(), - }), - config, - } - } - - // ============ L1: Exact Query ============ - - /// Look up an exact query result. - /// - /// Returns cached candidates if the same query was executed before - /// on the same document scope. - pub fn l1_get(&self, query: &str, scope_fp: &Fingerprint) -> Option> { - let query_fp = Fingerprint::from_str(query); - let l1 = read_lock(&self.l1)?; - let entry = l1.entries.get(&query_fp)?; - // Scope must match (same document set) - if &entry.scope_fp != scope_fp { - return None; - } - Some(entry.candidates.clone()) - } - - /// Store an L1 result. - pub fn l1_store( - &self, - query: &str, - scope_fp: Fingerprint, - candidates: Vec, - strategy: String, - ) { - let query_fp = Fingerprint::from_str(query); - if let Ok(mut l1) = self.l1.write() { - if l1.entries.len() >= self.config.l1_max { - Self::evict_lru_fingerprint(&mut l1); - } - l1.entries.insert( - query_fp, - L1Entry { - scope_fp, - candidates, - strategy, - created_at: Instant::now(), - }, - ); - l1.order.push_back(query_fp); - } - } - - // ============ L2: Path Pattern ============ - - /// Look up a cached navigation confidence for a document + node path. - /// - /// If a previous query successfully navigated through this path, - /// return the confidence score. - pub fn l2_get(&self, doc_key: &str, node_path: &str) -> Option { - let key = format!("{}:{}", doc_key, node_path); - let l2 = read_lock(&self.l2)?; - let entry = l2.entries.get(&key)?; - Some(entry.confidence) - } - - /// Record a successful navigation through a path. - /// - /// Call this after retrieval confirms a path was relevant. - pub fn l2_record(&self, doc_key: &str, node_path: &str, confidence: f32) { - let key = format!("{}:{}", doc_key, node_path); - if let Ok(mut l2) = self.l2.write() { - if let Some(entry) = l2.entries.get_mut(&key) { - // Update running average - entry.hit_count += 1; - entry.confidence = - entry.confidence + (confidence - entry.confidence) / entry.hit_count as f32; - } else { - if l2.entries.len() >= self.config.l2_max { - Self::evict_lru_string(&mut l2); - } - l2.entries.insert( - key.clone(), - L2Entry { - confidence, - hit_count: 1, - created_at: Instant::now(), - }, - ); - l2.order.push_back(key); - } - } - } - - /// Get top-N path hints for a document, sorted by confidence. - /// - /// Useful for bootstrapping new queries on a known document. - pub fn l2_top_paths(&self, doc_key: &str, n: usize) -> Vec<(String, f32)> { - let prefix = format!("{}:", doc_key); - let l2 = match read_lock(&self.l2) { - Some(guard) => guard, - None => return Vec::new(), - }; - - let mut paths: Vec<(String, f32)> = l2 - .entries - .iter() - .filter(|(k, _)| k.starts_with(&prefix)) - .map(|(k, v)| (k[prefix.len()..].to_string(), v.confidence)) - .collect(); - paths.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - paths.truncate(n); - paths - } - - // ============ L3: Strategy Score ============ - - /// Look up a cached strategy score for a node. - /// - /// Node scores from keyword/BM25 are content-dependent but - /// query-independent, so they can be shared across queries. - pub fn l3_get(&self, node_content_fp: &Fingerprint) -> Option<(f32, String)> { - let l3 = read_lock(&self.l3)?; - let entry = l3.entries.get(node_content_fp)?; - Some((entry.score, entry.strategy.clone())) - } - - /// Store a strategy score for a node. - pub fn l3_store(&self, node_content_fp: Fingerprint, score: f32, strategy: String) { - if let Ok(mut l3) = self.l3.write() { - if l3.entries.len() >= self.config.l3_max { - Self::evict_lru_fingerprint_l3(&mut l3); - } - l3.entries.insert( - node_content_fp, - L3Entry { - score, - strategy, - created_at: Instant::now(), - }, - ); - l3.order.push_back(node_content_fp); - } - } - - // ============ Stats ============ - - /// Get cache statistics. - pub fn stats(&self) -> ReasoningCacheStats { - let (l1_count, l2_count, l3_count) = ( - read_lock(&self.l1).map(|g| g.entries.len()).unwrap_or(0), - read_lock(&self.l2).map(|g| g.entries.len()).unwrap_or(0), - read_lock(&self.l3).map(|g| g.entries.len()).unwrap_or(0), - ); - ReasoningCacheStats { - l1_entries: l1_count, - l2_entries: l2_count, - l3_entries: l3_count, - } - } - - /// Clear all cache tiers. - pub fn clear(&self) { - if let Ok(mut l1) = self.l1.write() { - l1.entries.clear(); - l1.order.clear(); - } - if let Ok(mut l2) = self.l2.write() { - l2.entries.clear(); - l2.order.clear(); - } - if let Ok(mut l3) = self.l3.write() { - l3.entries.clear(); - l3.order.clear(); - } - } - - // ============ Eviction helpers ============ - - fn evict_lru_fingerprint(l1: &mut L1Store) { - if let Some(old) = l1.order.pop_front() { - l1.entries.remove(&old); - } - } - - fn evict_lru_string(l2: &mut L2Store) { - if let Some(old) = l2.order.pop_front() { - l2.entries.remove(&old); - } - } - - fn evict_lru_fingerprint_l3(l3: &mut L3Store) { - if let Some(old) = l3.order.pop_front() { - l3.entries.remove(&old); - } - } -} - -impl Default for ReasoningCache { - fn default() -> Self { - Self::new() - } -} - -/// Read from a RwLock, recovering from poison by taking the guard anyway. -/// -/// A poisoned lock means another thread panicked while holding it — the data -/// is still valid, just potentially in an inconsistent state. For a cache, -/// returning stale/empty data is always preferable to failing silently. -fn read_lock(lock: &RwLock) -> Option> { - match lock.read() { - Ok(guard) => Some(guard), - Err(poisoned) => { - warn!("ReasoningCache: recovering from poisoned lock"); - Some(poisoned.into_inner()) - } - } -} - -/// Cache statistics. -#[derive(Debug, Clone)] -pub struct ReasoningCacheStats { - /// L1 entries (exact query results). - pub l1_entries: usize, - /// L2 entries (path patterns). - pub l2_entries: usize, - /// L3 entries (strategy scores). - pub l3_entries: usize, -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_node_id(n: usize) -> NodeId { - let mut arena = indextree::Arena::new(); - NodeId(arena.new_node(n)) - } - - #[test] - fn test_l1_store_and_retrieve() { - let cache = ReasoningCache::new(); - let scope = Fingerprint::from_str("doc1"); - - let candidates = vec![CachedCandidate { - node_id: make_node_id(1), - score: 0.9, - depth: 2, - }]; - - cache.l1_store("what is rust?", scope, candidates.clone(), "keyword".into()); - let result = cache.l1_get("what is rust?", &scope); - assert!(result.is_some()); - assert_eq!(result.unwrap().len(), 1); - } - - #[test] - fn test_l1_miss_different_scope() { - let cache = ReasoningCache::new(); - let scope1 = Fingerprint::from_str("doc1"); - let scope2 = Fingerprint::from_str("doc2"); - - let candidates = vec![CachedCandidate { - node_id: make_node_id(1), - score: 0.9, - depth: 2, - }]; - - cache.l1_store("query", scope1, candidates, "keyword".into()); - assert!(cache.l1_get("query", &scope2).is_none()); - } - - #[test] - fn test_l2_record_and_get() { - let cache = ReasoningCache::new(); - - cache.l2_record("doc1", "3.2", 0.8); - let score = cache.l2_get("doc1", "3.2"); - assert!(score.is_some()); - assert!((score.unwrap() - 0.8).abs() < 0.01); - } - - #[test] - fn test_l2_running_average() { - let cache = ReasoningCache::new(); - - cache.l2_record("doc1", "3.2", 0.8); - cache.l2_record("doc1", "3.2", 0.6); - let score = cache.l2_get("doc1", "3.2").unwrap(); - // Running average: 0.8 + (0.6 - 0.8) / 2 = 0.7 - assert!((score - 0.7).abs() < 0.01); - } - - #[test] - fn test_l2_top_paths() { - let cache = ReasoningCache::new(); - - cache.l2_record("doc1", "3.1", 0.5); - cache.l2_record("doc1", "3.2", 0.9); - cache.l2_record("doc1", "2.1", 0.7); - - let top = cache.l2_top_paths("doc1", 2); - assert_eq!(top.len(), 2); - assert!((top[0].1 - 0.9).abs() < 0.01); // 3.2 is highest - } - - #[test] - fn test_l3_store_and_retrieve() { - let cache = ReasoningCache::new(); - let fp = Fingerprint::from_str("some node content"); - - cache.l3_store(fp, 0.85, "bm25".into()); - let (score, strategy) = cache.l3_get(&fp).unwrap(); - assert!((score - 0.85).abs() < 0.01); - assert_eq!(strategy, "bm25"); - } - - #[test] - fn test_clear() { - let cache = ReasoningCache::new(); - let scope = Fingerprint::from_str("doc1"); - - cache.l1_store("q", scope, vec![], "kw".into()); - cache.l2_record("doc1", "1", 0.5); - cache.l3_store(Fingerprint::from_str("c"), 0.5, "kw".into()); - - cache.clear(); - - let stats = cache.stats(); - assert_eq!(stats.l1_entries, 0); - assert_eq!(stats.l2_entries, 0); - assert_eq!(stats.l3_entries, 0); - } - - #[test] - fn test_l1_lru_eviction() { - let config = ReasoningCacheConfig { - l1_max: 2, - ..Default::default() - }; - let cache = ReasoningCache::with_config(config); - let scope = Fingerprint::from_str("doc"); - - cache.l1_store("q1", scope, vec![], "kw".into()); - cache.l1_store("q2", scope, vec![], "kw".into()); - cache.l1_store("q3", scope, vec![], "kw".into()); // evicts q1 - - assert!(cache.l1_get("q1", &scope).is_none()); - assert!(cache.l1_get("q2", &scope).is_some()); - assert!(cache.l1_get("q3", &scope).is_some()); - } - - #[test] - fn test_l2_lru_eviction() { - let config = ReasoningCacheConfig { - l2_max: 2, - ..Default::default() - }; - let cache = ReasoningCache::with_config(config); - - cache.l2_record("doc", "1", 0.5); - cache.l2_record("doc", "2", 0.6); - cache.l2_record("doc", "3", 0.7); // evicts "doc:1" - - assert!(cache.l2_get("doc", "1").is_none()); - assert!(cache.l2_get("doc", "2").is_some()); - assert!(cache.l2_get("doc", "3").is_some()); - } - - #[test] - fn test_l3_lru_eviction() { - let config = ReasoningCacheConfig { - l3_max: 2, - ..Default::default() - }; - let cache = ReasoningCache::with_config(config); - - let fp1 = Fingerprint::from_str("content_a"); - let fp2 = Fingerprint::from_str("content_b"); - let fp3 = Fingerprint::from_str("content_c"); - - cache.l3_store(fp1, 0.5, "kw".into()); - cache.l3_store(fp2, 0.6, "kw".into()); - cache.l3_store(fp3, 0.7, "kw".into()); // evicts fp1 - - assert!(cache.l3_get(&fp1).is_none()); - assert!(cache.l3_get(&fp2).is_some()); - assert!(cache.l3_get(&fp3).is_some()); - } - - #[test] - fn test_poisoned_lock_recovery() { - let cache = ReasoningCache::new(); - - // Verify normal operation: store and retrieve still works - let scope = Fingerprint::from_str("doc"); - cache.l1_store("query", scope, vec![], "kw".into()); - - let scope2 = Fingerprint::from_str("doc2"); - cache.l1_store("q2", scope2, vec![], "kw".into()); - assert!(cache.l1_get("q2", &scope2).is_some()); - - // Verify stats still works (internally uses read_lock) - let stats = cache.stats(); - assert!(stats.l1_entries >= 1); - } - - #[test] - fn test_poisoned_lock_read_recovery() { - use std::sync::Arc; - use std::thread; - - // Create a cache and populate it - let cache = Arc::new(ReasoningCache::new()); - let scope = Fingerprint::from_str("doc"); - cache.l1_store("query", scope, vec![], "kw".into()); - - // Poison the lock from another thread - let cache_clone = Arc::clone(&cache); - let handle = thread::spawn(move || { - // This will poison the L1 lock - let _guard = cache_clone.l1.write().unwrap(); - panic!("intentional panic to poison lock"); - }); - - // Wait for the panicking thread to finish - let _ = handle.join(); - - // The lock is now poisoned. Our read_lock() should recover from it. - // l1_get uses read_lock internally - let result = cache.l1_get("query", &scope); - // Should still return data (recovered from poison) - assert!(result.is_some()); - } -} diff --git a/vectorless-core/vectorless-retrieval/src/dispatcher.rs b/vectorless-core/vectorless-retrieval/src/dispatcher.rs deleted file mode 100644 index b2c4509..0000000 --- a/vectorless-core/vectorless-retrieval/src/dispatcher.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Retrieval dispatcher — the single entry point for all query operations. -//! -//! All queries go through the Orchestrator. There is no separate Worker path. -//! The Orchestrator internally decides whether to run the full analysis phase -//! based on user intent: -//! -//! - **User specified doc_ids** → Orchestrator skips analysis, spawns N Workers -//! directly (N=1 is a normal case, not special). -//! - **User unspecified (workspace)** → Orchestrator analyzes DocCards, selects -//! relevant docs, then spawns Workers. -//! -//! Post-processing (synthesis, dedup, rerank) is always unified through the -//! Orchestrator's output — never duplicated in Worker. - -use tracing::info; - -use vectorless_agent::config::{AgentConfig, Scope, WorkspaceContext}; -use vectorless_agent::orchestrator::Orchestrator; -use vectorless_agent::{Agent, EventEmitter, Output}; -use vectorless_error::{Error, Result}; -use vectorless_llm::LlmClient; -use vectorless_query::QueryPipeline; - -/// Dispatch a query to the Orchestrator. -/// -/// This is the single entry point from the client layer into the retrieval system. -/// It always goes through the Orchestrator — never directly to Worker. -/// -/// Flow: -/// 1. Query understanding via LLM (produces [`QueryPlan`]) -/// 2. Orchestrator dispatch (uses QueryPlan for strategy) -/// -/// - `Scope::Specified(docs)` → Orchestrator skips analysis, dispatches all docs directly. -/// - `Scope::Workspace(ws)` → Orchestrator runs full flow (analyze → dispatch → fuse → synthesize). -pub async fn dispatch( - query: &str, - scope: Scope<'_>, - config: &AgentConfig, - llm: &LlmClient, - emitter: &EventEmitter, -) -> Result { - let (ws, skip_analysis) = match scope { - Scope::Specified(docs) => { - info!( - docs = docs.len(), - "Dispatch (user-specified, skip analysis)" - ); - (WorkspaceContext::new(docs), true) - } - Scope::Workspace(ws) => { - info!(docs = ws.doc_count(), "Dispatch (workspace, full flow)"); - (ws, false) - } - }; - - // Step 1: Query understanding — LLM analyzes intent, concepts, complexity. - // This is required. "Model fails, we fail." — errors propagate. - info!("Starting query understanding..."); - let query_plan = QueryPipeline::understand(query, llm).await?; - - // Step 2: Dispatch to Orchestrator with the query plan. - let orchestrator = Orchestrator::new( - query, - &ws, - config.clone(), - llm.clone(), - emitter.clone(), - skip_analysis, - query_plan, - ); - orchestrator - .run() - .await - .map_err(|e| Error::Retrieval(e.to_string())) -} diff --git a/vectorless-core/vectorless-retrieval/src/lib.rs b/vectorless-core/vectorless-retrieval/src/lib.rs deleted file mode 100644 index 96d6f1d..0000000 --- a/vectorless-core/vectorless-retrieval/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Retrieval dispatch layer — the entry point for all query operations. -//! -//! This module sits between the client API and the agent execution layer. -//! It is responsible for: -//! -//! - **Dispatching** queries to the appropriate agent path (Worker vs Orchestrator) -//! - **Post-processing** agent output into client-facing results -//! - **Caching** query results (L1 exact, L2 path patterns, L3 strategy scores) -//! - **Streaming** retrieval events for async progress monitoring -//! -//! Call flow: -//! ```text -//! client → retrieval::dispatch() -//! ├── User specified doc_ids → parallel N × Worker -//! └── Workspace scope → Orchestrator (analyze → spawn → fusion) -//! ``` - -mod cache; -pub mod dispatcher; -pub mod postprocessor; -pub mod stream; -mod types; - -pub use stream::{RetrieveEvent, RetrieveEventReceiver}; -pub use types::{ - Confidence, EvidenceItem, QueryMetrics, QueryResultItem, ReasoningChain, RetrieveResponse, - SufficiencyLevel, -}; diff --git a/vectorless-core/vectorless-retrieval/src/postprocessor.rs b/vectorless-core/vectorless-retrieval/src/postprocessor.rs deleted file mode 100644 index 79da0eb..0000000 --- a/vectorless-core/vectorless-retrieval/src/postprocessor.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Post-processing of agent output into client-facing results. -//! -//! Converts raw agent [`Output`] into one or more [`QueryResultItem`]s. -//! When evidence comes from multiple documents (distinct `doc_name` values), -//! results are split by document so the caller can see per-doc attribution. - -use std::collections::BTreeMap; - -use crate::types::{Confidence, EvidenceItem, QueryMetrics, QueryResultItem}; -use vectorless_agent::config::{Evidence, Metrics, Output}; - -/// Convert agent output to query result items, split by document. -/// -/// Groups evidence by `doc_name` and creates one `QueryResultItem` per document. -/// For single-document queries (all evidence has the same or no `doc_name`), -/// returns a single item with the given `doc_id`. -/// -/// The synthesized answer is shared across all items (it was produced from -/// cross-document evidence). Each item gets its own subset of evidence. -pub fn to_results(output: &Output, doc_id: &str) -> Vec { - if output.evidence.is_empty() { - return vec![empty_item(doc_id, &output.answer, output.confidence)]; - } - - // Group evidence by doc_name - let groups = group_by_doc(&output.evidence); - - if groups.len() <= 1 { - // Single doc — return one item - return vec![build_item( - doc_id, - &output.answer, - output.confidence, - &output.evidence, - &output.metrics, - )]; - } - - // Multi-doc — one item per document - groups - .into_iter() - .map(|(name, refs)| { - let did = name.as_deref().unwrap_or(doc_id); - let evidence: Vec = refs.iter().map(|e| (*e).clone()).collect(); - build_item( - did, - &output.answer, - output.confidence, - &evidence, - &output.metrics, - ) - }) - .collect() -} - -/// Group evidence by `doc_name`, preserving order. -fn group_by_doc(evidence: &[Evidence]) -> BTreeMap, Vec<&Evidence>> { - let mut groups: BTreeMap, Vec<&Evidence>> = BTreeMap::new(); - for ev in evidence { - groups.entry(ev.doc_name.clone()).or_default().push(ev); - } - groups -} - -/// Build a single enriched result item. -fn build_item( - doc_id: &str, - answer: &str, - confidence: Confidence, - evidence: &[Evidence], - metrics: &Metrics, -) -> QueryResultItem { - let node_ids: Vec = evidence.iter().map(|e| e.source_path.clone()).collect(); - let evidence_items: Vec = evidence - .iter() - .map(|e| EvidenceItem { - title: e.node_title.clone(), - path: e.source_path.clone(), - content: e.content.clone(), - doc_name: e.doc_name.clone(), - }) - .collect(); - - let content = if answer.is_empty() { - evidence - .iter() - .map(|e| format!("## {}\n{}", e.node_title, e.content)) - .collect::>() - .join("\n\n---\n\n") - } else { - answer.to_string() - }; - - let evidence_count = evidence.len(); - - QueryResultItem { - doc_id: doc_id.to_string(), - node_ids, - content, - evidence: evidence_items, - metrics: Some(QueryMetrics { - llm_calls: metrics.llm_calls, - rounds_used: metrics.rounds_used, - nodes_visited: metrics.nodes_visited, - evidence_count, - evidence_chars: metrics.evidence_chars, - }), - confidence, - } -} - -/// Build an empty result item (no evidence). -fn empty_item(doc_id: &str, answer: &str, confidence: Confidence) -> QueryResultItem { - let content = if answer.is_empty() { - String::new() - } else { - answer.to_string() - }; - QueryResultItem { - doc_id: doc_id.to_string(), - node_ids: Vec::new(), - content, - evidence: Vec::new(), - metrics: None, - confidence, - } -} diff --git a/vectorless-core/vectorless-retrieval/src/stream.rs b/vectorless-core/vectorless-retrieval/src/stream.rs deleted file mode 100644 index 33aa75b..0000000 --- a/vectorless-core/vectorless-retrieval/src/stream.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Streaming retrieval events. -//! -//! When `RetrieveOptions::streaming` is enabled, retrieval emits -//! [`RetrieveEvent`]s incrementally as the pipeline progresses through -//! its stages (Analyze → Plan → Search → Evaluate). -//! -//! # Example -//! -//! ```rust,ignore -//! let options = RetrieveOptions::new().with_streaming(true); -//! let rx = client.query_stream(&tree, "query", &options).await?; -//! -//! while let Some(event) = rx.recv().await { -//! match event { -//! RetrieveEvent::Started { query, .. } => println!("Started: {query}"), -//! RetrieveEvent::StageCompleted { stage, .. } => println!("Done: {stage}"), -//! RetrieveEvent::Completed { response } => { -//! println!("Confidence: {}", response.confidence); -//! break; -//! } -//! RetrieveEvent::Error { message } => { -//! eprintln!("Error: {message}"); -//! break; -//! } -//! _ => {} -//! } -//! } -//! ``` - -use tokio::sync::mpsc; - -use super::types::{RetrieveResponse, SufficiencyLevel}; - -/// Events emitted during streaming retrieval. -/// -/// Each event represents a meaningful milestone in the retrieval pipeline. -/// The stream always terminates with either [`Completed`](RetrieveEvent::Completed) -/// or [`Error`](RetrieveEvent::Error). -#[derive(Debug, Clone)] -pub enum RetrieveEvent { - /// Retrieval pipeline started. - Started { - /// The query string. - query: String, - /// Planned retrieval strategy name. - strategy: String, - }, - - /// A pipeline stage completed. - StageCompleted { - /// Stage name (analyze, plan, search, evaluate). - stage: String, - /// Time spent in this stage (ms). - elapsed_ms: u64, - }, - - /// A node was visited during tree traversal. - NodeVisited { - /// Node ID. - node_id: String, - /// Node title. - title: String, - /// Relevance score (0.0 - 1.0). - score: f32, - }, - - /// Relevant content was found. - ContentFound { - /// Node ID. - node_id: String, - /// Node title. - title: String, - /// Short preview of the content. - preview: String, - /// Relevance score. - score: f32, - }, - - /// Pipeline is backtracking to an earlier stage. - Backtracking { - /// Stage backtracking from. - from: String, - /// Stage backtracking to. - to: String, - /// Reason for backtracking. - reason: String, - }, - - /// Sufficiency check result. - SufficiencyCheck { - /// Sufficiency level. - level: SufficiencyLevel, - /// Total tokens collected so far. - tokens: usize, - }, - - /// Retrieval completed successfully with final results. - Completed { - /// The full retrieval response. - response: RetrieveResponse, - }, - - /// An error occurred during retrieval. - Error { - /// Error message. - message: String, - }, -} - -/// Sender half for streaming retrieval events. -pub(crate) type RetrieveEventSender = mpsc::Sender; - -/// Receiver half for streaming retrieval events. -pub type RetrieveEventReceiver = mpsc::Receiver; - -/// Create a bounded channel for streaming retrieval events. -/// -/// The bound defaults to 64 events. The sender will apply backpressure -/// when the receiver cannot keep up, preventing unbounded memory growth. -pub(crate) fn channel(bound: usize) -> (RetrieveEventSender, RetrieveEventReceiver) { - mpsc::channel(bound) -} - -/// Default channel bound for streaming events. -pub const DEFAULT_STREAM_BOUND: usize = 64; diff --git a/vectorless-core/vectorless-retrieval/src/types.rs b/vectorless-core/vectorless-retrieval/src/types.rs deleted file mode 100644 index ca15ee5..0000000 --- a/vectorless-core/vectorless-retrieval/src/types.rs +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Core types for the retrieval system. - -use serde::{Deserialize, Serialize}; - -/// Re-export [`SufficiencyLevel`] from the document module. -pub use vectorless_document::SufficiencyLevel; - -/// Complete retrieval response. -#[derive(Debug, Clone)] -pub struct RetrieveResponse { - /// Retrieved results. - pub results: Vec, - - /// Aggregated content. - pub content: String, - - /// Overall confidence score. - pub confidence: f32, - - /// Whether information is sufficient. - pub is_sufficient: bool, - - /// Strategy that was used. - pub strategy_used: String, - - /// Reasoning chain explaining how results were found. - pub reasoning_chain: ReasoningChain, - - /// Total tokens used. - pub tokens_used: usize, -} - -impl Default for RetrieveResponse { - fn default() -> Self { - Self { - results: Vec::new(), - content: String::new(), - confidence: 0.0, - is_sufficient: false, - strategy_used: String::new(), - reasoning_chain: ReasoningChain::default(), - tokens_used: 0, - } - } -} - -impl RetrieveResponse { - /// Create a new empty response. - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Check if there are any results. - #[must_use] - pub fn is_empty(&self) -> bool { - self.results.is_empty() - } - - /// Get the number of results. - #[must_use] - pub fn len(&self) -> usize { - self.results.len() - } -} - -/// A single retrieval result. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RetrievalResult { - /// Node ID in the tree. - pub node_id: Option, - - /// Node title. - pub title: String, - - /// Node content (if included). - pub content: Option, - - /// Node summary (if included). - pub summary: Option, - - /// Relevance score (0.0 - 1.0). - pub score: f32, - - /// Depth in the tree. - pub depth: usize, - - /// Page range (for PDFs). - pub page_range: Option<(usize, usize)>, -} - -impl RetrievalResult { - /// Create a new retrieval result. - #[must_use] - pub fn new(title: impl Into) -> Self { - Self { - node_id: None, - title: title.into(), - content: None, - summary: None, - score: 1.0, - depth: 0, - page_range: None, - } - } - - /// Set the node ID. - #[must_use] - pub fn with_node_id(mut self, id: impl Into) -> Self { - self.node_id = Some(id.into()); - self - } - - /// Set the content. - #[must_use] - pub fn with_content(mut self, content: impl Into) -> Self { - self.content = Some(content.into()); - self - } - - /// Set the summary. - #[must_use] - pub fn with_summary(mut self, summary: impl Into) -> Self { - self.summary = Some(summary.into()); - self - } - - /// Set the score. - #[must_use] - pub fn with_score(mut self, score: f32) -> Self { - self.score = score; - self - } - - /// Set the depth. - #[must_use] - pub fn with_depth(mut self, depth: usize) -> Self { - self.depth = depth; - self - } - - /// Set the page range. - #[must_use] - pub fn with_page_range(mut self, start: usize, end: usize) -> Self { - self.page_range = Some((start, end)); - self - } -} - -/// Complete reasoning chain for a retrieval operation. -/// -/// Provides an ordered, auditable trace of every decision the engine made -/// from query analysis through final evaluation. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ReasoningChain { - /// Ordered reasoning steps. - pub steps: Vec, -} - -impl ReasoningChain { - /// Create an empty reasoning chain. - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Append a reasoning step. - pub fn push(&mut self, step: ReasoningStep) { - self.steps.push(step); - } - - /// Number of reasoning steps. - #[must_use] - pub fn len(&self) -> usize { - self.steps.len() - } - - /// Whether the chain is empty. - #[must_use] - pub fn is_empty(&self) -> bool { - self.steps.is_empty() - } -} - -/// A single step in the reasoning chain. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ReasoningStep { - /// Human-readable explanation of the decision. - pub reasoning: String, -} - -// ============================================================ -// Query result types (used by engine) -// ============================================================ - -/// Confidence score of the query result (0.0–1.0). -pub type Confidence = f32; - -/// A single piece of evidence with source attribution. -#[derive(Debug, Clone)] -pub struct EvidenceItem { - /// Section title where this evidence was found. - pub title: String, - /// Navigation path (e.g., "Root/Chapter 1/Section 1.2"). - pub path: String, - /// Raw evidence content. - pub content: String, - /// Source document name (set in multi-doc scenarios). - pub doc_name: Option, -} - -/// Query execution metrics. -#[derive(Debug, Clone, Default)] -pub struct QueryMetrics { - /// Number of LLM calls made. - pub llm_calls: u32, - /// Number of navigation rounds used. - pub rounds_used: u32, - /// Number of distinct nodes visited. - pub nodes_visited: usize, - /// Number of evidence items collected. - pub evidence_count: usize, - /// Total characters of collected evidence. - pub evidence_chars: usize, -} - -/// A single document's query result. -#[derive(Debug, Clone)] -pub struct QueryResultItem { - /// The document ID. - pub doc_id: String, - /// Matching node IDs (navigation paths). - pub node_ids: Vec, - /// Synthesized answer or raw evidence content. - pub content: String, - /// Evidence items that contributed to this result, with source attribution. - pub evidence: Vec, - /// Execution metrics for this query. - pub metrics: Option, - /// Confidence score (0.0–1.0) — derived from LLM evaluation. - pub confidence: Confidence, -} From 5997d058603b1649beb770ff04360ef74fc35398 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 16:16:12 +0800 Subject: [PATCH 03/30] refactor: remove unused validation module and simplify codebase BREAKING CHANGE: Removed the entire validator module from vectorless-config which was not being used. This affects the ConfigValidator implementation and related traits that were previously available. - Remove validator module from lib.rs - Delete entire validator.rs file with all validation logic - Remove unused re-exports in vectorless-llm/src/lib.rs - Remove unused imports and types throughout the codebase perf: optimize memo store by removing unused key builder - Remove MemoKeyBuilder struct and all related methods - Clean up unused Fingerprint import in store.rs - Remove age() method from MemoEntry as it was unused - Simplify test imports and remove related test refactor: update Python bindings with skip_from_py_object attribute - Add skip_from_py_object to all PyO3 class definitions - This optimizes Python object creation and prevents circular references - Affects Answer, Evidence, ReasoningTrace, Config, DocumentInfo, Concept - Affects Document, NodeInfo, MatchResult, FindResult, WordCount - Affects CollectedEvidence, TopicEntry, SectionSummary, TocEntry - Affects NodeStats, SimilarResult, SectionCard, DocCard, ConceptInfo - Affects Engine, VectorlessError, WeightedKeyword, EdgeEvidence - Affects GraphEdge, DocumentGraphNode, DocumentGraph, LlmMetricsReport - Affects RetrievalMetricsReport, and MetricsReport classes feat: introduce shared blackboard system for worker collaboration - Add Discovery and SharedBlackboard classes for inter-worker communication - Enable workers to share findings, leads, and cross-references - Provide formatted context views for individual workers - Implement discovery extraction from worker outputs feat: implement query reasoning pipeline replacing understanding - Replace QueryPlan with QueryAnalysis in dispatcher - Introduce QueryAnalyzer for multi-stage query analysis - Add reasoning types: Ambiguity, EntityRef, TemporalConstraint - Include RetrievalStrategy and QueryAnalysis components - Update dispatch function to use reasoning instead of understanding docs: update ask module documentation terminology - Change "query understanding" to "query reasoning" in docstrings - Reflect the shift from understanding to reasoning in comments --- vectorless-core/vectorless-config/src/lib.rs | 1 - .../vectorless-config/src/validator.rs | 323 --------------- .../vectorless-llm/src/fallback.rs | 44 -- vectorless-core/vectorless-llm/src/lib.rs | 2 - .../vectorless-llm/src/memo/store.rs | 91 +---- .../vectorless-llm/src/memo/types.rs | 5 - vectorless-core/vectorless-py/src/answer.rs | 6 +- vectorless-core/vectorless-py/src/config.rs | 2 +- vectorless-core/vectorless-py/src/document.rs | 32 +- vectorless-core/vectorless-py/src/engine.rs | 2 +- vectorless-core/vectorless-py/src/error.rs | 2 +- vectorless-core/vectorless-py/src/graph.rs | 10 +- vectorless-core/vectorless-py/src/metrics.rs | 6 +- vectorless/ask/__init__.py | 38 +- vectorless/ask/blackboard.py | 123 ++++++ vectorless/ask/dispatcher.py | 19 +- vectorless/ask/orchestrator.py | 259 ++++++++++-- vectorless/ask/plan.py | 24 ++ vectorless/ask/prompts.py | 15 +- vectorless/ask/reasoning/__init__.py | 27 ++ vectorless/ask/reasoning/analyzer.py | 380 ++++++++++++++++++ vectorless/ask/reasoning/prompts.py | 176 ++++++++ vectorless/ask/reasoning/types.py | 122 ++++++ vectorless/ask/understand.py | 172 +++----- vectorless/ask/verify/__init__.py | 15 + vectorless/ask/verify/prompts.py | 81 ++++ vectorless/ask/verify/types.py | 39 ++ vectorless/ask/verify/verifier.py | 232 +++++++++++ vectorless/ask/worker.py | 5 + 29 files changed, 1589 insertions(+), 664 deletions(-) delete mode 100644 vectorless-core/vectorless-config/src/validator.rs create mode 100644 vectorless/ask/blackboard.py create mode 100644 vectorless/ask/reasoning/__init__.py create mode 100644 vectorless/ask/reasoning/analyzer.py create mode 100644 vectorless/ask/reasoning/prompts.py create mode 100644 vectorless/ask/reasoning/types.py create mode 100644 vectorless/ask/verify/__init__.py create mode 100644 vectorless/ask/verify/prompts.py create mode 100644 vectorless/ask/verify/types.py create mode 100644 vectorless/ask/verify/verifier.py diff --git a/vectorless-core/vectorless-config/src/lib.rs b/vectorless-core/vectorless-config/src/lib.rs index 3021749..6812d67 100644 --- a/vectorless-core/vectorless-config/src/lib.rs +++ b/vectorless-core/vectorless-config/src/lib.rs @@ -7,7 +7,6 @@ //! not by directly interacting with this module. mod types; -mod validator; pub use types::Config; pub use types::DocumentGraphConfig; diff --git a/vectorless-core/vectorless-config/src/validator.rs b/vectorless-core/vectorless-config/src/validator.rs deleted file mode 100644 index c3f5542..0000000 --- a/vectorless-core/vectorless-config/src/validator.rs +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Configuration validation. -//! -//! This module provides comprehensive validation for configuration values, -//! including range checks, consistency checks, and dependency validation. - -use super::types::{Config, ConfigValidationError, Severity, ValidationError}; - -/// Configuration validator. -#[derive(Debug, Default)] -pub struct ConfigValidator { - /// Validation rules to apply. - rules: Vec>, -} - -impl ConfigValidator { - /// Create a new validator with default rules. - pub fn new() -> Self { - Self { - rules: vec![ - Box::new(RangeValidator), - Box::new(ConsistencyValidator), - Box::new(DependencyValidator), - ], - } - } - - /// Add a custom validation rule. - pub fn with_rule(mut self, rule: Box) -> Self { - self.rules.push(rule); - self - } - - /// Validate the configuration. - pub fn validate(&self, config: &Config) -> Result<(), ConfigValidationError> { - let mut errors = Vec::new(); - - for rule in &self.rules { - rule.validate(config, &mut errors); - } - - // Only fail on errors, not warnings or info - let has_errors = errors.iter().any(|e| e.severity == Severity::Error); - - if has_errors { - Err(ConfigValidationError { errors }) - } else { - Ok(()) - } - } -} - -/// Trait for validation rules. -pub trait ValidationRule: std::fmt::Debug + Send + Sync { - /// Validate the configuration, appending errors if found. - fn validate(&self, config: &Config, errors: &mut Vec); -} - -/// Validates value ranges. -#[derive(Debug)] -struct RangeValidator; - -impl ValidationRule for RangeValidator { - fn validate(&self, config: &Config, errors: &mut Vec) { - // Indexer ranges - if config.indexer.subsection_threshold == 0 { - errors.push(ValidationError::error( - "indexer.subsection_threshold", - "Subsection threshold must be greater than 0", - )); - } - - if config.indexer.subsection_threshold > 10000 { - errors.push( - ValidationError::warning( - "indexer.subsection_threshold", - "Subsection threshold is very high, may impact performance", - ) - .with_actual(config.indexer.subsection_threshold.to_string()), - ); - } - - // LLM slot token ranges - if config.llm.index.max_tokens == 0 { - errors.push(ValidationError::error( - "llm.index.max_tokens", - "Index max tokens must be greater than 0", - )); - } - - if config.llm.retrieval.max_tokens == 0 { - errors.push(ValidationError::error( - "llm.retrieval.max_tokens", - "Retrieval max tokens must be greater than 0", - )); - } - - // Retrieval ranges - if config.retrieval.top_k == 0 { - errors.push(ValidationError::error( - "retrieval.top_k", - "Top K must be greater than 0", - )); - } - - if config.retrieval.search.beam_width == 0 { - errors.push(ValidationError::error( - "retrieval.search.beam_width", - "Beam width must be greater than 0", - )); - } - - // Throttle ranges - if config.llm.throttle.max_concurrent_requests == 0 { - errors.push(ValidationError::error( - "llm.throttle.max_concurrent_requests", - "Max concurrent requests must be greater than 0", - )); - } - - if config.llm.throttle.requests_per_minute == 0 { - errors.push(ValidationError::error( - "llm.throttle.requests_per_minute", - "Requests per minute must be greater than 0", - )); - } - - // Fallback ranges - if config.llm.fallback.max_retries == 0 { - errors.push(ValidationError::warning( - "llm.fallback.max_retries", - "Max retries is 0, fallback will not retry", - )); - } - } -} - -/// Validates configuration consistency. -#[derive(Debug)] -struct ConsistencyValidator; - -impl ValidationRule for ConsistencyValidator { - fn validate(&self, config: &Config, errors: &mut Vec) { - // Check if index tokens are reasonable - if config.llm.index.max_tokens > config.indexer.max_segment_tokens { - errors.push( - ValidationError::warning( - "llm.index.max_tokens", - "Index max tokens exceeds max segment tokens", - ) - .with_expected(format!("<= {}", config.indexer.max_segment_tokens)) - .with_actual(config.llm.index.max_tokens.to_string()), - ); - } - - // Check if sufficiency thresholds are consistent - if config.retrieval.sufficiency.min_tokens > config.retrieval.sufficiency.target_tokens { - errors.push( - ValidationError::error( - "retrieval.sufficiency.min_tokens", - "Min tokens cannot exceed target tokens", - ) - .with_expected(format!("<= {}", config.retrieval.sufficiency.target_tokens)) - .with_actual(config.retrieval.sufficiency.min_tokens.to_string()), - ); - } - - if config.retrieval.sufficiency.target_tokens > config.retrieval.sufficiency.max_tokens { - errors.push( - ValidationError::error( - "retrieval.sufficiency.target_tokens", - "Target tokens cannot exceed max tokens", - ) - .with_expected(format!("<= {}", config.retrieval.sufficiency.max_tokens)) - .with_actual(config.retrieval.sufficiency.target_tokens.to_string()), - ); - } - } -} - -/// Validates configuration dependencies. -#[derive(Debug)] -struct DependencyValidator; - -impl ValidationRule for DependencyValidator { - fn validate(&self, config: &Config, errors: &mut Vec) { - // Check if API key is available when summaries are needed - if config.llm.api_key.is_none() { - if config.indexer.max_summary_tokens > 0 { - errors.push(ValidationError::info( - "llm.api_key", - "No API key configured, summary generation will be disabled", - )); - } - } - - // Check fallback configuration - if config.llm.fallback.enabled { - if config.llm.fallback.models.is_empty() && config.llm.fallback.endpoints.is_empty() { - errors.push(ValidationError::warning( - "llm.fallback.models", - "Fallback enabled but no fallback models or endpoints configured", - )); - } - - // Check retry behavior consistency - if matches!( - config.llm.fallback.on_rate_limit, - super::types::FallbackBehavior::Fallback - ) && config.llm.fallback.models.is_empty() - { - errors.push(ValidationError::error( - "llm.fallback.models", - "Rate limit behavior is 'fallback' but no fallback models configured", - )); - } - } - - // Check cache configuration - if config.retrieval.cache.max_entries == 0 { - errors.push(ValidationError::warning( - "retrieval.cache.max_entries", - "Cache disabled (max_entries = 0), performance may be impacted", - )); - } - - // Check strategy configuration - if config.retrieval.strategy.exploration_weight <= 0.0 { - errors.push( - ValidationError::error( - "retrieval.strategy.exploration_weight", - "Exploration weight must be positive", - ) - .with_actual(config.retrieval.strategy.exploration_weight.to_string()), - ); - } - - // Check similarity thresholds are ordered correctly - if config.retrieval.strategy.low_similarity_threshold - >= config.retrieval.strategy.high_similarity_threshold - { - errors.push( - ValidationError::error( - "retrieval.strategy.low_similarity_threshold", - "Low similarity threshold must be less than high similarity threshold", - ) - .with_expected(format!( - "< {}", - config.retrieval.strategy.high_similarity_threshold - )) - .with_actual( - config - .retrieval - .strategy - .low_similarity_threshold - .to_string(), - ), - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_validator_valid_config() { - let config = Config::default(); - let validator = ConfigValidator::new(); - // Default config should pass validation (no errors, warnings are ok) - let result = validator.validate(&config); - assert!(result.is_ok(), "Default config should pass validation"); - } - - #[test] - fn test_validator_catches_range_errors() { - let mut config = Config::default(); - config.retrieval.top_k = 0; - - let validator = ConfigValidator::new(); - let result = validator.validate(&config); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(err.errors.iter().any(|e| e.path.contains("top_k"))); - } - - #[test] - fn test_validator_catches_consistency_errors() { - let mut config = Config::default(); - config.retrieval.sufficiency.min_tokens = 3000; - config.retrieval.sufficiency.target_tokens = 2000; - - let validator = ConfigValidator::new(); - let result = validator.validate(&config); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(err.errors.iter().any(|e| e.path.contains("min_tokens"))); - } - - #[test] - fn test_validator_catches_dependency_warnings() { - let mut config = Config::default(); - config.llm.fallback.enabled = true; - config.llm.fallback.models.clear(); - - let validator = ConfigValidator::new(); - let result = validator.validate(&config); - - // Should succeed but with warnings - if let Err(err) = result { - assert!( - err.errors - .iter() - .any(|e| e.path.contains("llm.fallback.models")) - ); - } - } -} diff --git a/vectorless-core/vectorless-llm/src/fallback.rs b/vectorless-core/vectorless-llm/src/fallback.rs index d9a84c0..90cce99 100644 --- a/vectorless-core/vectorless-llm/src/fallback.rs +++ b/vectorless-core/vectorless-llm/src/fallback.rs @@ -28,50 +28,6 @@ use vectorless_config::{ FallbackBehavior, FallbackConfig as ConfigFallbackConfig, OnAllFailedBehavior, }; -/// Result from a fallback-aware LLM call. -#[derive(Debug, Clone)] -pub struct FallbackResult { - /// The actual result. - pub result: T, - /// Whether the result came from a fallback model/endpoint. - pub degraded: bool, - /// The model that was ultimately used. - pub model: String, - /// The endpoint that was ultimately used. - pub endpoint: String, - /// History of fallback attempts (for debugging). - pub fallback_history: Vec, -} - -impl FallbackResult { - /// Create a successful result without fallback. - pub fn success(result: T, model: String, endpoint: String) -> Self { - Self { - result, - degraded: false, - model, - endpoint, - fallback_history: Vec::new(), - } - } - - /// Create a result from a fallback. - pub fn from_fallback( - result: T, - model: String, - endpoint: String, - history: Vec, - ) -> Self { - Self { - result, - degraded: true, - model, - endpoint, - fallback_history: history, - } - } -} - /// A single step in the fallback chain. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FallbackStep { diff --git a/vectorless-core/vectorless-llm/src/lib.rs b/vectorless-core/vectorless-llm/src/lib.rs index 8bb01c3..02e6cca 100644 --- a/vectorless-core/vectorless-llm/src/lib.rs +++ b/vectorless-core/vectorless-llm/src/lib.rs @@ -41,5 +41,3 @@ pub use client::LlmClient; pub use error::LlmResult; pub use pool::LlmPool; -// Re-export vectorless_error types for internal use -pub(crate) use vectorless_error::{Error, Result}; diff --git a/vectorless-core/vectorless-llm/src/memo/store.rs b/vectorless-core/vectorless-llm/src/memo/store.rs index 5a87bd6..013f270 100644 --- a/vectorless-core/vectorless-llm/src/memo/store.rs +++ b/vectorless-core/vectorless-llm/src/memo/store.rs @@ -19,7 +19,6 @@ use tracing::{debug, info}; use super::types::{MemoEntry, MemoKey, MemoOpType, MemoStats, MemoValue}; use vectorless_error::Result; -use vectorless_utils::fingerprint::Fingerprint; /// Default TTL for cache entries (7 days). const DEFAULT_TTL: Duration = Duration::days(7); @@ -458,88 +457,11 @@ impl Default for MemoStore { } } -/// A helper for building memo keys with context. -pub struct MemoKeyBuilder { - model_id: Option, - version: u32, -} - -impl MemoKeyBuilder { - /// Create a new key builder. - pub fn new() -> Self { - Self { - model_id: None, - version: 1, - } - } - - /// Set the model identifier. - pub fn with_model(mut self, model_id: &str) -> Self { - self.model_id = Some(model_id.to_string()); - self - } - - /// Set the version. - pub fn with_version(mut self, version: u32) -> Self { - self.version = version; - self - } - - /// Build a summary key. - pub fn summary_key(&self, content_fp: &Fingerprint) -> MemoKey { - MemoKey { - op_type: super::types::MemoOpType::Summary, - input_fp: *content_fp, - model_id: self.model_id.clone(), - version: self.version, - context_fp: Fingerprint::zero(), - } - } - - /// Build a pilot decision key. - pub fn pilot_key(&self, context_fp: &Fingerprint, query_fp: &Fingerprint) -> MemoKey { - MemoKey { - op_type: super::types::MemoOpType::PilotDecision, - input_fp: *query_fp, - model_id: self.model_id.clone(), - version: self.version, - context_fp: *context_fp, - } - } - - /// Build a query analysis key. - pub fn query_analysis_key(&self, query_fp: &Fingerprint) -> MemoKey { - MemoKey { - op_type: super::types::MemoOpType::QueryAnalysis, - input_fp: *query_fp, - model_id: self.model_id.clone(), - version: self.version, - context_fp: Fingerprint::zero(), - } - } - - /// Build an extraction key. - pub fn extraction_key(&self, content_fp: &Fingerprint) -> MemoKey { - MemoKey { - op_type: super::types::MemoOpType::Extraction, - input_fp: *content_fp, - model_id: self.model_id.clone(), - version: self.version, - context_fp: Fingerprint::zero(), - } - } -} - -impl Default for MemoKeyBuilder { - fn default() -> Self { - Self::new() - } -} - #[cfg(test)] mod tests { use super::*; use tempfile::TempDir; + use vectorless_utils::fingerprint::Fingerprint; fn make_test_key() -> MemoKey { let fp = Fingerprint::from_str("test content"); @@ -667,15 +589,4 @@ mod tests { assert_eq!(stats.hits, 1); assert_eq!(stats.tokens_saved, 100); } - - #[test] - fn test_memo_key_builder() { - let builder = MemoKeyBuilder::new().with_model("gpt-4").with_version(2); - - let fp = Fingerprint::from_str("content"); - let key = builder.summary_key(&fp); - - assert_eq!(key.model_id, Some("gpt-4".to_string())); - assert_eq!(key.version, 2); - } } diff --git a/vectorless-core/vectorless-llm/src/memo/types.rs b/vectorless-core/vectorless-llm/src/memo/types.rs index 9e3cb86..c3f1100 100644 --- a/vectorless-core/vectorless-llm/src/memo/types.rs +++ b/vectorless-core/vectorless-llm/src/memo/types.rs @@ -294,11 +294,6 @@ impl MemoEntry { let now = Utc::now(); now - self.created_at > ttl } - - /// Get the age of this entry. - pub fn age(&self) -> chrono::Duration { - Utc::now() - self.created_at - } } /// Statistics for the memo store. diff --git a/vectorless-core/vectorless-py/src/answer.rs b/vectorless-core/vectorless-py/src/answer.rs index 9213139..9015118 100644 --- a/vectorless-core/vectorless-py/src/answer.rs +++ b/vectorless-core/vectorless-py/src/answer.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; use ::vectorless_engine::Answer; /// A reasoned answer with evidence and trace. -#[pyclass(name = "Answer")] +#[pyclass(name = "Answer", skip_from_py_object)] pub struct PyAnswer { pub(crate) inner: Answer, } @@ -71,7 +71,7 @@ impl PyAnswer { } /// A piece of evidence with source attribution. -#[pyclass(name = "Evidence")] +#[pyclass(name = "Evidence", skip_from_py_object)] pub struct PyEvidence { #[pyo3(get)] pub content: String, @@ -84,7 +84,7 @@ pub struct PyEvidence { } /// Reasoning trace — always present. -#[pyclass(name = "ReasoningTrace")] +#[pyclass(name = "ReasoningTrace", skip_from_py_object)] pub struct PyReasoningTrace { #[pyo3(get)] pub steps: Vec, diff --git a/vectorless-core/vectorless-py/src/config.rs b/vectorless-core/vectorless-py/src/config.rs index 6b043ea..21b67af 100644 --- a/vectorless-core/vectorless-py/src/config.rs +++ b/vectorless-core/vectorless-py/src/config.rs @@ -22,7 +22,7 @@ use pyo3::prelude::*; /// /// engine = Engine(api_key="sk-...", model="gpt-4o", config=config) /// ``` -#[pyclass(name = "Config")] +#[pyclass(name = "Config", skip_from_py_object)] pub struct PyConfig { pub(crate) inner: vectorless_engine::Config, } diff --git a/vectorless-core/vectorless-py/src/document.rs b/vectorless-core/vectorless-py/src/document.rs index 3895986..1d16d42 100644 --- a/vectorless-core/vectorless-py/src/document.rs +++ b/vectorless-core/vectorless-py/src/document.rs @@ -22,7 +22,7 @@ use super::error::VectorlessError; // ========================================================================= /// Information about an understood document. -#[pyclass(name = "DocumentInfo")] +#[pyclass(name = "DocumentInfo", skip_from_py_object)] pub struct PyDocumentInfo { pub(crate) inner: vectorless_engine::DocumentInfo, } @@ -81,7 +81,7 @@ impl PyDocumentInfo { } /// A key concept extracted from a document. -#[pyclass(name = "Concept")] +#[pyclass(name = "Concept", skip_from_py_object)] pub struct PyConcept { #[pyo3(get)] pub name: String, @@ -106,7 +106,7 @@ pub struct PyConcept { /// print(await doc.pwd()) /// print(await doc.cat(None)) /// ``` -#[pyclass(name = "Document")] +#[pyclass(name = "Document", skip_from_py_object)] pub struct PyDocument { inner: Arc>, } @@ -613,7 +613,7 @@ impl PyDocument { // ========================================================================= /// Information about a node in the document tree. -#[pyclass(name = "NodeInfo")] +#[pyclass(name = "NodeInfo", skip_from_py_object)] #[derive(Clone)] pub struct PyNodeInfo { #[pyo3(get)] @@ -647,7 +647,7 @@ impl From for PyNodeInfo { } /// A regex match within node content. -#[pyclass(name = "MatchResult")] +#[pyclass(name = "MatchResult", skip_from_py_object)] #[derive(Clone)] pub struct PyMatchResult { #[pyo3(get)] @@ -672,7 +672,7 @@ impl From for PyMatchResult { } /// A node found by search. -#[pyclass(name = "FindResult")] +#[pyclass(name = "FindResult", skip_from_py_object)] #[derive(Clone)] pub struct PyFindResult { #[pyo3(get)] @@ -697,7 +697,7 @@ impl From for PyFindResult { } /// Word/line/character count. -#[pyclass(name = "WordCount")] +#[pyclass(name = "WordCount", skip_from_py_object)] #[derive(Clone)] pub struct PyWordCount { #[pyo3(get)] @@ -719,7 +719,7 @@ impl From for PyWordCount { } /// Evidence collected during navigation. -#[pyclass(name = "CollectedEvidence")] +#[pyclass(name = "CollectedEvidence", skip_from_py_object)] #[derive(Clone)] pub struct PyCollectedEvidence { #[pyo3(get)] @@ -744,7 +744,7 @@ impl From for PyCollectedEvidence { } /// A topic entry from the reasoning index. -#[pyclass(name = "TopicEntry")] +#[pyclass(name = "TopicEntry", skip_from_py_object)] #[derive(Clone)] pub struct PyTopicEntry { #[pyo3(get)] @@ -766,7 +766,7 @@ impl From for PyTopicEntry { } /// A section summary from the reasoning index. -#[pyclass(name = "SectionSummary")] +#[pyclass(name = "SectionSummary", skip_from_py_object)] #[derive(Clone)] pub struct PySectionSummary { #[pyo3(get)] @@ -795,7 +795,7 @@ impl From for PySectionSummary { // ========================================================================= /// A single entry in the table of contents. -#[pyclass(name = "TocEntry")] +#[pyclass(name = "TocEntry", skip_from_py_object)] #[derive(Clone)] pub struct PyTocEntry { #[pyo3(get)] @@ -820,7 +820,7 @@ impl From for PyTocEntry { } /// Statistics about a node. -#[pyclass(name = "NodeStats")] +#[pyclass(name = "NodeStats", skip_from_py_object)] #[derive(Clone)] pub struct PyNodeStats { #[pyo3(get)] @@ -857,7 +857,7 @@ impl From for PyNodeStats { } /// A node found by semantic similarity. -#[pyclass(name = "SimilarResult")] +#[pyclass(name = "SimilarResult", skip_from_py_object)] #[derive(Clone)] pub struct PySimilarResult { #[pyo3(get)] @@ -882,7 +882,7 @@ impl From for PySimilarResult { } /// A top-level section in a document card. -#[pyclass(name = "SectionCard")] +#[pyclass(name = "SectionCard", skip_from_py_object)] #[derive(Clone)] pub struct PySectionCard { #[pyo3(get)] @@ -904,7 +904,7 @@ impl From for PySectionCard { } /// Document-level overview card. -#[pyclass(name = "DocCard")] +#[pyclass(name = "DocCard", skip_from_py_object)] #[derive(Clone)] pub struct PyDocCard { #[pyo3(get)] @@ -935,7 +935,7 @@ impl From for PyDocCard { } /// A key concept extracted from the document. -#[pyclass(name = "ConceptInfo")] +#[pyclass(name = "ConceptInfo", skip_from_py_object)] #[derive(Clone)] pub struct PyConceptInfo { #[pyo3(get)] diff --git a/vectorless-core/vectorless-py/src/engine.rs b/vectorless-core/vectorless-py/src/engine.rs index 3b297a0..107b5c3 100644 --- a/vectorless-core/vectorless-py/src/engine.rs +++ b/vectorless-core/vectorless-py/src/engine.rs @@ -102,7 +102,7 @@ fn run_metrics_report(engine: Arc) -> PyMetricsReport { /// # Forget a document /// await engine.forget(doc.doc_id) /// ``` -#[pyclass(name = "Engine")] +#[pyclass(name = "Engine", skip_from_py_object)] pub struct PyEngine { inner: Arc, } diff --git a/vectorless-core/vectorless-py/src/error.rs b/vectorless-core/vectorless-py/src/error.rs index c471561..9f59cc4 100644 --- a/vectorless-core/vectorless-py/src/error.rs +++ b/vectorless-core/vectorless-py/src/error.rs @@ -9,7 +9,7 @@ use pyo3::prelude::*; use ::vectorless_engine::Error as RustError; /// Python exception for vectorless errors. -#[pyclass(extends = PyException, subclass)] +#[pyclass(extends = PyException, subclass, skip_from_py_object)] pub struct VectorlessError { message: String, kind: String, diff --git a/vectorless-core/vectorless-py/src/graph.rs b/vectorless-core/vectorless-py/src/graph.rs index affba45..f4204b5 100644 --- a/vectorless-core/vectorless-py/src/graph.rs +++ b/vectorless-core/vectorless-py/src/graph.rs @@ -10,7 +10,7 @@ use ::vectorless_engine::{ }; /// A keyword with weight from document analysis. -#[pyclass(name = "WeightedKeyword")] +#[pyclass(name = "WeightedKeyword", skip_from_py_object)] pub struct PyWeightedKeyword { pub(crate) inner: WeightedKeyword, } @@ -36,7 +36,7 @@ impl PyWeightedKeyword { } /// Evidence for a cross-document connection. -#[pyclass(name = "EdgeEvidence")] +#[pyclass(name = "EdgeEvidence", skip_from_py_object)] pub struct PyEdgeEvidence { pub(crate) inner: EdgeEvidence, } @@ -74,7 +74,7 @@ impl PyEdgeEvidence { } /// An edge representing a relationship between two documents. -#[pyclass(name = "GraphEdge")] +#[pyclass(name = "GraphEdge", skip_from_py_object)] pub struct PyGraphEdge { pub(crate) inner: GraphEdge, } @@ -110,7 +110,7 @@ impl PyGraphEdge { } /// A node in the document graph representing an indexed document. -#[pyclass(name = "DocumentGraphNode")] +#[pyclass(name = "DocumentGraphNode", skip_from_py_object)] pub struct PyDocumentGraphNode { pub(crate) inner: DocumentGraphNode, } @@ -159,7 +159,7 @@ impl PyDocumentGraphNode { /// /// Automatically rebuilt after indexing. Connects documents /// that share keywords via Jaccard similarity. -#[pyclass(name = "DocumentGraph")] +#[pyclass(name = "DocumentGraph", skip_from_py_object)] pub struct PyDocumentGraph { pub(crate) inner: DocumentGraph, } diff --git a/vectorless-core/vectorless-py/src/metrics.rs b/vectorless-core/vectorless-py/src/metrics.rs index 19f9462..af33c78 100644 --- a/vectorless-core/vectorless-py/src/metrics.rs +++ b/vectorless-core/vectorless-py/src/metrics.rs @@ -8,7 +8,7 @@ use pyo3::prelude::*; use ::vectorless_engine::{LlmMetricsReport, MetricsReport, RetrievalMetricsReport}; /// LLM usage metrics report. -#[pyclass(name = "LlmMetricsReport")] +#[pyclass(name = "LlmMetricsReport", skip_from_py_object)] pub struct PyLlmMetricsReport { pub(crate) inner: LlmMetricsReport, } @@ -102,7 +102,7 @@ impl PyLlmMetricsReport { } /// Retrieval operation metrics report. -#[pyclass(name = "RetrievalMetricsReport")] +#[pyclass(name = "RetrievalMetricsReport", skip_from_py_object)] pub struct PyRetrievalMetricsReport { pub(crate) inner: RetrievalMetricsReport, } @@ -222,7 +222,7 @@ impl PyRetrievalMetricsReport { } /// Complete metrics report combining all subsystem metrics. -#[pyclass(name = "MetricsReport")] +#[pyclass(name = "MetricsReport", skip_from_py_object)] pub struct PyMetricsReport { pub(crate) inner: MetricsReport, } diff --git a/vectorless/ask/__init__.py b/vectorless/ask/__init__.py index 05096dc..62b5087 100644 --- a/vectorless/ask/__init__.py +++ b/vectorless/ask/__init__.py @@ -1,4 +1,4 @@ -"""Ask pipeline — query understanding, multi-agent retrieval, and answer synthesis.""" +"""Ask pipeline — query reasoning, multi-agent retrieval, and answer synthesis.""" from vectorless.ask.dispatcher import dispatch from vectorless.ask.evaluate import evaluate @@ -23,6 +23,24 @@ from vectorless.ask.understand import understand from vectorless.ask.worker import Worker +# New modules +from vectorless.ask.blackboard import Discovery, SharedBlackboard +from vectorless.ask.reasoning import ( + Ambiguity, + AmbiguityType, + EntityRef, + QueryAnalysis, + QueryAnalyzer, + RetrievalStrategy, + TemporalConstraint, +) +from vectorless.ask.verify import ( + DimensionScore, + VerificationDimension, + VerificationResult, + VerifyPipeline, +) + __all__ = [ # Core output types "Output", @@ -43,14 +61,30 @@ "Scope", "Specified", "Workspace", - # Query understanding + # Query understanding (legacy) "QueryIntent", "QueryPlan", "SubQuery", "Complexity", + # Query reasoning (new) + "QueryAnalysis", + "QueryAnalyzer", + "EntityRef", + "Ambiguity", + "AmbiguityType", + "TemporalConstraint", + "RetrievalStrategy", # Agents "Worker", "dispatch", "evaluate", "understand", + # Shared blackboard + "Discovery", + "SharedBlackboard", + # Verification + "VerifyPipeline", + "VerificationDimension", + "VerificationResult", + "DimensionScore", ] diff --git a/vectorless/ask/blackboard.py b/vectorless/ask/blackboard.py new file mode 100644 index 0000000..418b351 --- /dev/null +++ b/vectorless/ask/blackboard.py @@ -0,0 +1,123 @@ +"""Shared Blackboard — Worker-to-Worker information sharing. + +The SharedBlackboard enables Workers operating on different documents to +share discoveries via Orchestrator-mediated context. Workers write +discoveries; the Orchestrator formats them for subsequent Workers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Discovery: + """A finding from a Worker that may be relevant to other Workers.""" + worker_id: str # Doc name of the Worker that found this + doc_name: str + node_title: str + finding_type: str # "evidence", "lead", "dead_end", "cross_ref" + summary: str + relevance_to: list[str] = field(default_factory=list) # Other doc_names + + +@dataclass +class SharedBlackboard: + """Accumulates discoveries across Workers for cross-document context. + + Usage:: + + blackboard = SharedBlackboard() + blackboard.add_discovery(Discovery(...)) + context = blackboard.format_for_worker("doc_B.md") + """ + + discoveries: list[Discovery] = field(default_factory=list) + cross_references: dict[str, list[str]] = field(default_factory=dict) + dead_ends: set[str] = field(default_factory=set) + active_leads: list[str] = field(default_factory=list) + + def add_discovery(self, discovery: Discovery) -> None: + """Add a discovery to the blackboard.""" + self.discoveries.append(discovery) + + # Update cross-references + if discovery.finding_type == "cross_ref" and discovery.relevance_to: + src = discovery.doc_name + if src not in self.cross_references: + self.cross_references[src] = [] + self.cross_references[src].extend(discovery.relevance_to) + + # Track leads + if discovery.finding_type == "lead" and discovery.summary: + if discovery.summary not in self.active_leads: + self.active_leads.append(discovery.summary) + + # Track dead ends + if discovery.finding_type == "dead_end": + self.dead_ends.add(f"{discovery.doc_name}/{discovery.node_title}") + + def format_for_worker(self, worker_doc_name: str) -> str: + """Format a read-only view of the blackboard for a specific Worker. + + Includes discoveries from OTHER documents that are relevant. + Excludes the Worker's own previous discoveries. + """ + relevant = [ + d for d in self.discoveries + if d.doc_name != worker_doc_name + ] + + if not relevant and not self.active_leads: + return "" + + parts: list[str] = [] + + # Discoveries from other Workers + if relevant: + parts.append("Other Workers have found information in related documents:") + for d in relevant[:10]: + parts.append( + f" - [{d.doc_name}] {d.finding_type}: {d.summary}" + f"{' (relevant to: ' + ', '.join(d.relevance_to) + ')' if d.relevance_to else ''}" + ) + + # Active leads + if self.active_leads: + parts.append("\nActive leads to investigate:") + for lead in self.active_leads[:5]: + parts.append(f" - {lead}") + + # Dead ends + if self.dead_ends: + dead_end_list = ", ".join(sorted(self.dead_ends)[:5]) + parts.append(f"\nDead ends (avoid): {dead_end_list}") + + return "\n".join(parts) + + def format_for_all(self) -> str: + """Format the full blackboard for all Workers in parallel dispatch.""" + if not self.discoveries and not self.active_leads: + return "" + return self.format_for_worker("") + + +def extract_discoveries(worker_output, doc_name: str) -> list[Discovery]: + """Extract discoveries from a WorkerOutput for the blackboard. + + Converts Worker evidence into Discovery objects based on + evidence content and source paths. + """ + discoveries: list[Discovery] = [] + + for evidence in worker_output.evidence: + # Every evidence item is a potential cross-reference + discoveries.append(Discovery( + worker_id=doc_name, + doc_name=doc_name, + node_title=evidence.node_title, + finding_type="evidence", + summary=f"Found: {evidence.node_title} ({len(evidence.content)} chars)", + )) + + return discoveries diff --git a/vectorless/ask/dispatcher.py b/vectorless/ask/dispatcher.py index 486b598..ed35289 100644 --- a/vectorless/ask/dispatcher.py +++ b/vectorless/ask/dispatcher.py @@ -3,7 +3,7 @@ Mirrors vectorless-core/vectorless-agent/src/dispatcher.rs. All queries go through dispatch(): -1. Query understanding -> QueryPlan +1. Query reasoning -> QueryAnalysis (via QueryAnalyzer) 2. Scope resolution -> Specified | Workspace 3. Orchestrator.run() (always) 4. Return Output @@ -16,7 +16,7 @@ from vectorless.ask.types import DocCard, Output, Specified, Workspace from vectorless.ask.orchestrator import Orchestrator -from vectorless.ask.understand import understand +from vectorless.ask.reasoning import QueryAnalysis, QueryAnalyzer from vectorless.llm_client import LLMClient logger = logging.getLogger(__name__) @@ -35,13 +35,14 @@ async def dispatch( - Specified -> skip_analysis=True -> spawn Workers directly - Workspace -> skip_analysis=False -> analyze -> dispatch -> evaluate -> replan """ - # Step 1: Query understanding - logger.info("dispatch: query understanding started") - query_plan = await understand(query, llm) + # Step 1: Query reasoning (multi-stage analysis) + logger.info("dispatch: query reasoning started") + analyzer = QueryAnalyzer() + query_analysis = await analyzer.analyze(query, llm) logger.info( - "dispatch: query understanding complete (intent=%s, complexity=%s)", - query_plan.intent.value, - query_plan.complexity.value, + "dispatch: query reasoning complete (intent=%s, complexity=%s)", + query_analysis.intent.value, + query_analysis.complexity.value, ) # Step 2: Determine skip_analysis from scope @@ -55,7 +56,7 @@ async def dispatch( doc_loader=doc_loader, llm_client=llm, skip_analysis=skip_analysis, - query_plan=query_plan, + query_analysis=query_analysis, event_callback=event_callback, ) return await orchestrator.run() diff --git a/vectorless/ask/orchestrator.py b/vectorless/ask/orchestrator.py index 668b6fd..1529052 100644 --- a/vectorless/ask/orchestrator.py +++ b/vectorless/ask/orchestrator.py @@ -11,7 +11,7 @@ Flow: Orchestrator.run() Phase 1: analyze() → AnalyzeOutcome (dispatches or early return) - Phase 2: supervisor loop → dispatch Workers → evaluate → replan + Phase 2: supervisor loop → dispatch Workers → verify → replan Phase 3: finalize_output() → rerank → Output """ @@ -36,7 +36,10 @@ from vectorless.ask.evaluate import evaluate from vectorless.ask.worker import Worker from vectorless.llm_client import LLMClient -from vectorless.ask.plan import QueryPlan +from vectorless.ask.reasoning.types import QueryAnalysis +from vectorless.ask.reasoning.analyzer import QueryAnalyzer +from vectorless.ask.verify import VerifyPipeline, VerificationResult +from vectorless.ask.blackboard import SharedBlackboard, extract_discoveries from vectorless.ask.prompts import ( OrchestratorAnalysisParams, orchestrator_analysis, @@ -49,6 +52,7 @@ logger = logging.getLogger(__name__) MAX_SUPERVISOR_ITERATIONS = 3 +MAX_VERIFICATION_ITERATIONS = 2 # --------------------------------------------------------------------------- @@ -72,6 +76,7 @@ class _SupervisorOutcome: iteration: int eval_sufficient: bool llm_calls: int + verification_result: VerificationResult | None = None # --------------------------------------------------------------------------- @@ -90,7 +95,7 @@ class Orchestrator: doc_cards=[card1, card2], doc_loader=load_fn, llm_client=llm, - query_plan=plan, + query_analysis=analysis, ) output = await orch.run() """ @@ -103,7 +108,8 @@ def __init__( llm_client: LLMClient, *, skip_analysis: bool = False, - query_plan: QueryPlan | None = None, + query_plan: Any = None, # Deprecated: kept for backward compat + query_analysis: QueryAnalysis | None = None, max_rounds: int = 15, max_llm_calls: int = 0, event_callback: Any = None, # async callable: (dict) -> None @@ -113,11 +119,19 @@ def __init__( self._doc_loader = doc_loader self._llm = llm_client self._skip_analysis = skip_analysis - self._query_plan = query_plan self._max_rounds = max_rounds self._max_llm_calls = max_llm_calls self._emit = event_callback or _noop_emit + # Accept both old QueryPlan and new QueryAnalysis + if query_analysis is not None: + self._query_analysis = query_analysis + elif query_plan is not None: + # Backward compat: convert QueryPlan to QueryAnalysis + self._query_analysis = query_plan.to_query_analysis() + else: + self._query_analysis = None + async def run(self) -> Output: """Execute the Orchestrator: analyze → supervisor loop → finalize. @@ -129,8 +143,8 @@ async def run(self) -> Output: state = OrchestratorState() intent_context = "" - if self._query_plan: - intent_context = self._query_plan.intent_context() + if self._query_analysis: + intent_context = self._query_analysis.intent_context() logger.info( "Orchestrator starting (docs=%d, skip_analysis=%s)", @@ -155,17 +169,21 @@ async def run(self) -> Output: ) state.total_llm_calls += outcome.llm_calls - confidence = _compute_confidence( - eval_sufficient=outcome.eval_sufficient, - replan_rounds=outcome.iteration, - no_evidence=not state.all_evidence, - ) + # Use verification confidence if available, otherwise compute from eval + if outcome.verification_result is not None: + confidence = outcome.verification_result.overall_confidence + else: + confidence = _compute_confidence( + eval_sufficient=outcome.eval_sufficient, + replan_rounds=outcome.iteration, + no_evidence=not state.all_evidence, + ) # --- Phase 3: Finalize — rerank + assemble Output --- if state.all_evidence: return await self._finalize_output( state, - self._query_plan.intent if self._query_plan else None, + self._query_analysis.intent if self._query_analysis else None, confidence, ) @@ -282,58 +300,104 @@ async def _supervisor_loop( llm: LLMClient, state: OrchestratorState, ) -> _SupervisorOutcome: - """Run: dispatch → evaluate → replan loop.""" + """Run: dispatch → verify → re-analyze → replan loop. + + Integrates SharedBlackboard for cross-Worker discovery sharing + and VerifyPipeline for multi-dimensional evidence verification. + """ current_dispatches = initial_dispatches iteration = 0 eval_sufficient = False llm_calls = 0 + verification_result: VerificationResult | None = None + + # Initialize shared blackboard for multi-doc coordination + blackboard = SharedBlackboard() + verify_pipeline = VerifyPipeline() while iteration < MAX_SUPERVISOR_ITERATIONS: - # Dispatch current plan + # Dispatch current plan with adaptive strategy if current_dispatches: logger.info( "Dispatching %d Workers (iteration=%d)", len(current_dispatches), iteration, ) - await self._dispatch_and_collect( + await self._adaptive_dispatch( query, current_dispatches, cards, llm, state, + blackboard, iteration, ) - # No evidence — nothing to evaluate + # No evidence — nothing to verify if not state.all_evidence: logger.info("No evidence collected from any Worker") break - # Skip evaluation for user-specified documents (no replan needed) + # Skip verification for user-specified documents (no replan needed) if self._skip_analysis: eval_sufficient = bool(state.all_evidence) break - # Evaluate sufficiency + # Verify evidence using multi-dimensional pipeline + query_intent = "" + if self._query_analysis: + query_intent = self._query_analysis.intent.value + try: - eval_result = await evaluate(llm, query, state.all_evidence) + verification_result = await verify_pipeline.verify( + query=query, + evidence=state.all_evidence, + query_intent=query_intent, + iteration=iteration, + llm=llm, + ) + llm_calls += 1 except Exception as e: - logger.error("Cross-doc evaluation failed: %s", e) + logger.error("Verification failed: %s", e) break - llm_calls += 1 - if eval_result.sufficient: + logger.info( + "Verification result: passed=%s, confidence=%.2f, gaps=%d", + verification_result.passed, + verification_result.overall_confidence, + len(verification_result.gaps), + ) + + if verification_result.passed: eval_sufficient = True + break + + # Verification failed — check iteration limit + if iteration >= MAX_VERIFICATION_ITERATIONS - 1: logger.info( - "Evidence sufficient (evidence=%d, iteration=%d)", - len(state.all_evidence), iteration, + "Max verification iterations reached — returning with current confidence" ) break - # Insufficient — replan + # Re-analyze with gap context + if self._query_analysis and verification_result.gaps: + evidence_summary = _format_evidence_context(state.all_evidence) + analyzer = QueryAnalyzer() + try: + self._query_analysis = await analyzer.re_analyze( + analysis=self._query_analysis, + gaps=verification_result.gaps, + evidence_summary=evidence_summary, + llm=llm, + ) + llm_calls += 1 + except Exception as e: + logger.warning("Re-analysis failed: %s", e) + + # Replan with blackboard context logger.info( "Evidence insufficient (evidence=%d, iteration=%d) — replanning", len(state.all_evidence), iteration, ) + missing_info = "; ".join(verification_result.gaps) if verification_result.gaps else "" try: new_dispatches = await self._replan( - query, eval_result.missing_info, state, cards, llm, + query, missing_info, state, cards, llm, blackboard, ) except Exception as e: logger.error("Replan failed: %s", e) @@ -350,24 +414,60 @@ async def _supervisor_loop( iteration=iteration, eval_sufficient=eval_sufficient, llm_calls=llm_calls, + verification_result=verification_result, ) # ----------------------------------------------------------------------- - # Dispatch and collect — mirrors Rust orchestrator/dispatch.rs + # Adaptive dispatch — sequential or parallel based on iteration and doc count # ----------------------------------------------------------------------- - async def _dispatch_and_collect( + async def _adaptive_dispatch( + self, + query: str, + dispatches: list[DispatchEntry], + cards: list[DocCard], + llm: LLMClient, + state: OrchestratorState, + blackboard: SharedBlackboard, + iteration: int, + ) -> None: + """Dispatch Workers with adaptive strategy. + + - 1 document: parallel (no blackboard benefit) + - 2+ documents, first iteration: sequential (build blackboard) + - 2+ documents, subsequent iterations: parallel (blackboard pre-populated) + """ + if len(dispatches) == 1: + # Single doc: parallel (no blackboard benefit) + await self._dispatch_parallel( + query, dispatches, cards, llm, state, blackboard, "", + ) + elif iteration == 0: + # First iteration: sequential to build blackboard + await self._dispatch_sequential( + query, dispatches, cards, llm, state, blackboard, + ) + else: + # Subsequent iterations: parallel with full blackboard + shared_context = blackboard.format_for_all() + await self._dispatch_parallel( + query, dispatches, cards, llm, state, blackboard, shared_context, + ) + + async def _dispatch_parallel( self, query: str, dispatches: list[DispatchEntry], cards: list[DocCard], llm: LLMClient, state: OrchestratorState, + blackboard: SharedBlackboard, + shared_context: str, ) -> None: """Dispatch Workers in parallel and collect results.""" intent_context = "" - if self._query_plan: - intent_context = f"{self._query_plan.intent.value} — {self._query_plan.strategy_hint}" + if self._query_analysis: + intent_context = f"{self._query_analysis.intent.value} — {self._query_analysis.strategy.strategy_type}" async def run_worker(dispatch: DispatchEntry) -> tuple[int, WorkerOutput]: idx = dispatch.doc_idx @@ -391,6 +491,7 @@ async def run_worker(dispatch: DispatchEntry) -> tuple[int, WorkerOutput]: max_llm_calls=self._max_llm_calls, task=dispatch.task, intent_context=intent_context, + shared_context=shared_context, ) result = await worker.run() @@ -409,6 +510,67 @@ async def run_worker(dispatch: DispatchEntry) -> tuple[int, WorkerOutput]: continue idx, output = item state.collect_result(idx, output) + # Extract discoveries to blackboard + card = cards[idx] if idx < len(cards) else None + if card: + discoveries = extract_discoveries(output, card.name) + for d in discoveries: + blackboard.add_discovery(d) + + async def _dispatch_sequential( + self, + query: str, + dispatches: list[DispatchEntry], + cards: list[DocCard], + llm: LLMClient, + state: OrchestratorState, + blackboard: SharedBlackboard, + ) -> None: + """Dispatch Workers sequentially to build blackboard context.""" + intent_context = "" + if self._query_analysis: + intent_context = f"{self._query_analysis.intent.value} — {self._query_analysis.strategy.strategy_type}" + + for dispatch in dispatches: + idx = dispatch.doc_idx + if idx >= len(cards): + logger.warning("Document index %d out of range, skipping", idx) + continue + + card = cards[idx] + + try: + doc = await self._doc_loader(card.doc_id) + except Exception as e: + logger.warning("Failed to load document %s: %s", card.doc_id, e) + continue + + # Get context from blackboard for this Worker + shared_context = blackboard.format_for_worker(card.name) + + worker = Worker( + document=doc, + query=query, + llm_client=llm, + max_rounds=self._max_rounds, + max_llm_calls=self._max_llm_calls, + task=dispatch.task, + intent_context=intent_context, + shared_context=shared_context, + ) + + result = await worker.run() + logger.info( + "Worker completed for doc %d (%s): evidence=%d, rounds=%d", + idx, card.name, len(result.evidence), result.metrics.rounds_used, + ) + + state.collect_result(idx, result) + + # Extract discoveries to blackboard for subsequent Workers + discoveries = extract_discoveries(result, card.name) + for d in discoveries: + blackboard.add_discovery(d) # ----------------------------------------------------------------------- # Replan — mirrors Rust orchestrator/replan.rs @@ -421,17 +583,32 @@ async def _replan( state: OrchestratorState, cards: list[DocCard], llm: LLMClient, + blackboard: SharedBlackboard | None = None, ) -> list[DispatchEntry]: """Replan dispatch targets based on missing information.""" evidence_summary = _format_evidence_context(state.all_evidence) doc_cards_text = _format_doc_cards(cards) + # Include blackboard context in replan + keywords_text = "" + if blackboard and blackboard.active_leads: + keywords_text = "\n\nActive leads from other Workers:\n" + "\n".join( + f"- {lead}" for lead in blackboard.active_leads[:5] + ) + if blackboard and blackboard.cross_references: + cross_refs = [] + for src, targets in blackboard.cross_references.items(): + cross_refs.append(f" {src} → {', '.join(targets)}") + if cross_refs: + keywords_text += "\n\nCross-document references:\n" + "\n".join(cross_refs) + system, user = orchestrator_replan_prompt( query=query, missing_info=missing_info, evidence_summary=evidence_summary, dispatched_indices=state.dispatched, doc_cards=doc_cards_text, + keywords_text=keywords_text, ) try: @@ -454,9 +631,23 @@ async def _finalize_output( confidence: float, ) -> Output: """Rerank evidence and assemble the final Output.""" - from vectorless.ask.plan import QueryIntent - - effective_intent = intent or QueryIntent.FACTUAL + from vectorless.ask.plan import QueryIntent as PlanIntent + from vectorless.ask.reasoning.types import QueryIntent + + # Map reasoning QueryIntent to plan QueryIntent for rerank compat + if intent is not None: + intent_value = intent.value if hasattr(intent, "value") else str(intent) + _plan_intent_map = { + "factual": PlanIntent.FACTUAL, + "analytical": PlanIntent.ANALYTICAL, + "navigational": PlanIntent.NAVIGATIONAL, + "summary": PlanIntent.SUMMARY, + "comparative": PlanIntent.ANALYTICAL, + "procedural": PlanIntent.FACTUAL, + } + effective_intent = _plan_intent_map.get(intent_value, PlanIntent.FACTUAL) + else: + effective_intent = PlanIntent.FACTUAL reranked = rerank_process( evidence=state.all_evidence, diff --git a/vectorless/ask/plan.py b/vectorless/ask/plan.py index de5dc47..c6830e2 100644 --- a/vectorless/ask/plan.py +++ b/vectorless/ask/plan.py @@ -4,6 +4,10 @@ from dataclasses import dataclass, field from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vectorless.ask.reasoning.types import QueryAnalysis class QueryIntent(str, Enum): @@ -54,3 +58,23 @@ def intent_context(self) -> str: if self.rewritten: parts.append(f"Rewritten queries for matching: {'; '.join(self.rewritten)}") return "\n" + "\n".join(parts) + + def to_query_analysis(self) -> QueryAnalysis: + """Convert this QueryPlan to a QueryAnalysis for backward compatibility.""" + from vectorless.ask.reasoning.types import ( + QueryAnalysis as QA, + RetrievalStrategy, + ) + return QA( + original=self.original, + rewritten=self.rewritten, + intent=QueryIntent(self.intent.value), + complexity=Complexity(self.complexity.value), + keywords=self.keywords, + key_concepts=self.key_concepts, + strategy=RetrievalStrategy(strategy_type=self.strategy_hint or "focused"), + sub_queries=[ + SubQuery(query=sq.query, target_docs=sq.target_docs) + for sq in self.sub_queries + ], + ) diff --git a/vectorless/ask/prompts.py b/vectorless/ask/prompts.py index 3203c1a..0086fed 100644 --- a/vectorless/ask/prompts.py +++ b/vectorless/ask/prompts.py @@ -40,6 +40,7 @@ class NavigationParams: plan: str = "" intent_context: str = "" keyword_hints: str = "" + shared_context: str = "" _WORKER_NAVIGATION_SYSTEM = """\ @@ -158,12 +159,17 @@ def worker_navigation(params: NavigationParams) -> tuple[str, str]: f"\nQuery context: {params.intent_context}" if params.intent_context else "" ) + shared_context_section = ( + f"\nCross-document context (from other Workers):\n{params.shared_context}\n" + if params.shared_context else "" + ) + user = ( f"{last_feedback_section}" f"User question: {query}{task_section}{intent_section}\n" f"\nCurrent position: /{breadcrumb}\n" f"Collected evidence:\n" - f"{evidence_summary}{missing_section}{keyword_section}{visited_section}{plan_section}\n" + f"{evidence_summary}{missing_section}{keyword_section}{shared_context_section}{visited_section}{plan_section}\n" f"{history_section}" f"Remaining rounds: {remaining}/{max_rounds}\n" f"\nCommand:" @@ -182,6 +188,7 @@ class WorkerDispatchParams: task: str doc_name: str breadcrumb: str + shared_context: str = "" def worker_dispatch(params: WorkerDispatchParams) -> tuple[str, str]: @@ -218,11 +225,17 @@ def worker_dispatch(params: WorkerDispatchParams) -> tuple[str, str]: f"- When evidence is sufficient, use done." ) + shared_context_section = ( + f"\nCross-document context (from other Workers):\n{params.shared_context}\n" + if params.shared_context else "" + ) + user = ( f"Original question: {original_query}\n" f"Your task: {task}\n" f"Document: {doc_name}\n" f"Current position: /{breadcrumb}\n" + f"{shared_context_section}" f"\nCommand:" ) diff --git a/vectorless/ask/reasoning/__init__.py b/vectorless/ask/reasoning/__init__.py new file mode 100644 index 0000000..b1e1b7b --- /dev/null +++ b/vectorless/ask/reasoning/__init__.py @@ -0,0 +1,27 @@ +"""Query reasoning module — multi-stage query analysis pipeline.""" + +from vectorless.ask.reasoning.types import ( + Ambiguity, + AmbiguityType, + Complexity, + EntityRef, + QueryAnalysis, + QueryIntent, + RetrievalStrategy, + SubQuery, + TemporalConstraint, +) +from vectorless.ask.reasoning.analyzer import QueryAnalyzer + +__all__ = [ + "Ambiguity", + "AmbiguityType", + "Complexity", + "EntityRef", + "QueryAnalysis", + "QueryAnalyzer", + "QueryIntent", + "RetrievalStrategy", + "SubQuery", + "TemporalConstraint", +] diff --git a/vectorless/ask/reasoning/analyzer.py b/vectorless/ask/reasoning/analyzer.py new file mode 100644 index 0000000..91dfa4e --- /dev/null +++ b/vectorless/ask/reasoning/analyzer.py @@ -0,0 +1,380 @@ +"""QueryAnalyzer — multi-stage query reasoning pipeline. + +Replaces the single-call understand() with a multi-stage analysis that +adapts depth based on query complexity: +- Fast mode (SIMPLE): single LLM call, basic analysis +- Deep mode (MODERATE/COMPLEX): three sequential LLM calls +""" + +from __future__ import annotations + +import json +import logging +import re + +from vectorless.llm_client import LLMClient +from vectorless.ask.reasoning.types import ( + Ambiguity, + AmbiguityType, + Complexity, + EntityRef, + QueryAnalysis, + QueryIntent, + RetrievalStrategy, + SubQuery, + TemporalConstraint, +) +from vectorless.ask.reasoning.prompts import ( + stage1_classify_prompt, + stage2_deep_analysis_prompt, + stage3_strategy_prompt, + re_analyze_strategy_prompt, +) + +logger = logging.getLogger(__name__) + + +def _extract_keywords(query: str) -> list[str]: + """Extract keywords from query using stop word filtering.""" + stop_words = { + "what", "is", "the", "a", "an", "how", "does", "do", "are", + "in", "on", "at", "to", "for", "of", "with", "and", "or", + "this", "that", "it", "from", "by", "was", "were", "be", + "can", "could", "would", "should", "will", "has", "have", + "had", "not", "but", "if", "then", "than", "so", "as", + "there", "their", "they", "its", "about", "which", "when", + "who", "whom", "all", "each", "every", "both", "few", + "more", "most", "other", "some", "such", "no", "nor", + "only", "own", "same", "too", "very", "just", "because", + } + words = re.findall(r"\b\w+\b", query.lower()) + return list(dict.fromkeys(w for w in words if w not in stop_words and len(w) > 2)) + + +def _parse_json_response(response: str) -> dict: + """Parse LLM response as JSON, handling markdown-wrapped output.""" + trimmed = response.strip() + + # Try to extract JSON from markdown code blocks + if trimmed.startswith("```"): + match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) + if match: + trimmed = match.group(1).strip() + + # Try to find a { ... } block + start = trimmed.find("{") + if start != -1: + depth = 0 + for i in range(start, len(trimmed)): + if trimmed[i] == "{": + depth += 1 + elif trimmed[i] == "}": + depth -= 1 + if depth == 0: + candidate = trimmed[start : i + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError: + break + + # Last resort + return json.loads(trimmed) + + +def _parse_intent(raw: str) -> QueryIntent: + """Parse intent string to QueryIntent enum.""" + mapping = { + "factual": QueryIntent.FACTUAL, + "analytical": QueryIntent.ANALYTICAL, + "navigational": QueryIntent.NAVIGATIONAL, + "summary": QueryIntent.SUMMARY, + "comparative": QueryIntent.COMPARATIVE, + "procedural": QueryIntent.PROCEDURAL, + } + return mapping.get(raw.lower(), QueryIntent.FACTUAL) + + +def _parse_complexity(raw: str) -> Complexity: + """Parse complexity string to Complexity enum.""" + mapping = { + "simple": Complexity.SIMPLE, + "moderate": Complexity.MODERATE, + "complex": Complexity.COMPLEX, + } + return mapping.get(raw.lower(), Complexity.SIMPLE) + + +class QueryAnalyzer: + """Multi-stage query reasoning pipeline. + + Usage:: + + analyzer = QueryAnalyzer() + analysis = await analyzer.analyze("What is Q1 revenue?", llm) + # For re-analysis after verification failure: + analysis = await analyzer.re_analyze(analysis, gaps, evidence_summary, llm) + """ + + async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: + """Analyze a user query using multi-stage LLM pipeline. + + Fast mode (complexity=SIMPLE after Stage 1): single LLM call. + Deep mode (MODERATE/COMPLEX): three sequential LLM calls. + + Raises on LLM failure — no silent degradation. + """ + keywords = _extract_keywords(query) + + # Stage 1: Classify + Decompose + system, user = stage1_classify_prompt(query, keywords) + response = await llm.complete(system, user) + + if not response.strip(): + raise ValueError( + "Query analysis failed: LLM returned an empty response. " + "Check your API key, model, and endpoint configuration." + ) + + stage1 = _parse_json_response(response) + intent = _parse_intent(stage1.get("intent", "factual")) + complexity = _parse_complexity(stage1.get("complexity", "simple")) + key_concepts = stage1.get("key_concepts", []) + rewritten = _parse_rewritten(stage1.get("rewritten")) + sub_queries = _parse_sub_queries(stage1.get("sub_queries", [])) + + # Fast mode: SIMPLE queries don't need deep analysis + if complexity == Complexity.SIMPLE: + return QueryAnalysis( + original=query, + rewritten=rewritten, + intent=intent, + complexity=complexity, + keywords=keywords, + key_concepts=key_concepts, + sub_queries=sub_queries, + strategy=RetrievalStrategy( + strategy_type=_map_strategy(stage1.get("strategy_hint", "focused")), + ), + ) + + # Deep mode: Stage 2 + Stage 3 + stage1_summary = { + "intent": intent.value, + "complexity": complexity.value, + "key_concepts": key_concepts, + "rewritten": rewritten, + "sub_queries": [sq.query for sq in sub_queries], + } + + # Stage 2: Deep Analysis + entities: list[EntityRef] = [] + ambiguities: list[Ambiguity] = [] + temporal_constraints: list[TemporalConstraint] = [] + + try: + system2, user2 = stage2_deep_analysis_prompt(query, stage1_summary) + response2 = await llm.complete(system2, user2) + stage2 = _parse_json_response(response2) + entities = _parse_entities(stage2.get("entities", [])) + ambiguities = _parse_ambiguities(stage2.get("ambiguities", [])) + temporal_constraints = _parse_temporal(stage2.get("temporal_constraints", [])) + # Update key_concepts if stage 2 provides them + if stage2.get("key_concepts"): + key_concepts = stage2["key_concepts"] + except Exception as e: + logger.warning("Stage 2 (deep analysis) failed: %s — continuing with partial results", e) + + # Stage 3: Strategy Formation + strategy = RetrievalStrategy( + strategy_type=_map_strategy(stage1.get("strategy_hint", "focused")), + ) + + stage2_summary = { + "entities": [{"name": e.name, "type": e.entity_type} for e in entities], + "ambiguities": [{"type": a.ambiguity_type.value, "desc": a.description} for a in ambiguities], + "key_concepts": key_concepts, + } + + try: + system3, user3 = stage3_strategy_prompt(query, stage1_summary, stage2_summary) + response3 = await llm.complete(system3, user3) + stage3 = _parse_json_response(response3) + strategy = _parse_strategy(stage3) + except Exception as e: + logger.warning("Stage 3 (strategy formation) failed: %s — using default strategy", e) + + return QueryAnalysis( + original=query, + rewritten=rewritten, + intent=intent, + complexity=complexity, + keywords=keywords, + key_concepts=key_concepts, + entities=entities, + ambiguities=ambiguities, + temporal_constraints=temporal_constraints, + sub_queries=sub_queries, + strategy=strategy, + ) + + async def re_analyze( + self, + analysis: QueryAnalysis, + gaps: list[str], + evidence_summary: str, + llm: LLMClient, + ) -> QueryAnalysis: + """Re-analyze for verification-driven re-retrieval. + + Runs only Stage 3 (strategy update) with gap context. + Increments iteration. Always deep (1 LLM call). + """ + current = { + "intent": analysis.intent.value, + "complexity": analysis.complexity.value, + "strategy": analysis.strategy.strategy_type, + "entities": [{"name": e.name, "type": e.entity_type} for e in analysis.entities], + } + + system, user = re_analyze_strategy_prompt( + query=analysis.original, + current_analysis=current, + gaps=gaps, + evidence_summary=evidence_summary, + ) + + try: + response = await llm.complete(system, user) + stage3 = _parse_json_response(response) + new_strategy = _parse_strategy(stage3) + except Exception as e: + logger.warning("Re-analyze strategy update failed: %s — keeping current strategy", e) + new_strategy = analysis.strategy + + return QueryAnalysis( + original=analysis.original, + rewritten=analysis.rewritten, + intent=analysis.intent, + complexity=analysis.complexity, + keywords=analysis.keywords, + key_concepts=analysis.key_concepts, + entities=analysis.entities, + ambiguities=analysis.ambiguities, + temporal_constraints=analysis.temporal_constraints, + sub_queries=analysis.sub_queries, + strategy=new_strategy, + iteration=analysis.iteration + 1, + additional_context="; ".join(gaps), + previous_evidence_summary=evidence_summary, + ) + + +# --------------------------------------------------------------------------- +# Parsing helpers +# --------------------------------------------------------------------------- + +def _parse_rewritten(raw: str | list | None) -> list[str]: + """Extract rewritten queries from LLM response.""" + if raw is None: + return [] + if isinstance(raw, list): + return [r.strip() for r in raw if isinstance(r, str) and r.strip()] + if isinstance(raw, str) and raw.strip(): + return [raw.strip()] + return [] + + +def _parse_sub_queries(raw: list) -> list[SubQuery]: + """Parse sub_queries from LLM response.""" + if not isinstance(raw, list): + return [] + result = [] + for item in raw: + if isinstance(item, str) and item.strip(): + result.append(SubQuery(query=item.strip())) + elif isinstance(item, dict): + result.append(SubQuery( + query=item.get("query", ""), + intent=_parse_intent(item.get("intent", "factual")), + target_docs=item.get("target_docs"), + )) + return result + + +def _parse_entities(raw: list) -> list[EntityRef]: + """Parse entities from Stage 2 response.""" + if not isinstance(raw, list): + return [] + result = [] + for item in raw: + if not isinstance(item, dict): + continue + result.append(EntityRef( + name=item.get("name", ""), + entity_type=item.get("type", "concept"), + aliases=item.get("aliases", []), + definition_hint=item.get("definition_hint", ""), + )) + return result + + +def _parse_ambiguities(raw: list) -> list[Ambiguity]: + """Parse ambiguities from Stage 2 response.""" + if not isinstance(raw, list): + return [] + result = [] + for item in raw: + if not isinstance(item, dict): + continue + amb_type_str = item.get("type", "lexical") + try: + amb_type = AmbiguityType(amb_type_str) + except ValueError: + amb_type = AmbiguityType.LEXICAL + result.append(Ambiguity( + ambiguity_type=amb_type, + description=item.get("description", ""), + possible_interpretations=item.get("interpretations", []), + resolution_query=item.get("resolution_query", ""), + )) + return result + + +def _parse_temporal(raw: list) -> list[TemporalConstraint]: + """Parse temporal constraints from Stage 2 response.""" + if not isinstance(raw, list): + return [] + result = [] + for item in raw: + if not isinstance(item, dict): + continue + result.append(TemporalConstraint( + raw=item.get("raw", ""), + resolved=item.get("resolved"), + is_relative=bool(item.get("is_relative", False)), + )) + return result + + +def _parse_strategy(raw: dict) -> RetrievalStrategy: + """Parse strategy from Stage 3 response.""" + if not isinstance(raw, dict): + return RetrievalStrategy() + return RetrievalStrategy( + strategy_type=raw.get("strategy_type", "focused"), + sub_strategies=raw.get("sub_strategies", []), + target_sections=raw.get("target_sections", []), + requires_cross_doc=bool(raw.get("requires_cross_doc", False)), + estimated_depth=raw.get("estimated_depth", "medium"), + ) + + +def _map_strategy(hint: str) -> str: + """Map strategy_hint from Stage 1 to strategy_type.""" + mapping = { + "focused": "focused", + "exploratory": "exploratory", + "comparative": "comparative", + "summary": "summary", + } + return mapping.get(hint.lower(), "focused") diff --git a/vectorless/ask/reasoning/prompts.py b/vectorless/ask/reasoning/prompts.py new file mode 100644 index 0000000..49e3d9d --- /dev/null +++ b/vectorless/ask/reasoning/prompts.py @@ -0,0 +1,176 @@ +"""Prompt templates for query reasoning stages.""" + +from __future__ import annotations + + +def stage1_classify_prompt(query: str, keywords: list[str]) -> tuple[str, str]: + """Stage 1: Classify intent, complexity, and decompose into sub-queries. + + Returns (system, user) prompt pair. + """ + system = ( + 'You are a query analysis engine. Analyze the user\'s query and respond with a JSON object:\n' + '\n' + '{\n' + ' "intent": one of "factual", "analytical", "navigational", "summary", ' + '"comparative", "procedural",\n' + ' "complexity": one of "simple", "moderate", "complex",\n' + ' "key_concepts": array of main concepts/entities (distinct from keywords),\n' + ' "rewritten": optional rewritten version of the query for better retrieval ' + '(null if not needed),\n' + ' "sub_queries": array of sub-query strings if decomposable (empty if not)\n' + '}\n' + '\n' + 'Intent guidelines:\n' + '- "factual": looking for specific facts or definitions\n' + '- "analytical": requires analysis, comparison, or evaluation\n' + '- "navigational": looking for where to find something\n' + '- "summary": wants a summary or overview\n' + '- "comparative": explicit cross-reference between items\n' + '- "procedural": how-to or step-by-step instructions\n' + '\n' + 'Complexity guidelines:\n' + '- "simple": single concept, direct answer expected\n' + '- "moderate": multi-concept, requires some synthesis\n' + '- "complex": requires multi-step reasoning, cross-referencing\n' + '\n' + 'Respond with ONLY the JSON object, no additional text.' + ) + + user = f"Query: {query}\nExtracted keywords: [{', '.join(keywords)}]" + return system, user + + +def stage2_deep_analysis_prompt( + query: str, + stage1_result: dict, +) -> tuple[str, str]: + """Stage 2: Deep analysis — entities, ambiguities, temporal constraints. + + Returns (system, user) prompt pair. + """ + import json + + system = ( + 'You are a deep query analysis engine. Given a query and initial classification, ' + 'perform deep analysis and respond with a JSON object:\n' + '\n' + '{\n' + ' "entities": [\n' + ' {"name": "...", "type": "person|org|product|concept", ' + '"aliases": [...], "definition_hint": "..."}\n' + ' ],\n' + ' "ambiguities": [\n' + ' {"type": "lexical|scope|reference|temporal", ' + '"description": "...", "interpretations": [...], "resolution_query": "..."}\n' + ' ],\n' + ' "temporal_constraints": [\n' + ' {"raw": "...", "resolved": "... or null", "is_relative": true/false}\n' + ' ],\n' + ' "key_concepts": ["concept1", "concept2"]\n' + '}\n' + '\n' + 'Entity guidelines:\n' + '- Extract named entities (people, organizations, products, technical terms)\n' + '- Include common aliases or abbreviations\n' + '- definition_hint should be a brief description\n' + '\n' + 'Ambiguity guidelines:\n' + '- Only flag genuine ambiguities that affect retrieval\n' + '- "lexical": word has multiple meanings\n' + '- "scope": unclear what scope/level of detail is wanted\n' + '- "reference": unclear what a pronoun/reference points to\n' + '- "temporal": unclear time period\n' + '\n' + 'Respond with ONLY the JSON object.' + ) + + user = ( + f"Query: {query}\n" + f"Initial classification: {json.dumps(stage1_result, ensure_ascii=False)}" + ) + return system, user + + +def stage3_strategy_prompt( + query: str, + stage1_result: dict, + stage2_result: dict, +) -> tuple[str, str]: + """Stage 3: Strategy formation — how to retrieve. + + Returns (system, user) prompt pair. + """ + import json + + system = ( + 'You are a retrieval strategy planner. Given a query, its analysis, and entity information, ' + 'formulate a retrieval strategy and respond with a JSON object:\n' + '\n' + '{\n' + ' "strategy_type": "focused|exploratory|comparative|summary",\n' + ' "sub_strategies": ["strategy1", "strategy2"],\n' + ' "target_sections": ["likely section titles or topics to look for"],\n' + ' "requires_cross_doc": true/false,\n' + ' "estimated_depth": "shallow|medium|deep"\n' + '}\n' + '\n' + 'Strategy guidelines:\n' + '- "focused": single topic, targeted retrieval\n' + '- "exploratory": broad scan needed, multiple angles\n' + '- "comparative": cross-reference between items/documents\n' + '- "summary": aggregate information from multiple sources\n' + '\n' + 'sub_strategies are specific approaches within the main strategy.\n' + 'target_sections are hints about what document sections to look for.\n' + '\n' + 'Respond with ONLY the JSON object.' + ) + + user = ( + f"Query: {query}\n" + f"Classification: {json.dumps(stage1_result, ensure_ascii=False)}\n" + f"Deep analysis: {json.dumps(stage2_result, ensure_ascii=False)}" + ) + return system, user + + +def re_analyze_strategy_prompt( + query: str, + current_analysis: dict, + gaps: list[str], + evidence_summary: str, +) -> tuple[str, str]: + """Re-analyze: update strategy based on verification gaps. + + Only runs Stage 3 (strategy update) with gap context. + Returns (system, user) prompt pair. + """ + import json + + system = ( + 'You are a retrieval strategy planner. The current retrieval strategy has gaps. ' + 'Given the original query, current analysis, identified gaps in evidence, and ' + 'a summary of evidence collected so far, update the retrieval strategy.\n' + '\n' + 'Respond with a JSON object:\n' + '{\n' + ' "strategy_type": "focused|exploratory|comparative|summary",\n' + ' "sub_strategies": ["strategy1", "strategy2"],\n' + ' "target_sections": ["sections to look for based on gaps"],\n' + ' "requires_cross_doc": true/false,\n' + ' "estimated_depth": "shallow|medium|deep"\n' + '}\n' + '\n' + 'Focus the updated strategy on addressing the identified gaps.\n' + 'Respond with ONLY the JSON object.' + ) + + gaps_text = "\n".join(f"- {g}" for g in gaps) + user = ( + f"Query: {query}\n" + f"Current analysis: {json.dumps(current_analysis, ensure_ascii=False)}\n" + f"\nEvidence gaps:\n{gaps_text}\n" + f"\nEvidence summary:\n{evidence_summary}" + ) + return system, user diff --git a/vectorless/ask/reasoning/types.py b/vectorless/ask/reasoning/types.py new file mode 100644 index 0000000..8872115 --- /dev/null +++ b/vectorless/ask/reasoning/types.py @@ -0,0 +1,122 @@ +"""Query reasoning types — rich analysis replacing QueryPlan. + +Unlike QueryPlan which captures a shallow snapshot, QueryAnalysis is a living +object that can be re-invoked during retrieval with additional context from +verification gaps and evidence summaries. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class QueryIntent(str, Enum): + """Detected intent of a user query.""" + FACTUAL = "factual" + ANALYTICAL = "analytical" + NAVIGATIONAL = "navigational" + SUMMARY = "summary" + COMPARATIVE = "comparative" + PROCEDURAL = "procedural" + + +class Complexity(str, Enum): + """Estimated query complexity.""" + SIMPLE = "simple" + MODERATE = "moderate" + COMPLEX = "complex" + + +class AmbiguityType(str, Enum): + """Type of ambiguity detected in a query.""" + LEXICAL = "lexical" + SCOPE = "scope" + REFERENCE = "reference" + TEMPORAL = "temporal" + + +@dataclass +class Ambiguity: + """An ambiguity detected in the query.""" + ambiguity_type: AmbiguityType + description: str + possible_interpretations: list[str] + resolution_query: str + + +@dataclass +class TemporalConstraint: + """A temporal constraint extracted from the query.""" + raw: str + resolved: str | None = None + is_relative: bool = False + + +@dataclass +class EntityRef: + """A named entity referenced in the query.""" + name: str + entity_type: str # "person", "org", "product", "concept" + aliases: list[str] = field(default_factory=list) + definition_hint: str = "" + + +@dataclass +class RetrievalStrategy: + """Strategy for how to retrieve information.""" + strategy_type: str = "focused" # "focused", "exploratory", "comparative", "summary" + sub_strategies: list[str] = field(default_factory=list) + target_sections: list[str] = field(default_factory=list) + requires_cross_doc: bool = False + estimated_depth: str = "medium" # "shallow", "medium", "deep" + + +@dataclass +class SubQuery: + """A decomposed sub-query from a complex query.""" + query: str + intent: QueryIntent = QueryIntent.FACTUAL + target_docs: list[str] | None = None + + +@dataclass +class QueryAnalysis: + """Rich analysis of a user query. + + Produced by QueryAnalyzer, consumed by Orchestrator and Workers. + Can be re-invoked during retrieval with additional context. + """ + original: str + rewritten: list[str] = field(default_factory=list) + intent: QueryIntent = QueryIntent.FACTUAL + complexity: Complexity = Complexity.SIMPLE + keywords: list[str] = field(default_factory=list) + key_concepts: list[str] = field(default_factory=list) + entities: list[EntityRef] = field(default_factory=list) + ambiguities: list[Ambiguity] = field(default_factory=list) + temporal_constraints: list[TemporalConstraint] = field(default_factory=list) + sub_queries: list[SubQuery] = field(default_factory=list) + strategy: RetrievalStrategy = field(default_factory=RetrievalStrategy) + iteration: int = 0 + additional_context: str = "" + previous_evidence_summary: str = "" + + def intent_context(self) -> str: + """Format intent context string for prompts. + + Backward-compatible with QueryPlan.intent_context(). + """ + parts = [f"Query intent: {self.intent.value} (complexity: {self.complexity.value})"] + if self.key_concepts: + parts.append(f"Key concepts: {', '.join(self.key_concepts)}") + if self.strategy.strategy_type: + parts.append(f"Retrieval strategy: {self.strategy.strategy_type}") + if self.rewritten: + parts.append(f"Rewritten queries for matching: {'; '.join(self.rewritten)}") + if self.entities: + entity_names = ", ".join(e.name for e in self.entities[:5]) + parts.append(f"Key entities: {entity_names}") + if self.additional_context: + parts.append(f"Additional context: {self.additional_context}") + return "\n" + "\n".join(parts) diff --git a/vectorless/ask/understand.py b/vectorless/ask/understand.py index 75c0d62..c24db91 100644 --- a/vectorless/ask/understand.py +++ b/vectorless/ask/understand.py @@ -1,16 +1,19 @@ """Query understanding — LLM-driven analysis of user queries. Mirrors vectorless-core/vectorless-query/src/understand.rs. + +This module is now a thin wrapper around the multi-stage QueryAnalyzer. +Kept for backward compatibility — new code should use QueryAnalyzer directly. """ from __future__ import annotations -import json import logging -import re +import warnings from vectorless.llm_client import LLMClient -from vectorless.ask.plan import Complexity, QueryIntent, QueryPlan, SubQuery +from vectorless.ask.plan import QueryPlan +from vectorless.ask.reasoning.analyzer import QueryAnalyzer logger = logging.getLogger(__name__) @@ -21,139 +24,62 @@ async def understand( ) -> QueryPlan: """Analyze a user query using LLM to produce a structured QueryPlan. - Two-phase: - 1. Extract keywords locally (BM25-style, no LLM) - 2. Call LLM for intent classification, concepts, strategy, complexity + Delegates to QueryAnalyzer.analyze() and converts the result to QueryPlan + for backward compatibility. Raises on LLM failure — no silent degradation. """ - keywords = _extract_keywords(query) - - system = ( - 'You are a query analysis engine. Analyze the user\'s query and respond with a JSON object containing:\n' - '\n' - '- "intent": one of "factual", "analytical", "navigational", "summary"\n' - '- "key_concepts": array of the main concepts/entities in the query (distinct from keywords)\n' - '- "strategy_hint": one of "focused" (single-topic), "exploratory" (broad scan), ' - '"comparative" (cross-reference), or "summary" (aggregate)\n' - '- "complexity": one of "simple", "moderate", "complex"\n' - '- "rewritten": optional rewritten version of the query for better retrieval (null if not needed)\n' - '- "sub_queries": array of sub-query strings if the query can be decomposed (empty array if not)\n' - '\n' - 'Respond with ONLY the JSON object, no additional text.' + warnings.warn( + "understand() is deprecated — use QueryAnalyzer.analyze() directly", + DeprecationWarning, + stacklevel=2, ) - user = f"Query: {query}\nExtracted keywords: [{', '.join(keywords)}]" - - response = await llm.complete(system, user) - - if not response.strip(): - raise ValueError( - "Query understanding failed: LLM returned an empty response. " - "Check your API key, model, and endpoint configuration." - ) - - analysis = _parse_analysis(response) + analyzer = QueryAnalyzer() + analysis = await analyzer.analyze(query, llm) + # Convert QueryAnalysis back to QueryPlan return QueryPlan( - original=query, - intent=_parse_intent(analysis.get("intent", "factual")), - keywords=keywords, - key_concepts=analysis.get("key_concepts", []), - strategy_hint=analysis.get("strategy_hint", ""), - complexity=_parse_complexity(analysis.get("complexity", "simple")), - rewritten=_filter_rewritten(analysis.get("rewritten")), - sub_queries=_parse_sub_queries(analysis.get("sub_queries", [])), + original=analysis.original, + intent=_map_intent(analysis.intent), + keywords=analysis.keywords, + key_concepts=analysis.key_concepts, + strategy_hint=analysis.strategy.strategy_type, + complexity=_map_complexity(analysis.complexity), + rewritten=analysis.rewritten, + sub_queries=_map_sub_queries(analysis.sub_queries), ) -def _extract_keywords(query: str) -> list[str]: - """Extract keywords from query using stop word filtering.""" - stop_words = { - "what", "is", "the", "a", "an", "how", "does", "do", "are", - "in", "on", "at", "to", "for", "of", "with", "and", "or", - "this", "that", "it", "from", "by", "was", "were", "be", - "can", "could", "would", "should", "will", "has", "have", - "had", "not", "but", "if", "then", "than", "so", "as", - "there", "their", "they", "its", "about", "which", "when", - "who", "whom", "all", "each", "every", "both", "few", - "more", "most", "other", "some", "such", "no", "nor", - "only", "own", "same", "too", "very", "just", "because", - } - words = re.findall(r"\b\w+\b", query.lower()) - return list(dict.fromkeys(w for w in words if w not in stop_words and len(w) > 2)) - - -def _parse_analysis(response: str) -> dict: - """Parse LLM response as JSON, handling markdown-wrapped output.""" - trimmed = response.strip() - - # Try to extract JSON from markdown code blocks - if trimmed.startswith("```"): - match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) - if match: - trimmed = match.group(1).strip() - - # Try to find a { ... } block - start = trimmed.find("{") - if start != -1: - depth = 0 - for i in range(start, len(trimmed)): - if trimmed[i] == "{": - depth += 1 - elif trimmed[i] == "}": - depth -= 1 - if depth == 0: - candidate = trimmed[start : i + 1] - try: - return json.loads(candidate) - except json.JSONDecodeError: - break - - # Last resort - return json.loads(trimmed) - - -def _parse_intent(raw: str) -> QueryIntent: - """Parse intent string to QueryIntent enum.""" +def _map_intent(intent): + """Map reasoning QueryIntent to plan QueryIntent.""" + from vectorless.ask.plan import QueryIntent as PlanIntent mapping = { - "factual": QueryIntent.FACTUAL, - "analytical": QueryIntent.ANALYTICAL, - "navigational": QueryIntent.NAVIGATIONAL, - "summary": QueryIntent.SUMMARY, + "factual": PlanIntent.FACTUAL, + "analytical": PlanIntent.ANALYTICAL, + "navigational": PlanIntent.NAVIGATIONAL, + "summary": PlanIntent.SUMMARY, + "comparative": PlanIntent.ANALYTICAL, # Map new to existing + "procedural": PlanIntent.FACTUAL, # Map new to existing } - return mapping.get(raw.lower(), QueryIntent.FACTUAL) + return mapping.get(intent.value, PlanIntent.FACTUAL) -def _parse_complexity(raw: str) -> Complexity: - """Parse complexity string to Complexity enum.""" +def _map_complexity(complexity): + """Map reasoning Complexity to plan Complexity.""" + from vectorless.ask.plan import Complexity as PlanComplexity mapping = { - "simple": Complexity.SIMPLE, - "moderate": Complexity.MODERATE, - "complex": Complexity.COMPLEX, + "simple": PlanComplexity.SIMPLE, + "moderate": PlanComplexity.MODERATE, + "complex": PlanComplexity.COMPLEX, } - return mapping.get(raw.lower(), Complexity.SIMPLE) - - -def _filter_rewritten(raw: str | None) -> list[str]: - """Extract rewritten queries from LLM response.""" - if raw is None or not isinstance(raw, str) or not raw.strip(): - return [] - return [raw.strip()] - - -def _parse_sub_queries(raw: list) -> list[SubQuery]: - """Parse sub_queries from LLM response.""" - if not isinstance(raw, list): - return [] - result = [] - for item in raw: - if isinstance(item, str) and item.strip(): - result.append(SubQuery(query=item.strip())) - elif isinstance(item, dict): - result.append(SubQuery( - query=item.get("query", ""), - intent=_parse_intent(item.get("intent", "factual")), - target_docs=item.get("target_docs"), - )) - return result + return mapping.get(complexity.value, PlanComplexity.SIMPLE) + + +def _map_sub_queries(sub_queries): + """Map reasoning SubQueries to plan SubQueries.""" + from vectorless.ask.plan import SubQuery as PlanSubQuery + return [ + PlanSubQuery(query=sq.query, target_docs=sq.target_docs) + for sq in sub_queries + ] diff --git a/vectorless/ask/verify/__init__.py b/vectorless/ask/verify/__init__.py new file mode 100644 index 0000000..128daba --- /dev/null +++ b/vectorless/ask/verify/__init__.py @@ -0,0 +1,15 @@ +"""Verification module — multi-dimensional evidence verification pipeline.""" + +from vectorless.ask.verify.types import ( + DimensionScore, + VerificationDimension, + VerificationResult, +) +from vectorless.ask.verify.verifier import VerifyPipeline + +__all__ = [ + "DimensionScore", + "VerifyPipeline", + "VerificationDimension", + "VerificationResult", +] diff --git a/vectorless/ask/verify/prompts.py b/vectorless/ask/verify/prompts.py new file mode 100644 index 0000000..fd992e7 --- /dev/null +++ b/vectorless/ask/verify/prompts.py @@ -0,0 +1,81 @@ +"""Verification prompt templates.""" + +from __future__ import annotations + + +def verify_prompt( + query: str, + evidence_text: str, + query_intent: str, + iteration: int, +) -> tuple[str, str]: + """Build the verification prompt — single combined LLM call for all 4 dimensions. + + Returns (system, user) prompt pair. + """ + system = ( + "You are an evidence verification engine. Assess whether the collected evidence " + "can answer the user's question across four dimensions. Respond with a JSON object:\n" + "\n" + "{\n" + ' "dimensions": {\n' + ' "factual_accuracy": {\n' + ' "score": 0.0-1.0,\n' + ' "reasoning": "...",\n' + ' "evidence_refs": ["node_title_1", "node_title_2"]\n' + " },\n" + ' "completeness": {\n' + ' "score": 0.0-1.0,\n' + ' "reasoning": "...",\n' + ' "evidence_refs": ["node_title_1"]\n' + " },\n" + ' "relevance": {\n' + ' "score": 0.0-1.0,\n' + ' "reasoning": "...",\n' + ' "evidence_refs": ["node_title_1"]\n' + " },\n" + ' "coherence": {\n' + ' "score": 0.0-1.0,\n' + ' "reasoning": "...",\n' + ' "evidence_refs": ["node_title_1"]\n' + " }\n" + " },\n" + ' "passed": true/false,\n' + ' "overall_confidence": 0.0-1.0,\n' + ' "gaps": ["specific gap 1", "specific gap 2"],\n' + ' "re_retrieval_hints": ["what to search for next"]\n' + "}\n" + "\n" + "Dimension guidelines:\n" + "- factual_accuracy: Do the evidence texts support the claims needed to answer " + "the question? Are facts verifiable from the evidence?\n" + "- completeness: Does the evidence cover ALL aspects of the question? " + "If the question has multiple parts, are all addressed?\n" + "- relevance: Is the evidence directly on-topic for the question?\n" + "- coherence: Can the evidence pieces be logically assembled into an answer?\n" + "\n" + "Scoring guidelines:\n" + "- 0.0-0.3: Severely lacking\n" + "- 0.3-0.5: Partially addressed\n" + "- 0.5-0.7: Adequately addressed\n" + "- 0.7-0.9: Well addressed\n" + "- 0.9-1.0: Comprehensively addressed\n" + "\n" + 'Set "passed" to true ONLY if all dimension scores are >= 0.5.\n' + '"gaps" should list specific information still missing.\n' + '"re_retrieval_hints" should describe what additional searches would help.\n' + "\n" + "Respond with ONLY the JSON object." + ) + + iteration_note = f"\n(This is verification iteration {iteration + 1})" if iteration > 0 else "" + + user = ( + f"Question: {query}\n" + f"Query intent: {query_intent}{iteration_note}\n\n" + f"Collected evidence:\n" + f"{evidence_text}\n\n" + f"Verify the evidence." + ) + + return system, user diff --git a/vectorless/ask/verify/types.py b/vectorless/ask/verify/types.py new file mode 100644 index 0000000..0cce9d0 --- /dev/null +++ b/vectorless/ask/verify/types.py @@ -0,0 +1,39 @@ +"""Verification types — dimensions, scores, and results.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class VerificationDimension(str, Enum): + """Dimensions along which answers are verified.""" + FACTUAL_ACCURACY = "factual_accuracy" # Does evidence support the claims? + COMPLETENESS = "completeness" # Does it cover all query aspects? + RELEVANCE = "relevance" # Is it on-topic? + COHERENCE = "coherence" # Is the reasoning trace logical? + + +@dataclass +class DimensionScore: + """Score for a single verification dimension.""" + dimension: VerificationDimension + score: float # 0.0 - 1.0 + reasoning: str # LLM explanation + evidence_refs: list[str] = field(default_factory=list) # node_titles + + +@dataclass +class VerificationResult: + """Result of the verification pipeline.""" + passed: bool + overall_confidence: float + dimension_scores: list[DimensionScore] = field(default_factory=list) + gaps: list[str] = field(default_factory=list) + re_retrieval_hints: list[str] = field(default_factory=list) + iteration: int = 0 + + @property + def needs_re_retrieval(self) -> bool: + """Whether re-retrieval should be triggered.""" + return not self.passed and bool(self.re_retrieval_hints) diff --git a/vectorless/ask/verify/verifier.py b/vectorless/ask/verify/verifier.py new file mode 100644 index 0000000..944b21c --- /dev/null +++ b/vectorless/ask/verify/verifier.py @@ -0,0 +1,232 @@ +"""VerifyPipeline — multi-dimensional evidence verification. + +Single combined LLM call assessing all 4 dimensions simultaneously. +Configurable thresholds per intent. Max 2 verification iterations. +""" + +from __future__ import annotations + +import json +import logging +import re + +from vectorless.llm_client import LLMClient +from vectorless.ask.types import Evidence +from vectorless.ask.verify.types import ( + DimensionScore, + VerificationDimension, + VerificationResult, +) +from vectorless.ask.verify.prompts import verify_prompt + +logger = logging.getLogger(__name__) + +MAX_VERIFICATION_ITERATIONS = 2 + +# Per-intent, per-dimension thresholds +_DEFAULT_THRESHOLD = 0.5 +_INTENT_THRESHOLDS: dict[str, dict[str, float]] = { + "factual": { + "factual_accuracy": 0.7, + "completeness": 0.5, + "relevance": 0.6, + "coherence": 0.5, + }, + "analytical": { + "factual_accuracy": 0.6, + "completeness": 0.7, + "relevance": 0.6, + "coherence": 0.7, + }, + "comparative": { + "factual_accuracy": 0.6, + "completeness": 0.7, + "relevance": 0.6, + "coherence": 0.7, + }, + "summary": { + "factual_accuracy": 0.5, + "completeness": 0.7, + "relevance": 0.5, + "coherence": 0.6, + }, + "procedural": { + "factual_accuracy": 0.7, + "completeness": 0.7, + "relevance": 0.6, + "coherence": 0.7, + }, + "navigational": { + "factual_accuracy": 0.5, + "completeness": 0.5, + "relevance": 0.7, + "coherence": 0.5, + }, +} + + +def _format_evidence(evidence: list[Evidence]) -> str: + """Format evidence for the verification prompt.""" + if not evidence: + return "(no evidence)" + return "\n\n".join( + f"[{e.node_title}] (from {e.doc_name or 'unknown'})\n{e.content}" + for e in evidence + ) + + +def _parse_json_response(response: str) -> dict: + """Parse LLM response as JSON, handling markdown-wrapped output.""" + trimmed = response.strip() + + if trimmed.startswith("```"): + match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) + if match: + trimmed = match.group(1).strip() + + start = trimmed.find("{") + if start != -1: + depth = 0 + for i in range(start, len(trimmed)): + if trimmed[i] == "{": + depth += 1 + elif trimmed[i] == "}": + depth -= 1 + if depth == 0: + candidate = trimmed[start : i + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError: + break + + return json.loads(trimmed) + + +class VerifyPipeline: + """Multi-dimensional evidence verification pipeline. + + Usage:: + + pipeline = VerifyPipeline() + result = await pipeline.verify( + query="What is Q1 revenue?", + evidence=collected_evidence, + query_intent="factual", + iteration=0, + llm=llm_client, + ) + """ + + async def verify( + self, + query: str, + evidence: list[Evidence], + query_intent: str, + iteration: int, + llm: LLMClient, + ) -> VerificationResult: + """Run verification on collected evidence. + + Single combined LLM call assessing all 4 dimensions. + Returns VerificationResult with pass/fail, scores, and gaps. + """ + evidence_text = _format_evidence(evidence) + + if not evidence: + return VerificationResult( + passed=False, + overall_confidence=0.0, + gaps=["No evidence collected"], + re_retrieval_hints=[query], + iteration=iteration, + ) + + system, user = verify_prompt( + query=query, + evidence_text=evidence_text, + query_intent=query_intent, + iteration=iteration, + ) + + response = await llm.complete(system, user) + + if not response.strip(): + return VerificationResult( + passed=False, + overall_confidence=0.0, + gaps=["Verification LLM returned empty response"], + re_retrieval_hints=[], + iteration=iteration, + ) + + try: + data = _parse_json_response(response) + except (json.JSONDecodeError, ValueError) as e: + logger.warning("Verification response parse failed: %s", e) + return VerificationResult( + passed=False, + overall_confidence=0.0, + gaps=["Failed to parse verification response"], + re_retrieval_hints=[], + iteration=iteration, + ) + + return self._build_result(data, query_intent, iteration) + + def _build_result( + self, + data: dict, + query_intent: str, + iteration: int, + ) -> VerificationResult: + """Build VerificationResult from parsed LLM response.""" + dimensions_data = data.get("dimensions", {}) + thresholds = _INTENT_THRESHOLDS.get(query_intent, {}) + + scores: list[DimensionScore] = [] + low_dimensions: list[str] = [] + gaps: list[str] = [] + re_retrieval_hints: list[str] = [] + + for dim in VerificationDimension: + dim_data = dimensions_data.get(dim.value, {}) + raw_score = float(dim_data.get("score", 0.0)) + score = max(0.0, min(1.0, raw_score)) + reasoning = dim_data.get("reasoning", "") + refs = dim_data.get("evidence_refs", []) + + scores.append(DimensionScore( + dimension=dim, + score=score, + reasoning=reasoning, + evidence_refs=refs if isinstance(refs, list) else [], + )) + + threshold = thresholds.get(dim.value, _DEFAULT_THRESHOLD) + if score < threshold: + low_dimensions.append(dim.value) + if reasoning: + gaps.append(f"[{dim.value}] {reasoning}") + + passed = len(low_dimensions) == 0 + overall_confidence = max(0.0, min(1.0, float(data.get("overall_confidence", 0.5)))) + + # Synthesize re_retrieval_hints from gaps or explicit hints + raw_hints = data.get("re_retrieval_hints", []) + if isinstance(raw_hints, list): + re_retrieval_hints = [str(h) for h in raw_hints if h] + elif not passed: + re_retrieval_hints = gaps + + # Override passed if LLM says passed but dimensions disagree + if data.get("passed", False) and low_dimensions: + passed = False + + return VerificationResult( + passed=passed, + overall_confidence=overall_confidence, + dimension_scores=scores, + gaps=gaps, + re_retrieval_hints=re_retrieval_hints, + iteration=iteration, + ) diff --git a/vectorless/ask/worker.py b/vectorless/ask/worker.py index 3882b92..18085c2 100644 --- a/vectorless/ask/worker.py +++ b/vectorless/ask/worker.py @@ -837,6 +837,7 @@ def __init__( max_llm_calls: int = 0, task: str | None = None, intent_context: str = "", + shared_context: str = "", ) -> None: self._doc = document self._query = query @@ -845,6 +846,7 @@ def __init__( self._max_llm_calls = max_llm_calls self._task = task self._intent_context = intent_context + self._shared_context = shared_context async def run(self) -> WorkerOutput: """Execute the Worker navigation loop and return collected evidence.""" @@ -855,6 +857,7 @@ async def run(self) -> WorkerOutput: max_rounds = self._max_rounds max_llm = self._max_llm_calls intent_context = self._intent_context + shared_context = self._shared_context state = WorkerState(remaining=max_rounds, max_rounds=max_rounds) @@ -902,6 +905,7 @@ async def run(self) -> WorkerOutput: task=task or query, doc_name=await doc.doc_name(), breadcrumb=state.path_str(), + shared_context=shared_context, )) else: visited_titles = await _visited_titles(state, doc) @@ -919,6 +923,7 @@ async def run(self) -> WorkerOutput: plan=state.plan, intent_context=intent_context, keyword_hints=keyword_hints, + shared_context=shared_context, )) # LLM decision From ad42c1a739328d15e51334182dd1c8566b2e88af Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 16:48:37 +0800 Subject: [PATCH 04/30] refactor(config): restructure configuration modules and remove retrieval config - Move indexer, llm_pool, metrics, and storage modules from types/ directory to src/ directory root - Remove retrieval module as it's no longer needed in core configuration - Update lib.rs to export modules directly instead of through types mod - Add comprehensive Config struct with validation capabilities - Include ConfigValidationError and ValidationError types for proper configuration validation - Add tests for configuration defaults and validation feat(blackboard): enhance discovery extraction with cross-document refs - Extract cross-references from evidence content using regex patterns - Add "cross_ref" and "lead" discovery types for document references - Track evidence-referenced documents and generate lead discoveries refactor(analyzer): consolidate JSON parsing utilities - Move _parse_json_response function to shared utils module - Import parse_json_response from vectorless.ask.utils - Update QueryAnalyzer to use consolidated JSON parsing utility feat(analyzer): add analysis completion tracking - Add analysis_complete flag to QueryAnalysis to track whether deep analysis stages completed successfully - Set analysis_complete=False when deep analysis stages fail - Propagate completion status through re_analyze method refactor(python): remove deprecated retrieval configuration - Remove set_top_k and set_max_iterations methods from PyConfig - Update Config documentation to reflect removal of retrieval params - Remove RetrievalConfig from python exports refactor(utils): centralize JSON response parsing logic - Create parse_json_response utility function in utils module - Consolidate JSON parsing logic from analyzer and verifier modules - Handle markdown-wrapped JSON and extract JSON blocks properly feat(verify): improve evidence reference formatting and scoring - Update verify prompt to use "doc_name/node_title" format for evidence references - Modify DimensionScore to accept both "doc_name/node_title" and "node_title" formats - Calculate overall confidence from dimension scores instead of relying on LLM self-assessment --- .../src/{types => }/indexer.rs | 0 vectorless-core/vectorless-config/src/lib.rs | 357 ++++++++++++++++- .../src/{types => }/llm_pool.rs | 0 .../src/{types => }/metrics.rs | 0 .../src/{types => }/storage.rs | 0 .../vectorless-config/src/types/mod.rs | 375 ------------------ .../vectorless-config/src/types/retrieval.rs | 170 -------- vectorless-core/vectorless-py/src/config.rs | 15 +- vectorless/ask/blackboard.py | 39 +- vectorless/ask/plan.py | 24 +- vectorless/ask/reasoning/analyzer.py | 47 +-- vectorless/ask/reasoning/types.py | 1 + vectorless/ask/utils.py | 33 ++ vectorless/ask/verify/prompts.py | 8 +- vectorless/ask/verify/types.py | 2 +- vectorless/ask/verify/verifier.py | 36 +- vectorless/config/__init__.py | 2 - vectorless/config/loading.py | 25 +- vectorless/config/models.py | 11 - vectorless/rerank/quality.py | 2 +- vectorless/rerank/synthesize.py | 2 +- 21 files changed, 453 insertions(+), 696 deletions(-) rename vectorless-core/vectorless-config/src/{types => }/indexer.rs (100%) rename vectorless-core/vectorless-config/src/{types => }/llm_pool.rs (100%) rename vectorless-core/vectorless-config/src/{types => }/metrics.rs (100%) rename vectorless-core/vectorless-config/src/{types => }/storage.rs (100%) delete mode 100644 vectorless-core/vectorless-config/src/types/mod.rs delete mode 100644 vectorless-core/vectorless-config/src/types/retrieval.rs create mode 100644 vectorless/ask/utils.py diff --git a/vectorless-core/vectorless-config/src/types/indexer.rs b/vectorless-core/vectorless-config/src/indexer.rs similarity index 100% rename from vectorless-core/vectorless-config/src/types/indexer.rs rename to vectorless-core/vectorless-config/src/indexer.rs diff --git a/vectorless-core/vectorless-config/src/lib.rs b/vectorless-core/vectorless-config/src/lib.rs index 6812d67..78d6a13 100644 --- a/vectorless-core/vectorless-config/src/lib.rs +++ b/vectorless-core/vectorless-config/src/lib.rs @@ -6,14 +6,351 @@ //! Users configure vectorless via [`EngineBuilder`](vectorless_engine::EngineBuilder) methods, //! not by directly interacting with this module. -mod types; - -pub use types::Config; -pub use types::DocumentGraphConfig; -pub use types::LlmMetricsConfig; -pub use types::MetricsConfig; -pub use types::RetrievalMetricsConfig; -pub use types::{ - CompressionAlgorithm, FallbackBehavior, FallbackConfig, IndexerConfig, LlmConfig, - OnAllFailedBehavior, RetrievalConfig, RetryConfig, SlotConfig, StorageConfig, ThrottleConfig, +mod indexer; +mod llm_pool; +mod metrics; +mod storage; + +pub use indexer::IndexerConfig; +pub use llm_pool::{ + FallbackBehavior, FallbackConfig, LlmConfig, OnAllFailedBehavior, RetryConfig, SlotConfig, + ThrottleConfig, }; +pub use metrics::{LlmMetricsConfig, MetricsConfig, RetrievalMetricsConfig}; +pub use storage::{CompressionAlgorithm, StorageConfig}; +pub use vectorless_graph::DocumentGraphConfig; + +use serde::{Deserialize, Serialize}; + +/// Main configuration for vectorless. +/// +/// Users typically configure via [`EngineBuilder`](vectorless_engine::EngineBuilder): +/// +/// ```rust,no_run +/// use vectorless::client::EngineBuilder; +/// +/// # async fn example() -> Result<(), vectorless::BuildError> { +/// let engine = EngineBuilder::new() +/// .with_key("sk-...") +/// .with_model("gpt-4o") +/// .with_endpoint("https://api.openai.com/v1") +/// .build() +/// .await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// Advanced users can construct this programmatically: +/// +/// ```rust,ignore +/// use vectorless::config::{Config, LlmConfig, SlotConfig}; +/// +/// let config = Config::new().with_llm( +/// LlmConfig::new("gpt-4o") +/// .with_api_key("sk-...") +/// .with_endpoint("https://api.openai.com/v1") +/// .with_index(SlotConfig::fast().with_model("gpt-4o-mini")) +/// ); +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + /// LLM configuration (model, credentials, retry, throttle, fallback). + #[serde(default)] + pub llm: LlmConfig, + + /// Metrics configuration. + #[serde(default)] + pub metrics: MetricsConfig, + + /// Indexer configuration. + #[serde(default)] + pub indexer: IndexerConfig, + + /// Storage configuration. + #[serde(default)] + pub storage: StorageConfig, + + /// Document graph configuration. + #[serde(default)] + pub graph: DocumentGraphConfig, +} + +impl Default for Config { + fn default() -> Self { + Self { + llm: LlmConfig::default(), + metrics: MetricsConfig::default(), + indexer: IndexerConfig::default(), + storage: StorageConfig::default(), + graph: DocumentGraphConfig::default(), + } + } +} + +impl Config { + /// Create a new configuration with defaults. + pub fn new() -> Self { + Self::default() + } + + /// Set the LLM configuration. + pub fn with_llm(mut self, llm: LlmConfig) -> Self { + self.llm = llm; + self + } + + /// Set the metrics configuration. + pub fn with_metrics(mut self, metrics: MetricsConfig) -> Self { + self.metrics = metrics; + self + } + + /// Set the indexer configuration. + pub fn with_indexer(mut self, indexer: IndexerConfig) -> Self { + self.indexer = indexer; + self + } + + /// Set the storage configuration. + pub fn with_storage(mut self, storage: StorageConfig) -> Self { + self.storage = storage; + self + } + + /// Set the document graph configuration. + pub fn with_graph(mut self, graph: DocumentGraphConfig) -> Self { + self.graph = graph; + self + } + + /// Validate the configuration. + pub fn validate(&self) -> Result<(), ConfigValidationError> { + let mut errors = Vec::new(); + + // Validate indexer + if self.indexer.subsection_threshold == 0 { + errors.push(ValidationError::error( + "indexer.subsection_threshold", + "Subsection threshold must be greater than 0", + )); + } + + // Validate LLM slot tokens + if self.llm.index.max_tokens == 0 { + errors.push(ValidationError::error( + "llm.index.max_tokens", + "Index max tokens must be greater than 0", + )); + } + + if self.llm.retrieval.max_tokens == 0 { + errors.push(ValidationError::error( + "llm.retrieval.max_tokens", + "Retrieval max tokens must be greater than 0", + )); + } + + // Validate throttle + if self.llm.throttle.max_concurrent_requests == 0 { + errors.push(ValidationError::error( + "llm.throttle.max_concurrent_requests", + "Max concurrent requests must be greater than 0", + )); + } + + // Validate graph + if self.graph.min_keyword_jaccard < 0.0 || self.graph.min_keyword_jaccard > 1.0 { + errors.push(ValidationError::error( + "graph.min_keyword_jaccard", + "Must be between 0.0 and 1.0", + )); + } + if self.graph.max_edges_per_node == 0 { + errors.push(ValidationError::error( + "graph.max_edges_per_node", + "Must be greater than 0", + )); + } + + // Validate fallback + if self.llm.fallback.enabled && self.llm.fallback.models.is_empty() { + errors.push(ValidationError::warning( + "llm.fallback.models", + "Fallback enabled but no fallback models configured", + )); + } + + if errors.is_empty() { + Ok(()) + } else { + Err(ConfigValidationError { errors }) + } + } +} + +/// Configuration validation error. +#[derive(Debug, Clone)] +pub struct ConfigValidationError { + /// Validation errors. + pub errors: Vec, +} + +impl std::fmt::Display for ConfigValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Configuration validation failed with {} error(s)", + self.errors.len() + ) + } +} + +impl std::error::Error for ConfigValidationError {} + +/// A single validation error. +#[derive(Debug, Clone)] +pub struct ValidationError { + /// Field path (e.g., "retrieval.content.token_budget"). + pub path: String, + + /// Error message. + pub message: String, + + /// Expected value/range. + pub expected: Option, + + /// Actual value. + pub actual: Option, + + /// Severity level. + pub severity: Severity, +} + +impl ValidationError { + /// Create an error-level validation error. + pub fn error(path: impl Into, message: impl Into) -> Self { + Self { + path: path.into(), + message: message.into(), + expected: None, + actual: None, + severity: Severity::Error, + } + } + + /// Create a warning-level validation error. + pub fn warning(path: impl Into, message: impl Into) -> Self { + Self { + path: path.into(), + message: message.into(), + expected: None, + actual: None, + severity: Severity::Warning, + } + } + + /// Create an info-level validation error. + pub fn info(path: impl Into, message: impl Into) -> Self { + Self { + path: path.into(), + message: message.into(), + expected: None, + actual: None, + severity: Severity::Info, + } + } + + /// Set the expected value. + pub fn with_expected(mut self, expected: impl Into) -> Self { + self.expected = Some(expected.into()); + self + } + + /// Set the actual value. + pub fn with_actual(mut self, actual: impl Into) -> Self { + self.actual = Some(actual.into()); + self + } +} + +impl std::fmt::Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let severity = match self.severity { + Severity::Error => "ERROR", + Severity::Warning => "WARNING", + Severity::Info => "INFO", + }; + write!(f, "[{}] {}: {}", severity, self.path, self.message)?; + if let Some(ref expected) = self.expected { + write!(f, " (expected: {})", expected)?; + } + if let Some(ref actual) = self.actual { + write!(f, " (actual: {})", actual)?; + } + Ok(()) + } +} + +/// Validation severity level. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Severity { + /// Error - must fix. + Error, + /// Warning - should fix. + Warning, + /// Info - suggestion. + Info, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_defaults() { + let config = Config::default(); + assert!(config.llm.model.is_empty()); + assert!(config.llm.index.model.is_none()); + assert_eq!(config.indexer.subsection_threshold, 300); + assert!(config.metrics.enabled); + } + + #[test] + fn test_llm_config_defaults() { + let config = LlmConfig::default(); + assert!(config.index.model.is_none()); + assert!(config.retrieval.model.is_none()); + assert_eq!(config.retry.max_attempts, 3); + assert_eq!(config.throttle.max_concurrent_requests, 10); + } + + #[test] + fn test_config_validation_success() { + let config = Config::default(); + assert!(config.validate().is_ok()); + } + + #[test] + fn test_config_validation_errors() { + let mut config = Config::default(); + config.indexer.subsection_threshold = 0; + + let result = config.validate(); + assert!(result.is_err()); + + let err = result.unwrap_err(); + assert!(!err.errors.is_empty()); + } + + #[test] + fn test_validation_error_display() { + let err = ValidationError::error("test.field", "Invalid value") + .with_expected(">= 1") + .with_actual("0"); + + let display = format!("{}", err); + assert!(display.contains("ERROR")); + assert!(display.contains("test.field")); + assert!(display.contains("expected")); + } +} diff --git a/vectorless-core/vectorless-config/src/types/llm_pool.rs b/vectorless-core/vectorless-config/src/llm_pool.rs similarity index 100% rename from vectorless-core/vectorless-config/src/types/llm_pool.rs rename to vectorless-core/vectorless-config/src/llm_pool.rs diff --git a/vectorless-core/vectorless-config/src/types/metrics.rs b/vectorless-core/vectorless-config/src/metrics.rs similarity index 100% rename from vectorless-core/vectorless-config/src/types/metrics.rs rename to vectorless-core/vectorless-config/src/metrics.rs diff --git a/vectorless-core/vectorless-config/src/types/storage.rs b/vectorless-core/vectorless-config/src/storage.rs similarity index 100% rename from vectorless-core/vectorless-config/src/types/storage.rs rename to vectorless-core/vectorless-config/src/storage.rs diff --git a/vectorless-core/vectorless-config/src/types/mod.rs b/vectorless-core/vectorless-config/src/types/mod.rs deleted file mode 100644 index 717b137..0000000 --- a/vectorless-core/vectorless-config/src/types/mod.rs +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Configuration type definitions. - -mod indexer; -mod llm_pool; -mod metrics; -mod retrieval; -mod storage; - -use serde::{Deserialize, Serialize}; - -pub use indexer::IndexerConfig; -pub use llm_pool::{ - FallbackBehavior, FallbackConfig, LlmConfig, OnAllFailedBehavior, RetryConfig, SlotConfig, - ThrottleConfig, -}; -pub use metrics::{LlmMetricsConfig, MetricsConfig, RetrievalMetricsConfig}; -pub use retrieval::RetrievalConfig; -pub use storage::{CompressionAlgorithm, StorageConfig}; -pub use vectorless_graph::DocumentGraphConfig; - -/// Main configuration for vectorless. -/// -/// Users typically configure via [`EngineBuilder`](vectorless_engine::EngineBuilder): -/// -/// ```rust,no_run -/// use vectorless::client::EngineBuilder; -/// -/// # async fn example() -> Result<(), vectorless::BuildError> { -/// let engine = EngineBuilder::new() -/// .with_key("sk-...") -/// .with_model("gpt-4o") -/// .with_endpoint("https://api.openai.com/v1") -/// .build() -/// .await?; -/// # Ok(()) -/// # } -/// ``` -/// -/// Advanced users can construct this programmatically: -/// -/// ```rust,ignore -/// use vectorless::config::{Config, LlmConfig, SlotConfig}; -/// -/// let config = Config::new().with_llm( -/// LlmConfig::new("gpt-4o") -/// .with_api_key("sk-...") -/// .with_endpoint("https://api.openai.com/v1") -/// .with_index(SlotConfig::fast().with_model("gpt-4o-mini")) -/// ); -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Config { - /// LLM configuration (model, credentials, retry, throttle, fallback). - #[serde(default)] - pub llm: LlmConfig, - - /// Metrics configuration. - #[serde(default)] - pub metrics: MetricsConfig, - - /// Indexer configuration. - #[serde(default)] - pub indexer: IndexerConfig, - - /// Retrieval strategy configuration (search, content aggregation, etc.). - #[serde(default)] - pub retrieval: RetrievalConfig, - - /// Storage configuration. - #[serde(default)] - pub storage: StorageConfig, - - /// Document graph configuration. - #[serde(default)] - pub graph: DocumentGraphConfig, -} - -impl Default for Config { - fn default() -> Self { - Self { - llm: LlmConfig::default(), - metrics: MetricsConfig::default(), - indexer: IndexerConfig::default(), - retrieval: RetrievalConfig::default(), - storage: StorageConfig::default(), - graph: DocumentGraphConfig::default(), - } - } -} - -impl Config { - /// Create a new configuration with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the LLM configuration. - pub fn with_llm(mut self, llm: LlmConfig) -> Self { - self.llm = llm; - self - } - - /// Set the metrics configuration. - pub fn with_metrics(mut self, metrics: MetricsConfig) -> Self { - self.metrics = metrics; - self - } - - /// Set the indexer configuration. - pub fn with_indexer(mut self, indexer: IndexerConfig) -> Self { - self.indexer = indexer; - self - } - - /// Set the retrieval configuration. - pub fn with_retrieval(mut self, retrieval: RetrievalConfig) -> Self { - self.retrieval = retrieval; - self - } - - /// Set the storage configuration. - pub fn with_storage(mut self, storage: StorageConfig) -> Self { - self.storage = storage; - self - } - - /// Set the document graph configuration. - pub fn with_graph(mut self, graph: DocumentGraphConfig) -> Self { - self.graph = graph; - self - } - - /// Validate the configuration. - pub fn validate(&self) -> Result<(), ConfigValidationError> { - let mut errors = Vec::new(); - - // Validate indexer - if self.indexer.subsection_threshold == 0 { - errors.push(ValidationError::error( - "indexer.subsection_threshold", - "Subsection threshold must be greater than 0", - )); - } - - // Validate LLM slot tokens - if self.llm.index.max_tokens == 0 { - errors.push(ValidationError::error( - "llm.index.max_tokens", - "Index max tokens must be greater than 0", - )); - } - - if self.llm.retrieval.max_tokens == 0 { - errors.push(ValidationError::error( - "llm.retrieval.max_tokens", - "Retrieval max tokens must be greater than 0", - )); - } - - // Validate retrieval - if self.retrieval.top_k == 0 { - errors.push(ValidationError::error( - "retrieval.top_k", - "Top K must be greater than 0", - )); - } - - // Validate throttle - if self.llm.throttle.max_concurrent_requests == 0 { - errors.push(ValidationError::error( - "llm.throttle.max_concurrent_requests", - "Max concurrent requests must be greater than 0", - )); - } - - // Validate graph - if self.graph.min_keyword_jaccard < 0.0 || self.graph.min_keyword_jaccard > 1.0 { - errors.push(ValidationError::error( - "graph.min_keyword_jaccard", - "Must be between 0.0 and 1.0", - )); - } - if self.graph.max_edges_per_node == 0 { - errors.push(ValidationError::error( - "graph.max_edges_per_node", - "Must be greater than 0", - )); - } - - // Validate fallback - if self.llm.fallback.enabled && self.llm.fallback.models.is_empty() { - errors.push(ValidationError::warning( - "llm.fallback.models", - "Fallback enabled but no fallback models configured", - )); - } - - if errors.is_empty() { - Ok(()) - } else { - Err(ConfigValidationError { errors }) - } - } -} - -/// Configuration validation error. -#[derive(Debug, Clone)] -pub struct ConfigValidationError { - /// Validation errors. - pub errors: Vec, -} - -impl std::fmt::Display for ConfigValidationError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Configuration validation failed with {} error(s)", - self.errors.len() - ) - } -} - -impl std::error::Error for ConfigValidationError {} - -/// A single validation error. -#[derive(Debug, Clone)] -pub struct ValidationError { - /// Field path (e.g., "retrieval.content.token_budget"). - pub path: String, - - /// Error message. - pub message: String, - - /// Expected value/range. - pub expected: Option, - - /// Actual value. - pub actual: Option, - - /// Severity level. - pub severity: Severity, -} - -impl ValidationError { - /// Create an error-level validation error. - pub fn error(path: impl Into, message: impl Into) -> Self { - Self { - path: path.into(), - message: message.into(), - expected: None, - actual: None, - severity: Severity::Error, - } - } - - /// Create a warning-level validation error. - pub fn warning(path: impl Into, message: impl Into) -> Self { - Self { - path: path.into(), - message: message.into(), - expected: None, - actual: None, - severity: Severity::Warning, - } - } - - /// Create an info-level validation error. - pub fn info(path: impl Into, message: impl Into) -> Self { - Self { - path: path.into(), - message: message.into(), - expected: None, - actual: None, - severity: Severity::Info, - } - } - - /// Set the expected value. - pub fn with_expected(mut self, expected: impl Into) -> Self { - self.expected = Some(expected.into()); - self - } - - /// Set the actual value. - pub fn with_actual(mut self, actual: impl Into) -> Self { - self.actual = Some(actual.into()); - self - } -} - -impl std::fmt::Display for ValidationError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let severity = match self.severity { - Severity::Error => "ERROR", - Severity::Warning => "WARNING", - Severity::Info => "INFO", - }; - write!(f, "[{}] {}: {}", severity, self.path, self.message)?; - if let Some(ref expected) = self.expected { - write!(f, " (expected: {})", expected)?; - } - if let Some(ref actual) = self.actual { - write!(f, " (actual: {})", actual)?; - } - Ok(()) - } -} - -/// Validation severity level. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Severity { - /// Error - must fix. - Error, - /// Warning - should fix. - Warning, - /// Info - suggestion. - Info, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_config_defaults() { - let config = Config::default(); - assert!(config.llm.model.is_empty()); - assert!(config.llm.index.model.is_none()); - assert_eq!(config.retrieval.top_k, 3); - assert_eq!(config.indexer.subsection_threshold, 300); - assert!(config.metrics.enabled); - } - - #[test] - fn test_llm_config_defaults() { - let config = LlmConfig::default(); - assert!(config.index.model.is_none()); - assert!(config.retrieval.model.is_none()); - assert_eq!(config.retry.max_attempts, 3); - assert_eq!(config.throttle.max_concurrent_requests, 10); - } - - #[test] - fn test_config_validation_success() { - let config = Config::default(); - assert!(config.validate().is_ok()); - } - - #[test] - fn test_config_validation_errors() { - let mut config = Config::default(); - config.retrieval.top_k = 0; - - let result = config.validate(); - assert!(result.is_err()); - - let err = result.unwrap_err(); - assert!(!err.errors.is_empty()); - } - - #[test] - fn test_validation_error_display() { - let err = ValidationError::error("test.field", "Invalid value") - .with_expected(">= 1") - .with_actual("0"); - - let display = format!("{}", err); - assert!(display.contains("ERROR")); - assert!(display.contains("test.field")); - assert!(display.contains("expected")); - } -} diff --git a/vectorless-core/vectorless-config/src/types/retrieval.rs b/vectorless-core/vectorless-config/src/types/retrieval.rs deleted file mode 100644 index c430098..0000000 --- a/vectorless-core/vectorless-config/src/types/retrieval.rs +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Retrieval strategy configuration types. -//! -//! LLM configuration (model, api_key, endpoint) is managed centrally -//! in [`LlmConfig`](super::LlmConfig). This module only contains -//! retrieval strategy parameters. - -use serde::{Deserialize, Serialize}; - -use super::storage::{CacheConfig, StrategyConfig, SufficiencyConfig}; - -/// Retrieval strategy configuration. -/// -/// Controls how documents are searched and retrieved, independent -/// of which LLM model is used for navigation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RetrievalConfig { - /// Number of top-k results to return. - #[serde(default = "default_top_k")] - pub top_k: usize, - - /// Search algorithm configuration. - #[serde(default)] - pub search: SearchConfig, - - /// Sufficiency checker configuration. - #[serde(default)] - pub sufficiency: SufficiencyConfig, - - /// Cache configuration. - #[serde(default)] - pub cache: CacheConfig, - - /// Strategy-specific configuration. - #[serde(default)] - pub strategy: StrategyConfig, -} - -fn default_top_k() -> usize { - 3 -} - -impl Default for RetrievalConfig { - fn default() -> Self { - Self { - top_k: default_top_k(), - search: SearchConfig::default(), - sufficiency: SufficiencyConfig::default(), - cache: CacheConfig::default(), - strategy: StrategyConfig::default(), - } - } -} - -impl RetrievalConfig { - /// Create a new retrieval config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the top_k. - pub fn with_top_k(mut self, top_k: usize) -> Self { - self.top_k = top_k; - self - } -} - -/// Search algorithm configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SearchConfig { - /// Number of top-k results to return. - #[serde(default = "default_search_top_k")] - pub top_k: usize, - - /// Beam width for multi-path search. - #[serde(default = "default_beam_width")] - pub beam_width: usize, - - /// Maximum iterations for search algorithms. - #[serde(default = "default_max_iterations")] - pub max_iterations: usize, - - /// Minimum score to include a path. - #[serde(default = "default_min_score")] - pub min_score: f32, - - /// Fallback chain: algorithms tried in order until min_score is met. - /// Options: "beam", "mcts", "pure_pilot". - /// Default: ["beam", "mcts", "pure_pilot"] - #[serde(default = "default_fallback_chain")] - pub fallback_chain: Vec, -} - -fn default_search_top_k() -> usize { - 5 -} - -fn default_beam_width() -> usize { - 3 -} - -fn default_max_iterations() -> usize { - 10 -} - -fn default_min_score() -> f32 { - 0.1 -} -fn default_fallback_chain() -> Vec { - vec!["beam".into(), "mcts".into(), "pure_pilot".into()] -} - -impl Default for SearchConfig { - fn default() -> Self { - Self { - top_k: default_search_top_k(), - beam_width: default_beam_width(), - max_iterations: default_max_iterations(), - min_score: default_min_score(), - fallback_chain: default_fallback_chain(), - } - } -} - -impl SearchConfig { - /// Create new search config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the top_k. - pub fn with_top_k(mut self, top_k: usize) -> Self { - self.top_k = top_k; - self - } - - /// Set the beam width. - pub fn with_beam_width(mut self, width: usize) -> Self { - self.beam_width = width; - self - } - - /// Set the max iterations. - pub fn with_max_iterations(mut self, max: usize) -> Self { - self.max_iterations = max; - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_retrieval_config_defaults() { - let config = RetrievalConfig::default(); - assert_eq!(config.top_k, 3); - assert_eq!(config.search.top_k, 5); - } - - #[test] - fn test_search_config_defaults() { - let config = SearchConfig::default(); - assert_eq!(config.top_k, 5); - assert_eq!(config.beam_width, 3); - assert_eq!(config.max_iterations, 10); - } -} diff --git a/vectorless-core/vectorless-py/src/config.rs b/vectorless-core/vectorless-py/src/config.rs index 21b67af..a145caf 100644 --- a/vectorless-core/vectorless-py/src/config.rs +++ b/vectorless-core/vectorless-py/src/config.rs @@ -7,7 +7,7 @@ use pyo3::prelude::*; /// Advanced configuration for Engine internals. /// -/// Create a Config to customize storage, retrieval, concurrency, +/// Create a Config to customize storage, concurrency, /// and other engine parameters beyond the basic builder API. /// /// Example: @@ -17,7 +17,6 @@ use pyo3::prelude::*; /// /// config = Config() /// config.set_workspace_dir("/data/vectorless") -/// config.set_top_k(10) /// config.set_max_concurrent_requests(20) /// /// engine = Engine(api_key="sk-...", model="gpt-4o", config=config) @@ -44,13 +43,6 @@ impl PyConfig { self.inner.storage.workspace_dir = std::path::PathBuf::from(dir); } - /// Set the number of top-k results to return from queries. - /// - /// Default: 3 - fn set_top_k(&mut self, k: usize) { - self.inner.retrieval.top_k = k; - } - /// Set the maximum concurrent LLM API calls. /// /// Default: 10 @@ -65,11 +57,6 @@ impl PyConfig { self.inner.llm.throttle.requests_per_minute = rpm; } - /// Set the maximum iterations for retrieval search. - fn set_max_iterations(&mut self, max: usize) { - self.inner.retrieval.search.max_iterations = max; - } - /// Enable or disable metrics collection. /// /// Default: True diff --git a/vectorless/ask/blackboard.py b/vectorless/ask/blackboard.py index 418b351..df86ead 100644 --- a/vectorless/ask/blackboard.py +++ b/vectorless/ask/blackboard.py @@ -105,19 +105,50 @@ def format_for_all(self) -> str: def extract_discoveries(worker_output, doc_name: str) -> list[Discovery]: """Extract discoveries from a WorkerOutput for the blackboard. - Converts Worker evidence into Discovery objects based on - evidence content and source paths. + Converts Worker evidence into Discovery objects: + - "evidence": direct evidence findings + - "cross_ref": evidence mentioning other documents by name + - "dead_end": nodes visited but no evidence collected (trace steps with empty results) """ discoveries: list[Discovery] = [] + evidence_docs_referenced: set[str] = set() for evidence in worker_output.evidence: - # Every evidence item is a potential cross-reference + # Check if evidence content references other documents + referenced_docs: list[str] = [] + if evidence.content: + # Simple heuristic: look for document-like references + # (file.md, file.txt, "document X", etc.) + import re + doc_refs = re.findall( + r'(?:see|refer to|in|from|document(?:ed)? in)\s+["\']?([\w\-\.]+\.(?:md|txt|pdf|doc))["\']?', + evidence.content, + re.IGNORECASE, + ) + if doc_refs: + referenced_docs = doc_refs + evidence_docs_referenced.update(doc_refs) + + finding_type = "cross_ref" if referenced_docs else "evidence" discoveries.append(Discovery( worker_id=doc_name, doc_name=doc_name, node_title=evidence.node_title, - finding_type="evidence", + finding_type=finding_type, summary=f"Found: {evidence.node_title} ({len(evidence.content)} chars)", + relevance_to=referenced_docs, + )) + + # Generate "lead" discoveries from cross-references found in evidence + if evidence_docs_referenced: + lead_docs = sorted(evidence_docs_referenced) + discoveries.append(Discovery( + worker_id=doc_name, + doc_name=doc_name, + node_title="cross_document_leads", + finding_type="lead", + summary=f"Evidence references other documents: {', '.join(lead_docs[:5])}", + relevance_to=lead_docs, )) return discoveries diff --git a/vectorless/ask/plan.py b/vectorless/ask/plan.py index c6830e2..07734d8 100644 --- a/vectorless/ask/plan.py +++ b/vectorless/ask/plan.py @@ -1,30 +1,20 @@ -"""Query plan types — mirrors vectorless-query types.""" +"""Query plan types — kept for backward compatibility. + +New code should use vectorless.ask.reasoning.types directly. +""" from __future__ import annotations from dataclasses import dataclass, field -from enum import Enum from typing import TYPE_CHECKING +# Re-export canonical enums from reasoning.types +from vectorless.ask.reasoning.types import QueryIntent, Complexity + if TYPE_CHECKING: from vectorless.ask.reasoning.types import QueryAnalysis -class QueryIntent(str, Enum): - """Detected intent of a user query.""" - FACTUAL = "factual" - ANALYTICAL = "analytical" - NAVIGATIONAL = "navigational" - SUMMARY = "summary" - - -class Complexity(str, Enum): - """Estimated query complexity.""" - SIMPLE = "simple" - MODERATE = "moderate" - COMPLEX = "complex" - - @dataclass class SubQuery: """A decomposed sub-query from a complex query.""" diff --git a/vectorless/ask/reasoning/analyzer.py b/vectorless/ask/reasoning/analyzer.py index 91dfa4e..0d1cb41 100644 --- a/vectorless/ask/reasoning/analyzer.py +++ b/vectorless/ask/reasoning/analyzer.py @@ -13,6 +13,7 @@ import re from vectorless.llm_client import LLMClient +from vectorless.ask.utils import parse_json_response from vectorless.ask.reasoning.types import ( Ambiguity, AmbiguityType, @@ -51,36 +52,6 @@ def _extract_keywords(query: str) -> list[str]: return list(dict.fromkeys(w for w in words if w not in stop_words and len(w) > 2)) -def _parse_json_response(response: str) -> dict: - """Parse LLM response as JSON, handling markdown-wrapped output.""" - trimmed = response.strip() - - # Try to extract JSON from markdown code blocks - if trimmed.startswith("```"): - match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) - if match: - trimmed = match.group(1).strip() - - # Try to find a { ... } block - start = trimmed.find("{") - if start != -1: - depth = 0 - for i in range(start, len(trimmed)): - if trimmed[i] == "{": - depth += 1 - elif trimmed[i] == "}": - depth -= 1 - if depth == 0: - candidate = trimmed[start : i + 1] - try: - return json.loads(candidate) - except json.JSONDecodeError: - break - - # Last resort - return json.loads(trimmed) - - def _parse_intent(raw: str) -> QueryIntent: """Parse intent string to QueryIntent enum.""" mapping = { @@ -135,7 +106,7 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: "Check your API key, model, and endpoint configuration." ) - stage1 = _parse_json_response(response) + stage1 = parse_json_response(response) intent = _parse_intent(stage1.get("intent", "factual")) complexity = _parse_complexity(stage1.get("complexity", "simple")) key_concepts = stage1.get("key_concepts", []) @@ -158,6 +129,8 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: ) # Deep mode: Stage 2 + Stage 3 + analysis_complete = True + stage1_summary = { "intent": intent.value, "complexity": complexity.value, @@ -174,7 +147,7 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: try: system2, user2 = stage2_deep_analysis_prompt(query, stage1_summary) response2 = await llm.complete(system2, user2) - stage2 = _parse_json_response(response2) + stage2 = parse_json_response(response2) entities = _parse_entities(stage2.get("entities", [])) ambiguities = _parse_ambiguities(stage2.get("ambiguities", [])) temporal_constraints = _parse_temporal(stage2.get("temporal_constraints", [])) @@ -183,6 +156,7 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: key_concepts = stage2["key_concepts"] except Exception as e: logger.warning("Stage 2 (deep analysis) failed: %s — continuing with partial results", e) + analysis_complete = False # Stage 3: Strategy Formation strategy = RetrievalStrategy( @@ -198,10 +172,11 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: try: system3, user3 = stage3_strategy_prompt(query, stage1_summary, stage2_summary) response3 = await llm.complete(system3, user3) - stage3 = _parse_json_response(response3) + stage3 = parse_json_response(response3) strategy = _parse_strategy(stage3) except Exception as e: logger.warning("Stage 3 (strategy formation) failed: %s — using default strategy", e) + analysis_complete = False return QueryAnalysis( original=query, @@ -215,6 +190,7 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: temporal_constraints=temporal_constraints, sub_queries=sub_queries, strategy=strategy, + analysis_complete=analysis_complete, ) async def re_analyze( @@ -243,13 +219,15 @@ async def re_analyze( evidence_summary=evidence_summary, ) + strategy_ok = True try: response = await llm.complete(system, user) - stage3 = _parse_json_response(response) + stage3 = parse_json_response(response) new_strategy = _parse_strategy(stage3) except Exception as e: logger.warning("Re-analyze strategy update failed: %s — keeping current strategy", e) new_strategy = analysis.strategy + strategy_ok = False return QueryAnalysis( original=analysis.original, @@ -266,6 +244,7 @@ async def re_analyze( iteration=analysis.iteration + 1, additional_context="; ".join(gaps), previous_evidence_summary=evidence_summary, + analysis_complete=analysis.analysis_complete and strategy_ok, ) diff --git a/vectorless/ask/reasoning/types.py b/vectorless/ask/reasoning/types.py index 8872115..f5adc1a 100644 --- a/vectorless/ask/reasoning/types.py +++ b/vectorless/ask/reasoning/types.py @@ -101,6 +101,7 @@ class QueryAnalysis: iteration: int = 0 additional_context: str = "" previous_evidence_summary: str = "" + analysis_complete: bool = True # False if deep analysis stages failed def intent_context(self) -> str: """Format intent context string for prompts. diff --git a/vectorless/ask/utils.py b/vectorless/ask/utils.py new file mode 100644 index 0000000..f057cc9 --- /dev/null +++ b/vectorless/ask/utils.py @@ -0,0 +1,33 @@ +"""Shared utilities for the ask pipeline.""" + +from __future__ import annotations + +import json +import re + + +def parse_json_response(response: str) -> dict: + """Parse LLM response as JSON, handling markdown-wrapped output.""" + trimmed = response.strip() + + if trimmed.startswith("```"): + match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) + if match: + trimmed = match.group(1).strip() + + start = trimmed.find("{") + if start != -1: + depth = 0 + for i in range(start, len(trimmed)): + if trimmed[i] == "{": + depth += 1 + elif trimmed[i] == "}": + depth -= 1 + if depth == 0: + candidate = trimmed[start : i + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError: + break + + return json.loads(trimmed) diff --git a/vectorless/ask/verify/prompts.py b/vectorless/ask/verify/prompts.py index fd992e7..3b3492d 100644 --- a/vectorless/ask/verify/prompts.py +++ b/vectorless/ask/verify/prompts.py @@ -22,22 +22,22 @@ def verify_prompt( ' "factual_accuracy": {\n' ' "score": 0.0-1.0,\n' ' "reasoning": "...",\n' - ' "evidence_refs": ["node_title_1", "node_title_2"]\n' + ' "evidence_refs": ["doc_name/node_title", "doc_name/node_title"]\n' " },\n" ' "completeness": {\n' ' "score": 0.0-1.0,\n' ' "reasoning": "...",\n' - ' "evidence_refs": ["node_title_1"]\n' + ' "evidence_refs": ["doc_name/node_title"]\n' " },\n" ' "relevance": {\n' ' "score": 0.0-1.0,\n' ' "reasoning": "...",\n' - ' "evidence_refs": ["node_title_1"]\n' + ' "evidence_refs": ["doc_name/node_title"]\n' " },\n" ' "coherence": {\n' ' "score": 0.0-1.0,\n' ' "reasoning": "...",\n' - ' "evidence_refs": ["node_title_1"]\n' + ' "evidence_refs": ["doc_name/node_title"]\n' " }\n" " },\n" ' "passed": true/false,\n' diff --git a/vectorless/ask/verify/types.py b/vectorless/ask/verify/types.py index 0cce9d0..0f11d83 100644 --- a/vectorless/ask/verify/types.py +++ b/vectorless/ask/verify/types.py @@ -20,7 +20,7 @@ class DimensionScore: dimension: VerificationDimension score: float # 0.0 - 1.0 reasoning: str # LLM explanation - evidence_refs: list[str] = field(default_factory=list) # node_titles + evidence_refs: list[str] = field(default_factory=list) # "doc_name/node_title" or "node_title" @dataclass diff --git a/vectorless/ask/verify/verifier.py b/vectorless/ask/verify/verifier.py index 944b21c..a37d0f4 100644 --- a/vectorless/ask/verify/verifier.py +++ b/vectorless/ask/verify/verifier.py @@ -8,10 +8,10 @@ import json import logging -import re from vectorless.llm_client import LLMClient from vectorless.ask.types import Evidence +from vectorless.ask.utils import parse_json_response from vectorless.ask.verify.types import ( DimensionScore, VerificationDimension, @@ -75,33 +75,6 @@ def _format_evidence(evidence: list[Evidence]) -> str: ) -def _parse_json_response(response: str) -> dict: - """Parse LLM response as JSON, handling markdown-wrapped output.""" - trimmed = response.strip() - - if trimmed.startswith("```"): - match = re.search(r"```(?:json)?\s*\n?(.*?)```", trimmed, re.DOTALL) - if match: - trimmed = match.group(1).strip() - - start = trimmed.find("{") - if start != -1: - depth = 0 - for i in range(start, len(trimmed)): - if trimmed[i] == "{": - depth += 1 - elif trimmed[i] == "}": - depth -= 1 - if depth == 0: - candidate = trimmed[start : i + 1] - try: - return json.loads(candidate) - except json.JSONDecodeError: - break - - return json.loads(trimmed) - - class VerifyPipeline: """Multi-dimensional evidence verification pipeline. @@ -160,7 +133,7 @@ async def verify( ) try: - data = _parse_json_response(response) + data = parse_json_response(response) except (json.JSONDecodeError, ValueError) as e: logger.warning("Verification response parse failed: %s", e) return VerificationResult( @@ -209,7 +182,10 @@ def _build_result( gaps.append(f"[{dim.value}] {reasoning}") passed = len(low_dimensions) == 0 - overall_confidence = max(0.0, min(1.0, float(data.get("overall_confidence", 0.5)))) + # Compute confidence from dimension scores rather than LLM self-assessment + overall_confidence = ( + sum(s.score for s in scores) / len(scores) if scores else 0.0 + ) # Synthesize re_retrieval_hints from gaps or explicit hints raw_hints = data.get("re_retrieval_hints", []) diff --git a/vectorless/config/__init__.py b/vectorless/config/__init__.py index 7a925a6..4eca0f2 100644 --- a/vectorless/config/__init__.py +++ b/vectorless/config/__init__.py @@ -5,7 +5,6 @@ EngineConfig, LlmConfig, MetricsConfig, - RetrievalConfig, RetryConfig, StorageConfig, ThrottleConfig, @@ -15,7 +14,6 @@ "EngineConfig", "LlmConfig", "MetricsConfig", - "RetrievalConfig", "RetryConfig", "StorageConfig", "ThrottleConfig", diff --git a/vectorless/config/loading.py b/vectorless/config/loading.py index 445ffbe..6ededbd 100644 --- a/vectorless/config/loading.py +++ b/vectorless/config/loading.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Optional -from vectorless.config.models import EngineConfig, LlmConfig, RetrievalConfig, StorageConfig +from vectorless.config.models import EngineConfig, LlmConfig, StorageConfig if sys.version_info >= (3, 11): import tomllib @@ -27,13 +27,10 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: VECTORLESS_MODEL -> llm.model VECTORLESS_ENDPOINT -> llm.endpoint VECTORLESS_WORKSPACE_DIR -> storage.workspace_dir - VECTORLESS_TOP_K -> retrieval.top_k - VECTORLESS_MAX_ITERATIONS -> retrieval.max_iterations VECTORLESS_METRICS_ENABLED -> metrics.enabled """ llm = LlmConfig() storage = StorageConfig() - retrieval = RetrievalConfig() metrics_enabled: Optional[bool] = None env_map = { @@ -41,8 +38,6 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: f"{prefix}MODEL": ("llm.model", str), f"{prefix}ENDPOINT": ("llm.endpoint", str), f"{prefix}WORKSPACE_DIR": ("storage.workspace_dir", str), - f"{prefix}TOP_K": ("retrieval.top_k", int), - f"{prefix}MAX_ITERATIONS": ("retrieval.max_iterations", int), f"{prefix}METRICS_ENABLED": ("metrics.enabled", bool), } @@ -57,7 +52,7 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: kwargs[path] = type_fn(value) # Apply to sub-models - if f"llm.api_key" in kwargs: + if "llm.api_key" in kwargs: llm = LlmConfig( api_key=kwargs["llm.api_key"], model=kwargs.get("llm.model", llm.model), @@ -72,12 +67,6 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: if "storage.workspace_dir" in kwargs: storage = StorageConfig(workspace_dir=kwargs["storage.workspace_dir"]) - if "retrieval.top_k" in kwargs or "retrieval.max_iterations" in kwargs: - retrieval = RetrievalConfig( - top_k=kwargs.get("retrieval.top_k", retrieval.top_k), - max_iterations=kwargs.get("retrieval.max_iterations", retrieval.max_iterations), - ) - if "metrics.enabled" in kwargs: from vectorless.config.models import MetricsConfig @@ -87,7 +76,7 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: metrics = MetricsConfig() - return EngineConfig(llm=llm, storage=storage, retrieval=retrieval, metrics=metrics) + return EngineConfig(llm=llm, storage=storage, metrics=metrics) def load_config_from_file(path: Path) -> EngineConfig: @@ -104,10 +93,6 @@ def load_config_from_file(path: Path) -> EngineConfig: max_concurrent_requests = 10 requests_per_minute = 500 - [retrieval] - top_k = 5 - max_iterations = 10 - [storage] workspace_dir = "~/.vectorless" @@ -152,16 +137,12 @@ def load_config( # Layer 2: environment variables env_config = load_config_from_env(prefix=env_prefix) - # Merge: only override if env var was actually set - env_defaults = load_config_from_env.__wrapped__ if hasattr(load_config_from_env, "__wrapped__") else None # noqa: E501 base = EngineConfig() env_data: dict[str, Any] = {} if env_config.llm.api_key != base.llm.api_key or env_config.llm.model != base.llm.model: env_data["llm"] = env_config.llm.model_dump() if env_config.storage.workspace_dir != base.storage.workspace_dir: env_data["storage"] = env_config.storage.model_dump() - if env_config.retrieval != base.retrieval: - env_data["retrieval"] = env_config.retrieval.model_dump() config_data.update(env_data) diff --git a/vectorless/config/models.py b/vectorless/config/models.py index eafc2a2..52d5894 100644 --- a/vectorless/config/models.py +++ b/vectorless/config/models.py @@ -40,13 +40,6 @@ class MetricsConfig(BaseModel): enabled: bool = True -class RetrievalConfig(BaseModel): - """Retrieval behavior configuration.""" - - top_k: int = Field(default=3, ge=1) - max_iterations: int = Field(default=10, ge=1) - - class StorageConfig(BaseModel): """Storage and workspace configuration.""" @@ -62,7 +55,6 @@ class EngineConfig(BaseModel): config = EngineConfig( llm=LlmConfig(model="gpt-4o", api_key="sk-..."), - retrieval=RetrievalConfig(top_k=5), ) # Convert to Rust Config for Engine construction @@ -71,7 +63,6 @@ class EngineConfig(BaseModel): llm: LlmConfig = LlmConfig() metrics: MetricsConfig = MetricsConfig() - retrieval: RetrievalConfig = RetrievalConfig() storage: StorageConfig = StorageConfig() def to_rust_config(self) -> RustConfig: @@ -81,8 +72,6 @@ def to_rust_config(self) -> RustConfig: """ cfg = RustConfig() cfg.set_workspace_dir(self.storage.workspace_dir) - cfg.set_top_k(self.retrieval.top_k) - cfg.set_max_iterations(self.retrieval.max_iterations) cfg.set_max_concurrent_requests(self.llm.throttle.max_concurrent_requests) cfg.set_requests_per_minute(self.llm.throttle.requests_per_minute) cfg.set_metrics_enabled(self.metrics.enabled) diff --git a/vectorless/rerank/quality.py b/vectorless/rerank/quality.py index 3b8247a..2aea94b 100644 --- a/vectorless/rerank/quality.py +++ b/vectorless/rerank/quality.py @@ -12,7 +12,7 @@ from vectorless.ask.types import Evidence from vectorless.llm_client import LLMClient -from vectorless.ask.plan import QueryIntent +from vectorless.ask.reasoning.types import QueryIntent logger = logging.getLogger(__name__) diff --git a/vectorless/rerank/synthesize.py b/vectorless/rerank/synthesize.py index 72279ae..a1b18fe 100644 --- a/vectorless/rerank/synthesize.py +++ b/vectorless/rerank/synthesize.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from vectorless.ask.types import Evidence -from vectorless.ask.plan import QueryIntent +from vectorless.ask.reasoning.types import QueryIntent # --------------------------------------------------------------------------- From 0ee2f3a2846e8ae99d7806a986af307df1c5a908 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 16:57:16 +0800 Subject: [PATCH 05/30] refactor(config): remove sufficiency and strategy configs from storage module - Remove SufficiencyConfig struct and related default functions - Remove CacheConfig struct and related default functions - Remove StrategyConfig and all related strategy configuration structs - Update module documentation to reflect removal of sufficiency types - Clean up tests by removing sufficiency and strategy config tests - Keep only storage-related configuration types in the module --- .../vectorless-config/src/storage.rs | 451 +----------------- 1 file changed, 3 insertions(+), 448 deletions(-) diff --git a/vectorless-core/vectorless-config/src/storage.rs b/vectorless-core/vectorless-config/src/storage.rs index b13304e..f66684d 100644 --- a/vectorless-core/vectorless-config/src/storage.rs +++ b/vectorless-core/vectorless-config/src/storage.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Storage and sufficiency configuration types. +//! Storage configuration types. use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -234,395 +234,6 @@ pub enum CompressionAlgorithm { Zstd, } -/// Sufficiency checker configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SufficiencyConfig { - /// Minimum tokens for sufficiency. - #[serde(default = "default_min_tokens")] - pub min_tokens: usize, - - /// Target tokens for full sufficiency. - #[serde(default = "default_target_tokens")] - pub target_tokens: usize, - - /// Maximum tokens before stopping. - #[serde(default = "default_max_tokens")] - pub max_tokens: usize, - - /// Minimum content length (characters). - #[serde(default = "default_min_content_length")] - pub min_content_length: usize, - - /// Confidence threshold for LLM judge. - #[serde(default = "default_confidence_threshold")] - pub confidence_threshold: f32, -} - -fn default_min_tokens() -> usize { - 500 -} - -fn default_target_tokens() -> usize { - 2000 -} - -fn default_max_tokens() -> usize { - 4000 -} - -fn default_min_content_length() -> usize { - 200 -} - -fn default_confidence_threshold() -> f32 { - 0.7 -} - -impl Default for SufficiencyConfig { - fn default() -> Self { - Self { - min_tokens: default_min_tokens(), - target_tokens: default_target_tokens(), - max_tokens: default_max_tokens(), - min_content_length: default_min_content_length(), - confidence_threshold: default_confidence_threshold(), - } - } -} - -impl SufficiencyConfig { - /// Create new sufficiency config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the minimum tokens. - pub fn with_min_tokens(mut self, tokens: usize) -> Self { - self.min_tokens = tokens; - self - } - - /// Set the target tokens. - pub fn with_target_tokens(mut self, tokens: usize) -> Self { - self.target_tokens = tokens; - self - } - - /// Set the maximum tokens. - pub fn with_max_tokens(mut self, tokens: usize) -> Self { - self.max_tokens = tokens; - self - } - - /// Set the confidence threshold. - pub fn with_confidence_threshold(mut self, threshold: f32) -> Self { - self.confidence_threshold = threshold; - self - } -} - -/// Cache configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CacheConfig { - /// Maximum number of cache entries. - #[serde(default = "default_max_entries")] - pub max_entries: usize, - - /// Time-to-live for cache entries (seconds). - #[serde(default = "default_ttl_secs")] - pub ttl_secs: u64, -} - -fn default_max_entries() -> usize { - 1000 -} - -fn default_ttl_secs() -> u64 { - 3600 -} - -impl Default for CacheConfig { - fn default() -> Self { - Self { - max_entries: default_max_entries(), - ttl_secs: default_ttl_secs(), - } - } -} - -impl CacheConfig { - /// Create new cache config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the maximum entries. - pub fn with_max_entries(mut self, max: usize) -> Self { - self.max_entries = max; - self - } - - /// Set the TTL in seconds. - pub fn with_ttl_secs(mut self, secs: u64) -> Self { - self.ttl_secs = secs; - self - } -} - -/// Strategy-specific configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StrategyConfig { - /// MCTS exploration weight (sqrt(2) ≈ 1.414). - #[serde(default = "default_exploration_weight")] - pub exploration_weight: f32, - - /// Semantic similarity threshold. - #[serde(default = "default_similarity_threshold")] - pub similarity_threshold: f32, - - /// High similarity threshold for "answer" decision. - #[serde(default = "default_high_similarity_threshold")] - pub high_similarity_threshold: f32, - - /// Low similarity threshold for "explore" decision. - #[serde(default = "default_low_similarity_threshold")] - pub low_similarity_threshold: f32, - - /// Hybrid strategy configuration (BM25 + LLM refinement). - #[serde(default)] - pub hybrid: HybridStrategyConfig, - - /// Cross-document strategy configuration. - #[serde(default)] - pub cross_document: CrossDocumentStrategyConfig, - - /// Page-range strategy configuration. - #[serde(default)] - pub page_range: PageRangeStrategyConfig, -} - -/// Hybrid strategy configuration (BM25 pre-filter + LLM refinement). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HybridStrategyConfig { - /// Enable hybrid strategy. - #[serde(default = "default_true")] - pub enabled: bool, - - /// BM25 pre-filter: keep top N% of candidates. - #[serde(default = "default_pre_filter_ratio")] - pub pre_filter_ratio: f32, - - /// Minimum candidates to pass to LLM. - #[serde(default = "default_min_candidates")] - pub min_candidates: usize, - - /// Maximum candidates for LLM refinement. - #[serde(default = "default_max_candidates")] - pub max_candidates: usize, - - /// BM25 score for auto-accept (skip LLM). - #[serde(default = "default_auto_accept_threshold")] - pub auto_accept_threshold: f32, - - /// BM25 score for auto-reject (skip LLM). - #[serde(default = "default_auto_reject_threshold")] - pub auto_reject_threshold: f32, - - /// Weight for BM25 score in final scoring. - #[serde(default = "default_bm25_weight")] - pub bm25_weight: f32, - - /// Weight for LLM score in final scoring. - #[serde(default = "default_llm_weight")] - pub llm_weight: f32, -} - -fn default_true() -> bool { - true -} -fn default_pre_filter_ratio() -> f32 { - 0.3 -} -fn default_min_candidates() -> usize { - 2 -} -fn default_max_candidates() -> usize { - 5 -} -fn default_auto_accept_threshold() -> f32 { - 0.85 -} -fn default_auto_reject_threshold() -> f32 { - 0.15 -} -fn default_bm25_weight() -> f32 { - 0.4 -} -fn default_llm_weight() -> f32 { - 0.6 -} - -impl Default for HybridStrategyConfig { - fn default() -> Self { - Self { - enabled: true, - pre_filter_ratio: default_pre_filter_ratio(), - min_candidates: default_min_candidates(), - max_candidates: default_max_candidates(), - auto_accept_threshold: default_auto_accept_threshold(), - auto_reject_threshold: default_auto_reject_threshold(), - bm25_weight: default_bm25_weight(), - llm_weight: default_llm_weight(), - } - } -} - -/// Cross-document strategy configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CrossDocumentStrategyConfig { - /// Enable cross-document strategy. - #[serde(default = "default_true")] - pub enabled: bool, - - /// Maximum documents to search. - #[serde(default = "default_max_documents")] - pub max_documents: usize, - - /// Maximum results per document. - #[serde(default = "default_max_results_per_doc")] - pub max_results_per_doc: usize, - - /// Maximum total results. - #[serde(default = "default_max_total_results")] - pub max_total_results: usize, - - /// Minimum score threshold. - #[serde(default = "default_min_score")] - pub min_score: f32, - - /// Merge strategy: TopK, BestPerDocument, WeightedByRelevance. - #[serde(default = "default_merge_strategy")] - pub merge_strategy: String, - - /// Search documents in parallel. - #[serde(default = "default_true")] - pub parallel_search: bool, -} - -fn default_max_documents() -> usize { - 10 -} -fn default_max_results_per_doc() -> usize { - 3 -} -fn default_max_total_results() -> usize { - 10 -} -fn default_min_score() -> f32 { - 0.3 -} -fn default_merge_strategy() -> String { - "TopK".to_string() -} - -impl Default for CrossDocumentStrategyConfig { - fn default() -> Self { - Self { - enabled: true, - max_documents: default_max_documents(), - max_results_per_doc: default_max_results_per_doc(), - max_total_results: default_max_total_results(), - min_score: default_min_score(), - merge_strategy: default_merge_strategy(), - parallel_search: true, - } - } -} - -/// Page-range strategy configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PageRangeStrategyConfig { - /// Enable page-range strategy. - #[serde(default = "default_true")] - pub enabled: bool, - - /// Include nodes that span across the boundary. - #[serde(default = "default_true")] - pub include_boundary_nodes: bool, - - /// Expand range by N pages for context. - #[serde(default)] - pub expand_context_pages: usize, - - /// Minimum overlap ratio for node inclusion. - #[serde(default = "default_min_overlap_ratio")] - pub min_overlap_ratio: f32, -} - -fn default_min_overlap_ratio() -> f32 { - 0.1 -} - -impl Default for PageRangeStrategyConfig { - fn default() -> Self { - Self { - enabled: true, - include_boundary_nodes: true, - expand_context_pages: 0, - min_overlap_ratio: default_min_overlap_ratio(), - } - } -} - -fn default_exploration_weight() -> f32 { - 1.414 -} - -fn default_similarity_threshold() -> f32 { - 0.5 -} - -fn default_high_similarity_threshold() -> f32 { - 0.8 -} - -fn default_low_similarity_threshold() -> f32 { - 0.3 -} - -impl Default for StrategyConfig { - fn default() -> Self { - Self { - exploration_weight: default_exploration_weight(), - similarity_threshold: default_similarity_threshold(), - high_similarity_threshold: default_high_similarity_threshold(), - low_similarity_threshold: default_low_similarity_threshold(), - hybrid: HybridStrategyConfig::default(), - cross_document: CrossDocumentStrategyConfig::default(), - page_range: PageRangeStrategyConfig::default(), - } - } -} - -impl StrategyConfig { - /// Create new strategy config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the exploration weight. - pub fn with_exploration_weight(mut self, weight: f32) -> Self { - self.exploration_weight = weight; - self - } - - /// Set the similarity threshold. - pub fn with_similarity_threshold(mut self, threshold: f32) -> Self { - self.similarity_threshold = threshold; - self - } -} - #[cfg(test)] mod tests { use super::*; @@ -630,8 +241,6 @@ mod tests { #[test] fn test_storage_config_defaults() { let config = StorageConfig::default(); - // Default workspace should be under .vectorless/workspaces/ (Unix) - // or vectorless/workspaces/ (Windows via AppData) let path_str = config.workspace_dir.to_string_lossy(); if cfg!(windows) { assert!( @@ -680,63 +289,9 @@ mod tests { #[test] fn test_compression_config_level_clamp() { let config = CompressionConfig::new().with_level(15); - assert_eq!(config.level, 9); // clamped to max + assert_eq!(config.level, 9); let config = CompressionConfig::new().with_level(0); - assert_eq!(config.level, 1); // clamped to min - } - - #[test] - fn test_sufficiency_config_defaults() { - let config = SufficiencyConfig::default(); - assert_eq!(config.min_tokens, 500); - assert_eq!(config.target_tokens, 2000); - assert_eq!(config.max_tokens, 4000); - } - - #[test] - fn test_cache_config_defaults() { - let config = CacheConfig::default(); - assert_eq!(config.max_entries, 1000); - assert_eq!(config.ttl_secs, 3600); - } - - #[test] - fn test_strategy_config_defaults() { - let config = StrategyConfig::default(); - assert!((config.exploration_weight - 1.414).abs() < 0.001); - assert_eq!(config.similarity_threshold, 0.5); - assert!(config.hybrid.enabled); - assert!(config.cross_document.enabled); - assert!(config.page_range.enabled); - } - - #[test] - fn test_hybrid_strategy_config_defaults() { - let config = HybridStrategyConfig::default(); - assert!(config.enabled); - assert!((config.pre_filter_ratio - 0.3).abs() < f32::EPSILON); - assert_eq!(config.min_candidates, 2); - assert_eq!(config.max_candidates, 5); - assert!((config.auto_accept_threshold - 0.85).abs() < f32::EPSILON); - } - - #[test] - fn test_cross_document_strategy_config_defaults() { - let config = CrossDocumentStrategyConfig::default(); - assert!(config.enabled); - assert_eq!(config.max_documents, 10); - assert_eq!(config.max_results_per_doc, 3); - assert_eq!(config.merge_strategy, "TopK"); - assert!(config.parallel_search); - } - - #[test] - fn test_page_range_strategy_config_defaults() { - let config = PageRangeStrategyConfig::default(); - assert!(config.enabled); - assert!(config.include_boundary_nodes); - assert_eq!(config.expand_context_pages, 0); - assert!((config.min_overlap_ratio - 0.1).abs() < f32::EPSILON); + assert_eq!(config.level, 1); } } From 6675b92fc3615bc7132c5d813b60e8af5ab25535 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 16:58:38 +0800 Subject: [PATCH 06/30] feat(ask): enhance JSON parsing with proper error handling - Add explicit ValueError documentation in docstring when JSON parsing fails - Implement proper exception handling for JSONDecodeError - Wrap JSON parsing in try-catch block and raise descriptive ValueError - Include original exception in the raised error using 'from e' syntax --- vectorless/ask/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vectorless/ask/utils.py b/vectorless/ask/utils.py index f057cc9..269ccba 100644 --- a/vectorless/ask/utils.py +++ b/vectorless/ask/utils.py @@ -7,7 +7,10 @@ def parse_json_response(response: str) -> dict: - """Parse LLM response as JSON, handling markdown-wrapped output.""" + """Parse LLM response as JSON, handling markdown-wrapped output. + + Raises ``ValueError`` if the response cannot be parsed as JSON. + """ trimmed = response.strip() if trimmed.startswith("```"): @@ -30,4 +33,7 @@ def parse_json_response(response: str) -> dict: except json.JSONDecodeError: break - return json.loads(trimmed) + try: + return json.loads(trimmed) + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse LLM response as JSON: {e}") from e From 2a583c24b369f0aabe1fd34d14329f2355da5427 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:00:16 +0800 Subject: [PATCH 07/30] refactor(vectorless-document): remove unused ReferenceResolver struct Remove the ReferenceResolver struct that was caching resolved references for batch resolution. The implementation is no longer needed and has been removed from the codebase. --- .../vectorless-document/src/reference.rs | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/vectorless-core/vectorless-document/src/reference.rs b/vectorless-core/vectorless-document/src/reference.rs index 10d08e4..74a10c0 100644 --- a/vectorless-core/vectorless-document/src/reference.rs +++ b/vectorless-core/vectorless-document/src/reference.rs @@ -387,47 +387,6 @@ impl ReferenceExtractor { } } -/// Reference resolver for batch resolution. -/// -/// Caches resolved references for efficient reuse. -#[derive(Debug, Clone, Default)] -pub struct ReferenceResolver { - /// Cache of resolved references by ref_text. - cache: std::collections::HashMap>, -} - -impl ReferenceResolver { - /// Create a new reference resolver. - pub fn new() -> Self { - Self::default() - } - - /// Resolve references in batch and cache results. - pub fn resolve_batch( - &mut self, - references: &[NodeReference], - tree: &super::DocumentTree, - index: &super::RetrievalIndex, - ) { - for r#ref in references { - if !self.cache.contains_key(&r#ref.ref_text) { - let resolved = ReferenceExtractor::resolve_reference(r#ref, tree, index); - self.cache.insert(r#ref.ref_text.clone(), resolved); - } - } - } - - /// Get a cached resolution. - pub fn get(&self, ref_text: &str) -> Option> { - self.cache.get(ref_text).copied() - } - - /// Clear the cache. - pub fn clear(&mut self) { - self.cache.clear(); - } -} - #[cfg(test)] mod tests { use super::*; From 96e3f05f994fae1cf726b2bf48c4901d58ff7030 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:13:14 +0800 Subject: [PATCH 08/30] refactor(engine): remove unused ask method and related types from engine module Remove the `ask` method documentation reference from the Engine module's main documentation block, along with unused imports for Answer, Evidence, and ReasoningTrace types that were only used by the removed ask functionality. BREAKING CHANGE: The ask method has been removed from the Engine API. --- .../vectorless-engine/src/engine.rs | 26 +--- .../vectorless-engine/src/indexer.rs | 8 - .../vectorless-engine/src/types.rs | 31 +--- .../vectorless-engine/src/workspace.rs | 26 ---- vectorless/jupyter.py | 137 ------------------ 5 files changed, 2 insertions(+), 226 deletions(-) delete mode 100644 vectorless/jupyter.py diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index fa81082..e5fcb9b 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -6,7 +6,6 @@ //! The Engine provides a unified API for the Document Understanding Engine: //! //! - [`ingest`](Engine::ingest) — Understand a document (parse, analyze, persist) -//! - [`ask`](Engine::ask) — Ask a question (returns answer + evidence + trace) //! - [`forget`](Engine::forget) — Remove a document //! - [`list_documents`](Engine::list_documents) — List all understood documents //! @@ -51,7 +50,7 @@ use tracing::{info, warn}; use vectorless_config::Config; use vectorless_document::{ - Answer, Document as UnderstandingDocument, DocumentTree, Evidence, IngestInput, ReasoningTrace, + Document as UnderstandingDocument, DocumentTree, IngestInput, }; use vectorless_error::{Error, Result}; use vectorless_events::EventEmitter; @@ -551,29 +550,6 @@ impl Engine { // Internal // ============================================================ - /// Load documents by ID, returning loaded artifacts and failures. - async fn load_documents( - &self, - doc_ids: &[String], - ) -> Result<(Vec, Vec)> { - let mut documents = Vec::new(); - let mut failed = Vec::new(); - for doc_id in doc_ids { - match self.workspace.load(doc_id).await { - Ok(Some(doc)) => { - documents.push(Self::persisted_to_understanding_document(doc)); - } - Ok(None) => { - failed.push(FailedItem::new(doc_id, "Document not found")); - } - Err(e) => { - failed.push(FailedItem::new(doc_id, &e.to_string())); - } - } - } - Ok((documents, failed)) - } - /// Run a future with an optional timeout. /// If `timeout_secs` is `Some`, wraps the future in `tokio::time::timeout`. async fn with_timeout(&self, timeout_secs: Option, fut: F) -> Result diff --git a/vectorless-core/vectorless-engine/src/indexer.rs b/vectorless-core/vectorless-engine/src/indexer.rs index 20490f6..5319dc9 100644 --- a/vectorless-core/vectorless-engine/src/indexer.rs +++ b/vectorless-core/vectorless-engine/src/indexer.rs @@ -59,14 +59,6 @@ impl IndexerClient { } } - /// Create with a custom executor factory (for testing). - pub(crate) fn with_factory(factory: Arc PipelineExecutor + Send + Sync>) -> Self { - Self { - executor_factory: factory, - events: EventEmitter::new(), - } - } - /// Create with event emitter. pub fn with_events(mut self, events: EventEmitter) -> Self { self.events = events; diff --git a/vectorless-core/vectorless-engine/src/types.rs b/vectorless-core/vectorless-engine/src/types.rs index 247191e..925d647 100644 --- a/vectorless-core/vectorless-engine/src/types.rs +++ b/vectorless-core/vectorless-engine/src/types.rs @@ -415,27 +415,6 @@ pub struct DocumentInfo { pub line_count: Option, } -impl DocumentInfo { - /// Create a new document info. - pub fn new(id: impl Into, name: impl Into) -> Self { - Self { - id: id.into(), - name: name.into(), - format: String::new(), - description: None, - source_path: None, - page_count: None, - line_count: None, - } - } - - /// Set the format. - pub fn with_format(mut self, format: impl Into) -> Self { - self.format = format.into(); - self - } -} - #[cfg(test)] mod tests { use super::*; @@ -482,15 +461,7 @@ mod tests { assert!(result.single().is_some()); assert_eq!(result.single().unwrap().doc_id, "doc-1"); } - - #[test] - fn test_document_info() { - let info = DocumentInfo::new("doc-1", "Test").with_format("markdown"); - - assert_eq!(info.id, "doc-1"); - assert_eq!(info.format, "markdown"); - } - + #[test] fn test_index_result() { let item = IndexItem::new("doc-1", "Test", DocumentFormat::Markdown, None, None); diff --git a/vectorless-core/vectorless-engine/src/workspace.rs b/vectorless-core/vectorless-engine/src/workspace.rs index e05a175..ab62078 100644 --- a/vectorless-core/vectorless-engine/src/workspace.rs +++ b/vectorless-core/vectorless-engine/src/workspace.rs @@ -144,32 +144,6 @@ impl WorkspaceClient { Ok(self.workspace.contains(doc_id).await) } - /// List all documents in the workspace. - /// - /// # Errors - /// - /// Returns an error if the workspace read fails. - pub async fn list(&self) -> Result> { - let doc_ids = self.workspace.list_documents().await; - let mut result = Vec::with_capacity(doc_ids.len()); - - for id in &doc_ids { - if let Some(meta) = self.workspace.get_meta(id).await { - result.push(DocumentInfo { - id: meta.id, - name: meta.doc_name, - format: meta.doc_type, - description: meta.doc_description, - source_path: meta.path, - page_count: meta.page_count, - line_count: meta.line_count, - }); - } - } - - Ok(result) - } - /// Get document info by ID. /// /// # Errors diff --git a/vectorless/jupyter.py b/vectorless/jupyter.py deleted file mode 100644 index 2ae81cd..0000000 --- a/vectorless/jupyter.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Jupyter rich display integration for Vectorless results.""" - -from __future__ import annotations - -import html as html_module -from typing import Any, List, Optional - -from vectorless.ask.types import Output - - -class QueryResultDisplay: - """Rich display for query results in Jupyter notebooks. - - Implements _repr_html_(), _repr_markdown_(), and _repr_json_() - for automatic rendering. - """ - - def __init__(self, result: Output) -> None: - self._result = result - - def _repr_html_(self) -> str: - result = self._result - escaped_answer = html_module.escape(result.answer[:500]) - confidence_bar = _confidence_bar(result.confidence) - evidence_html = _evidence_list_html(result.evidence) - - return ( - f"
" - f"
" - f"

Result

{confidence_bar}
" - f"

{escaped_answer}

" - f"{evidence_html}" - f"
" - ) - - def _repr_markdown_(self) -> str: - result = self._result - lines = [ - f"## Result (confidence: {result.confidence:.2f})\n", - f"{result.answer}\n", - ] - if result.evidence: - lines.append("**Evidence:**\n") - for ev in result.evidence: - doc_label = f" [{ev.doc_name}]" if ev.doc_name else "" - lines.append(f"- **{ev.node_title}** ({ev.source_path}){doc_label}") - lines.append("") - return "\n".join(lines) - - def _repr_json_(self) -> dict: - result = self._result - return { - "answer": result.answer, - "confidence": result.confidence, - "evidence": [ - { - "title": e.node_title, - "path": e.source_path, - "content": e.content, - "doc_name": e.doc_name, - } - for e in result.evidence - ], - "metrics": { - "llm_calls": result.metrics.llm_calls, - "rounds_used": result.metrics.rounds_used, - "nodes_visited": result.metrics.nodes_visited, - "evidence_chars": result.metrics.evidence_chars, - }, - } - - -class DocumentGraphDisplay: - """Rich display for document relationship graphs.""" - - def __init__(self, graph: Any) -> None: - self._graph = graph - - def _repr_html_(self) -> str: - node_count = self._graph.node_count() if self._graph else 0 - edge_count = self._graph.edge_count() if self._graph else 0 - doc_ids = self._graph.doc_ids() if self._graph else [] - - rows = [] - for doc_id in doc_ids: - node = self._graph.get_node(doc_id) - if node: - rows.append( - f"{html_module.escape(node.doc_id)}" - f"{html_module.escape(node.title)}" - f"{node.node_count}" - ) - - return ( - f"
" - f"

Document Graph

" - f"

{node_count} nodes, {edge_count} edges

" - f"" - f"" - f"" - f"" - f"" - f"{''.join(rows)}
Doc IDTitleNodes
" - ) - - -def _confidence_bar(confidence: float) -> str: - """Generate an HTML confidence indicator bar.""" - pct = int(confidence * 100) - if confidence >= 0.8: - color = "#4caf50" - elif confidence >= 0.5: - color = "#ff9800" - else: - color = "#f44336" - return ( - f"
" - f"{pct}%" - f"
" - f"
" - f"
" - ) - - -def _evidence_list_html(evidence: list) -> str: - """Generate HTML for evidence items.""" - if not evidence: - return "" - items = [] - for ev in evidence[:5]: - items.append( - f"
  • {html_module.escape(ev.node_title)} " - f"{html_module.escape(ev.source_path)}
    • " - ) - extra = f" (+{len(evidence) - 5} more)" if len(evidence) > 5 else "" - return f"
      {''.join(items)}{extra}
    " From bc801131fd9d30e1c347842c42c6af62d81bf16b Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:25:21 +0800 Subject: [PATCH 09/30] refactor(storage): remove memory backend and simplify persistence layer - Remove unused FieldKey and Field structs from bm25 module - Remove entire memory backend implementation including tests - Remove memory backend from storage backend module exports - Remove example documentation from storage lib.rs - Remove unused file system imports from persistence module - Remove PersistenceOptions struct and related configuration methods - Remove file-based save/load functions with atomic write logic - Remove index save/load functions that used file operations - Update tests to use bytes-based serialization instead of file-based - Simplify checksum verification tests to work with byte arrays --- .../vectorless-scoring/src/bm25.rs | 14 - .../vectorless-storage/src/backend/memory.rs | 173 ------- .../vectorless-storage/src/backend/mod.rs | 1 - vectorless-core/vectorless-storage/src/lib.rs | 25 +- .../vectorless-storage/src/persistence.rs | 476 +----------------- 5 files changed, 17 insertions(+), 672 deletions(-) delete mode 100644 vectorless-core/vectorless-storage/src/backend/memory.rs diff --git a/vectorless-core/vectorless-scoring/src/bm25.rs b/vectorless-core/vectorless-scoring/src/bm25.rs index 8bc2008..f372105 100644 --- a/vectorless-core/vectorless-scoring/src/bm25.rs +++ b/vectorless-core/vectorless-scoring/src/bm25.rs @@ -95,20 +95,6 @@ impl FieldDocument { } } -/// Key for field-specific document storage. -#[derive(Debug, Clone, Hash, Eq, PartialEq)] -struct FieldKey { - doc_id: K, - field: Field, -} - -#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] -enum Field { - Title, - Summary, - Content, -} - /// BM25 engine with per-field weighting support. /// /// This wraps the `bm25` crate's Embedder and Scorer to provide: diff --git a/vectorless-core/vectorless-storage/src/backend/memory.rs b/vectorless-core/vectorless-storage/src/backend/memory.rs deleted file mode 100644 index 197ddac..0000000 --- a/vectorless-core/vectorless-storage/src/backend/memory.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! In-memory storage backend (for testing). - -use std::collections::HashMap; -use std::sync::RwLock; - -use super::StorageBackend; -use vectorless_error::Result; - -/// In-memory storage backend. -/// -/// Stores all data in a `HashMap`. Useful for testing and scenarios -/// where persistence is not required. -/// -/// # Thread Safety -/// -/// Uses `RwLock` for thread-safe access to the internal map. -#[derive(Debug, Default)] -pub struct MemoryBackend { - /// Internal storage. - data: RwLock>>, -} - -impl MemoryBackend { - /// Create a new in-memory backend. - pub fn new() -> Self { - Self::default() - } - - /// Create a new in-memory backend with pre-seeded data. - pub fn with_data(data: HashMap>) -> Self { - Self { - data: RwLock::new(data), - } - } -} - -impl StorageBackend for MemoryBackend { - fn get(&self, key: &str) -> Result>> { - let data = self.data.read().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - Ok(data.get(key).cloned()) - } - - fn put(&self, key: &str, value: &[u8]) -> Result<()> { - let mut data = self.data.write().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - data.insert(key.to_string(), value.to_vec()); - Ok(()) - } - - fn delete(&self, key: &str) -> Result { - let mut data = self.data.write().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - Ok(data.remove(key).is_some()) - } - - fn exists(&self, key: &str) -> Result { - let data = self.data.read().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - Ok(data.contains_key(key)) - } - - fn keys(&self) -> Result> { - let data = self.data.read().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - Ok(data.keys().cloned().collect()) - } - - fn len(&self) -> Result { - let data = self.data.read().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - Ok(data.len()) - } - - fn clear(&self) -> Result<()> { - let mut data = self.data.write().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - data.clear(); - Ok(()) - } - - fn batch_put(&self, items: &[(&str, &[u8])]) -> Result<()> { - let mut data = self.data.write().map_err(|_| { - vectorless_error::Error::Cache("Memory backend lock poisoned".to_string()) - })?; - for (key, value) in items { - data.insert(key.to_string(), value.to_vec()); - } - Ok(()) - } - - fn backend_name(&self) -> &'static str { - "memory" - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_memory_backend_basic() { - let backend = MemoryBackend::new(); - - // Put and get - backend.put("key1", b"value1").unwrap(); - let value = backend.get("key1").unwrap(); - assert_eq!(value, Some(b"value1".to_vec())); - - // Non-existent key - let missing = backend.get("missing").unwrap(); - assert!(missing.is_none()); - } - - #[test] - fn test_memory_backend_delete() { - let backend = MemoryBackend::new(); - - backend.put("key1", b"value1").unwrap(); - assert!(backend.exists("key1").unwrap()); - - let deleted = backend.delete("key1").unwrap(); - assert!(deleted); - assert!(!backend.exists("key1").unwrap()); - - // Delete non-existent - let not_deleted = backend.delete("missing").unwrap(); - assert!(!not_deleted); - } - - #[test] - fn test_memory_backend_keys() { - let backend = MemoryBackend::new(); - - backend.put("key1", b"v1").unwrap(); - backend.put("key2", b"v2").unwrap(); - backend.put("key3", b"v3").unwrap(); - - let keys = backend.keys().unwrap(); - assert_eq!(keys.len(), 3); - } - - #[test] - fn test_memory_backend_clear() { - let backend = MemoryBackend::new(); - - backend.put("key1", b"v1").unwrap(); - backend.put("key2", b"v2").unwrap(); - - backend.clear().unwrap(); - assert!(backend.is_empty().unwrap()); - } - - #[test] - fn test_memory_backend_with_data() { - let mut initial = HashMap::new(); - initial.insert("k1".to_string(), b"v1".to_vec()); - initial.insert("k2".to_string(), b"v2".to_vec()); - - let backend = MemoryBackend::with_data(initial); - assert_eq!(backend.len().unwrap(), 2); - } -} diff --git a/vectorless-core/vectorless-storage/src/backend/mod.rs b/vectorless-core/vectorless-storage/src/backend/mod.rs index a8bc805..14ceccb 100644 --- a/vectorless-core/vectorless-storage/src/backend/mod.rs +++ b/vectorless-core/vectorless-storage/src/backend/mod.rs @@ -27,7 +27,6 @@ //! ``` mod file; -mod memory; mod trait_def; pub use file::FileBackend; diff --git a/vectorless-core/vectorless-storage/src/lib.rs b/vectorless-core/vectorless-storage/src/lib.rs index ca7c27f..2d6a698 100644 --- a/vectorless-core/vectorless-storage/src/lib.rs +++ b/vectorless-core/vectorless-storage/src/lib.rs @@ -8,30 +8,7 @@ //! - **Persistence** — Save/load document trees and metadata with atomic writes //! - **Cache** — LRU cache for loaded documents //! - **Lock** — File locking for multi-process safety -//! - **Backend** — Storage backend abstraction (file, memory, etc.) -//! -//! # Example -//! -//! ```rust,no_run -//! use vectorless::storage::{Workspace, PersistedDocument, DocumentMeta}; -//! use vectorless::document::DocumentTree; -//! -//! # #[tokio::main] -//! # async fn main() -> vectorless::error::Result<()> { -//! // Create a workspace -//! let workspace = Workspace::new("./my_workspace").await?; -//! -//! // Add a document -//! let meta = DocumentMeta::new("doc-1", "My Document", "md"); -//! let tree = DocumentTree::new("Root", "Content"); -//! let doc = PersistedDocument::new(meta, tree); -//! workspace.add(&doc).await?; -//! -//! // Load it back (uses LRU cache) -//! let loaded = workspace.load_and_cache("doc-1").await?.unwrap(); -//! # Ok(()) -//! # } -//! ``` +//! - **Backend** — Storage backend abstraction (file, memory, etc.) pub mod backend; pub mod cache; diff --git a/vectorless-core/vectorless-storage/src/persistence.rs b/vectorless-core/vectorless-storage/src/persistence.rs index e0a8376..f7fd0a4 100644 --- a/vectorless-core/vectorless-storage/src/persistence.rs +++ b/vectorless-core/vectorless-storage/src/persistence.rs @@ -11,9 +11,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use std::fs::File; -use std::io::{BufReader, BufWriter, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use vectorless_document::{DocumentTree, NavigationIndex, ReasoningIndex}; use vectorless_error::Error; @@ -285,43 +283,6 @@ struct PersistedWrapper { payload: serde_json::Value, } -/// Options for save/load operations. -#[derive(Debug, Clone)] -pub struct PersistenceOptions { - /// Use atomic writes (temp file + rename). - pub atomic_writes: bool, - /// Verify checksums on load. - pub verify_checksum: bool, -} - -impl Default for PersistenceOptions { - fn default() -> Self { - Self { - atomic_writes: true, - verify_checksum: true, - } - } -} - -impl PersistenceOptions { - /// Create new options with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set atomic writes option. - pub fn with_atomic_writes(mut self, enabled: bool) -> Self { - self.atomic_writes = enabled; - self - } - - /// Set checksum verification option. - pub fn with_verify_checksum(mut self, enabled: bool) -> Self { - self.verify_checksum = enabled; - self - } -} - /// Calculate SHA-256 checksum of data. fn calculate_checksum(data: &[u8]) -> String { let mut hasher = Sha256::new(); @@ -329,264 +290,6 @@ fn calculate_checksum(data: &[u8]) -> String { format!("{:x}", hasher.finalize()) } -/// Save a document to a JSON file with atomic write and checksum. -/// -/// # Atomic Write -/// -/// When `atomic_writes` is enabled (default), this function: -/// 1. Writes to a temporary file (`.tmp` suffix) -/// 2. Renames temp file to target (atomic on most filesystems) -/// -/// This prevents data corruption if the process crashes during write. -/// -/// # Errors -/// -/// Returns an error if: -/// - Serialization fails -/// - Cannot create temp file -/// - Write fails -/// - Rename fails -pub fn save_document(path: &Path, doc: &PersistedDocument) -> Result<()> { - save_document_with_options(path, doc, &PersistenceOptions::default()) -} - -/// Save a document with custom options. -pub fn save_document_with_options( - path: &Path, - doc: &PersistedDocument, - options: &PersistenceOptions, -) -> Result<()> { - // Serialize to serde_json::Value first (avoids HashMap key ordering drift) - let payload_value = - serde_json::to_value(doc).map_err(|e| Error::Serialization(e.to_string()))?; - - // Calculate checksum on the Value's canonical bytes - let payload_bytes = - serde_json::to_vec(&payload_value).map_err(|e| Error::Serialization(e.to_string()))?; - let checksum = calculate_checksum(&payload_bytes); - - // Create wrapper - let wrapper = PersistedWrapper { - version: FORMAT_VERSION, - checksum, - payload: payload_value, - }; - - // Serialize wrapper - let json = - serde_json::to_string_pretty(&wrapper).map_err(|e| Error::Serialization(e.to_string()))?; - - if options.atomic_writes { - // Atomic write: write to temp file, then rename - let temp_path = path.with_extension("tmp"); - - // Ensure parent directory exists - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(Error::Io)?; - } - - // Write to temp file - { - let file = File::create(&temp_path).map_err(Error::Io)?; - let mut writer = BufWriter::new(file); - writer.write_all(json.as_bytes()).map_err(Error::Io)?; - writer.flush().map_err(Error::Io)?; - } - - // Atomic rename - std::fs::rename(&temp_path, path).map_err(Error::Io)?; - } else { - // Direct write (not atomic) - std::fs::write(path, json).map_err(Error::Io)?; - } - - Ok(()) -} - -/// Load a document from a JSON file with checksum verification. -/// -/// # Checksum Verification -/// -/// When `verify_checksum` is enabled (default), this function: -/// 1. Reads the file -/// 2. Parses the wrapper -/// 3. Re-serializes the payload -/// 4. Verifies the checksum matches -/// -/// # Errors -/// -/// Returns an error if: -/// - File doesn't exist -/// - Parse fails -/// - Checksum mismatch -/// - Version mismatch (future: migration) -pub fn load_document(path: &Path) -> Result { - load_document_with_options(path, &PersistenceOptions::default()) -} - -/// Load a document with custom options. -pub fn load_document_with_options( - path: &Path, - options: &PersistenceOptions, -) -> Result { - if !path.exists() { - return Err(Error::DocumentNotFound(path.display().to_string())); - } - - let file = File::open(path).map_err(Error::Io)?; - let reader = BufReader::new(file); - - // Parse wrapper (payload is serde_json::Value) - let wrapper: PersistedWrapper = serde_json::from_reader(reader) - .map_err(|e| Error::Parse(format!("Failed to parse document: {}", e)))?; - - // Check version - if wrapper.version != FORMAT_VERSION { - return Err(Error::Parse(format!( - "Unsupported format version: {} (expected {})", - wrapper.version, FORMAT_VERSION - ))); - } - - // Verify checksum if enabled - if options.verify_checksum { - let payload_bytes = serde_json::to_vec(&wrapper.payload) - .map_err(|e| Error::Serialization(e.to_string()))?; - - let expected_checksum = calculate_checksum(&payload_bytes); - - if wrapper.checksum != expected_checksum { - return Err(Error::Parse(format!( - "Checksum mismatch: expected {}, got {}", - expected_checksum, wrapper.checksum - ))); - } - } - - // Deserialize Value to target type - let doc: PersistedDocument = serde_json::from_value(wrapper.payload) - .map_err(|e| Error::Parse(format!("Failed to deserialize document: {}", e)))?; - - // Check schema version — warn on stale documents, fail on future versions - if doc.schema_version == 0 { - tracing::warn!( - doc_id = %doc.meta.id, - "Document was created before schema versioning — consider re-indexing" - ); - } else if doc.schema_version > SCHEMA_VERSION { - return Err(Error::Parse(format!( - "Document schema version {} is newer than supported {} — please upgrade vectorless", - doc.schema_version, SCHEMA_VERSION - ))); - } - - Ok(doc) -} - -/// Save the workspace index (metadata for all documents). -pub fn save_index(path: &Path, entries: &[DocumentMeta]) -> Result<()> { - save_index_with_options(path, entries, &PersistenceOptions::default()) -} - -/// Save the workspace index with custom options. -pub fn save_index_with_options( - path: &Path, - entries: &[DocumentMeta], - options: &PersistenceOptions, -) -> Result<()> { - // Serialize to serde_json::Value first - let payload_value = - serde_json::to_value(entries).map_err(|e| Error::Serialization(e.to_string()))?; - - let payload_bytes = - serde_json::to_vec(&payload_value).map_err(|e| Error::Serialization(e.to_string()))?; - - let checksum = calculate_checksum(&payload_bytes); - - let wrapper = PersistedWrapper { - version: FORMAT_VERSION, - checksum, - payload: payload_value, - }; - - let json = - serde_json::to_string_pretty(&wrapper).map_err(|e| Error::Serialization(e.to_string()))?; - - if options.atomic_writes { - let temp_path = path.with_extension("tmp"); - - // Ensure parent directory exists - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(Error::Io)?; - } - - // Write to temp file - { - let file = File::create(&temp_path).map_err(Error::Io)?; - let mut writer = BufWriter::new(file); - writer.write_all(json.as_bytes()).map_err(Error::Io)?; - writer.flush().map_err(Error::Io)?; - } - - // Atomic rename - std::fs::rename(&temp_path, path).map_err(Error::Io)?; - } else { - std::fs::write(path, json).map_err(Error::Io)?; - } - - Ok(()) -} - -/// Load the workspace index. -pub fn load_index(path: &Path) -> Result> { - load_index_with_options(path, &PersistenceOptions::default()) -} - -/// Load the workspace index with custom options. -pub fn load_index_with_options( - path: &Path, - options: &PersistenceOptions, -) -> Result> { - if !path.exists() { - return Ok(Vec::new()); - } - - let file = File::open(path).map_err(Error::Io)?; - let reader = BufReader::new(file); - - let wrapper: PersistedWrapper = serde_json::from_reader(reader) - .map_err(|e| Error::Parse(format!("Failed to parse index: {}", e)))?; - - // Check version - if wrapper.version != FORMAT_VERSION { - return Err(Error::Parse(format!( - "Unsupported format version: {} (expected {})", - wrapper.version, FORMAT_VERSION - ))); - } - - // Verify checksum if enabled - if options.verify_checksum { - let payload_bytes = serde_json::to_vec(&wrapper.payload) - .map_err(|e| Error::Serialization(e.to_string()))?; - - let expected_checksum = calculate_checksum(&payload_bytes); - - if wrapper.checksum != expected_checksum { - return Err(Error::Parse(format!( - "Checksum mismatch: expected {}, got {}", - expected_checksum, wrapper.checksum - ))); - } - } - - // Deserialize Value to target type - let entries: Vec = serde_json::from_value(wrapper.payload) - .map_err(|e| Error::Parse(format!("Failed to deserialize index: {}", e)))?; - - Ok(entries) -} - // ============================================================================ // Bytes-based serialization (for StorageBackend integration) // ============================================================================ @@ -674,71 +377,9 @@ pub fn load_document_from_bytes_with_options( Ok(doc) } -/// Serialize an index to bytes. -pub fn save_index_to_bytes(entries: &[DocumentMeta]) -> Result> { - let payload_value = - serde_json::to_value(entries).map_err(|e| Error::Serialization(e.to_string()))?; - - let payload_bytes = - serde_json::to_vec(&payload_value).map_err(|e| Error::Serialization(e.to_string()))?; - let checksum = calculate_checksum(&payload_bytes); - - let wrapper = PersistedWrapper { - version: FORMAT_VERSION, - checksum, - payload: payload_value, - }; - - serde_json::to_vec(&wrapper).map_err(|e| Error::Serialization(e.to_string())) -} - -/// Deserialize an index from bytes. -pub fn load_index_from_bytes(data: &[u8]) -> Result> { - load_index_from_bytes_with_options(data, true) -} - -/// Deserialize an index from bytes with optional checksum verification. -pub fn load_index_from_bytes_with_options( - data: &[u8], - verify_checksum: bool, -) -> Result> { - let wrapper: PersistedWrapper = serde_json::from_slice(data) - .map_err(|e| Error::Parse(format!("Failed to parse index: {}", e)))?; - - // Check version - if wrapper.version != FORMAT_VERSION { - return Err(Error::VersionMismatch(format!( - "Expected version {}, got {}", - FORMAT_VERSION, wrapper.version - ))); - } - - // Verify checksum if enabled - if verify_checksum { - let payload_bytes = serde_json::to_vec(&wrapper.payload) - .map_err(|e| Error::Serialization(e.to_string()))?; - - let expected_checksum = calculate_checksum(&payload_bytes); - - if wrapper.checksum != expected_checksum { - return Err(Error::ChecksumMismatch(format!( - "Expected {}, got {}", - expected_checksum, wrapper.checksum - ))); - } - } - - // Deserialize Value to target type - let entries: Vec = serde_json::from_value(wrapper.payload) - .map_err(|e| Error::Parse(format!("Failed to deserialize index: {}", e)))?; - - Ok(entries) -} - #[cfg(test)] mod tests { use super::*; - use tempfile::TempDir; fn create_test_doc(id: &str) -> PersistedDocument { let meta = DocumentMeta::new(id, "Test Doc", "md"); @@ -747,120 +388,35 @@ mod tests { } #[test] - fn test_save_and_load_document() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("test.json"); - + fn test_save_and_load_bytes() { let doc = create_test_doc("doc-1"); - save_document(&path, &doc).unwrap(); - - let loaded = load_document(&path).unwrap(); + let bytes = save_document_to_bytes(&doc).unwrap(); + let loaded = load_document_from_bytes(&bytes).unwrap(); assert_eq!(loaded.meta.id, "doc-1"); assert_eq!(loaded.meta.name, "Test Doc"); } #[test] - fn test_atomic_write() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("atomic.json"); - - let doc = create_test_doc("doc-atomic"); - let options = PersistenceOptions::new().with_atomic_writes(true); - save_document_with_options(&path, &doc, &options).unwrap(); - - // Temp file should not exist after save - assert!(!path.with_extension("tmp").exists()); - - let loaded = load_document(&path).unwrap(); - assert_eq!(loaded.meta.id, "doc-atomic"); - } - - #[test] - fn test_checksum_verification() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("checksum.json"); + fn test_checksum_verification_bytes() { + let doc = create_test_doc("doc-check"); + let bytes = save_document_to_bytes(&doc).unwrap(); - let doc = create_test_doc("doc-checksum"); - save_document(&path, &doc).unwrap(); + // Corrupt a byte + let mut corrupted = bytes.clone(); + corrupted[10] ^= 0xFF; - // Corrupt the file - let content = std::fs::read_to_string(&path).unwrap(); - let corrupted = content.replace("doc-checksum", "doc-corrupted"); - std::fs::write(&path, corrupted).unwrap(); - - // Load should fail with checksum error - let result = load_document(&path); + let result = load_document_from_bytes(&corrupted); assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(matches!(err, Error::Parse(_))); } #[test] - fn test_checksum_disabled() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("no-checksum.json"); - + fn test_checksum_disabled_bytes() { let doc = create_test_doc("doc-no-check"); - save_document(&path, &doc).unwrap(); + let bytes = save_document_to_bytes(&doc).unwrap(); - // Load with checksum disabled should succeed - let options = PersistenceOptions::new().with_verify_checksum(false); - let result = load_document_with_options(&path, &options); + // Load with checksum disabled should succeed even for raw bytes + let result = load_document_from_bytes_with_options(&bytes, false); assert!(result.is_ok()); - let loaded = result.unwrap(); - assert_eq!(loaded.meta.id, "doc-no-check"); - - // Now corrupt the checksum field specifically - let content = std::fs::read_to_string(&path).unwrap(); - // Change the checksum value but keep the payload intact - let payload_value = serde_json::to_value(&doc).unwrap(); - let corrupted = content.replace( - &calculate_checksum(&serde_json::to_vec(&payload_value).unwrap()), - "0000000000000000000000000000000000000000000000000000000000000000", - ); - std::fs::write(&path, corrupted).unwrap(); - - // Load with checksum disabled should still succeed - let result = load_document_with_options(&path, &options); - assert!(result.is_ok()); - - // Load with checksum enabled should fail - let options_enabled = PersistenceOptions::new().with_verify_checksum(true); - let result = load_document_with_options(&path, &options_enabled); - assert!(result.is_err()); - } - - #[test] - fn test_load_nonexistent() { - let result = load_document(Path::new("/nonexistent/path.json")); - assert!(result.is_err()); - assert!(result.unwrap_err().is_not_found()); - } - - #[test] - fn test_save_and_load_index() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("meta.bin"); - - let mut entries = Vec::new(); - entries.push(DocumentMeta::new("doc-1", "Doc 1", "md")); - entries.push(DocumentMeta::new("doc-2", "Doc 2", "pdf")); - - save_index(&path, &entries).unwrap(); - - let loaded = load_index(&path).unwrap(); - assert_eq!(loaded.len(), 2); - assert_eq!(loaded[0].id, "doc-1"); - assert_eq!(loaded[1].format, "pdf"); - } - - #[test] - fn test_load_empty_index() { - let temp = TempDir::new().unwrap(); - let path = temp.path().join("nonexistent.json"); - - let loaded = load_index(&path).unwrap(); - assert!(loaded.is_empty()); } #[test] @@ -875,6 +431,6 @@ mod tests { assert_eq!(checksum1, checksum2); assert_ne!(checksum1, checksum3); - assert_eq!(checksum1.len(), 64); // SHA-256 produces 64 hex chars + assert_eq!(checksum1.len(), 64); } } From 63f81f8d6bef7807ba933f23522cbc2068c4c13b Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:25:58 +0800 Subject: [PATCH 10/30] refactor(workspace): remove unused test module Removed the unused test module from workspace.rs that contained a helper function for creating test documents which was no longer being used in the codebase. --- vectorless-core/vectorless-storage/src/workspace.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/vectorless-core/vectorless-storage/src/workspace.rs b/vectorless-core/vectorless-storage/src/workspace.rs index adb2d4d..d09e390 100644 --- a/vectorless-core/vectorless-storage/src/workspace.rs +++ b/vectorless-core/vectorless-storage/src/workspace.rs @@ -655,15 +655,3 @@ impl Workspace { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - use vectorless_document::DocumentTree; - - fn create_test_doc(id: &str) -> PersistedDocument { - let meta = super::super::persistence::DocumentMeta::new(id, "Test Doc", "md"); - let tree = DocumentTree::new("Root", "Content"); - PersistedDocument::new(meta, tree) - } -} From 55008413f8d11505ce201a6d5cb33361059036c8 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:42:37 +0800 Subject: [PATCH 11/30] feat: upgrade minimum Python version to 3.11 BREAKING CHANGE: drop support for Python 3.10 and require Python >= 3.11 - Update pyproject.toml to require Python >= 3.11 - Remove Python 3.10 classifier from package metadata - Remove conditional tomli dependency since tomllib is built-in from 3.11+ - Update mypy and ruff configurations to target Python 3.11 - Simplify TOML loading code by removing Python version checks refactor: improve type safety with protocol-based typing - Replace loose `Any` and `Callable` types with structured protocols - Add DocLoader and EventCallback protocols for better type checking - Update dispatcher and orchestrator to use typed parameters - Remove dynamic imports for tomllib in favor of built-in module refactor: modernize optional type annotations - Convert Optional[T] to T | None union syntax throughout codebase - Update engine class constructors and methods with modern typing - Standardize nullability patterns across all modules perf: replace asyncio.gather with TaskGroup for better error handling - Use TaskGroup instead of gather(return_exceptions=True) for worker tasks - Maintain same fault-tolerance while improving async execution - Update batch compilation to use TaskGroup for better resource management --- pyproject.toml | 8 ++--- vectorless/ask/dispatcher.py | 6 ++-- vectorless/ask/orchestrator.py | 24 ++++++++++--- vectorless/ask/protocols.py | 20 +++++++++++ vectorless/cli/workspace.py | 11 +----- vectorless/config/loading.py | 24 ++++--------- vectorless/engine.py | 62 ++++++++++++++++++---------------- 7 files changed, 85 insertions(+), 70 deletions(-) create mode 100644 vectorless/ask/protocols.py diff --git a/pyproject.toml b/pyproject.toml index 9f7bd10..2e8d972 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "vectorless" dynamic = ["version"] description = "Document Understanding Engine for AI" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" license = { text = "Apache-2.0" } authors = [ { name = "Vectorless", email = "beautifularea@gmail.com" } @@ -17,7 +17,6 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -32,7 +31,6 @@ dependencies = [ "click>=8.0", "litellm>=1.50", "instructor>=1.0", - "tomli>=2.0; python_version < '3.11'", # 3.10 only, 3.11+ has tomllib built-in ] [project.optional-dependencies] @@ -78,13 +76,13 @@ asyncio_mode = "auto" testpaths = ["tests"] [tool.mypy] -python_version = "3.10" +python_version = "3.11" warn_return_any = true warn_unused_configs = true [tool.ruff] line-length = 100 -target-version = "py310" +target-version = "py311" [tool.ruff.lint] select = ["E", "F", "W", "I", "N", "UP", "B"] diff --git a/vectorless/ask/dispatcher.py b/vectorless/ask/dispatcher.py index ed35289..4d8906a 100644 --- a/vectorless/ask/dispatcher.py +++ b/vectorless/ask/dispatcher.py @@ -12,8 +12,8 @@ from __future__ import annotations import logging -from typing import Any, Callable +from vectorless.ask.protocols import DocLoader, EventCallback from vectorless.ask.types import DocCard, Output, Specified, Workspace from vectorless.ask.orchestrator import Orchestrator from vectorless.ask.reasoning import QueryAnalysis, QueryAnalyzer @@ -26,8 +26,8 @@ async def dispatch( query: str, scope: Specified | Workspace, llm: LLMClient, - doc_loader: Callable[[str], Any], - event_callback: Any = None, + doc_loader: DocLoader, + event_callback: EventCallback | None = None, ) -> Output: """Unified entry point — mirrors Rust dispatcher::dispatch(). diff --git a/vectorless/ask/orchestrator.py b/vectorless/ask/orchestrator.py index 1529052..7d6b8b2 100644 --- a/vectorless/ask/orchestrator.py +++ b/vectorless/ask/orchestrator.py @@ -23,6 +23,7 @@ from dataclasses import dataclass from typing import Any +from vectorless.ask.protocols import DocLoader, EventCallback from vectorless.ask.types import ( DispatchEntry, DocCard, @@ -104,7 +105,7 @@ def __init__( self, query: str, doc_cards: list[DocCard], - doc_loader: Any, # async callable: (doc_id: str) -> PyDocument + doc_loader: DocLoader, llm_client: LLMClient, *, skip_analysis: bool = False, @@ -112,7 +113,7 @@ def __init__( query_analysis: QueryAnalysis | None = None, max_rounds: int = 15, max_llm_calls: int = 0, - event_callback: Any = None, # async callable: (dict) -> None + event_callback: EventCallback | None = None, ) -> None: self._query = query self._doc_cards = doc_cards @@ -502,11 +503,24 @@ async def run_worker(dispatch: DispatchEntry) -> tuple[int, WorkerOutput]: return (idx, result) tasks = [run_worker(d) for d in dispatches] - results = await asyncio.gather(*tasks, return_exceptions=True) - for item in results: + # Use TaskGroup with per-task exception wrapping to maintain + # the same fault-tolerance as gather(return_exceptions=True) + task_results: list[tuple[int, WorkerOutput] | Exception] = [] + async with asyncio.TaskGroup() as tg: + async def _safe_run(d: DispatchEntry) -> None: + try: + result = await run_worker(d) + task_results.append(result) + except Exception as e: + task_results.append(e) + logger.warning("Worker failed: %s", e) + + for d in dispatches: + tg.create_task(_safe_run(d)) + + for item in task_results: if isinstance(item, Exception): - logger.warning("Worker failed: %s", item) continue idx, output = item state.collect_result(idx, output) diff --git a/vectorless/ask/protocols.py b/vectorless/ask/protocols.py new file mode 100644 index 0000000..a24a60f --- /dev/null +++ b/vectorless/ask/protocols.py @@ -0,0 +1,20 @@ +"""Protocol definitions for callable parameters. + +Replaces loose `Any` / `Callable` typing with structural typing via Protocol. +""" + +from __future__ import annotations + +from typing import Any, Protocol + + +class DocLoader(Protocol): + """Async callable that loads a navigable document by ID.""" + + async def __call__(self, doc_id: str) -> Any: ... + + +class EventCallback(Protocol): + """Async callable that receives event dicts.""" + + async def __call__(self, event: dict) -> Any: ... diff --git a/vectorless/cli/workspace.py b/vectorless/cli/workspace.py index b93898c..55ed38c 100644 --- a/vectorless/cli/workspace.py +++ b/vectorless/cli/workspace.py @@ -115,20 +115,11 @@ def load_config(workspace: str) -> Dict[str, Any]: Returns: Configuration dict. """ - import sys - config_path = Path(workspace) / CONFIG_FILE if not config_path.exists(): return {} - if sys.version_info >= (3, 11): - import tomllib - else: - try: - import tomli as tomllib # type: ignore[no-redef] - except ImportError: - # Fallback: parse as plain text (comments-only files) - return {} + import tomllib with open(config_path, "rb") as f: return tomllib.load(f) diff --git a/vectorless/config/loading.py b/vectorless/config/loading.py index 6ededbd..0ddf586 100644 --- a/vectorless/config/loading.py +++ b/vectorless/config/loading.py @@ -3,20 +3,12 @@ from __future__ import annotations import os -import sys +import tomllib from pathlib import Path -from typing import Any, Optional +from typing import Any from vectorless.config.models import EngineConfig, LlmConfig, StorageConfig -if sys.version_info >= (3, 11): - import tomllib -else: - try: - import tomli as tomllib # type: ignore[no-redef] - except ImportError: - tomllib = None # type: ignore[assignment] - def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: """Load configuration from environment variables. @@ -31,7 +23,7 @@ def load_config_from_env(prefix: str = "VECTORLESS_") -> EngineConfig: """ llm = LlmConfig() storage = StorageConfig() - metrics_enabled: Optional[bool] = None + metrics_enabled: bool | None = None env_map = { f"{prefix}API_KEY": ("llm.api_key", str), @@ -101,8 +93,7 @@ def load_config_from_file(path: Path) -> EngineConfig: """ if tomllib is None: raise ImportError( - "TOML parsing requires 'tomli' on Python < 3.11. " - "Install with: pip install vectorless[cli]" + "TOML parsing requires Python >= 3.11 (tomllib built-in)." ) with open(path, "rb") as f: @@ -112,9 +103,9 @@ def load_config_from_file(path: Path) -> EngineConfig: def load_config( - config_file: Optional[Path] = None, + config_file: Path | None = None, env_prefix: str = "VECTORLESS_", - overrides: Optional[dict[str, Any]] = None, + overrides: dict[str, Any] | None = None, ) -> EngineConfig: """Load configuration with layered precedence. @@ -128,8 +119,7 @@ def load_config( if config_file is not None and config_file.exists(): if tomllib is None: raise ImportError( - "TOML parsing requires 'tomli' on Python < 3.11. " - "Install with: pip install vectorless[cli]" + "TOML parsing requires Python >= 3.11 (tomllib built-in)." ) with open(config_file, "rb") as f: file_data = tomllib.load(f) diff --git a/vectorless/engine.py b/vectorless/engine.py index fd681a3..31d91a0 100644 --- a/vectorless/engine.py +++ b/vectorless/engine.py @@ -10,7 +10,7 @@ import asyncio import logging from pathlib import Path -from typing import Any, Callable, List, Optional, Union +from typing import Any from vectorless._internal._core import Engine as RustEngine from vectorless.ask.dispatcher import dispatch @@ -57,12 +57,12 @@ class Engine: def __init__( self, - api_key: Optional[str] = None, - model: Optional[str] = None, - endpoint: Optional[str] = None, - config: Optional[EngineConfig] = None, - config_file: Optional[Union[str, Path]] = None, - events: Optional[EventEmitter] = None, + api_key: str | None = None, + model: str | None = None, + endpoint: str | None = None, + config: EngineConfig | None = None, + config_file: str | Path | None = None, + events: EventEmitter | None = None, ) -> None: self._events = events or EventEmitter() @@ -89,7 +89,7 @@ def __init__( ) @classmethod - def from_env(cls, events: Optional[EventEmitter] = None) -> Engine: + def from_env(cls, events: EventEmitter | None = None) -> Engine: """Create an Engine from environment variables only.""" config = load_config_from_env() return cls(config=config, events=events) @@ -97,8 +97,8 @@ def from_env(cls, events: Optional[EventEmitter] = None) -> Engine: @classmethod def from_config_file( cls, - path: Union[str, Path], - events: Optional[EventEmitter] = None, + path: str | Path, + events: EventEmitter | None = None, ) -> Engine: """Create an Engine from a TOML config file.""" config = load_config_from_file(Path(path)) @@ -106,10 +106,10 @@ def from_config_file( def _resolve_config( self, - api_key: Optional[str], - model: Optional[str], - endpoint: Optional[str], - config_file: Optional[Union[str, Path]], + api_key: str | None, + model: str | None, + endpoint: str | None, + config_file: str | Path | None, ) -> EngineConfig: overrides: dict[str, Any] = {} llm_overrides: dict[str, Any] = {} @@ -131,13 +131,13 @@ def _resolve_config( async def compile( self, - path: Optional[Union[str, Path]] = None, - paths: Optional[List[Union[str, Path]]] = None, - directory: Optional[Union[str, Path]] = None, - content: Optional[str] = None, - bytes_data: Optional[bytes] = None, + path: str | Path | None = None, + paths: list[str | Path] | None = None, + directory: str | Path | None = None, + content: str | None = None, + bytes_data: bytes | None = None, format: str = "markdown", - name: Optional[str] = None, + name: str | None = None, mode: str = "default", force: bool = False, ) -> IndexResultWrapper: @@ -219,7 +219,7 @@ async def compile( async def compile_batch( self, - paths: List[Union[str, Path]], + paths: list[str | Path], *, mode: str = "default", jobs: int = 1, @@ -237,7 +237,7 @@ async def compile_batch( """ semaphore = asyncio.Semaphore(jobs) - async def _index_one(p: Union[str, Path]) -> object: + async def _index_one(p: str | Path) -> object: async with semaphore: self._events.emit_index( IndexEventData(event_type=IndexEventType.STARTED, path=str(p)) @@ -253,7 +253,9 @@ async def _index_one(p: Union[str, Path]) -> object: ) return doc_info - results = await asyncio.gather(*[_index_one(p) for p in paths]) + async with asyncio.TaskGroup() as tg: + tasks = [tg.create_task(_index_one(p)) for p in paths] + results = [t.result() for t in tasks] return IndexResultWrapper.from_doc_infos(list(results)) # ── Querying (Python strategy layer) ──────────────────────── @@ -262,8 +264,8 @@ async def ask( self, question: str, *, - doc_ids: Optional[List[str]] = None, - timeout_secs: Optional[int] = None, + doc_ids: list[str] | None = None, + timeout_secs: int | None = None, ) -> Output: """Ask a question and get results with source attribution. @@ -304,8 +306,8 @@ async def query_stream( self, question: str, *, - doc_ids: Optional[List[str]] = None, - timeout_secs: Optional[int] = None, + doc_ids: list[str] | None = None, + timeout_secs: int | None = None, ) -> StreamingQueryResult: """Stream query progress as an async iterator. @@ -326,8 +328,8 @@ async def query_stream( async def _ask_python( self, question: str, - doc_ids: Optional[List[str]], - event_queue: Optional[asyncio.Queue] = None, + doc_ids: list[str] | None, + event_queue: asyncio.Queue | None = None, ) -> Output: """Run the full Python strategy: dispatch → Output. @@ -409,7 +411,7 @@ async def clear_all(self) -> int: # ── Graph (Rust) ──────────────────────────────────────────── - async def get_graph(self) -> Optional[DocumentGraphWrapper]: + async def get_graph(self) -> DocumentGraphWrapper | None: """Get the cross-document relationship graph.""" graph = await self._rust.get_graph() if graph is None: From 24120bb56b3fef2928cce6640b0ae10316a067d1 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 17:56:32 +0800 Subject: [PATCH 12/30] feat: update project description and Python version requirement - Change project description from "Document Understanding Engine for AI" to "Knowing by reasoning, not vectors." in both Cargo.toml and pyproject.toml - Update Python version requirement from 3.9+ to 3.11+ in installation docs --- Cargo.toml | 2 +- docs/docs/installation.mdx | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2cf9ec8..ed68677 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ resolver = "2" [workspace.package] version = "0.1.12" -description = "Document Understanding Engine for AI" +description = "Knowing by reasoning, not vectors." edition = "2024" authors = ["zTgx "] license = "Apache-2.0" diff --git a/docs/docs/installation.mdx b/docs/docs/installation.mdx index 15c23aa..43cd2f6 100644 --- a/docs/docs/installation.mdx +++ b/docs/docs/installation.mdx @@ -6,7 +6,7 @@ sidebar_position: 2 ## Prerequisites -- Python 3.9+ +- Python 3.11+ - An LLM API key (OpenAI, or any OpenAI-compatible endpoint) ## Install diff --git a/pyproject.toml b/pyproject.toml index 2e8d972..e894c05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "vectorless" dynamic = ["version"] -description = "Document Understanding Engine for AI" +description = "Knowing by reasoning, not vectors." readme = "README.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } From 04247bfb9236ee0a18bc56649e0e7d71227087db Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 18:10:16 +0800 Subject: [PATCH 13/30] refactor(core): remove unused query types and events from engine BREAKING CHANGE: Removed QueryResult, QueryResultItem, QueryMetrics, EvidenceItem, and Confidence types from vectorless-engine as query logic has been moved to Python layer. Also removed QueryEvent enum and related functionality from events module since query handling is now managed externally. --- vectorless-core/vectorless-engine/src/lib.rs | 11 +- .../vectorless-engine/src/types.rs | 158 ------------------ .../vectorless-events/src/emitter.rs | 50 +----- vectorless-core/vectorless-events/src/lib.rs | 2 +- .../vectorless-events/src/types.rs | 53 +----- vectorless-core/vectorless-py/Cargo.toml | 1 + vectorless-core/vectorless-py/src/answer.rs | 2 +- 7 files changed, 9 insertions(+), 268 deletions(-) diff --git a/vectorless-core/vectorless-engine/src/lib.rs b/vectorless-core/vectorless-engine/src/lib.rs index 526c9fd..8cd8cca 100644 --- a/vectorless-core/vectorless-engine/src/lib.rs +++ b/vectorless-core/vectorless-engine/src/lib.rs @@ -35,10 +35,7 @@ pub use index_context::IndexContext; // Result & Info Types // ============================================================ -pub use types::{ - Confidence, EvidenceItem, FailedItem, IndexItem, IndexMode, IndexOptions, IndexResult, - QueryMetrics, QueryResult, QueryResultItem, -}; +pub use types::{FailedItem, IndexItem, IndexMode, IndexOptions, IndexResult}; // ============================================================ // Parser Types (needed for IndexContext::from_content) @@ -52,11 +49,9 @@ pub use vectorless_document::DocumentFormat; pub use vectorless_config::Config; pub use vectorless_document::DocumentTree; -pub use vectorless_document::{ - Answer, Concept, DocumentInfo, Evidence, IngestInput, ReasoningTrace, TraceStep, -}; +pub use vectorless_document::{Concept, DocumentInfo, IngestInput}; pub use vectorless_error::{Error, Result}; -pub use vectorless_events::{EventEmitter, IndexEvent, QueryEvent, WorkspaceEvent}; +pub use vectorless_events::{EventEmitter, IndexEvent, WorkspaceEvent}; pub use vectorless_graph::{ DocumentGraph, DocumentGraphNode, EdgeEvidence, GraphEdge, WeightedKeyword, }; diff --git a/vectorless-core/vectorless-engine/src/types.rs b/vectorless-core/vectorless-engine/src/types.rs index 925d647..562510a 100644 --- a/vectorless-core/vectorless-engine/src/types.rs +++ b/vectorless-core/vectorless-engine/src/types.rs @@ -252,140 +252,6 @@ impl IndexItem { } } -// ============================================================ -// Query Types — defined locally (strategy layer moved to Python) -// ============================================================ - -/// Confidence level of a query result. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct Confidence(pub f64); - -impl Confidence { - /// Create a new confidence value (0.0 - 1.0). - pub fn new(value: f64) -> Self { - Self(value.clamp(0.0, 1.0)) - } - - /// Get the raw value. - pub fn value(&self) -> f64 { - self.0 - } -} - -/// A piece of evidence supporting a query result. -#[derive(Debug, Clone)] -pub struct EvidenceItem { - /// Title of the source section. - pub title: String, - /// Path within the document. - pub path: String, - /// Content of the evidence. - pub content: String, -} - -/// Metrics for a single query result. -#[derive(Debug, Clone, Default)] -pub struct QueryMetrics { - /// Number of LLM calls made. - pub llm_calls: usize, - /// Number of navigation rounds used. - pub rounds_used: usize, - /// Number of document nodes visited. - pub nodes_visited: usize, - /// Number of evidence items collected. - pub evidence_count: usize, - /// Total characters in evidence. - pub evidence_chars: usize, -} - -/// A single query result item. -#[derive(Debug, Clone)] -pub struct QueryResultItem { - /// Document ID. - pub doc_id: String, - /// Node IDs that contributed evidence. - pub node_ids: Vec, - /// Result content. - pub content: String, - /// Supporting evidence. - pub evidence: Vec, - /// Optional metrics. - pub metrics: Option, - /// Confidence score. - pub confidence: f64, -} - -/// Result of a document query. -/// -/// Contains results from one or more documents. For single-document queries, -/// `items` has one entry. For multi-document or workspace queries, it has -/// one entry per document that matched. -#[derive(Debug, Clone)] -pub struct QueryResult { - /// Query results per document. - pub items: Vec, - - /// Documents that failed during multi-doc query. - pub failed: Vec, -} - -impl QueryResult { - /// Create a new query result (empty). - pub fn new() -> Self { - Self { - items: Vec::new(), - failed: Vec::new(), - } - } - - /// Create a query result with items. - pub fn new_with_items(items: Vec) -> Self { - Self { - items, - failed: Vec::new(), - } - } - - /// Create a query result with a single item. - pub fn from_single(item: QueryResultItem) -> Self { - Self { - items: vec![item], - failed: Vec::new(), - } - } - - /// Create with both successes and failures. - pub fn with_partial(items: Vec, failed: Vec) -> Self { - Self { items, failed } - } - - /// Check if the result is empty. - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } - - /// Get the number of result items. - pub fn len(&self) -> usize { - self.items.len() - } - - /// Get the first (single-doc) result item, if any. - pub fn single(&self) -> Option<&QueryResultItem> { - self.items.first() - } - - /// Whether any documents failed. - pub fn has_failures(&self) -> bool { - !self.failed.is_empty() - } -} - -impl Default for QueryResult { - fn default() -> Self { - Self::new() - } -} - // ============================================================ // Document Info Types // ============================================================ @@ -438,30 +304,6 @@ mod tests { assert_eq!(default.timeout_secs, None); } - #[test] - fn test_query_result() { - let result = QueryResult::new(); - assert!(result.is_empty()); - assert_eq!(result.len(), 0); - } - - #[test] - fn test_query_result_single() { - let item = QueryResultItem { - doc_id: "doc-1".into(), - node_ids: vec!["n1".into()], - content: "content".into(), - evidence: vec![], - metrics: None, - confidence: 0.9, - }; - let result = QueryResult::from_single(item); - assert!(!result.is_empty()); - assert_eq!(result.len(), 1); - assert!(result.single().is_some()); - assert_eq!(result.single().unwrap().doc_id, "doc-1"); - } - #[test] fn test_index_result() { let item = IndexItem::new("doc-1", "Test", DocumentFormat::Markdown, None, None); diff --git a/vectorless-core/vectorless-events/src/emitter.rs b/vectorless-core/vectorless-events/src/emitter.rs index 7804a25..9867e5c 100644 --- a/vectorless-core/vectorless-events/src/emitter.rs +++ b/vectorless-core/vectorless-events/src/emitter.rs @@ -10,14 +10,11 @@ use std::sync::Arc; use parking_lot::RwLock; -use super::types::{IndexEvent, QueryEvent, WorkspaceEvent}; +use super::types::{IndexEvent, WorkspaceEvent}; /// Type alias for sync index handler. pub(crate) type IndexHandler = Box; -/// Type alias for sync query handler. -pub(crate) type QueryHandler = Box; - /// Type alias for sync workspace handler. pub(crate) type WorkspaceHandler = Box; @@ -26,9 +23,6 @@ struct EventEmitterInner { /// Index event handlers. index_handlers: Vec, - /// Query event handlers. - query_handlers: Vec, - /// Workspace event handlers. workspace_handlers: Vec, } @@ -37,7 +31,6 @@ impl Default for EventEmitterInner { fn default() -> Self { Self { index_handlers: Vec::new(), - query_handlers: Vec::new(), workspace_handlers: Vec::new(), } } @@ -80,15 +73,6 @@ impl EventEmitter { self } - /// Add a query event handler. - pub fn on_query(self, handler: F) -> Self - where - F: Fn(&QueryEvent) + Send + Sync + 'static, - { - self.inner.write().query_handlers.push(Box::new(handler)); - self - } - /// Add a workspace event handler. pub fn on_workspace(self, handler: F) -> Self where @@ -109,14 +93,6 @@ impl EventEmitter { } } - /// Emit a query event. - pub fn emit_query(&self, event: QueryEvent) { - let inner = self.inner.read(); - for handler in &inner.query_handlers { - handler(&event); - } - } - /// Emit a workspace event. pub fn emit_workspace(&self, event: WorkspaceEvent) { let inner = self.inner.read(); @@ -128,9 +104,7 @@ impl EventEmitter { /// Check if there are any handlers registered. pub fn has_handlers(&self) -> bool { let inner = self.inner.read(); - !inner.index_handlers.is_empty() - || !inner.query_handlers.is_empty() - || !inner.workspace_handlers.is_empty() + !inner.index_handlers.is_empty() || !inner.workspace_handlers.is_empty() } /// Merge another emitter into this one. @@ -140,9 +114,6 @@ impl EventEmitter { inner .index_handlers .extend(other_inner.index_handlers.drain(..)); - inner - .query_handlers - .extend(other_inner.query_handlers.drain(..)); inner .workspace_handlers .extend(other_inner.workspace_handlers.drain(..)); @@ -173,7 +144,6 @@ impl std::fmt::Debug for EventEmitter { let inner = self.inner.read(); f.debug_struct("EventEmitter") .field("index_handlers", &inner.index_handlers.len()) - .field("query_handlers", &inner.query_handlers.len()) .field("workspace_handlers", &inner.workspace_handlers.len()) .finish() } @@ -203,22 +173,6 @@ mod tests { assert_eq!(counter.load(Ordering::SeqCst), 2); } - #[test] - fn test_event_emitter_query() { - let counter = Arc::new(AtomicUsize::new(0)); - let counter_clone = counter.clone(); - - let emitter = EventEmitter::new().on_query(move |_e| { - counter_clone.fetch_add(1, Ordering::SeqCst); - }); - - emitter.emit_query(QueryEvent::Started { - query: "test".to_string(), - }); - - assert_eq!(counter.load(Ordering::SeqCst), 1); - } - #[test] fn test_event_emitter_has_handlers() { let empty = EventEmitter::new(); diff --git a/vectorless-core/vectorless-events/src/lib.rs b/vectorless-core/vectorless-events/src/lib.rs index e8e55df..cbb6e2f 100644 --- a/vectorless-core/vectorless-events/src/lib.rs +++ b/vectorless-core/vectorless-events/src/lib.rs @@ -28,4 +28,4 @@ mod emitter; mod types; pub use emitter::EventEmitter; -pub use types::{IndexEvent, QueryEvent, WorkspaceEvent}; +pub use types::{IndexEvent, WorkspaceEvent}; diff --git a/vectorless-core/vectorless-events/src/types.rs b/vectorless-core/vectorless-events/src/types.rs index 45bb5ad..28c7aaa 100644 --- a/vectorless-core/vectorless-events/src/types.rs +++ b/vectorless-core/vectorless-events/src/types.rs @@ -3,11 +3,10 @@ //! Event types for client operations. //! -//! Provides enums for indexing, query, and workspace events +//! Provides enums for indexing and workspace events //! that can be observed via [`EventEmitter`](super::EventEmitter). use vectorless_document::DocumentFormat; -use vectorless_document::SufficiencyLevel; /// Indexing operation events. #[derive(Debug, Clone)] @@ -57,56 +56,6 @@ pub enum IndexEvent { }, } -/// Query operation events. -#[derive(Debug, Clone)] -pub enum QueryEvent { - /// Search started. - Started { - /// The query string. - query: String, - }, - - /// Node visited during search. - NodeVisited { - /// Node ID. - node_id: String, - /// Node title. - title: String, - /// Relevance score. - score: f32, - }, - - /// Candidate result found. - CandidateFound { - /// Node ID. - node_id: String, - /// Relevance score. - score: f32, - }, - - /// Sufficiency check result. - SufficiencyCheck { - /// Sufficiency level. - level: SufficiencyLevel, - /// Total tokens collected. - tokens: usize, - }, - - /// Query completed. - Complete { - /// Total results found. - total_results: usize, - /// Overall confidence score. - confidence: f32, - }, - - /// Error occurred during query. - Error { - /// Error message. - message: String, - }, -} - /// Workspace operation events. #[derive(Debug, Clone)] pub enum WorkspaceEvent { diff --git a/vectorless-core/vectorless-py/Cargo.toml b/vectorless-core/vectorless-py/Cargo.toml index 9c7b66a..25642b1 100644 --- a/vectorless-core/vectorless-py/Cargo.toml +++ b/vectorless-core/vectorless-py/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["cdylib"] pyo3 = { workspace = true } pyo3-async-runtimes = { workspace = true } tokio = { version = "1", features = ["rt-multi-thread", "sync"] } +vectorless-document = { path = "../vectorless-document" } vectorless-engine = { path = "../vectorless-engine" } vectorless-primitives = { path = "../vectorless-primitives" } diff --git a/vectorless-core/vectorless-py/src/answer.rs b/vectorless-core/vectorless-py/src/answer.rs index 9015118..e6f351d 100644 --- a/vectorless-core/vectorless-py/src/answer.rs +++ b/vectorless-core/vectorless-py/src/answer.rs @@ -5,7 +5,7 @@ use pyo3::prelude::*; -use ::vectorless_engine::Answer; +use ::vectorless_document::Answer; /// A reasoned answer with evidence and trace. #[pyclass(name = "Answer", skip_from_py_object)] From 340f22a2a90afc194c79541f41b03fce7b18849e Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 18:29:48 +0800 Subject: [PATCH 14/30] refactor(vectorless-document): remove unused exports and types from understanding module BREAKING CHANGE: Removed Answer, Evidence, ReasoningTrace, and TraceStep types from the understanding module as they were no longer used. Also removed SufficiencyLevel from format exports. - Remove Answer, Evidence, ReasoningTrace, TraceStep from understanding exports - Remove SufficiencyLevel from format exports - Clean up related documentation comments --- .../vectorless-document/src/lib.rs | 6 +- .../vectorless-document/src/understanding.rs | 113 +----------------- .../vectorless-engine/src/builder.rs | 53 +------- .../vectorless-engine/src/engine.rs | 34 ------ .../vectorless-engine/src/index_context.rs | 51 -------- 5 files changed, 5 insertions(+), 252 deletions(-) diff --git a/vectorless-core/vectorless-document/src/lib.rs b/vectorless-core/vectorless-document/src/lib.rs index 85ea1ff..956dfde 100644 --- a/vectorless-core/vectorless-document/src/lib.rs +++ b/vectorless-core/vectorless-document/src/lib.rs @@ -27,7 +27,7 @@ mod toc; mod tree; pub mod understanding; -pub use format::{DocumentFormat, SufficiencyLevel}; +pub use format::DocumentFormat; pub use navigation::{ChildRoute, DocCard, NavEntry, NavigationIndex, SectionCard}; pub use node::{NodeId, TreeNode}; pub use reasoning::{ @@ -38,6 +38,4 @@ pub use reference::ReferenceExtractor; pub use structure::{DocumentStructure, StructureNode}; pub use toc::{TocConfig, TocEntry, TocNode, TocView}; pub use tree::{DocumentTree, RetrievalIndex}; -pub use understanding::{ - Answer, Concept, Document, DocumentInfo, Evidence, IngestInput, ReasoningTrace, TraceStep, -}; +pub use understanding::{Concept, Document, DocumentInfo, IngestInput}; diff --git a/vectorless-core/vectorless-document/src/understanding.rs b/vectorless-core/vectorless-document/src/understanding.rs index 4be8e29..2b97ce3 100644 --- a/vectorless-core/vectorless-document/src/understanding.rs +++ b/vectorless-core/vectorless-document/src/understanding.rs @@ -7,9 +7,6 @@ //! - [`Document`] — the unified post-ingest artifact (internal first-class citizen) //! - [`DocumentInfo`] — what `ingest()` returns to users //! - [`Concept`] — key concept extracted from a document -//! - [`Answer`] — what `ask()` returns -//! - [`Evidence`] — proof trail for an answer -//! - [`ReasoningTrace`] / [`TraceStep`] — always-mandatory reasoning trace use serde::{Deserialize, Serialize}; @@ -21,10 +18,9 @@ use super::toc::TocNode; /// A understood document — the core artifact of the understand phase. /// -/// This is what `ingest()` produces internally and what `ask()` consumes. +/// This is what `ingest()` produces internally. /// It unifies tree + navigation index + reasoning index + summary + concepts -/// into a single first-class type, replacing the previous loose coupling of -/// `DocContext { &tree, &nav, &reasoning }`. +/// into a single first-class type. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Document { /// Unique document identifier. @@ -157,89 +153,6 @@ pub struct Concept { pub sections: Vec, } -// --------------------------------------------------------------------------- -// Answer — what ask() returns -// --------------------------------------------------------------------------- - -/// The result of `ask()` — a reasoned answer with evidence and trace. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Answer { - /// The answer content. - pub content: String, - /// Evidence supporting the answer. - pub evidence: Vec, - /// Confidence score (0.0–1.0). - pub confidence: f32, - /// Reasoning trace — how the agent arrived at this answer. Always present. - pub trace: ReasoningTrace, -} - -// --------------------------------------------------------------------------- -// Evidence -// --------------------------------------------------------------------------- - -/// A piece of evidence supporting an answer — with source attribution. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Evidence { - /// Original document text. - pub content: String, - /// Navigation path (e.g., "Root/Chapter 3/Section 3.2"). - pub source_path: String, - /// Which document this evidence came from. - pub doc_name: String, - /// Relevance to the question (0.0–1.0). - pub relevance: f32, -} - -// --------------------------------------------------------------------------- -// ReasoningTrace — always mandatory -// --------------------------------------------------------------------------- - -/// Reasoning trace — how the agent arrived at the answer. Always present. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ReasoningTrace { - /// The steps the agent took. - pub steps: Vec, -} - -impl ReasoningTrace { - /// Create an empty trace. - pub fn empty() -> Self { - Self { steps: Vec::new() } - } - - /// Create a trace with a single step. - pub fn single(action: impl Into, observation: impl Into, round: u32) -> Self { - Self { - steps: vec![TraceStep { - action: action.into(), - observation: observation.into(), - round, - }], - } - } - - /// Add a step to the trace. - pub fn push(&mut self, action: impl Into, observation: impl Into, round: u32) { - self.steps.push(TraceStep { - action: action.into(), - observation: observation.into(), - round, - }); - } -} - -/// A single step in the reasoning trace. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TraceStep { - /// What the agent did (e.g., "cd Chapter 3"). - pub action: String, - /// What the agent observed (e.g., "Found 5 sections about..."). - pub observation: String, - /// Which round this step was in. - pub round: u32, -} - // --------------------------------------------------------------------------- // IngestInput — what ingest() takes // --------------------------------------------------------------------------- @@ -271,28 +184,6 @@ pub enum IngestInput { mod tests { use super::*; - #[test] - fn test_reasoning_trace_empty() { - let trace = ReasoningTrace::empty(); - assert!(trace.steps.is_empty()); - } - - #[test] - fn test_reasoning_trace_single() { - let trace = ReasoningTrace::single("cd Chapter 3", "Found 5 sections", 1); - assert_eq!(trace.steps.len(), 1); - assert_eq!(trace.steps[0].action, "cd Chapter 3"); - assert_eq!(trace.steps[0].round, 1); - } - - #[test] - fn test_reasoning_trace_push() { - let mut trace = ReasoningTrace::empty(); - trace.push("ls", "Root with 3 children", 0); - trace.push("cd Chapter 2", "Found target section", 1); - assert_eq!(trace.steps.len(), 2); - } - #[test] fn test_concept_serialization() { let concept = Concept { diff --git a/vectorless-core/vectorless-engine/src/builder.rs b/vectorless-core/vectorless-engine/src/builder.rs index 7527575..e20048b 100644 --- a/vectorless-core/vectorless-engine/src/builder.rs +++ b/vectorless-core/vectorless-engine/src/builder.rs @@ -17,42 +17,6 @@ use super::engine::Engine; /// /// `api_key`, `model` and `endpoint` are **required** for simple usage. /// Advanced users can provide a pre-built [`Config`] via [`with_config`](EngineBuilder::with_config). -/// -/// # Example (simple) -/// -/// ```rust,no_run -/// use vectorless::client::EngineBuilder; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), vectorless::BuildError> { -/// let client = EngineBuilder::new() -/// .with_key("sk-...") -/// .with_model("gpt-4o") -/// .with_endpoint("https://api.xxx.com/v1") -/// .build() -/// .await?; -/// Ok(()) -/// } -/// ``` -/// -/// # Example (advanced) -/// -/// ```rust,ignore -/// use vectorless::client::EngineBuilder; -/// use vectorless::config::{Config, LlmConfig, SlotConfig}; -/// -/// let config = Config::new().with_llm( -/// LlmConfig::new("gpt-4o") -/// .with_api_key("sk-...") -/// .with_endpoint("https://api.openai.com/v1") -/// .with_index(SlotConfig::fast().with_model("gpt-4o-mini")) -/// ); -/// -/// let engine = EngineBuilder::new() -/// .with_config(config) -/// .build() -/// .await?; -/// ``` #[derive(Debug)] pub struct EngineBuilder { /// Custom configuration for advanced tuning. @@ -143,22 +107,7 @@ impl EngineBuilder { /// - Workspace creation fails /// - Required `api_key` or `model` is missing /// - /// # Example - /// - /// ```rust,no_run - /// use vectorless::client::EngineBuilder; - /// - /// # #[tokio::main] - /// # async fn main() -> Result<(), vectorless::BuildError> { - /// let engine = EngineBuilder::new() - /// .with_key("sk-...") - /// .with_model("gpt-4o") - /// .with_endpoint("https://api.openai.com/v1") - /// .build() - /// .await?; - /// # Ok(()) - /// # } - /// ``` + /// Build the Engine client. pub async fn build(self) -> Result { // Load user-provided or default configuration let mut config = self.config.unwrap_or_default(); diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index e5fcb9b..8e81c8b 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -8,40 +8,6 @@ //! - [`ingest`](Engine::ingest) — Understand a document (parse, analyze, persist) //! - [`forget`](Engine::forget) — Remove a document //! - [`list_documents`](Engine::list_documents) — List all understood documents -//! -//! # Example -//! -//! ```rust,no_run -//! use vectorless::{EngineBuilder, IngestInput}; -//! -//! # #[tokio::main] -//! # async fn main() -> Result<(), Box> { -//! let engine = EngineBuilder::new() -//! .with_key("sk-...") -//! .with_model("gpt-4o") -//! .with_endpoint("https://api.openai.com/v1") -//! .build() -//! .await?; -//! -//! // Understand a document -//! let doc = engine.ingest(IngestInput::Path("./document.md".into())).await?; -//! println!("{}: {}", doc.name, doc.summary); -//! -//! // Ask a question -//! let answer = engine.ask("What is this?", &[doc.doc_id.clone()]).await?; -//! println!("{}", answer.content); -//! -//! // List all understood documents -//! let docs = engine.list_documents().await?; -//! for d in &docs { -//! println!("{}: {}", d.name, d.summary); -//! } -//! -//! // Forget a document -//! engine.forget(&doc.doc_id).await?; -//! # Ok(()) -//! # } -//! ``` use std::{collections::HashMap, sync::Arc}; diff --git a/vectorless-core/vectorless-engine/src/index_context.rs b/vectorless-core/vectorless-engine/src/index_context.rs index df109db..104e1b9 100644 --- a/vectorless-core/vectorless-engine/src/index_context.rs +++ b/vectorless-core/vectorless-engine/src/index_context.rs @@ -7,34 +7,6 @@ //! - **File path** — Load and parse a file from disk //! - **Content string** — Parse content directly (HTML, Markdown, text) //! - **Byte data** — Parse binary data (PDF, DOCX) -//! -//! # Single document -//! -//! ```rust,no_run -//! use vectorless::client::IndexContext; -//! -//! let ctx = IndexContext::from_path("./document.md"); -//! ``` -//! -//! # Multiple documents -//! -//! ```rust,no_run -//! use vectorless::client::IndexContext; -//! -//! let ctx = IndexContext::from_paths(vec!["./doc1.md", "./doc2.pdf"]); -//! ``` -//! -//! # From directory -//! -//! ```rust,no_run -//! use vectorless::client::IndexContext; -//! -//! // Non-recursive (top-level only) -//! let ctx = IndexContext::from_dir("./documents", false); -//! -//! // Recursive (includes subdirectories) -//! let ctx = IndexContext::from_dir("./documents", true); -//! ``` use std::path::PathBuf; @@ -74,29 +46,6 @@ pub(crate) enum IndexSource { /// Supports single or multiple document sources. When multiple sources /// are provided, each is indexed independently and the results are /// collected into [`IndexResult`](super::IndexResult). -/// -/// # Examples -/// -/// ```rust,no_run -/// use vectorless::client::IndexContext; -/// use vectorless::client::DocumentFormat; -/// -/// # #[tokio::main] -/// # async fn main() -> Result<(), Box> { -/// # let engine = vectorless::EngineBuilder::new().build().await?; -/// // Single file -/// let result = engine.index(IndexContext::from_path("./doc.md")).await?; -/// -/// // Multiple files -/// let result = engine.index( -/// IndexContext::from_paths(vec!["./doc1.md", "./doc2.pdf"]) -/// ).await?; -/// -/// // Entire directory -/// let result = engine.index(IndexContext::from_dir("./docs", false)).await?; -/// # Ok(()) -/// # } -/// ``` #[derive(Debug, Clone)] pub struct IndexContext { /// Document sources (supports multiple). From e43213f22c1ab497504b14bba6171c4d732a0ea1 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 18:42:50 +0800 Subject: [PATCH 15/30] refactor(core): remove SufficiencyLevel enum and consolidate utility functions BREAKING CHANGE: Remove SufficiencyLevel enum from vectorless-document crate and consolidate keyword extraction and evidence formatting utilities into shared module. - Remove SufficiencyLevel enum from format.rs as it's no longer used - Move extract_keywords function to vectorless/ask/utils.py as single source of truth - Move format_evidence function to vectorless/ask/utils.py as single source of truth - Replace in-memory response cache with bounded LRU cache in LLMClient - Add structured error types for ask pipeline operations - Remove Answer-related Python bindings that were unused --- .../vectorless-document/src/format.rs | 24 +-- vectorless-core/vectorless-py/src/answer.rs | 103 ------------- vectorless-core/vectorless-py/src/lib.rs | 15 -- vectorless/__init__.py | 9 -- vectorless/ask/errors.py | 38 +++++ vectorless/ask/orchestrator.py | 31 +--- vectorless/ask/reasoning/analyzer.py | 21 +-- vectorless/ask/utils.py | 56 ++++++- vectorless/ask/verify/verifier.py | 14 +- vectorless/ask/worker.py | 10 +- vectorless/llm_client.py | 140 +++++++++--------- 11 files changed, 172 insertions(+), 289 deletions(-) delete mode 100644 vectorless-core/vectorless-py/src/answer.rs create mode 100644 vectorless/ask/errors.py diff --git a/vectorless-core/vectorless-document/src/format.rs b/vectorless-core/vectorless-document/src/format.rs index 78f6e52..8901dc4 100644 --- a/vectorless-core/vectorless-document/src/format.rs +++ b/vectorless-core/vectorless-document/src/format.rs @@ -1,10 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Document format and sufficiency types. -//! -//! These types are used across multiple modules and are defined here -//! to avoid circular dependencies between crates. +//! Document format types. use serde::{Deserialize, Serialize}; @@ -41,22 +38,3 @@ impl DocumentFormat { /// discover indexable files. pub const SUPPORTED_EXTENSIONS: &'static [&'static str] = &["md", "pdf"]; } - -/// Sufficiency level for incremental retrieval. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SufficiencyLevel { - /// Information is sufficient, stop retrieving. - Sufficient, - - /// Partial information, can continue if needed. - PartialSufficient, - - /// Information is insufficient, continue retrieving. - Insufficient, -} - -impl Default for SufficiencyLevel { - fn default() -> Self { - Self::Insufficient - } -} diff --git a/vectorless-core/vectorless-py/src/answer.rs b/vectorless-core/vectorless-py/src/answer.rs deleted file mode 100644 index e6f351d..0000000 --- a/vectorless-core/vectorless-py/src/answer.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Answer Python wrapper. - -use pyo3::prelude::*; - -use ::vectorless_document::Answer; - -/// A reasoned answer with evidence and trace. -#[pyclass(name = "Answer", skip_from_py_object)] -pub struct PyAnswer { - pub(crate) inner: Answer, -} - -#[pymethods] -impl PyAnswer { - /// The answer content. - #[getter] - fn content(&self) -> &str { - &self.inner.content - } - - /// Evidence supporting the answer. - #[getter] - fn evidence(&self) -> Vec { - self.inner - .evidence - .iter() - .map(|e| PyEvidence { - content: e.content.clone(), - source_path: e.source_path.clone(), - doc_name: e.doc_name.clone(), - relevance: e.relevance, - }) - .collect() - } - - /// Confidence score (0.0–1.0). - #[getter] - fn confidence(&self) -> f32 { - self.inner.confidence - } - - /// Reasoning trace — how the agent arrived at this answer. - #[getter] - fn trace(&self) -> PyReasoningTrace { - PyReasoningTrace { - steps: self - .inner - .trace - .steps - .iter() - .map(|s| PyTraceStep { - action: s.action.clone(), - observation: s.observation.clone(), - round: s.round, - }) - .collect(), - } - } - - fn __repr__(&self) -> String { - format!( - "Answer(confidence={:.2}, evidence={}, trace_steps={})", - self.inner.confidence, - self.inner.evidence.len(), - self.inner.trace.steps.len() - ) - } -} - -/// A piece of evidence with source attribution. -#[pyclass(name = "Evidence", skip_from_py_object)] -pub struct PyEvidence { - #[pyo3(get)] - pub content: String, - #[pyo3(get)] - pub source_path: String, - #[pyo3(get)] - pub doc_name: String, - #[pyo3(get)] - pub relevance: f32, -} - -/// Reasoning trace — always present. -#[pyclass(name = "ReasoningTrace", skip_from_py_object)] -pub struct PyReasoningTrace { - #[pyo3(get)] - pub steps: Vec, -} - -/// A single step in the reasoning trace. -#[pyclass(name = "TraceStep", skip_from_py_object)] -#[derive(Clone)] -pub struct PyTraceStep { - #[pyo3(get)] - pub action: String, - #[pyo3(get)] - pub observation: String, - #[pyo3(get)] - pub round: u32, -} diff --git a/vectorless-core/vectorless-py/src/lib.rs b/vectorless-core/vectorless-py/src/lib.rs index 9d1f8b0..5c3c0ad 100644 --- a/vectorless-core/vectorless-py/src/lib.rs +++ b/vectorless-core/vectorless-py/src/lib.rs @@ -5,7 +5,6 @@ use pyo3::prelude::*; -mod answer; mod config; mod document; mod engine; @@ -13,7 +12,6 @@ mod error; mod graph; mod metrics; -use answer::{PyAnswer, PyEvidence, PyReasoningTrace, PyTraceStep}; use config::PyConfig; use document::{ PyCollectedEvidence, PyConcept, PyConceptInfo, PyDocCard, PyDocument, PyDocumentInfo, @@ -26,15 +24,6 @@ use graph::{PyDocumentGraph, PyDocumentGraphNode, PyEdgeEvidence, PyGraphEdge, P use metrics::{PyLlmMetricsReport, PyMetricsReport, PyRetrievalMetricsReport}; /// Vectorless — Document Understanding Engine for AI. -/// -/// ```python -/// from vectorless import Engine -/// -/// engine = Engine(api_key="sk-...", model="gpt-4o") -/// doc = await engine.ingest("./report.pdf") -/// answer = await engine.ask("What is the revenue?", doc_ids=[doc.doc_id]) -/// print(answer.content) -/// ``` #[pymodule] fn _vectorless(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -55,10 +44,6 @@ fn _vectorless(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/vectorless/__init__.py b/vectorless/__init__.py index 6464e73..973d508 100644 --- a/vectorless/__init__.py +++ b/vectorless/__init__.py @@ -21,7 +21,6 @@ # Rust types re-exported for convenience from vectorless._vectorless import ( - Answer, CollectedEvidence, Concept, Config, @@ -29,17 +28,14 @@ DocumentGraphNode, DocumentInfo, EdgeEvidence, - Evidence, FindResult, GraphEdge, LlmMetricsReport, MatchResult, MetricsReport, NodeInfo, - ReasoningTrace, RetrievalMetricsReport, SectionSummary, - TraceStep, TopicEntry, VectorlessError, WeightedKeyword, @@ -68,11 +64,6 @@ "CollectedEvidence", "TopicEntry", "SectionSummary", - # Answer types - "Answer", - "Evidence", - "ReasoningTrace", - "TraceStep", # Graph types "DocumentGraph", "DocumentGraphNode", diff --git a/vectorless/ask/errors.py b/vectorless/ask/errors.py new file mode 100644 index 0000000..daf7fe9 --- /dev/null +++ b/vectorless/ask/errors.py @@ -0,0 +1,38 @@ +"""Structured error types for the ask pipeline. + +Replaces bare ``except Exception`` with a typed hierarchy. +""" + +from __future__ import annotations + + +class AskError(Exception): + """Base error for all ask pipeline failures.""" + + +class LLMFailureError(AskError): + """LLM call failed after retries.""" + + def __init__(self, message: str, *, attempts: int = 0) -> None: + super().__init__(message) + self.attempts = attempts + + +class ParseError(AskError): + """Failed to parse LLM output into structured data.""" + + def __init__(self, message: str, *, raw_output: str = "") -> None: + super().__init__(message) + self.raw_output = raw_output + + +class BudgetExceededError(AskError): + """Token or call budget exceeded.""" + + +class NavigationError(AskError): + """Document navigation failure (load, cd, cat, etc.).""" + + +class VerificationError(AskError): + """Verification pipeline failure.""" diff --git a/vectorless/ask/orchestrator.py b/vectorless/ask/orchestrator.py index 7d6b8b2..8172e2f 100644 --- a/vectorless/ask/orchestrator.py +++ b/vectorless/ask/orchestrator.py @@ -19,11 +19,11 @@ import asyncio import logging -import re from dataclasses import dataclass from typing import Any from vectorless.ask.protocols import DocLoader, EventCallback +from vectorless.ask.utils import extract_keywords, format_evidence from vectorless.ask.types import ( DispatchEntry, DocCard, @@ -226,7 +226,7 @@ async def _analyze( doc_cards_text = _format_doc_cards(cards) # Cross-document keyword search - keywords = _extract_keywords(query) + keywords = extract_keywords(query) find_text = await self._cross_doc_find(cards, keywords) # Build analysis prompt with query understanding context @@ -376,7 +376,7 @@ async def _supervisor_loop( # Re-analyze with gap context if self._query_analysis and verification_result.gaps: - evidence_summary = _format_evidence_context(state.all_evidence) + evidence_summary = format_evidence(state.all_evidence) analyzer = QueryAnalyzer() try: self._query_analysis = await analyzer.re_analyze( @@ -600,7 +600,7 @@ async def _replan( blackboard: SharedBlackboard | None = None, ) -> list[DispatchEntry]: """Replan dispatch targets based on missing information.""" - evidence_summary = _format_evidence_context(state.all_evidence) + evidence_summary = format_evidence(state.all_evidence) doc_cards_text = _format_doc_cards(cards) # Include blackboard context in replan @@ -707,19 +707,6 @@ def _compute_confidence( return max(0.1, 0.4 - replan_rounds * 0.1) -def _extract_keywords(query: str) -> list[str]: - """Extract simple keywords from a query.""" - stop_words = { - "what", "is", "the", "a", "an", "how", "does", "do", "are", - "in", "on", "at", "to", "for", "of", "with", "and", "or", - "this", "that", "it", "from", "by", "was", "were", "be", - "can", "could", "would", "should", "will", "has", "have", - "had", "not", "but", "if", "then", "than", "so", "as", - } - words = re.findall(r"\b\w+\b", query.lower()) - return [w for w in words if w not in stop_words and len(w) > 2] - - def _format_doc_cards(cards: list[DocCard]) -> str: """Format document cards for the analysis prompt.""" lines = [] @@ -730,13 +717,3 @@ def _format_doc_cards(cards: list[DocCard]) -> str: f"({card.section_count} sections){concepts}" ) return "\n".join(lines) - - -def _format_evidence_context(evidence: list[Evidence]) -> str: - """Format collected evidence for the replan prompt.""" - if not evidence: - return "(no evidence collected)" - return "\n\n".join( - f"[{e.node_title}] (from {e.doc_name or 'unknown'})\n{e.content}" - for e in evidence - ) diff --git a/vectorless/ask/reasoning/analyzer.py b/vectorless/ask/reasoning/analyzer.py index 0d1cb41..e51b93e 100644 --- a/vectorless/ask/reasoning/analyzer.py +++ b/vectorless/ask/reasoning/analyzer.py @@ -13,7 +13,7 @@ import re from vectorless.llm_client import LLMClient -from vectorless.ask.utils import parse_json_response +from vectorless.ask.utils import extract_keywords, parse_json_response from vectorless.ask.reasoning.types import ( Ambiguity, AmbiguityType, @@ -35,23 +35,6 @@ logger = logging.getLogger(__name__) -def _extract_keywords(query: str) -> list[str]: - """Extract keywords from query using stop word filtering.""" - stop_words = { - "what", "is", "the", "a", "an", "how", "does", "do", "are", - "in", "on", "at", "to", "for", "of", "with", "and", "or", - "this", "that", "it", "from", "by", "was", "were", "be", - "can", "could", "would", "should", "will", "has", "have", - "had", "not", "but", "if", "then", "than", "so", "as", - "there", "their", "they", "its", "about", "which", "when", - "who", "whom", "all", "each", "every", "both", "few", - "more", "most", "other", "some", "such", "no", "nor", - "only", "own", "same", "too", "very", "just", "because", - } - words = re.findall(r"\b\w+\b", query.lower()) - return list(dict.fromkeys(w for w in words if w not in stop_words and len(w) > 2)) - - def _parse_intent(raw: str) -> QueryIntent: """Parse intent string to QueryIntent enum.""" mapping = { @@ -94,7 +77,7 @@ async def analyze(self, query: str, llm: LLMClient) -> QueryAnalysis: Raises on LLM failure — no silent degradation. """ - keywords = _extract_keywords(query) + keywords = extract_keywords(query) # Stage 1: Classify + Decompose system, user = stage1_classify_prompt(query, keywords) diff --git a/vectorless/ask/utils.py b/vectorless/ask/utils.py index 269ccba..0c929fb 100644 --- a/vectorless/ask/utils.py +++ b/vectorless/ask/utils.py @@ -1,10 +1,64 @@ -"""Shared utilities for the ask pipeline.""" +"""Shared utilities for the ask pipeline. + +Single source of truth for keyword extraction, evidence formatting, +and JSON parsing. All modules should import from here instead of +defining their own copies. +""" from __future__ import annotations import json import re +from vectorless.ask.types import Evidence + + +# --------------------------------------------------------------------------- +# Keyword extraction — unified stop word list +# --------------------------------------------------------------------------- + +_STOP_WORDS = frozenset({ + "what", "is", "the", "a", "an", "how", "does", "do", "are", + "in", "on", "at", "to", "for", "of", "with", "and", "or", + "this", "that", "it", "from", "by", "was", "were", "be", + "can", "could", "would", "should", "will", "has", "have", + "had", "not", "but", "if", "then", "than", "so", "as", + "there", "their", "they", "its", "about", "which", "when", + "who", "whom", "all", "each", "every", "both", "few", + "more", "most", "other", "some", "such", "no", "nor", + "only", "own", "same", "too", "very", "just", "because", +}) + + +def extract_keywords(query: str) -> list[str]: + """Extract keywords from a query using stop word filtering. + + Returns deduplicated keywords in order of first appearance. + """ + words = re.findall(r"\b\w+\b", query.lower()) + return list(dict.fromkeys(w for w in words if w not in _STOP_WORDS and len(w) > 2)) + + +# --------------------------------------------------------------------------- +# Evidence formatting — single source of truth +# --------------------------------------------------------------------------- + +def format_evidence(evidence: list[Evidence]) -> str: + """Format evidence with source attribution. + + Used by orchestrator, verifier, and evaluation modules. + """ + if not evidence: + return "(no evidence)" + return "\n\n".join( + f"[{e.node_title}] (from {e.doc_name or 'unknown'})\n{e.content}" + for e in evidence + ) + + +# --------------------------------------------------------------------------- +# JSON parsing — single source of truth +# --------------------------------------------------------------------------- def parse_json_response(response: str) -> dict: """Parse LLM response as JSON, handling markdown-wrapped output. diff --git a/vectorless/ask/verify/verifier.py b/vectorless/ask/verify/verifier.py index a37d0f4..7d71b6d 100644 --- a/vectorless/ask/verify/verifier.py +++ b/vectorless/ask/verify/verifier.py @@ -11,7 +11,7 @@ from vectorless.llm_client import LLMClient from vectorless.ask.types import Evidence -from vectorless.ask.utils import parse_json_response +from vectorless.ask.utils import format_evidence, parse_json_response from vectorless.ask.verify.types import ( DimensionScore, VerificationDimension, @@ -65,16 +65,6 @@ } -def _format_evidence(evidence: list[Evidence]) -> str: - """Format evidence for the verification prompt.""" - if not evidence: - return "(no evidence)" - return "\n\n".join( - f"[{e.node_title}] (from {e.doc_name or 'unknown'})\n{e.content}" - for e in evidence - ) - - class VerifyPipeline: """Multi-dimensional evidence verification pipeline. @@ -103,7 +93,7 @@ async def verify( Single combined LLM call assessing all 4 dimensions. Returns VerificationResult with pass/fail, scores, and gaps. """ - evidence_text = _format_evidence(evidence) + evidence_text = format_evidence(evidence) if not evidence: return VerificationResult( diff --git a/vectorless/ask/worker.py b/vectorless/ask/worker.py index 18085c2..f249abd 100644 --- a/vectorless/ask/worker.py +++ b/vectorless/ask/worker.py @@ -14,6 +14,7 @@ from typing import Any from vectorless.ask.types import TraceStep, Evidence, WorkerOutput, WorkerState +from vectorless.ask.utils import extract_keywords from vectorless.llm_client import LLMClient from vectorless.ask.tools import compare_nodes, summarize_section, trace_reasoning from vectorless.ask.prompts import ( @@ -999,14 +1000,7 @@ async def run(self) -> WorkerOutput: async def _build_keyword_hints(self, doc: Any, query: str) -> str: """Build keyword hints from the document's reasoning index.""" - # Extract simple keywords from the query - stop_words = { - "what", "is", "the", "a", "an", "how", "does", "do", "are", - "in", "on", "at", "to", "for", "of", "with", "and", "or", - "this", "that", "it", "from", "by", "was", "were", "be", - } - words = re.findall(r"\b\w+\b", query.lower()) - keywords = [w for w in words if w not in stop_words and len(w) > 2] + keywords = extract_keywords(query) if not keywords: return "" diff --git a/vectorless/llm_client.py b/vectorless/llm_client.py index 4b99b8d..66aaa2b 100644 --- a/vectorless/llm_client.py +++ b/vectorless/llm_client.py @@ -8,7 +8,8 @@ - Structured JSON output via instructor + Pydantic - Automatic retry with feedback on validation failure - Per-request timeout -- In-memory response cache (optional, per-session dedup) +- LRU response cache (bounded, per-session dedup) +- Per-call api_base (no global state mutation) """ from __future__ import annotations @@ -16,11 +17,15 @@ import hashlib import json import logging -from typing import Any, Optional, Type, TypeVar +from collections import OrderedDict +from typing import Any, Type, TypeVar import litellm from pydantic import BaseModel +from vectorless.ask.errors import LLMFailureError, ParseError +from vectorless.ask.utils import parse_json_response + logger = logging.getLogger(__name__) T = TypeVar("T", bound=BaseModel) @@ -31,6 +36,36 @@ DEFAULT_MAX_RETRIES = 2 DEFAULT_TIMEOUT = 120.0 # seconds +DEFAULT_CACHE_SIZE = 256 + +# --------------------------------------------------------------------------- +# LRU Cache +# --------------------------------------------------------------------------- + + +class _LRUCache: + """Bounded LRU cache for LLM response dedup.""" + + def __init__(self, max_size: int = DEFAULT_CACHE_SIZE) -> None: + self._cache: OrderedDict[str, str] = OrderedDict() + self._max_size = max_size + + def get(self, key: str) -> str | None: + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key] + return None + + def put(self, key: str, value: str) -> None: + if key in self._cache: + self._cache.move_to_end(key) + self._cache[key] = value + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + def clear(self) -> None: + self._cache.clear() + # --------------------------------------------------------------------------- # LLMClient @@ -60,24 +95,21 @@ def __init__( self, api_key: str, model: str, - endpoint: Optional[str] = None, + endpoint: str | None = None, *, max_retries: int = DEFAULT_MAX_RETRIES, timeout: float = DEFAULT_TIMEOUT, enable_cache: bool = True, + cache_size: int = DEFAULT_CACHE_SIZE, ) -> None: self._model = model self._api_key = api_key self._endpoint = endpoint self._max_retries = max_retries self._timeout = timeout - self._cache: dict[str, str] = {} if enable_cache else {} + self._cache = _LRUCache(cache_size) if enable_cache else None self._cache_enabled = enable_cache - # Configure litellm defaults - if endpoint: - litellm.api_base = endpoint - @property def model(self) -> str: return self._model @@ -90,7 +122,7 @@ async def complete( user: str, *, temperature: float = 0.0, - timeout: Optional[float] = None, + timeout: float | None = None, ) -> str: """Send a completion request and return the assistant message text. @@ -104,8 +136,10 @@ async def complete( The assistant's text response. """ cache_key = self._cache_key(system, user, temperature) - if self._cache_enabled and cache_key in self._cache: - return self._cache[cache_key] + if self._cache_enabled and self._cache is not None: + cached = self._cache.get(cache_key) + if cached is not None: + return cached response = await self._call_with_retry( system=system, @@ -114,8 +148,8 @@ async def complete( timeout=timeout or self._timeout, ) - if self._cache_enabled: - self._cache[cache_key] = response + if self._cache_enabled and self._cache is not None: + self._cache.put(cache_key, response) return response @@ -125,14 +159,14 @@ async def complete_json( user: str, *, temperature: float = 0.0, - timeout: Optional[float] = None, + timeout: float | None = None, ) -> dict[str, Any]: - """Send a completion request and parse the response as JSON. - - Falls back to regex extraction if the response is not valid JSON. - """ + """Send a completion request and parse the response as JSON.""" text = await self.complete(system, user, temperature=temperature, timeout=timeout) - return _extract_json(text) + try: + return parse_json_response(text) + except ValueError as e: + raise ParseError(str(e), raw_output=text) from e async def complete_structured( self, @@ -140,9 +174,9 @@ async def complete_structured( user: str, response_model: Type[T], *, - max_retries: Optional[int] = None, + max_retries: int | None = None, temperature: float = 0.0, - timeout: Optional[float] = None, + timeout: float | None = None, ) -> T: """Send a completion request with structured output via instructor. @@ -186,7 +220,7 @@ async def complete_with_messages( messages: list[dict[str, str]], *, temperature: float = 0.0, - timeout: Optional[float] = None, + timeout: float | None = None, ) -> str: """Send a completion request with pre-built messages.""" response = await litellm.acompletion( @@ -209,12 +243,14 @@ async def _call_with_retry( timeout: float, ) -> str: """Call litellm.acompletion with retry on transient errors.""" + import asyncio + messages = [ {"role": "system", "content": system}, {"role": "user", "content": user}, ] - last_error: Optional[Exception] = None + last_error: Exception | None = None for attempt in range(1 + self._max_retries): try: response = await litellm.acompletion( @@ -230,16 +266,22 @@ async def _call_with_retry( last_error = e logger.warning("LLM rate limit hit, attempt %d/%d: %s", attempt + 1, self._max_retries + 1, e) if attempt < self._max_retries: - import asyncio await asyncio.sleep(2 ** attempt) except litellm.Timeout as e: last_error = e logger.warning("LLM timeout, attempt %d/%d", attempt + 1, self._max_retries + 1) + if attempt < self._max_retries: + await asyncio.sleep(1.0) except litellm.APIConnectionError as e: last_error = e logger.warning("LLM connection error, attempt %d/%d: %s", attempt + 1, self._max_retries + 1, e) + if attempt < self._max_retries: + await asyncio.sleep(1.0) - raise LLMError(f"LLM call failed after {self._max_retries + 1} attempts: {last_error}") from last_error + raise LLMFailureError( + f"LLM call failed after {self._max_retries + 1} attempts: {last_error}", + attempts=self._max_retries + 1, + ) from last_error def _cache_key(self, system: str, user: str, temperature: float) -> str: raw = f"{self._model}:{temperature}:{system}|||{user}" @@ -247,51 +289,5 @@ def _cache_key(self, system: str, user: str, temperature: float) -> str: def clear_cache(self) -> None: """Clear the in-memory response cache.""" - if self._cache_enabled: + if self._cache is not None: self._cache.clear() - - -# --------------------------------------------------------------------------- -# Exceptions -# --------------------------------------------------------------------------- - - -class LLMError(Exception): - """Raised when an LLM call fails after all retries.""" - - -# --------------------------------------------------------------------------- -# JSON extraction fallback -# --------------------------------------------------------------------------- - - -def _extract_json(text: str) -> dict[str, Any]: - """Extract a JSON object from LLM output. - - Handles: - - Plain JSON - - JSON wrapped in ```json ... ``` code blocks - - JSON with leading/trailing text - """ - import re - - match = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL) - if match: - text = match.group(1).strip() - - start = text.find("{") - if start != -1: - depth = 0 - for i in range(start, len(text)): - if text[i] == "{": - depth += 1 - elif text[i] == "}": - depth -= 1 - if depth == 0: - candidate = text[start : i + 1] - try: - return json.loads(candidate) - except json.JSONDecodeError: - break - - return json.loads(text.strip()) From f1f10cc1919ab4061080bb4499ea2965714e2b54 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 19:35:45 +0800 Subject: [PATCH 16/30] feat(compiler): rename index module to compiler with updated types - rename "vectorless-index" crate to "vectorless-compiler" - update IndexMode enum to SourceFormat - rename IndexInput to CompilerInput and PipelineResult to CompileResult - update IndexContext to CompileContext and related stage names - rename IndexStage trait to CompileStage across all modules - update documentation to reflect document compilation instead of indexing --- Cargo.toml | 2 +- .../Cargo.toml | 2 +- .../src/config.rs | 20 +- .../src/incremental/detector.rs | 0 .../src/incremental/mod.rs | 0 .../src/incremental/resolver.rs | 0 .../src/incremental/updater.rs | 0 .../src/lib.rs | 25 +- .../src/parse/markdown/config.rs | 0 .../src/parse/markdown/frontmatter.rs | 0 .../src/parse/markdown/mod.rs | 0 .../src/parse/markdown/parser.rs | 0 .../src/parse/mod.rs | 0 .../src/parse/pdf/mod.rs | 0 .../src/parse/pdf/parser.rs | 0 .../src/parse/pdf/types.rs | 0 .../src/parse/toc/assigner.rs | 0 .../src/parse/toc/detector.rs | 0 .../src/parse/toc/mod.rs | 0 .../src/parse/toc/parser.rs | 0 .../src/parse/toc/processor.rs | 0 .../src/parse/toc/repairer.rs | 0 .../src/parse/toc/structure_extractor.rs | 0 .../src/parse/toc/types.rs | 0 .../src/parse/toc/verifier.rs | 0 .../src/parse/types.rs | 0 .../src/pipeline/checkpoint.rs | 0 .../src/pipeline/context.rs | 32 +- .../src/pipeline/executor.rs | 24 +- .../src/pipeline/metrics.rs | 0 .../src/pipeline/mod.rs | 4 +- .../src/pipeline/orchestrator.rs | 48 +- .../src/pipeline/policy.rs | 0 .../src/stages/build.rs | 10 +- .../src/stages/concept.rs | 14 +- .../src/stages/enhance.rs | 10 +- .../src/stages/enrich.rs | 10 +- .../src/stages/mod.rs | 14 +- .../src/stages/navigation.rs | 64 +- .../src/stages/optimize.rs | 16 +- .../src/stages/parse.rs | 36 +- .../src/stages/reasoning.rs | 32 +- .../src/stages/split.rs | 8 +- .../src/stages/validate.rs | 20 +- .../src/stages/verify_ingest.rs | 8 +- .../src/summary/full.rs | 0 .../src/summary/lazy.rs | 0 .../src/summary/mod.rs | 0 .../src/summary/selective.rs | 0 .../src/summary/strategy.rs | 0 vectorless-core/vectorless-engine/Cargo.toml | 2 +- .../{index_context.rs => compile_input.rs} | 92 +- .../vectorless-engine/src/engine.rs | 120 +- .../vectorless-engine/src/indexer.rs | 36 +- vectorless-core/vectorless-engine/src/lib.rs | 14 +- .../vectorless-engine/src/types.rs | 62 +- vectorless-core/vectorless-py/src/document.rs | 2 +- vectorless-core/vectorless-py/src/engine.rs | 23 +- vectorless/ask/__init__.py | 13 + vectorless/ask/blackboard.py | 3 + vectorless/ask/dispatcher.py | 9 +- vectorless/ask/events.py | 24 + vectorless/ask/orchestrator.py | 145 ++- vectorless/ask/protocols.py | 135 +- vectorless/ask/worker.py | 1114 ----------------- vectorless/ask/worker/__init__.py | 6 + vectorless/ask/worker/agent.py | 327 +++++ vectorless/ask/worker/commands.py | 779 ++++++++++++ vectorless/ask/worker/parse.py | 148 +++ vectorless/engine.py | 28 +- vectorless/types/__init__.py | 8 +- vectorless/types/results.py | 26 +- 72 files changed, 1965 insertions(+), 1550 deletions(-) rename vectorless-core/{vectorless-index => vectorless-compiler}/Cargo.toml (97%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/config.rs (96%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/incremental/detector.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/incremental/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/incremental/resolver.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/incremental/updater.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/lib.rs (76%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/markdown/config.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/markdown/frontmatter.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/markdown/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/markdown/parser.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/pdf/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/pdf/parser.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/pdf/types.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/assigner.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/detector.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/parser.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/processor.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/repairer.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/structure_extractor.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/types.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/toc/verifier.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/parse/types.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/checkpoint.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/context.rs (94%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/executor.rs (90%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/metrics.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/mod.rs (83%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/orchestrator.rs (96%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/pipeline/policy.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/build.rs (97%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/concept.rs (93%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/enhance.rs (98%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/enrich.rs (96%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/mod.rs (90%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/navigation.rs (90%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/optimize.rs (97%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/parse.rs (81%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/reasoning.rs (95%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/split.rs (97%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/validate.rs (94%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/stages/verify_ingest.rs (90%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/summary/full.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/summary/lazy.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/summary/mod.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/summary/selective.rs (100%) rename vectorless-core/{vectorless-index => vectorless-compiler}/src/summary/strategy.rs (100%) rename vectorless-core/vectorless-engine/src/{index_context.rs => compile_input.rs} (73%) create mode 100644 vectorless/ask/events.py delete mode 100644 vectorless/ask/worker.py create mode 100644 vectorless/ask/worker/__init__.py create mode 100644 vectorless/ask/worker/agent.py create mode 100644 vectorless/ask/worker/commands.py create mode 100644 vectorless/ask/worker/parse.py diff --git a/Cargo.toml b/Cargo.toml index ed68677..b895873 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ # "vectorless-core/vectorless-agent", # "vectorless-core/vectorless-retrieval", # "vectorless-core/vectorless-rerank", - "vectorless-core/vectorless-index", + "vectorless-core/vectorless-compiler", "vectorless-core/vectorless-primitives", "vectorless-core/vectorless-engine", "vectorless-core/vectorless-py", diff --git a/vectorless-core/vectorless-index/Cargo.toml b/vectorless-core/vectorless-compiler/Cargo.toml similarity index 97% rename from vectorless-core/vectorless-index/Cargo.toml rename to vectorless-core/vectorless-compiler/Cargo.toml index 587b54b..7bc8fcd 100644 --- a/vectorless-core/vectorless-index/Cargo.toml +++ b/vectorless-core/vectorless-compiler/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "vectorless-index" +name = "vectorless-compiler" version.workspace = true edition.workspace = true authors.workspace = true diff --git a/vectorless-core/vectorless-index/src/config.rs b/vectorless-core/vectorless-compiler/src/config.rs similarity index 96% rename from vectorless-core/vectorless-index/src/config.rs rename to vectorless-core/vectorless-compiler/src/config.rs index e9133c4..3bce23a 100644 --- a/vectorless-core/vectorless-index/src/config.rs +++ b/vectorless-core/vectorless-compiler/src/config.rs @@ -4,7 +4,7 @@ //! Configuration types for the index pipeline. //! //! This module contains all configuration types used by the indexing pipeline: -//! - [`IndexMode`] - Document format selection +//! - [`SourceFormat`] - Document format selection //! - [`PipelineOptions`] - Full pipeline configuration //! - [`OptimizationConfig`] - Tree optimization settings //! - [`ThinningConfig`] - Node merging settings @@ -19,7 +19,7 @@ use std::path::PathBuf; /// Index mode for document processing. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IndexMode { +pub enum SourceFormat { /// Auto-detect format from file extension. Auto, /// Force Markdown format. @@ -28,7 +28,7 @@ pub enum IndexMode { Pdf, } -impl Default for IndexMode { +impl Default for SourceFormat { fn default() -> Self { Self::Auto } @@ -191,7 +191,7 @@ impl SplitConfig { #[derive(Debug, Clone)] pub struct PipelineOptions { /// Index mode. - pub mode: IndexMode, + pub mode: SourceFormat, /// Whether to generate node IDs. pub generate_ids: bool, @@ -238,7 +238,7 @@ pub struct PipelineOptions { impl Default for PipelineOptions { fn default() -> Self { Self { - mode: IndexMode::Auto, + mode: SourceFormat::Auto, generate_ids: true, summary_strategy: SummaryStrategy::full(), thinning: ThinningConfig::default(), @@ -262,7 +262,7 @@ impl PipelineOptions { } /// Set the index mode. - pub fn with_mode(mut self, mode: IndexMode) -> Self { + pub fn with_mode(mut self, mode: SourceFormat) -> Self { self.mode = mode; self } @@ -352,8 +352,8 @@ mod tests { #[test] fn test_index_mode_default() { - let mode = IndexMode::default(); - assert_eq!(mode, IndexMode::Auto); + let mode = SourceFormat::default(); + assert_eq!(mode, SourceFormat::Auto); } #[test] @@ -380,10 +380,10 @@ mod tests { #[test] fn test_pipeline_options_builder() { let options = PipelineOptions::new() - .with_mode(IndexMode::Markdown) + .with_mode(SourceFormat::Markdown) .with_generate_ids(false); - assert_eq!(options.mode, IndexMode::Markdown); + assert_eq!(options.mode, SourceFormat::Markdown); assert!(!options.generate_ids); } } diff --git a/vectorless-core/vectorless-index/src/incremental/detector.rs b/vectorless-core/vectorless-compiler/src/incremental/detector.rs similarity index 100% rename from vectorless-core/vectorless-index/src/incremental/detector.rs rename to vectorless-core/vectorless-compiler/src/incremental/detector.rs diff --git a/vectorless-core/vectorless-index/src/incremental/mod.rs b/vectorless-core/vectorless-compiler/src/incremental/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/incremental/mod.rs rename to vectorless-core/vectorless-compiler/src/incremental/mod.rs diff --git a/vectorless-core/vectorless-index/src/incremental/resolver.rs b/vectorless-core/vectorless-compiler/src/incremental/resolver.rs similarity index 100% rename from vectorless-core/vectorless-index/src/incremental/resolver.rs rename to vectorless-core/vectorless-compiler/src/incremental/resolver.rs diff --git a/vectorless-core/vectorless-index/src/incremental/updater.rs b/vectorless-core/vectorless-compiler/src/incremental/updater.rs similarity index 100% rename from vectorless-core/vectorless-index/src/incremental/updater.rs rename to vectorless-core/vectorless-compiler/src/incremental/updater.rs diff --git a/vectorless-core/vectorless-index/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs similarity index 76% rename from vectorless-core/vectorless-index/src/lib.rs rename to vectorless-core/vectorless-compiler/src/lib.rs index 048158c..41ce84d 100644 --- a/vectorless-core/vectorless-index/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -1,11 +1,11 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Index Pipeline module. +//! Document Compiler module. //! -//! This module provides a modular, extensible document indexing pipeline. +//! This module provides a modular document compilation pipeline that transforms +//! documents (Markdown, PDF) into agent-friendly intermediate artifacts. //! -//! # Architecture //! //! ```text //! Priority 10: ┌──────────┐ @@ -39,21 +39,6 @@ //! //! Checkpointing is available when `PipelineOptions::checkpoint_dir` is set. //! State is saved after each stage group and resumed on restart. -//! -//! # Usage -//! -//! ```rust,ignore -//! use vectorless::index::{PipelineExecutor, IndexInput, PipelineOptions}; -//! use vectorless::index::summary::SummaryStrategy; -//! -//! let options = PipelineOptions::new() -//! .with_summary_strategy(SummaryStrategy::selective(100, true)); -//! -//! let result = PipelineExecutor::new() -//! .with_options(options) -//! .execute(input) -//! .await?; -//! ``` pub mod config; pub mod incremental; @@ -63,10 +48,10 @@ pub mod stages; pub mod summary; // Re-export main types from pipeline -pub use pipeline::{IndexInput, IndexMetrics, PipelineExecutor, PipelineResult}; +pub use pipeline::{CompilerInput, IndexMetrics, PipelineExecutor, CompileResult}; // Re-export config types -pub use config::{IndexMode, PipelineOptions, ThinningConfig}; +pub use config::{SourceFormat, PipelineOptions, ThinningConfig}; pub use vectorless_document::ReasoningIndexConfig; // Re-export summary diff --git a/vectorless-core/vectorless-index/src/parse/markdown/config.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/config.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/markdown/config.rs rename to vectorless-core/vectorless-compiler/src/parse/markdown/config.rs diff --git a/vectorless-core/vectorless-index/src/parse/markdown/frontmatter.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/frontmatter.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/markdown/frontmatter.rs rename to vectorless-core/vectorless-compiler/src/parse/markdown/frontmatter.rs diff --git a/vectorless-core/vectorless-index/src/parse/markdown/mod.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/markdown/mod.rs rename to vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs diff --git a/vectorless-core/vectorless-index/src/parse/markdown/parser.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/markdown/parser.rs rename to vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs diff --git a/vectorless-core/vectorless-index/src/parse/mod.rs b/vectorless-core/vectorless-compiler/src/parse/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/mod.rs rename to vectorless-core/vectorless-compiler/src/parse/mod.rs diff --git a/vectorless-core/vectorless-index/src/parse/pdf/mod.rs b/vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/pdf/mod.rs rename to vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs diff --git a/vectorless-core/vectorless-index/src/parse/pdf/parser.rs b/vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/pdf/parser.rs rename to vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs diff --git a/vectorless-core/vectorless-index/src/parse/pdf/types.rs b/vectorless-core/vectorless-compiler/src/parse/pdf/types.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/pdf/types.rs rename to vectorless-core/vectorless-compiler/src/parse/pdf/types.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/assigner.rs b/vectorless-core/vectorless-compiler/src/parse/toc/assigner.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/assigner.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/assigner.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/detector.rs b/vectorless-core/vectorless-compiler/src/parse/toc/detector.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/detector.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/detector.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/mod.rs b/vectorless-core/vectorless-compiler/src/parse/toc/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/mod.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/mod.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/parser.rs b/vectorless-core/vectorless-compiler/src/parse/toc/parser.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/parser.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/parser.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/processor.rs b/vectorless-core/vectorless-compiler/src/parse/toc/processor.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/processor.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/processor.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/repairer.rs b/vectorless-core/vectorless-compiler/src/parse/toc/repairer.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/repairer.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/repairer.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/structure_extractor.rs b/vectorless-core/vectorless-compiler/src/parse/toc/structure_extractor.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/structure_extractor.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/structure_extractor.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/types.rs b/vectorless-core/vectorless-compiler/src/parse/toc/types.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/types.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/types.rs diff --git a/vectorless-core/vectorless-index/src/parse/toc/verifier.rs b/vectorless-core/vectorless-compiler/src/parse/toc/verifier.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/toc/verifier.rs rename to vectorless-core/vectorless-compiler/src/parse/toc/verifier.rs diff --git a/vectorless-core/vectorless-index/src/parse/types.rs b/vectorless-core/vectorless-compiler/src/parse/types.rs similarity index 100% rename from vectorless-core/vectorless-index/src/parse/types.rs rename to vectorless-core/vectorless-compiler/src/parse/types.rs diff --git a/vectorless-core/vectorless-index/src/pipeline/checkpoint.rs b/vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs similarity index 100% rename from vectorless-core/vectorless-index/src/pipeline/checkpoint.rs rename to vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs diff --git a/vectorless-core/vectorless-index/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs similarity index 94% rename from vectorless-core/vectorless-index/src/pipeline/context.rs rename to vectorless-core/vectorless-compiler/src/pipeline/context.rs index 9bc0110..2ee4ad5 100644 --- a/vectorless-core/vectorless-index/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -15,7 +15,7 @@ use super::metrics::IndexMetrics; /// Input for the index pipeline. #[derive(Debug, Clone)] -pub enum IndexInput { +pub enum CompilerInput { /// Index from file path. File(PathBuf), @@ -40,7 +40,7 @@ pub enum IndexInput { }, } -impl IndexInput { +impl CompilerInput { /// Create input from file path. pub fn file(path: impl Into) -> Self { Self::File(path.into()) @@ -211,12 +211,12 @@ impl SummaryCache { /// Index context passed between stages. #[derive(Debug)] -pub struct IndexContext { +pub struct CompileContext { /// Document ID. pub doc_id: String, /// Source input. - pub input: IndexInput, + pub input: CompilerInput, /// Document format. pub format: DocumentFormat, @@ -245,10 +245,10 @@ pub struct IndexContext { /// Summary cache for lazy generation. pub summary_cache: SummaryCache, - /// Pre-computed reasoning index (built by ReasoningIndexStage). + /// Pre-computed reasoning index (built by ReasoningCompileStage). pub reasoning_index: Option, - /// Navigation index for Agent-based retrieval (built by NavigationIndexStage). + /// Navigation index for Agent-based retrieval (built by NavigationCompileStage). pub navigation_index: Option, /// Key concepts extracted from the document (built by ConceptExtractionStage). @@ -274,9 +274,9 @@ pub struct IndexContext { pub line_count: Option, } -impl IndexContext { +impl CompileContext { /// Create a new context from input. - pub fn new(input: IndexInput, options: PipelineOptions) -> Self { + pub fn new(input: CompilerInput, options: PipelineOptions) -> Self { let source_hash = Self::compute_source_hash(&input); Self { doc_id: uuid::Uuid::new_v4().to_string(), @@ -303,17 +303,17 @@ impl IndexContext { } /// Compute SHA-256 hash of the source content. - fn compute_source_hash(input: &IndexInput) -> String { + fn compute_source_hash(input: &CompilerInput) -> String { use sha2::{Digest, Sha256}; let hash = match input { - IndexInput::File(path) => { + CompilerInput::File(path) => { // Hash the file path as proxy — actual content may not be readable yet // (the parse stage reads it). This is sufficient for checkpoint invalidation // since a different file path implies different content. Sha256::digest(path.to_string_lossy().as_bytes()) } - IndexInput::Content { content, .. } => Sha256::digest(content.as_bytes()), - IndexInput::Bytes { data, .. } => Sha256::digest(data), + CompilerInput::Content { content, .. } => Sha256::digest(content.as_bytes()), + CompilerInput::Bytes { data, .. } => Sha256::digest(data), }; format!("{:x}", hash) } @@ -377,8 +377,8 @@ impl IndexContext { } /// Finalize and build the result. - pub fn finalize(self) -> PipelineResult { - PipelineResult { + pub fn finalize(self) -> CompileResult { + CompileResult { doc_id: self.doc_id, name: self.name, format: self.format, @@ -398,7 +398,7 @@ impl IndexContext { /// Final result from the index pipeline. #[derive(Debug)] -pub struct PipelineResult { +pub struct CompileResult { /// Document ID. pub doc_id: String, @@ -439,7 +439,7 @@ pub struct PipelineResult { pub concepts: Vec, } -impl PipelineResult { +impl CompileResult { /// Check if the result has a tree. pub fn has_tree(&self) -> bool { self.tree.is_some() diff --git a/vectorless-core/vectorless-index/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs similarity index 90% rename from vectorless-core/vectorless-index/src/pipeline/executor.rs rename to vectorless-core/vectorless-compiler/src/pipeline/executor.rs index 1695688..c60d0f0 100644 --- a/vectorless-core/vectorless-index/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -13,11 +13,11 @@ use vectorless_llm::LlmClient; use super::super::PipelineOptions; use super::super::stages::{ - BuildStage, ConceptExtractionStage, EnhanceStage, EnrichStage, IndexStage, - NavigationIndexStage, OptimizeStage, ParseStage, ReasoningIndexStage, SplitStage, + BuildStage, ConceptExtractionStage, EnhanceStage, EnrichStage, CompileStage, + NavigationCompileStage, OptimizeStage, ParseStage, ReasoningCompileStage, SplitStage, ValidateStage, VerifyStage, }; -use super::context::{IndexInput, PipelineResult}; +use super::context::{CompilerInput, CompileResult}; use super::orchestrator::PipelineOrchestrator; /// Pipeline executor for document indexing. @@ -67,9 +67,9 @@ impl PipelineExecutor { .stage_with_priority(ValidateStage::new(), 22) .stage_with_priority(SplitStage::new(), 25) .stage_with_priority(EnrichStage::new(), 40) - .stage_with_priority(ReasoningIndexStage::new(), 45) + .stage_with_priority(ReasoningCompileStage::new(), 45) .stage_with_priority(ConceptExtractionStage::new(), 47) - .stage_with_priority(NavigationIndexStage::new(), 50) + .stage_with_priority(NavigationCompileStage::new(), 50) .stage_with_priority(VerifyStage, 55) .stage_with_priority(OptimizeStage::new(), 60); @@ -102,9 +102,9 @@ impl PipelineExecutor { .stage_with_priority(SplitStage::new(), 25) .stage_with_priority(EnhanceStage::with_llm_client(client.clone()), 30) .stage_with_priority(EnrichStage::new(), 40) - .stage_with_priority(ReasoningIndexStage::new(), 45) + .stage_with_priority(ReasoningCompileStage::new(), 45) .stage_with_priority(ConceptExtractionStage::with_llm_client(client), 47) - .stage_with_priority(NavigationIndexStage::new(), 50) + .stage_with_priority(NavigationCompileStage::new(), 50) .stage_with_priority(VerifyStage, 55) .stage_with_priority(OptimizeStage::new(), 60); @@ -133,7 +133,7 @@ impl PipelineExecutor { /// Add a stage with default priority. /// /// The stage will be added after existing stages with the same priority. - pub fn add_stage(mut self, stage: impl IndexStage + 'static) -> Self { + pub fn add_stage(mut self, stage: impl CompileStage + 'static) -> Self { self.orchestrator = self.orchestrator.stage(stage); self } @@ -143,7 +143,7 @@ impl PipelineExecutor { /// Lower priority = earlier execution. pub fn add_stage_with_priority( mut self, - stage: impl IndexStage + 'static, + stage: impl CompileStage + 'static, priority: i32, ) -> Self { self.orchestrator = self.orchestrator.stage_with_priority(stage, priority); @@ -155,7 +155,7 @@ impl PipelineExecutor { /// The stage will run after all specified dependencies. pub fn add_stage_with_deps( mut self, - stage: impl IndexStage + 'static, + stage: impl CompileStage + 'static, priority: i32, depends_on: &[&str], ) -> Self { @@ -180,9 +180,9 @@ impl PipelineExecutor { /// Stages are executed in dependency-resolved order. pub async fn execute( &mut self, - input: IndexInput, + input: CompilerInput, options: PipelineOptions, - ) -> Result { + ) -> Result { info!( "Starting index pipeline with {} stages", self.orchestrator.stage_count() diff --git a/vectorless-core/vectorless-index/src/pipeline/metrics.rs b/vectorless-core/vectorless-compiler/src/pipeline/metrics.rs similarity index 100% rename from vectorless-core/vectorless-index/src/pipeline/metrics.rs rename to vectorless-core/vectorless-compiler/src/pipeline/metrics.rs diff --git a/vectorless-core/vectorless-index/src/pipeline/mod.rs b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs similarity index 83% rename from vectorless-core/vectorless-index/src/pipeline/mod.rs rename to vectorless-core/vectorless-compiler/src/pipeline/mod.rs index e6e3752..31b7035 100644 --- a/vectorless-core/vectorless-index/src/pipeline/mod.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs @@ -4,7 +4,7 @@ //! Pipeline execution module. //! //! This module provides the core pipeline infrastructure: -//! - [`IndexContext`] - Context passed between stages +//! - [`CompileContext`] - Context passed between stages //! - [`PipelineExecutor`] - Executes the indexing pipeline //! - [`PipelineOrchestrator`] - Flexible stage orchestration with dependencies //! - [`IndexMetrics`] - Performance metrics collection @@ -18,7 +18,7 @@ mod metrics; mod orchestrator; mod policy; -pub use context::{IndexContext, IndexInput, PipelineResult, StageResult}; +pub use context::{CompileContext, CompilerInput, CompileResult, StageResult}; pub use executor::PipelineExecutor; pub use metrics::IndexMetrics; pub use policy::{FailurePolicy, StageRetryConfig}; diff --git a/vectorless-core/vectorless-index/src/pipeline/orchestrator.rs b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs similarity index 96% rename from vectorless-core/vectorless-index/src/pipeline/orchestrator.rs rename to vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs index 9421d2c..c3c607d 100644 --- a/vectorless-core/vectorless-index/src/pipeline/orchestrator.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs @@ -30,15 +30,15 @@ use tracing::{debug, error, info, warn}; use vectorless_error::Result; use super::super::PipelineOptions; -use super::super::stages::IndexStage; +use super::super::stages::CompileStage; use super::checkpoint::{CheckpointContextData, CheckpointManager, PipelineCheckpoint}; -use super::context::{IndexContext, IndexInput, PipelineResult, StageResult}; +use super::context::{CompileContext, CompilerInput, CompileResult, StageResult}; use super::policy::FailurePolicy; /// Stage entry with metadata for orchestration. struct StageEntry { /// The stage implementation. - stage: Box, + stage: Box, /// Priority (lower = earlier execution). priority: i32, /// Names of stages this depends on. @@ -124,7 +124,7 @@ impl PipelineOrchestrator { /// Dependencies are automatically read from the stage's `depends_on()` method. pub fn stage(mut self, stage: S) -> Self where - S: IndexStage + 'static, + S: CompileStage + 'static, { let deps = stage.depends_on(); self.stages.push(StageEntry { @@ -142,7 +142,7 @@ impl PipelineOrchestrator { /// Default priority is 100. pub fn stage_with_priority(mut self, stage: S, priority: i32) -> Self where - S: IndexStage + 'static, + S: CompileStage + 'static, { let deps = stage.depends_on(); self.stages.push(StageEntry { @@ -164,7 +164,7 @@ impl PipelineOrchestrator { explicit_depends_on: &[&str], ) -> Self where - S: IndexStage + 'static, + S: CompileStage + 'static, { let trait_deps = stage.depends_on(); let mut all_deps: Vec = trait_deps.into_iter().map(|s| s.to_string()).collect(); @@ -346,8 +346,8 @@ impl PipelineOrchestrator { /// Execute a stage with its failure policy applied. async fn execute_stage_with_policy( - stage: &mut Box, - ctx: &mut IndexContext, + stage: &mut Box, + ctx: &mut CompileContext, ) -> Result { let policy = stage.failure_policy(); let stage_name = stage.name().to_string(); @@ -406,7 +406,7 @@ impl PipelineOrchestrator { result: Result, stage_name: &str, policy: &FailurePolicy, - ctx: &mut IndexContext, + ctx: &mut CompileContext, ) -> Result<()> { match result { Ok(result) => { @@ -438,9 +438,9 @@ impl PipelineOrchestrator { /// Failure policies are applied per-stage. pub async fn execute( &mut self, - input: IndexInput, + input: CompilerInput, options: PipelineOptions, - ) -> Result { + ) -> Result { let total_start = Instant::now(); info!( "Starting orchestrated pipeline with {} stages", @@ -471,8 +471,8 @@ impl PipelineOrchestrator { // Create context let mut opts = options; let existing_tree = opts.existing_tree.take(); - let mut ctx = IndexContext::new(input, opts); - // Inject shared LLM client into context for stages that need it (e.g. ReasoningIndexStage) + let mut ctx = CompileContext::new(input, opts); + // Inject shared LLM client into context for stages that need it (e.g. ReasoningCompileStage) if let Some(client) = self.llm_client.take() { ctx = ctx.with_llm_client(client); } @@ -556,7 +556,7 @@ impl PipelineOrchestrator { .copied(); // For each stage, prepare (stage, context) pair. - // Swap out stages from self.stages to get owned Box. + // Swap out stages from self.stages to get owned Box. let mut entries: Vec = Vec::with_capacity(group.stage_indices.len()); for &idx in &group.stage_indices { @@ -571,7 +571,7 @@ impl PipelineOrchestrator { } else { // Reader gets a cloned context let mut clone = - IndexContext::new(IndexInput::content(""), ctx.options.clone()); + CompileContext::new(CompilerInput::content(""), ctx.options.clone()); clone.tree = ctx.tree.clone(); clone.existing_tree = ctx.existing_tree.clone(); clone.doc_id = ctx.doc_id.clone(); @@ -610,7 +610,7 @@ impl PipelineOrchestrator { // Execute writer on main ctx concurrently with readers. // Move each reader's stage+ctx into an owned async block. - // All futures are !Send (Box), but join_all + // All futures are !Send (Box), but join_all // works fine on the same thread. let reader_futs: Vec< @@ -740,7 +740,7 @@ impl PipelineOrchestrator { /// /// Reads the reader's AccessPattern to know which fields to copy, /// and merges additive metrics (LLM calls, tokens, etc.). - fn merge_reader_outputs(ctx: &mut IndexContext, reader: &ParallelEntry) { + fn merge_reader_outputs(ctx: &mut CompileContext, reader: &ParallelEntry) { if reader.access.writes_reasoning_index { if let Some(ref rctx) = reader.ctx { ctx.reasoning_index = rctx.reasoning_index.clone(); @@ -791,7 +791,7 @@ impl PipelineOrchestrator { } /// Save a checkpoint of the current pipeline state. - fn save_checkpoint(ctx: &IndexContext) { + fn save_checkpoint(ctx: &CompileContext) { let checkpoint_dir = match ctx.options.checkpoint_dir { Some(ref dir) => dir.clone(), None => return, @@ -841,12 +841,12 @@ impl PipelineOrchestrator { struct NopStage; #[async_trait::async_trait] -impl IndexStage for NopStage { +impl CompileStage for NopStage { fn name(&self) -> &'static str { "_nop" } - async fn execute(&mut self, _ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, _ctx: &mut CompileContext) -> Result { Ok(StageResult::success("_nop")) } } @@ -860,10 +860,10 @@ struct ParallelEntry { /// Index into orchestrator's stages vec (for swapping back). idx: usize, /// The owned stage implementation. - stage: Box, + stage: Box, /// Cloned context for reader stages; None for the tree writer /// (which uses the main ctx directly). - ctx: Option, + ctx: Option, /// Stage name (captured before swap). name: String, /// Failure policy (captured before swap). @@ -1016,12 +1016,12 @@ mod tests { } #[async_trait::async_trait] - impl IndexStage for MockStage { + impl CompileStage for MockStage { fn name(&self) -> &str { &self.name } - async fn execute(&mut self, _ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, _ctx: &mut CompileContext) -> Result { Ok(StageResult::success(&self.name)) } } diff --git a/vectorless-core/vectorless-index/src/pipeline/policy.rs b/vectorless-core/vectorless-compiler/src/pipeline/policy.rs similarity index 100% rename from vectorless-core/vectorless-index/src/pipeline/policy.rs rename to vectorless-core/vectorless-compiler/src/pipeline/policy.rs diff --git a/vectorless-core/vectorless-index/src/stages/build.rs b/vectorless-core/vectorless-compiler/src/stages/build.rs similarity index 97% rename from vectorless-core/vectorless-index/src/stages/build.rs rename to vectorless-core/vectorless-compiler/src/stages/build.rs index 29eb687..283dfb5 100644 --- a/vectorless-core/vectorless-index/src/stages/build.rs +++ b/vectorless-core/vectorless-compiler/src/stages/build.rs @@ -12,9 +12,9 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use super::{IndexStage, StageResult}; +use super::{CompileStage, StageResult}; use crate::ThinningConfig; -use crate::pipeline::IndexContext; +use crate::pipeline::CompileContext; /// Build stage - constructs a tree from raw nodes. pub struct BuildStage; @@ -158,7 +158,7 @@ impl BuildStage { } /// Build tree from raw nodes. - fn build_tree(&self, raw_nodes: Vec, ctx: &mut IndexContext) -> DocumentTree { + fn build_tree(&self, raw_nodes: Vec, ctx: &mut CompileContext) -> DocumentTree { let root_title = ctx.name.clone(); let root_content = String::new(); @@ -243,7 +243,7 @@ impl Default for BuildStage { } #[async_trait] -impl IndexStage for BuildStage { +impl CompileStage for BuildStage { fn name(&self) -> &'static str { "build" } @@ -252,7 +252,7 @@ impl IndexStage for BuildStage { vec!["parse"] } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); // Take raw nodes from context diff --git a/vectorless-core/vectorless-index/src/stages/concept.rs b/vectorless-core/vectorless-compiler/src/stages/concept.rs similarity index 93% rename from vectorless-core/vectorless-index/src/stages/concept.rs rename to vectorless-core/vectorless-compiler/src/stages/concept.rs index d6a1f52..13cc301 100644 --- a/vectorless-core/vectorless-index/src/stages/concept.rs +++ b/vectorless-core/vectorless-compiler/src/stages/concept.rs @@ -13,8 +13,8 @@ use vectorless_error::Result; use vectorless_llm::LlmClient; use super::async_trait; -use super::{AccessPattern, IndexStage, StageResult}; -use crate::pipeline::IndexContext; +use super::{AccessPattern, CompileStage, StageResult}; +use crate::pipeline::CompileContext; /// Maximum number of top keywords to send to the LLM for concept extraction. const MAX_TOPICS: usize = 20; @@ -46,7 +46,7 @@ impl ConceptExtractionStage { } #[async_trait] -impl IndexStage for ConceptExtractionStage { +impl CompileStage for ConceptExtractionStage { fn name(&self) -> &str { "concept_extraction" } @@ -67,7 +67,7 @@ impl IndexStage for ConceptExtractionStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let concepts = if let Some(ref client) = self.llm_client { extract_with_llm(ctx, client).await } else { @@ -83,7 +83,7 @@ impl IndexStage for ConceptExtractionStage { } /// Extract concepts using LLM from topics and summaries. -async fn extract_with_llm(ctx: &mut IndexContext, client: &LlmClient) -> Vec { +async fn extract_with_llm(ctx: &mut CompileContext, client: &LlmClient) -> Vec { let (topics, section_titles) = gather_source_data(ctx); if topics.is_empty() { @@ -143,7 +143,7 @@ async fn extract_with_llm(ctx: &mut IndexContext, client: &LlmClient) -> Vec Vec { +fn extract_from_topics(ctx: &mut CompileContext) -> Vec { let (topics, section_titles) = gather_source_data(ctx); topics @@ -158,7 +158,7 @@ fn extract_from_topics(ctx: &mut IndexContext) -> Vec { } /// Gather top topics and section titles from the pipeline context. -fn gather_source_data(ctx: &IndexContext) -> (Vec<(String, f32)>, Vec) { +fn gather_source_data(ctx: &CompileContext) -> (Vec<(String, f32)>, Vec) { // Collect top keywords by weight let mut topics: Vec<(String, f32)> = Vec::new(); diff --git a/vectorless-core/vectorless-index/src/stages/enhance.rs b/vectorless-core/vectorless-compiler/src/stages/enhance.rs similarity index 98% rename from vectorless-core/vectorless-index/src/stages/enhance.rs rename to vectorless-core/vectorless-compiler/src/stages/enhance.rs index 9613674..9e8bf73 100644 --- a/vectorless-core/vectorless-index/src/stages/enhance.rs +++ b/vectorless-core/vectorless-compiler/src/stages/enhance.rs @@ -16,8 +16,8 @@ use vectorless_llm::LlmClient; use vectorless_llm::memo::{MemoKey, MemoStore}; use vectorless_utils::fingerprint::Fingerprint; -use super::{IndexStage, StageResult}; -use crate::pipeline::{FailurePolicy, IndexContext, StageRetryConfig}; +use super::{CompileStage, StageResult}; +use crate::pipeline::{FailurePolicy, CompileContext, StageRetryConfig}; use crate::summary::{LlmSummaryGenerator, SummaryGenerator, SummaryStrategy}; /// A node that needs LLM summary generation. @@ -111,7 +111,7 @@ impl EnhanceStage { } /// Check if summary generation is needed based on strategy. - fn needs_summaries(&self, ctx: &IndexContext) -> bool { + fn needs_summaries(&self, ctx: &CompileContext) -> bool { match &ctx.options.summary_strategy { SummaryStrategy::None => false, SummaryStrategy::Lazy { .. } => false, // Generated on-demand at query time @@ -127,7 +127,7 @@ impl Default for EnhanceStage { } #[async_trait] -impl IndexStage for EnhanceStage { +impl CompileStage for EnhanceStage { fn name(&self) -> &'static str { "enhance" } @@ -149,7 +149,7 @@ impl IndexStage for EnhanceStage { ) } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); info!( diff --git a/vectorless-core/vectorless-index/src/stages/enrich.rs b/vectorless-core/vectorless-compiler/src/stages/enrich.rs similarity index 96% rename from vectorless-core/vectorless-index/src/stages/enrich.rs rename to vectorless-core/vectorless-compiler/src/stages/enrich.rs index e14611e..2490e16 100644 --- a/vectorless-core/vectorless-index/src/stages/enrich.rs +++ b/vectorless-core/vectorless-compiler/src/stages/enrich.rs @@ -10,8 +10,8 @@ use tracing::{debug, info}; use vectorless_document::{DocumentTree, NodeId, ReferenceExtractor, TocView}; use vectorless_error::Result; -use super::{AccessPattern, IndexStage, StageResult}; -use crate::pipeline::IndexContext; +use super::{AccessPattern, CompileStage, StageResult}; +use crate::pipeline::CompileContext; /// Enrich stage - adds metadata to the tree. pub struct EnrichStage; @@ -78,7 +78,7 @@ impl EnrichStage { } /// Generate document description from root summary. - fn generate_description(&self, ctx: &mut IndexContext) { + fn generate_description(&self, ctx: &mut CompileContext) { if !ctx.options.generate_description { return; } @@ -145,7 +145,7 @@ impl Default for EnrichStage { } #[async_trait] -impl IndexStage for EnrichStage { +impl CompileStage for EnrichStage { fn name(&self) -> &'static str { "enrich" } @@ -163,7 +163,7 @@ impl IndexStage for EnrichStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); let tree = ctx diff --git a/vectorless-core/vectorless-index/src/stages/mod.rs b/vectorless-core/vectorless-compiler/src/stages/mod.rs similarity index 90% rename from vectorless-core/vectorless-index/src/stages/mod.rs rename to vectorless-core/vectorless-compiler/src/stages/mod.rs index a5bab45..c8b5360 100644 --- a/vectorless-core/vectorless-index/src/stages/mod.rs +++ b/vectorless-core/vectorless-compiler/src/stages/mod.rs @@ -19,15 +19,15 @@ pub use build::BuildStage; pub use concept::ConceptExtractionStage; pub use enhance::EnhanceStage; pub use enrich::EnrichStage; -pub use navigation::NavigationIndexStage; +pub use navigation::NavigationCompileStage; pub use optimize::OptimizeStage; pub use parse::ParseStage; -pub use reasoning::ReasoningIndexStage; +pub use reasoning::ReasoningCompileStage; pub use split::SplitStage; pub use validate::ValidateStage; pub use verify_ingest::VerifyStage; -use super::pipeline::{FailurePolicy, IndexContext, StageResult}; +use super::pipeline::{FailurePolicy, CompileContext, StageResult}; pub use async_trait::async_trait; use vectorless_error::Result; @@ -67,21 +67,21 @@ pub struct AccessPattern { /// struct MyStage; /// /// #[async_trait] -/// impl IndexStage for MyStage { +/// impl CompileStage for MyStage { /// fn name(&self) -> &str { "my_stage" } /// /// fn depends_on(&self) -> Vec<&'static str> { /// vec!["parse", "build"] /// } /// -/// async fn execute(&mut self, ctx: &mut IndexContext) -> Result { +/// async fn execute(&mut self, ctx: &mut CompileContext) -> Result { /// // Process the context... /// Ok(StageResult::success("my_stage")) /// } /// } /// ``` #[async_trait] -pub trait IndexStage: Send + Sync { +pub trait CompileStage: Send + Sync { /// Stage name (must be unique within pipeline). fn name(&self) -> &str; @@ -89,7 +89,7 @@ pub trait IndexStage: Send + Sync { /// /// This method receives a mutable reference to the shared context, /// allowing stages to read from and write to it. - async fn execute(&mut self, ctx: &mut IndexContext) -> Result; + async fn execute(&mut self, ctx: &mut CompileContext) -> Result; /// Whether this stage is optional (can be skipped on failure). /// diff --git a/vectorless-core/vectorless-index/src/stages/navigation.rs b/vectorless-core/vectorless-compiler/src/stages/navigation.rs similarity index 90% rename from vectorless-core/vectorless-index/src/stages/navigation.rs rename to vectorless-core/vectorless-compiler/src/stages/navigation.rs index 8c25a41..6602fce 100644 --- a/vectorless-core/vectorless-index/src/stages/navigation.rs +++ b/vectorless-core/vectorless-compiler/src/stages/navigation.rs @@ -3,7 +3,7 @@ //! Navigation Index Stage — Build the Agent navigation index from the document tree. //! -//! This stage runs after EnrichStage and ReasoningIndexStage. It reads the +//! This stage runs after EnrichStage and ReasoningCompileStage. It reads the //! enhanced TreeNode fields (summary, description, routing_keywords, leaf_count) //! and builds a [`NavigationIndex`] containing compact [`NavEntry`] and //! [`ChildRoute`] records for every non-leaf node. @@ -21,8 +21,8 @@ use vectorless_document::{ChildRoute, DocumentTree, NavEntry, NavigationIndex, N use vectorless_error::Result; use super::async_trait; -use super::{AccessPattern, IndexStage, StageResult}; -use crate::pipeline::IndexContext; +use super::{AccessPattern, CompileStage, StageResult}; +use crate::pipeline::CompileContext; /// Navigation Index Stage — builds the Agent navigation index. /// @@ -32,9 +32,9 @@ use crate::pipeline::IndexContext; /// /// The resulting [`NavigationIndex`] is stored in `ctx.navigation_index` and /// serialized as part of [`PersistedDocument`](vectorless_storage::persistence::PersistedDocument). -pub struct NavigationIndexStage; +pub struct NavigationCompileStage; -impl NavigationIndexStage { +impl NavigationCompileStage { /// Create a new navigation index stage. pub fn new() -> Self { Self @@ -118,14 +118,14 @@ impl NavigationIndexStage { } } -impl Default for NavigationIndexStage { +impl Default for NavigationCompileStage { fn default() -> Self { Self::new() } } #[async_trait] -impl IndexStage for NavigationIndexStage { +impl CompileStage for NavigationCompileStage { fn name(&self) -> &'static str { "navigation_index" } @@ -146,7 +146,7 @@ impl IndexStage for NavigationIndexStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); let tree = match ctx.tree.as_ref() { @@ -320,7 +320,7 @@ mod tests { let root = tree.root(); // Root has 3 leaves: 1.1, 1.2, 2.1 - assert_eq!(NavigationIndexStage::count_leaves(&tree, root), 3); + assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 3); } #[test] @@ -328,7 +328,7 @@ mod tests { let tree = DocumentTree::new("Root", "content"); let root = tree.root(); - assert_eq!(NavigationIndexStage::count_leaves(&tree, root), 1); + assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 1); } #[test] @@ -336,7 +336,7 @@ mod tests { let tree = build_test_tree(); let root = tree.root(); - let entry = NavigationIndexStage::build_nav_entry(&tree, root, 3); + let entry = NavigationCompileStage::build_nav_entry(&tree, root, 3); assert_eq!(entry.overview, "A comprehensive guide"); assert_eq!(entry.leaf_count, 3); assert_eq!(entry.level, 0); @@ -347,7 +347,7 @@ mod tests { let tree = DocumentTree::new("Root", "content"); let root = tree.root(); - let entry = NavigationIndexStage::build_nav_entry(&tree, root, 1); + let entry = NavigationCompileStage::build_nav_entry(&tree, root, 1); assert_eq!(entry.overview, "Root"); } @@ -357,14 +357,14 @@ mod tests { let root = tree.root(); let children: Vec<_> = tree.children_iter(root).collect(); - let route = NavigationIndexStage::build_child_route(&tree, children[0], 2); + let route = NavigationCompileStage::build_child_route(&tree, children[0], 2); assert_eq!(route.title, "Section 1"); assert_eq!(route.leaf_count, 2); } #[test] fn test_stage_config() { - let stage = NavigationIndexStage::new(); + let stage = NavigationCompileStage::new(); assert_eq!(stage.name(), "navigation_index"); assert!(stage.is_optional()); assert_eq!(stage.depends_on(), vec!["enrich"]); @@ -391,14 +391,14 @@ mod tests { tree.set_summary(sec1, "Getting started"); // Build context with the tree - let mut ctx = IndexContext::new( - crate::pipeline::IndexInput::content("test"), + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), crate::config::PipelineOptions::default(), ); ctx.tree = Some(tree); // Execute the stage - let mut stage = NavigationIndexStage::new(); + let mut stage = NavigationCompileStage::new(); let result = stage.execute(&mut ctx).await; assert!(result.is_ok()); @@ -439,13 +439,13 @@ mod tests { // Single node = root is leaf → no non-leaf nodes → empty index let tree = DocumentTree::new("Root", "content"); - let mut ctx = IndexContext::new( - crate::pipeline::IndexInput::content("test"), + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), crate::config::PipelineOptions::default(), ); ctx.tree = Some(tree); - let mut stage = NavigationIndexStage::new(); + let mut stage = NavigationCompileStage::new(); let result = stage.execute(&mut ctx).await; assert!(result.is_ok()); @@ -458,13 +458,13 @@ mod tests { #[tokio::test] async fn test_execute_no_tree() { - let ctx = IndexContext::new( - crate::pipeline::IndexInput::content("test"), + let ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), crate::config::PipelineOptions::default(), ); // ctx.tree is None - let mut stage = NavigationIndexStage::new(); + let mut stage = NavigationCompileStage::new(); // Can't move ctx since tree is None, construct manually let mut ctx = ctx; ctx.tree = None; @@ -481,7 +481,7 @@ mod tests { let root = tree.root(); let child = tree.add_child(root, "Child", "this is a long content string that exceeds 100 characters and should be truncated when used as a fallback description for the child route"); - let route = NavigationIndexStage::build_child_route(&tree, child, 1); + let route = NavigationCompileStage::build_child_route(&tree, child, 1); assert_eq!(route.title, "Child"); // description should be truncated content, not the full string assert!(route.description.len() <= 100); @@ -497,7 +497,7 @@ mod tests { // Clear any auto-generated content tree.set_summary(child, ""); - let route = NavigationIndexStage::build_child_route(&tree, child, 1); + let route = NavigationCompileStage::build_child_route(&tree, child, 1); assert_eq!(route.title, "Orphan Section"); // Fallback: description = title when no summary and no content assert_eq!(route.description, "Orphan Section"); @@ -510,7 +510,7 @@ mod tests { let child = tree.add_child(root, "Child", "some content"); tree.set_summary(child, "A concise summary"); - let route = NavigationIndexStage::build_child_route(&tree, child, 1); + let route = NavigationCompileStage::build_child_route(&tree, child, 1); assert_eq!(route.description, "A concise summary"); } @@ -524,14 +524,14 @@ mod tests { tree.set_summary(root, "Root overview"); tree.set_summary(sec1, "Section overview"); - let root_entry = NavigationIndexStage::build_nav_entry(&tree, root, 3); + let root_entry = NavigationCompileStage::build_nav_entry(&tree, root, 3); assert_eq!(root_entry.level, 0); - let sec1_entry = NavigationIndexStage::build_nav_entry(&tree, sec1, 1); + let sec1_entry = NavigationCompileStage::build_nav_entry(&tree, sec1, 1); assert_eq!(sec1_entry.level, 1); // Leaf node should still return valid NavEntry if called - let leaf_entry = NavigationIndexStage::build_nav_entry(&tree, sec1_1, 1); + let leaf_entry = NavigationCompileStage::build_nav_entry(&tree, sec1_1, 1); assert_eq!(leaf_entry.level, 2); assert_eq!(leaf_entry.overview, "S1.1"); // no summary → fallback to title } @@ -549,11 +549,11 @@ mod tests { let _s2a = tree.add_child(sec2, "S2.A", "leaf"); // sec1 subtree has 3 leaves - assert_eq!(NavigationIndexStage::count_leaves(&tree, sec1), 3); + assert_eq!(NavigationCompileStage::count_leaves(&tree, sec1), 3); // sec2 subtree has 1 leaf - assert_eq!(NavigationIndexStage::count_leaves(&tree, sec2), 1); + assert_eq!(NavigationCompileStage::count_leaves(&tree, sec2), 1); // root has 4 leaves total - assert_eq!(NavigationIndexStage::count_leaves(&tree, root), 4); + assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 4); } /// Helper to check success without destructuring. diff --git a/vectorless-core/vectorless-index/src/stages/optimize.rs b/vectorless-core/vectorless-compiler/src/stages/optimize.rs similarity index 97% rename from vectorless-core/vectorless-index/src/stages/optimize.rs rename to vectorless-core/vectorless-compiler/src/stages/optimize.rs index 61ee470..4654024 100644 --- a/vectorless-core/vectorless-index/src/stages/optimize.rs +++ b/vectorless-core/vectorless-compiler/src/stages/optimize.rs @@ -7,11 +7,11 @@ use super::{AccessPattern, async_trait}; use std::time::Instant; use tracing::{debug, info}; -use crate::pipeline::IndexContext; +use crate::pipeline::CompileContext; use vectorless_document::NodeId; use vectorless_error::Result; -use super::{IndexStage, StageResult}; +use super::{CompileStage, StageResult}; /// Optimize stage - optimizes tree structure. pub struct OptimizeStage; @@ -156,7 +156,7 @@ impl Default for OptimizeStage { } #[async_trait] -impl IndexStage for OptimizeStage { +impl CompileStage for OptimizeStage { fn name(&self) -> &'static str { "optimize" } @@ -177,7 +177,7 @@ impl IndexStage for OptimizeStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); let config = &ctx.options.optimization; @@ -243,8 +243,8 @@ impl IndexStage for OptimizeStage { mod tests { use super::*; use crate::PipelineOptions; - use crate::pipeline::IndexContext; - use crate::pipeline::IndexInput; + use crate::pipeline::CompileContext; + use crate::pipeline::CompilerInput; use vectorless_document::DocumentTree; /// Create a tree with small leaf children under root for merge tests. @@ -436,8 +436,8 @@ mod tests { let mut options = PipelineOptions::default(); options.optimization.enabled = false; - let input = IndexInput::content("# Test\nHello"); - let mut ctx = IndexContext::new(input, options); + let input = CompilerInput::content("# Test\nHello"); + let mut ctx = CompileContext::new(input, options); ctx.tree = Some(DocumentTree::new("Root", "content")); let result = stage.execute(&mut ctx).await.unwrap(); diff --git a/vectorless-core/vectorless-index/src/stages/parse.rs b/vectorless-core/vectorless-compiler/src/stages/parse.rs similarity index 81% rename from vectorless-core/vectorless-index/src/stages/parse.rs rename to vectorless-core/vectorless-compiler/src/stages/parse.rs index 7dbaa07..7c90572 100644 --- a/vectorless-core/vectorless-index/src/stages/parse.rs +++ b/vectorless-core/vectorless-compiler/src/stages/parse.rs @@ -10,9 +10,9 @@ use tracing::{debug, info}; use vectorless_document::DocumentFormat; use vectorless_error::Result; -use super::{IndexStage, StageResult}; -use crate::IndexMode; -use crate::pipeline::{IndexContext, IndexInput}; +use super::{CompileStage, StageResult}; +use crate::SourceFormat; +use crate::pipeline::{CompileContext, CompilerInput}; /// Parse stage - extracts raw nodes from documents. pub struct ParseStage { @@ -34,20 +34,20 @@ impl ParseStage { } /// Detect document format from path and options. - fn detect_format(&self, ctx: &IndexContext) -> Result { + fn detect_format(&self, ctx: &CompileContext) -> Result { match ctx.options.mode { - IndexMode::Auto => match &ctx.input { - IndexInput::File(path) => { + SourceFormat::Auto => match &ctx.input { + CompilerInput::File(path) => { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); DocumentFormat::from_extension(ext).ok_or_else(|| { vectorless_error::Error::Parse(format!("Unknown format: {}", ext)) }) } - IndexInput::Content { format, .. } => Ok(*format), - IndexInput::Bytes { format, .. } => Ok(*format), + CompilerInput::Content { format, .. } => Ok(*format), + CompilerInput::Bytes { format, .. } => Ok(*format), }, - IndexMode::Markdown => Ok(DocumentFormat::Markdown), - IndexMode::Pdf => Ok(DocumentFormat::Pdf), + SourceFormat::Markdown => Ok(DocumentFormat::Markdown), + SourceFormat::Pdf => Ok(DocumentFormat::Pdf), } } } @@ -59,12 +59,12 @@ impl Default for ParseStage { } #[async_trait] -impl IndexStage for ParseStage { +impl CompileStage for ParseStage { fn name(&self) -> &'static str { "parse" } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); // Detect format @@ -72,9 +72,9 @@ impl IndexStage for ParseStage { ctx.format = format; let input_type = match &ctx.input { - IndexInput::File(_) => "file", - IndexInput::Content { .. } => "content", - IndexInput::Bytes { .. } => "bytes", + CompilerInput::File(_) => "file", + CompilerInput::Content { .. } => "content", + CompilerInput::Bytes { .. } => "bytes", }; info!( "[parse] Starting: format={:?}, input={}, llm={}", @@ -85,7 +85,7 @@ impl IndexStage for ParseStage { // Parse based on input type let result = match &ctx.input { - IndexInput::File(path) => { + CompilerInput::File(path) => { // Resolve path let path = path.canonicalize().unwrap_or_else(|_| path.clone()); ctx.source_path = Some(path.clone()); @@ -102,7 +102,7 @@ impl IndexStage for ParseStage { // Parse directly crate::parse::parse_file(&path, format, self.llm_client.clone()).await? } - IndexInput::Content { + CompilerInput::Content { content, name, format, @@ -115,7 +115,7 @@ impl IndexStage for ParseStage { // Parse content directly crate::parse::parse_content(content, *format, self.llm_client.clone()).await? } - IndexInput::Bytes { data, name, format } => { + CompilerInput::Bytes { data, name, format } => { // Set name ctx.name = name.clone(); diff --git a/vectorless-core/vectorless-index/src/stages/reasoning.rs b/vectorless-core/vectorless-compiler/src/stages/reasoning.rs similarity index 95% rename from vectorless-core/vectorless-index/src/stages/reasoning.rs rename to vectorless-core/vectorless-compiler/src/stages/reasoning.rs index da11d00..228a4cb 100644 --- a/vectorless-core/vectorless-index/src/stages/reasoning.rs +++ b/vectorless-core/vectorless-compiler/src/stages/reasoning.rs @@ -20,8 +20,8 @@ use vectorless_llm::LlmClient; use vectorless_scoring::extract_keywords; use super::async_trait; -use super::{AccessPattern, IndexStage, StageResult}; -use crate::pipeline::IndexContext; +use super::{AccessPattern, CompileStage, StageResult}; +use crate::pipeline::CompileContext; /// Reasoning Index Stage - builds a pre-computed reasoning index from the document tree. /// @@ -29,11 +29,11 @@ use crate::pipeline::IndexContext; /// - Topic-to-path mappings from titles and summaries /// - Summary shortcuts for high-frequency "overview" queries /// - Section map for fast ToC lookup -pub struct ReasoningIndexStage { +pub struct ReasoningCompileStage { config: ReasoningIndexConfig, } -impl ReasoningIndexStage { +impl ReasoningCompileStage { /// Create a new reasoning index stage with default config. pub fn new() -> Self { Self { @@ -294,14 +294,14 @@ impl ReasoningIndexStage { } } -impl Default for ReasoningIndexStage { +impl Default for ReasoningCompileStage { fn default() -> Self { Self::new() } } #[async_trait] -impl IndexStage for ReasoningIndexStage { +impl CompileStage for ReasoningCompileStage { fn name(&self) -> &'static str { "reasoning_index" } @@ -322,7 +322,7 @@ impl IndexStage for ReasoningIndexStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); // Check if enabled via pipeline options @@ -452,7 +452,7 @@ mod tests { #[test] fn test_extract_node_keywords() { let keywords = - ReasoningIndexStage::extract_node_keywords("Introduction to Machine Learning", 2); + ReasoningCompileStage::extract_node_keywords("Introduction to Machine Learning", 2); assert!(keywords.contains(&"introduction".to_string())); assert!(keywords.contains(&"machine".to_string())); assert!(keywords.contains(&"learning".to_string())); @@ -460,7 +460,7 @@ mod tests { #[test] fn test_extract_node_keywords_min_length() { - let keywords = ReasoningIndexStage::extract_node_keywords("A B CD", 2); + let keywords = ReasoningCompileStage::extract_node_keywords("A B CD", 2); assert!(!keywords.contains(&"a".to_string())); assert!(!keywords.contains(&"b".to_string())); assert!(keywords.contains(&"cd".to_string())); @@ -468,7 +468,7 @@ mod tests { #[test] fn test_stage_config_default() { - let stage = ReasoningIndexStage::new(); + let stage = ReasoningCompileStage::new(); assert!(stage.config.enabled); assert_eq!(stage.name(), "reasoning_index"); assert!(stage.is_optional()); @@ -493,7 +493,7 @@ mod tests { } let config = ReasoningIndexConfig::default(); - let (topic_paths, keyword_count) = ReasoningIndexStage::build_topic_paths(&tree, &config); + let (topic_paths, keyword_count) = ReasoningCompileStage::build_topic_paths(&tree, &config); assert!( keyword_count > 0, @@ -518,7 +518,7 @@ mod tests { let _c1 = tree.add_child(root, "rust ownership", "rust borrowing rules"); let config = ReasoningIndexConfig::default(); - let (topic_paths, _) = ReasoningIndexStage::build_topic_paths(&tree, &config); + let (topic_paths, _) = ReasoningCompileStage::build_topic_paths(&tree, &config); // All weights should be in 0.0-1.0 range for entries in topic_paths.values() { @@ -549,7 +549,7 @@ mod tests { let mut config = ReasoningIndexConfig::default(); config.max_keyword_entries = 5; - let (topic_paths, keyword_count) = ReasoningIndexStage::build_topic_paths(&tree, &config); + let (topic_paths, keyword_count) = ReasoningCompileStage::build_topic_paths(&tree, &config); assert!( keyword_count <= 5, @@ -574,7 +574,7 @@ mod tests { n.structure = "2".to_string(); } - let section_map = ReasoningIndexStage::build_section_map(&tree); + let section_map = ReasoningCompileStage::build_section_map(&tree); // Should index by title (lowercase) and structure index assert!(section_map.contains_key("introduction")); @@ -602,7 +602,7 @@ mod tests { n.summary = "second section summary".to_string(); } - let shortcut = ReasoningIndexStage::build_summary_shortcut(&tree); + let shortcut = ReasoningCompileStage::build_summary_shortcut(&tree); assert!(shortcut.is_some()); let sc = shortcut.unwrap(); @@ -626,7 +626,7 @@ mod tests { n.summary = "child summary 2".to_string(); } - let shortcut = ReasoningIndexStage::build_summary_shortcut(&tree); + let shortcut = ReasoningCompileStage::build_summary_shortcut(&tree); assert!(shortcut.is_some()); let sc = shortcut.unwrap(); diff --git a/vectorless-core/vectorless-index/src/stages/split.rs b/vectorless-core/vectorless-compiler/src/stages/split.rs similarity index 97% rename from vectorless-core/vectorless-index/src/stages/split.rs rename to vectorless-core/vectorless-compiler/src/stages/split.rs index c729214..01f6737 100644 --- a/vectorless-core/vectorless-index/src/stages/split.rs +++ b/vectorless-core/vectorless-compiler/src/stages/split.rs @@ -10,9 +10,9 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use super::{AccessPattern, IndexStage, StageResult, async_trait}; +use super::{AccessPattern, CompileStage, StageResult, async_trait}; use crate::config::SplitConfig; -use crate::pipeline::IndexContext; +use crate::pipeline::CompileContext; /// Split stage — breaks oversized leaf nodes into smaller children. /// @@ -208,7 +208,7 @@ impl Default for SplitStage { } #[async_trait] -impl IndexStage for SplitStage { +impl CompileStage for SplitStage { fn name(&self) -> &'static str { "split" } @@ -232,7 +232,7 @@ impl IndexStage for SplitStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); let tree = match ctx.tree.as_mut() { diff --git a/vectorless-core/vectorless-index/src/stages/validate.rs b/vectorless-core/vectorless-compiler/src/stages/validate.rs similarity index 94% rename from vectorless-core/vectorless-index/src/stages/validate.rs rename to vectorless-core/vectorless-compiler/src/stages/validate.rs index 5b165a2..9855504 100644 --- a/vectorless-core/vectorless-index/src/stages/validate.rs +++ b/vectorless-core/vectorless-compiler/src/stages/validate.rs @@ -9,8 +9,8 @@ use tracing::{debug, info, warn}; use vectorless_error::Result; -use super::{AccessPattern, IndexStage, StageResult, async_trait}; -use crate::pipeline::IndexContext; +use super::{AccessPattern, CompileStage, StageResult, async_trait}; +use crate::pipeline::CompileContext; /// Maximum allowed tree depth. const MAX_DEPTH: usize = 20; @@ -58,7 +58,7 @@ impl ValidateStage { } /// Run all validation checks and collect issues. - fn validate_tree(&self, ctx: &IndexContext) -> Vec { + fn validate_tree(&self, ctx: &CompileContext) -> Vec { let tree = match ctx.tree.as_ref() { Some(t) => t, None => { @@ -222,7 +222,7 @@ impl Default for ValidateStage { } #[async_trait] -impl IndexStage for ValidateStage { +impl CompileStage for ValidateStage { fn name(&self) -> &'static str { "validate" } @@ -246,7 +246,7 @@ impl IndexStage for ValidateStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { let start = Instant::now(); let node_count = ctx.tree.as_ref().map(|t| t.node_count()).unwrap_or(0); @@ -301,10 +301,10 @@ mod tests { use super::*; use vectorless_document::DocumentTree; - fn make_context_with_tree(tree: DocumentTree) -> IndexContext { - let input = crate::IndexInput::content("test"); + fn make_context_with_tree(tree: DocumentTree) -> CompileContext { + let input = crate::CompilerInput::content("test"); let options = crate::config::PipelineOptions::default(); - let mut ctx = IndexContext::new(input, options); + let mut ctx = CompileContext::new(input, options); ctx.tree = Some(tree); ctx } @@ -355,9 +355,9 @@ mod tests { #[test] fn test_validate_no_tree_error() { - let input = crate::IndexInput::content("test"); + let input = crate::CompilerInput::content("test"); let options = crate::config::PipelineOptions::default(); - let ctx = IndexContext::new(input, options); + let ctx = CompileContext::new(input, options); let stage = ValidateStage::new(); let issues = stage.validate_tree(&ctx); diff --git a/vectorless-core/vectorless-index/src/stages/verify_ingest.rs b/vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs similarity index 90% rename from vectorless-core/vectorless-index/src/stages/verify_ingest.rs rename to vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs index 33119d8..6f50252 100644 --- a/vectorless-core/vectorless-index/src/stages/verify_ingest.rs +++ b/vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs @@ -6,8 +6,8 @@ use tracing::{info, warn}; use super::async_trait; -use super::{AccessPattern, IndexStage}; -use crate::pipeline::{IndexContext, StageResult}; +use super::{AccessPattern, CompileStage}; +use crate::pipeline::{CompileContext, StageResult}; use vectorless_error::{Error, Result}; /// Verification stage — ensures ingest produced reliable output. @@ -21,7 +21,7 @@ use vectorless_error::{Error, Result}; pub struct VerifyStage; #[async_trait] -impl IndexStage for VerifyStage { +impl CompileStage for VerifyStage { fn name(&self) -> &str { "verify" } @@ -41,7 +41,7 @@ impl IndexStage for VerifyStage { } } - async fn execute(&mut self, ctx: &mut IndexContext) -> Result { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result { // Tree must exist and have nodes let tree = ctx .tree diff --git a/vectorless-core/vectorless-index/src/summary/full.rs b/vectorless-core/vectorless-compiler/src/summary/full.rs similarity index 100% rename from vectorless-core/vectorless-index/src/summary/full.rs rename to vectorless-core/vectorless-compiler/src/summary/full.rs diff --git a/vectorless-core/vectorless-index/src/summary/lazy.rs b/vectorless-core/vectorless-compiler/src/summary/lazy.rs similarity index 100% rename from vectorless-core/vectorless-index/src/summary/lazy.rs rename to vectorless-core/vectorless-compiler/src/summary/lazy.rs diff --git a/vectorless-core/vectorless-index/src/summary/mod.rs b/vectorless-core/vectorless-compiler/src/summary/mod.rs similarity index 100% rename from vectorless-core/vectorless-index/src/summary/mod.rs rename to vectorless-core/vectorless-compiler/src/summary/mod.rs diff --git a/vectorless-core/vectorless-index/src/summary/selective.rs b/vectorless-core/vectorless-compiler/src/summary/selective.rs similarity index 100% rename from vectorless-core/vectorless-index/src/summary/selective.rs rename to vectorless-core/vectorless-compiler/src/summary/selective.rs diff --git a/vectorless-core/vectorless-index/src/summary/strategy.rs b/vectorless-core/vectorless-compiler/src/summary/strategy.rs similarity index 100% rename from vectorless-core/vectorless-index/src/summary/strategy.rs rename to vectorless-core/vectorless-compiler/src/summary/strategy.rs diff --git a/vectorless-core/vectorless-engine/Cargo.toml b/vectorless-core/vectorless-engine/Cargo.toml index 5b57292..a813f00 100644 --- a/vectorless-core/vectorless-engine/Cargo.toml +++ b/vectorless-core/vectorless-engine/Cargo.toml @@ -14,7 +14,7 @@ vectorless-document = { path = "../vectorless-document" } vectorless-error = { path = "../vectorless-error" } vectorless-events = { path = "../vectorless-events" } vectorless-graph = { path = "../vectorless-graph" } -vectorless-index = { path = "../vectorless-index" } +vectorless-compiler = { path = "../vectorless-compiler" } vectorless-llm = { path = "../vectorless-llm" } vectorless-metrics = { path = "../vectorless-metrics" } vectorless-storage = { path = "../vectorless-storage" } diff --git a/vectorless-core/vectorless-engine/src/index_context.rs b/vectorless-core/vectorless-engine/src/compile_input.rs similarity index 73% rename from vectorless-core/vectorless-engine/src/index_context.rs rename to vectorless-core/vectorless-engine/src/compile_input.rs index 104e1b9..3c215b7 100644 --- a/vectorless-core/vectorless-engine/src/index_context.rs +++ b/vectorless-core/vectorless-engine/src/compile_input.rs @@ -1,9 +1,9 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Index context for document indexing operations. +//! Compile input for document compilation operations. //! -//! [`IndexContext`] supports single or multiple document sources: +//! [`CompileInput`] supports single or multiple document sources: //! - **File path** — Load and parse a file from disk //! - **Content string** — Parse content directly (HTML, Markdown, text) //! - **Byte data** — Parse binary data (PDF, DOCX) @@ -12,15 +12,15 @@ use std::path::PathBuf; use vectorless_document::DocumentFormat; -use super::types::{IndexMode, IndexOptions}; +use super::types::{CompileMode, CompileOptions}; // ============================================================ -// Index Source +// Compile Source // ============================================================ -/// The source of document content for indexing. +/// The source of document content for compilation. #[derive(Debug, Clone)] -pub(crate) enum IndexSource { +pub(crate) enum CompileSource { /// Load document from a file path. Path(PathBuf), @@ -38,35 +38,35 @@ pub(crate) enum IndexSource { } // ============================================================ -// Index Context +// Compile Input // ============================================================ -/// Context for document indexing operations. +/// Input for document compilation operations. /// /// Supports single or multiple document sources. When multiple sources -/// are provided, each is indexed independently and the results are -/// collected into [`IndexResult`](super::IndexResult). +/// are provided, each is compiled independently and the results are +/// collected into [`CompileOutput`](super::CompileOutput). #[derive(Debug, Clone)] -pub struct IndexContext { +pub struct CompileInput { /// Document sources (supports multiple). - pub(crate) sources: Vec, + pub(crate) sources: Vec, /// Optional document name for metadata (single-source only). pub(crate) name: Option, /// Indexing options. - pub(crate) options: IndexOptions, + pub(crate) options: CompileOptions, } -impl IndexContext { +impl CompileInput { /// Create from a single file path. /// /// The document format is automatically detected from the file extension. pub fn from_path(path: impl Into) -> Self { Self { - sources: vec![IndexSource::Path(path.into())], + sources: vec![CompileSource::Path(path.into())], name: None, - options: IndexOptions::default(), + options: CompileOptions::default(), } } @@ -75,10 +75,10 @@ impl IndexContext { Self { sources: paths .into_iter() - .map(|p| IndexSource::Path(p.into())) + .map(|p| CompileSource::Path(p.into())) .collect(), name: None, - options: IndexOptions::default(), + options: CompileOptions::default(), } } @@ -107,7 +107,7 @@ impl IndexContext { Self { sources, name: None, - options: IndexOptions::default(), + options: CompileOptions::default(), } } @@ -116,7 +116,7 @@ impl IndexContext { dir: &std::path::Path, extensions: &[&str], recursive: bool, - sources: &mut Vec, + sources: &mut Vec, ) { if let Ok(entries) = std::fs::read_dir(dir) { let mut subdirs = Vec::new(); @@ -128,7 +128,7 @@ impl IndexContext { } } else if let Some(ext) = path.extension().and_then(|e| e.to_str()) { if extensions.contains(&ext.to_lowercase().as_str()) { - sources.push(IndexSource::Path(path)); + sources.push(CompileSource::Path(path)); } } } @@ -141,24 +141,24 @@ impl IndexContext { /// Create from a content string. pub fn from_content(content: impl Into, format: DocumentFormat) -> Self { Self { - sources: vec![IndexSource::Content { + sources: vec![CompileSource::Content { data: content.into(), format, }], name: None, - options: IndexOptions::default(), + options: CompileOptions::default(), } } /// Create from binary data. pub fn from_bytes(bytes: Vec, format: DocumentFormat) -> Self { Self { - sources: vec![IndexSource::Bytes { + sources: vec![CompileSource::Bytes { data: bytes, format, }], name: None, - options: IndexOptions::default(), + options: CompileOptions::default(), } } @@ -169,13 +169,13 @@ impl IndexContext { } /// Set the indexing options. - pub fn with_options(mut self, options: IndexOptions) -> Self { + pub fn with_options(mut self, options: CompileOptions) -> Self { self.options = options; self } /// Set the indexing mode. - pub fn with_mode(mut self, mode: IndexMode) -> Self { + pub fn with_mode(mut self, mode: CompileMode) -> Self { self.options.mode = mode; self } @@ -196,41 +196,41 @@ impl IndexContext { } /// Get the indexing options. - pub fn options(&self) -> &IndexOptions { + pub fn options(&self) -> &CompileOptions { &self.options } } -impl From for IndexContext { +impl From for CompileInput { fn from(path: PathBuf) -> Self { Self::from_path(path) } } -impl From<&std::path::Path> for IndexContext { +impl From<&std::path::Path> for CompileInput { fn from(path: &std::path::Path) -> Self { Self::from_path(path.to_path_buf()) } } -impl From<&str> for IndexContext { +impl From<&str> for CompileInput { fn from(path: &str) -> Self { Self::from_path(path) } } -impl From for IndexContext { +impl From for CompileInput { fn from(path: String) -> Self { Self::from_path(path) } } -impl std::fmt::Display for IndexSource { +impl std::fmt::Display for CompileSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - IndexSource::Path(p) => write!(f, "path:{}", p.display()), - IndexSource::Content { format, .. } => write!(f, "content:{}", format.extension()), - IndexSource::Bytes { format, .. } => write!(f, "bytes:{}", format.extension()), + CompileSource::Path(p) => write!(f, "path:{}", p.display()), + CompileSource::Content { format, .. } => write!(f, "content:{}", format.extension()), + CompileSource::Bytes { format, .. } => write!(f, "bytes:{}", format.extension()), } } } @@ -241,44 +241,44 @@ mod tests { #[test] fn test_from_path() { - let ctx = IndexContext::from_path("./test.md"); + let ctx = CompileInput::from_path("./test.md"); assert_eq!(ctx.len(), 1); assert!(ctx.name.is_none()); } #[test] fn test_from_paths() { - let ctx = IndexContext::from_paths(vec!["./a.md", "./b.pdf"]); + let ctx = CompileInput::from_paths(vec!["./a.md", "./b.pdf"]); assert_eq!(ctx.len(), 2); } #[test] fn test_from_content() { - let ctx = IndexContext::from_content("# Title", DocumentFormat::Markdown); + let ctx = CompileInput::from_content("# Title", DocumentFormat::Markdown); assert_eq!(ctx.len(), 1); } #[test] fn test_from_bytes() { - let ctx = IndexContext::from_bytes(vec![1, 2, 3], DocumentFormat::Pdf); + let ctx = CompileInput::from_bytes(vec![1, 2, 3], DocumentFormat::Pdf); assert_eq!(ctx.len(), 1); } #[test] fn test_with_name() { - let ctx = IndexContext::from_path("./test.md").with_name("My Document"); + let ctx = CompileInput::from_path("./test.md").with_name("My Document"); assert_eq!(ctx.name(), Some("My Document")); } #[test] fn test_with_mode() { - let ctx = IndexContext::from_path("./test.md").with_mode(IndexMode::Force); - assert_eq!(ctx.options.mode, IndexMode::Force); + let ctx = CompileInput::from_path("./test.md").with_mode(CompileMode::Force); + assert_eq!(ctx.options.mode, CompileMode::Force); } #[test] fn test_from_path_trait() { - let ctx = IndexContext::from(PathBuf::from("./test.md")); + let ctx = CompileInput::from(PathBuf::from("./test.md")); assert_eq!(ctx.len(), 1); } @@ -300,11 +300,11 @@ mod tests { std::fs::write(tmp.join("sub/deep/ignore.dat"), b"xxx").unwrap(); // Non-recursive: only top-level - let ctx = IndexContext::from_dir(&tmp, false); + let ctx = CompileInput::from_dir(&tmp, false); assert_eq!(ctx.len(), 1); // only a.md // Recursive: all levels - let ctx = IndexContext::from_dir(&tmp, true); + let ctx = CompileInput::from_dir(&tmp, true); assert_eq!(ctx.len(), 3); // a.md, b.md, c.pdf let _ = std::fs::remove_dir_all(&tmp); diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index 8e81c8b..4082c17 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -5,7 +5,7 @@ //! //! The Engine provides a unified API for the Document Understanding Engine: //! -//! - [`ingest`](Engine::ingest) — Understand a document (parse, analyze, persist) +//! - [`compile`](Engine::compile) — Understand a document (parse, analyze, persist) //! - [`forget`](Engine::forget) — Remove a document //! - [`list_documents`](Engine::list_documents) — List all understood documents @@ -20,7 +20,7 @@ use vectorless_document::{ }; use vectorless_error::{Error, Result}; use vectorless_events::EventEmitter; -use vectorless_index::{ +use vectorless_compiler::{ PipelineOptions, incremental::{self, IndexAction}, }; @@ -28,9 +28,9 @@ use vectorless_metrics::MetricsHub; use vectorless_storage::{PersistedDocument, Workspace}; use super::{ - index_context::{IndexContext, IndexSource}, + compile_input::{CompileInput, CompileSource}, indexer::IndexerClient, - types::{FailedItem, IndexItem, IndexMode, IndexResult}, + types::{FailedItem, CompileArtifact, CompileMode, CompileOutput}, workspace::WorkspaceClient, }; @@ -93,16 +93,16 @@ impl Engine { } // ============================================================ - // Ingest Pipeline (private — called by ingest()) + // Compile Pipeline (private — called by compile()) // ============================================================ - /// Run the ingest pipeline: parse, compile, persist. + /// Run the compile pipeline: parse, compile, persist. /// - /// Accepts an [`IndexContext`] that specifies the source and options. + /// Accepts an [`CompileInput`] that specifies the source and options. /// Multiple sources are processed in parallel. - /// Returns an [`IndexResult`] containing the indexed document metadata. + /// Returns an [`CompileOutput`] containing the indexed document metadata. #[tracing::instrument(skip_all, fields(sources = ctx.sources.len()))] - async fn ingest_pipeline(&self, ctx: IndexContext) -> Result { + async fn compile_pipeline(&self, ctx: CompileInput) -> Result { if ctx.is_empty() { return Err(Error::Config("No document sources provided".into())); } @@ -144,7 +144,7 @@ impl Engine { }); } - Ok(IndexResult::with_partial(items, failed)) + Ok(CompileOutput::with_partial(items, failed)) }) .await } @@ -152,12 +152,12 @@ impl Engine { /// Process multiple sources in parallel. async fn process_sources( &self, - sources: &[IndexSource], - options: &super::types::IndexOptions, + sources: &[CompileSource], + options: &super::types::CompileOptions, name: Option<&str>, concurrency: usize, - ) -> (Vec, Vec) { - let results: Vec<(Vec, Vec)> = + ) -> (Vec, Vec) { + let results: Vec<(Vec, Vec)> = futures::stream::iter(sources.iter().cloned()) .map(|source| { let options = options.clone(); @@ -189,17 +189,17 @@ impl Engine { /// Returns `(items, failed)`. async fn process_source( &self, - source: &IndexSource, - options: &super::types::IndexOptions, + source: &CompileSource, + options: &super::types::CompileOptions, name: Option<&str>, - ) -> (Vec, Vec) { + ) -> (Vec, Vec) { let source_label = source.to_string(); match self.resolve_index_action(source, options).await { Ok(IndexAction::Skip(skip_info)) => { info!("Skipped (unchanged): {}", source_label); ( - vec![IndexItem::new( + vec![CompileArtifact::new( skip_info.doc_id, skip_info.name, skip_info.format, @@ -274,7 +274,7 @@ impl Engine { /// is not retryable. async fn index_with_retry( &self, - source: &IndexSource, + source: &CompileSource, name: Option<&str>, pipeline_options: PipelineOptions, existing_tree: Option<&DocumentTree>, @@ -313,7 +313,7 @@ impl Engine { unreachable!() } - /// Convert an [`IndexedDocument`] to an [`IndexItem`] and persist it. + /// Convert an [`IndexedDocument`] to an [`CompileArtifact`] and persist it. /// /// If `old_id` is provided, the old document is removed after a /// successful save (atomic save-first, then remove old). @@ -323,7 +323,7 @@ impl Engine { pipeline_options: &PipelineOptions, source_label: &str, old_id: Option<&str>, - ) -> (Vec, Vec) { + ) -> (Vec, Vec) { let item = Self::build_index_item(&doc); info!("[index] Persisting document '{}'...", doc.name,); @@ -347,9 +347,9 @@ impl Engine { (vec![item], Vec::new()) } - /// Build an [`IndexItem`] from an [`IndexedDocument`](super::indexed_document::IndexedDocument). - fn build_index_item(doc: &super::indexed_document::IndexedDocument) -> IndexItem { - IndexItem::new( + /// Build an [`CompileArtifact`] from an [`IndexedDocument`](super::indexed_document::IndexedDocument). + fn build_index_item(doc: &super::indexed_document::IndexedDocument) -> CompileArtifact { + CompileArtifact::new( doc.id.clone(), doc.name.clone(), doc.format.clone(), @@ -374,23 +374,23 @@ impl Engine { /// Returns a [`vectorless_document::DocumentInfo`] with summary, structure, and concepts. /// The engine builds a full understanding including tree, navigation index, /// reasoning index, summary, and key concepts. - pub async fn ingest(&self, input: IngestInput) -> Result { + pub async fn compile(&self, input: IngestInput) -> Result { let ctx = match &input { - IngestInput::Path(path) => IndexContext::from_path(path), + IngestInput::Path(path) => CompileInput::from_path(path), IngestInput::Bytes { data, format, .. } => { - IndexContext::from_bytes(data.clone(), *format) + CompileInput::from_bytes(data.clone(), *format) } - IngestInput::Text { content, .. } => IndexContext::from_content( + IngestInput::Text { content, .. } => CompileInput::from_content( content, - vectorless_index::parse::DocumentFormat::Markdown, + vectorless_compiler::parse::DocumentFormat::Markdown, ), }; - let result = self.ingest_pipeline(ctx).await?; + let result = self.compile_pipeline(ctx).await?; let doc_id = result .doc_id() - .ok_or_else(|| Error::Config("ingest produced no results".into()))? + .ok_or_else(|| Error::Config("compile produced no results".into()))? .to_string(); // Load the persisted document to build DocumentInfo @@ -398,7 +398,7 @@ impl Engine { .workspace .load(&doc_id) .await? - .ok_or_else(|| Error::Config("Document not found after ingest".into()))?; + .ok_or_else(|| Error::Config("Document not found after compile".into()))?; let doc = Self::persisted_to_understanding_document(persisted); Ok(doc.info()) @@ -538,26 +538,26 @@ impl Engine { /// This is the single source of truth for pipeline configuration. fn build_pipeline_options( &self, - options: &super::types::IndexOptions, - source: &IndexSource, + options: &super::types::CompileOptions, + source: &CompileSource, ) -> PipelineOptions { - use vectorless_index::{IndexMode, ReasoningIndexConfig, SummaryStrategy}; + use vectorless_compiler::{SourceFormat, ReasoningIndexConfig, SummaryStrategy}; let format = match source { - IndexSource::Path(path) => self + CompileSource::Path(path) => self .indexer .detect_format_from_path(path) - .unwrap_or(vectorless_index::parse::DocumentFormat::Markdown), - IndexSource::Content { format, .. } => *format, - IndexSource::Bytes { format, .. } => *format, + .unwrap_or(vectorless_compiler::parse::DocumentFormat::Markdown), + CompileSource::Content { format, .. } => *format, + CompileSource::Bytes { format, .. } => *format, }; let checkpoint_dir = Some(self.config.storage.checkpoint_dir.clone()); PipelineOptions { mode: match format { - vectorless_index::parse::DocumentFormat::Markdown => IndexMode::Markdown, - vectorless_index::parse::DocumentFormat::Pdf => IndexMode::Pdf, + vectorless_compiler::parse::DocumentFormat::Markdown => SourceFormat::Markdown, + vectorless_compiler::parse::DocumentFormat::Pdf => SourceFormat::Pdf, }, generate_ids: options.generate_ids, summary_strategy: if options.generate_summaries { @@ -581,19 +581,19 @@ impl Engine { /// Resolve what action to take for a source. async fn resolve_index_action( &self, - source: &IndexSource, - options: &super::types::IndexOptions, + source: &CompileSource, + options: &super::types::CompileOptions, ) -> Result { let workspace = &self.workspace; // Force mode always re-indexes from scratch - if options.mode == IndexMode::Force { + if options.mode == CompileMode::Force { return Ok(IndexAction::FullIndex { existing_id: None }); } // Only path sources support incremental indexing let path = match source { - IndexSource::Path(p) => p, + CompileSource::Path(p) => p, _ => return Ok(IndexAction::FullIndex { existing_id: None }), }; @@ -604,7 +604,7 @@ impl Engine { }; // Default mode: skip if already indexed (no content check) - if options.mode == IndexMode::Default { + if options.mode == CompileMode::Default { let info = workspace.get_document_info(&existing_id).await?; let (name, format_str, desc, pages) = match info { Some(i) => (i.name, i.format, i.description, i.page_count), @@ -613,8 +613,8 @@ impl Engine { return Ok(IndexAction::Skip(incremental::SkipInfo { doc_id: existing_id, name, - format: vectorless_index::parse::DocumentFormat::from_extension(&format_str) - .unwrap_or(vectorless_index::parse::DocumentFormat::Markdown), + format: vectorless_compiler::parse::DocumentFormat::from_extension(&format_str) + .unwrap_or(vectorless_compiler::parse::DocumentFormat::Markdown), description: desc, page_count: pages, })); @@ -632,8 +632,8 @@ impl Engine { }; let format = - vectorless_index::parse::DocumentFormat::from_extension(&stored_doc.meta.format) - .unwrap_or(vectorless_index::parse::DocumentFormat::Markdown); + vectorless_compiler::parse::DocumentFormat::from_extension(&stored_doc.meta.format) + .unwrap_or(vectorless_compiler::parse::DocumentFormat::Markdown); let pipeline_options = self.build_pipeline_options(options, source); // If logic fingerprint changed, remove old doc before full reprocess @@ -755,18 +755,18 @@ impl std::fmt::Debug for Engine { #[cfg(test)] mod tests { use super::*; - use crate::types::IndexMode; + use crate::types::CompileMode; // -- resolve_index_action Default mode ---------------------------------------------- // We can't call resolve_index_action without a workspace, but we can - // verify IndexMode equality logic used inside. + // verify CompileMode equality logic used inside. #[test] fn test_index_mode_force_skips_incremental() { - let mode = IndexMode::Force; - assert_eq!(mode, IndexMode::Force); - assert_ne!(mode, IndexMode::Default); - assert_ne!(mode, IndexMode::Incremental); + let mode = CompileMode::Force; + assert_eq!(mode, CompileMode::Force); + assert_ne!(mode, CompileMode::Default); + assert_ne!(mode, CompileMode::Incremental); } // -- build_index_item ---------------------------------------------------------------- @@ -775,7 +775,7 @@ mod tests { use crate::indexed_document::IndexedDocument; fn make_doc() -> IndexedDocument { - IndexedDocument::new("test-id", vectorless_index::parse::DocumentFormat::Markdown) + IndexedDocument::new("test-id", vectorless_compiler::parse::DocumentFormat::Markdown) .with_name("test.md") .with_description("test doc") .with_source_path(std::path::PathBuf::from("/tmp/test.md")) @@ -790,7 +790,7 @@ mod tests { assert_eq!(item.name, "test.md"); assert_eq!( item.format, - vectorless_index::parse::DocumentFormat::Markdown + vectorless_compiler::parse::DocumentFormat::Markdown ); assert_eq!(item.description, Some("test doc".to_string())); assert_eq!(item.source_path, Some("/tmp/test.md".to_string())); @@ -799,10 +799,10 @@ mod tests { #[test] fn test_build_index_item_no_source_path() { - let doc = IndexedDocument::new("id", vectorless_index::parse::DocumentFormat::Pdf); + let doc = IndexedDocument::new("id", vectorless_compiler::parse::DocumentFormat::Pdf); let item = Engine::build_index_item(&doc); assert_eq!(item.source_path, Some(String::new())); // unwrap_or_default - assert_eq!(item.format, vectorless_index::parse::DocumentFormat::Pdf); + assert_eq!(item.format, vectorless_compiler::parse::DocumentFormat::Pdf); } } diff --git a/vectorless-core/vectorless-engine/src/indexer.rs b/vectorless-core/vectorless-engine/src/indexer.rs index 5319dc9..816dd72 100644 --- a/vectorless-core/vectorless-engine/src/indexer.rs +++ b/vectorless-core/vectorless-engine/src/indexer.rs @@ -9,12 +9,12 @@ //! # Example //! //! ```rust,ignore -//! use vectorless::client::{IndexerClient, IndexContext}; +//! use vectorless::client::{IndexerClient, CompileInput}; //! //! let indexer = IndexerClient::new(executor); //! //! let result = indexer -//! .index(IndexContext::from_path("./document.md")) +//! .index(CompileInput::from_path("./document.md")) //! .await?; //! //! println!("Indexed: {} ({} nodes)", result.id, result.tree.as_ref().map(|t| t.node_count()).unwrap_or(0)); @@ -28,11 +28,11 @@ use uuid::Uuid; use vectorless_document::DocumentFormat; use vectorless_error::{Error, Result}; -use vectorless_index::{IndexInput, IndexMode, PipelineExecutor, PipelineOptions}; +use vectorless_compiler::{CompilerInput, SourceFormat, PipelineExecutor, PipelineOptions}; use vectorless_llm::LlmClient; use vectorless_storage::{DocumentMeta, PersistedDocument}; -use super::index_context::IndexSource; +use super::compile_input::CompileSource; use super::indexed_document::IndexedDocument; use vectorless_events::{EventEmitter, IndexEvent}; @@ -71,7 +71,7 @@ impl IndexerClient { /// (including checkpoint dir, reasoning config, etc.). pub async fn index( &self, - source: &IndexSource, + source: &CompileSource, name: Option<&str>, pipeline_options: PipelineOptions, ) -> Result { @@ -84,19 +84,19 @@ impl IndexerClient { /// The caller provides fully constructed [`PipelineOptions`]. pub async fn index_with_existing( &self, - source: &IndexSource, + source: &CompileSource, name: Option<&str>, mut pipeline_options: PipelineOptions, existing_tree: Option<&vectorless_document::DocumentTree>, ) -> Result { pipeline_options.existing_tree = existing_tree.cloned(); match source { - IndexSource::Path(path) => self.index_from_path(path, name, pipeline_options).await, - IndexSource::Content { data, format } => { + CompileSource::Path(path) => self.index_from_path(path, name, pipeline_options).await, + CompileSource::Content { data, format } => { self.index_from_content(data, *format, name, pipeline_options) .await } - IndexSource::Bytes { data, format } => { + CompileSource::Bytes { data, format } => { self.index_from_bytes(data, *format, name, pipeline_options) .await } @@ -132,7 +132,7 @@ impl IndexerClient { // Resolve format from pipeline options (set by Engine) — no re-detection let format = Self::format_from_mode(&pipeline_options.mode); - let input = IndexInput::file(&path); + let input = CompilerInput::file(&path); self.run_pipeline( input, format, @@ -164,7 +164,7 @@ impl IndexerClient { )); } - let input = IndexInput::content(content); + let input = CompilerInput::content(content); self.run_pipeline( input, format, @@ -202,7 +202,7 @@ impl IndexerClient { bytes.len() ); - let input = IndexInput::bytes(bytes); + let input = CompilerInput::bytes(bytes); self.run_pipeline( input, format, @@ -218,7 +218,7 @@ impl IndexerClient { #[tracing::instrument(skip_all, fields(format = ?format, source = %source_label))] async fn run_pipeline( &self, - input: IndexInput, + input: CompilerInput, format: DocumentFormat, source_label: &str, name: Option<&str>, @@ -245,7 +245,7 @@ impl IndexerClient { fn build_indexed_document( &self, doc_id: String, - result: vectorless_index::PipelineResult, + result: vectorless_compiler::CompileResult, format: DocumentFormat, name: Option<&str>, path: Option<&Path>, @@ -296,11 +296,11 @@ impl IndexerClient { /// /// Falls back to Markdown for `Auto` mode (the engine resolves /// `Auto` to a concrete format before calling the indexer). - fn format_from_mode(mode: &IndexMode) -> DocumentFormat { + fn format_from_mode(mode: &SourceFormat) -> DocumentFormat { match mode { - IndexMode::Markdown => DocumentFormat::Markdown, - IndexMode::Pdf => DocumentFormat::Pdf, - IndexMode::Auto => DocumentFormat::Markdown, + SourceFormat::Markdown => DocumentFormat::Markdown, + SourceFormat::Pdf => DocumentFormat::Pdf, + SourceFormat::Auto => DocumentFormat::Markdown, } } diff --git a/vectorless-core/vectorless-engine/src/lib.rs b/vectorless-core/vectorless-engine/src/lib.rs index 8cd8cca..b588592 100644 --- a/vectorless-core/vectorless-engine/src/lib.rs +++ b/vectorless-core/vectorless-engine/src/lib.rs @@ -1,18 +1,18 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! High-level client API for document indexing and retrieval. +//! High-level client API for document compilation and retrieval. //! //! This module provides the main entry point for using vectorless: -//! - [`Engine`] — The main client for indexing and querying documents +//! - [`Engine`] — The main client for compiling and querying documents //! - [`EngineBuilder`] — Builder pattern for client configuration -//! - [`IndexContext`] — Unified input for document indexing +//! - [`CompileInput`] — Unified input for document compilation //! //! Retrieval (ask) is handled by the Python strategy layer. mod builder; +mod compile_input; mod engine; -mod index_context; mod indexed_document; mod indexer; mod types; @@ -29,16 +29,16 @@ pub use engine::Engine; // Context Types // ============================================================ -pub use index_context::IndexContext; +pub use compile_input::CompileInput; // ============================================================ // Result & Info Types // ============================================================ -pub use types::{FailedItem, IndexItem, IndexMode, IndexOptions, IndexResult}; +pub use types::{FailedItem, CompileArtifact, CompileMode, CompileOptions, CompileOutput}; // ============================================================ -// Parser Types (needed for IndexContext::from_content) +// Parser Types (needed for CompileInput::from_content) // ============================================================ pub use vectorless_document::DocumentFormat; diff --git a/vectorless-core/vectorless-engine/src/types.rs b/vectorless-core/vectorless-engine/src/types.rs index 562510a..61c44b1 100644 --- a/vectorless-core/vectorless-engine/src/types.rs +++ b/vectorless-core/vectorless-engine/src/types.rs @@ -41,7 +41,7 @@ impl FailedItem { /// /// Controls how the indexer handles existing documents and re-indexing. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum IndexMode { +pub enum CompileMode { /// Default mode - skip if already indexed. /// /// If a document with the same source has already been indexed, @@ -58,15 +58,15 @@ pub enum IndexMode { /// Incremental mode - only re-index changed files. /// /// Re-index only if the file has been modified since the last index. - /// For content/bytes sources, this behaves like [`IndexMode::Default`]. + /// For content/bytes sources, this behaves like [`CompileMode::Default`]. Incremental, } /// Options for indexing a document. #[derive(Debug, Clone)] -pub struct IndexOptions { +pub struct CompileOptions { /// Indexing mode. - pub mode: IndexMode, + pub mode: CompileMode, /// Whether to generate summaries using LLM. pub generate_summaries: bool, @@ -86,10 +86,10 @@ pub struct IndexOptions { pub timeout_secs: Option, } -impl Default for IndexOptions { +impl Default for CompileOptions { fn default() -> Self { Self { - mode: IndexMode::Default, + mode: CompileMode::Default, generate_summaries: true, generate_ids: true, generate_description: true, @@ -99,7 +99,7 @@ impl Default for IndexOptions { } } -impl IndexOptions { +impl CompileOptions { /// Create new index options with defaults. pub fn new() -> Self { Self::default() @@ -121,10 +121,10 @@ impl IndexOptions { /// /// # Modes /// - /// - [`IndexMode::Default`] - Skip if already indexed - /// - [`IndexMode::Force`] - Always re-index - /// - [`IndexMode::Incremental`] - Only re-index changed files - pub fn with_mode(mut self, mode: IndexMode) -> Self { + /// - [`CompileMode::Default`] - Skip if already indexed + /// - [`CompileMode::Force`] - Always re-index + /// - [`CompileMode::Incremental`] - Only re-index changed files + pub fn with_mode(mut self, mode: CompileMode) -> Self { self.mode = mode; self } @@ -142,17 +142,17 @@ impl IndexOptions { /// Result of a document indexing operation. #[derive(Debug, Clone)] -pub struct IndexResult { +pub struct CompileOutput { /// Successfully indexed items. - pub items: Vec, + pub items: Vec, /// Items that failed to index (partial success). pub failed: Vec, } -impl IndexResult { +impl CompileOutput { /// Create a new index result. - pub fn new(items: Vec) -> Self { + pub fn new(items: Vec) -> Self { Self { items, failed: Vec::new(), @@ -160,7 +160,7 @@ impl IndexResult { } /// Create with both successes and failures. - pub fn with_partial(items: Vec, failed: Vec) -> Self { + pub fn with_partial(items: Vec, failed: Vec) -> Self { Self { items, failed } } @@ -196,7 +196,7 @@ impl IndexResult { /// A single indexed document item. #[derive(Debug, Clone)] -pub struct IndexItem { +pub struct CompileArtifact { /// The unique document ID. pub doc_id: String, /// The document name. @@ -213,7 +213,7 @@ pub struct IndexItem { pub metrics: Option, } -impl IndexItem { +impl CompileArtifact { /// Create a new index item. pub fn new( doc_id: impl Into, @@ -287,27 +287,27 @@ mod tests { #[test] fn test_index_options() { - let options = IndexOptions::new() + let options = CompileOptions::new() .with_summaries() - .with_mode(IndexMode::Force); + .with_mode(CompileMode::Force); assert!(options.generate_summaries); - assert_eq!(options.mode, IndexMode::Force); + assert_eq!(options.mode, CompileMode::Force); } #[test] fn test_index_options_timeout() { - let opts = IndexOptions::new().with_timeout_secs(30); + let opts = CompileOptions::new().with_timeout_secs(30); assert_eq!(opts.timeout_secs, Some(30)); - let default = IndexOptions::default(); + let default = CompileOptions::default(); assert_eq!(default.timeout_secs, None); } #[test] fn test_index_result() { - let item = IndexItem::new("doc-1", "Test", DocumentFormat::Markdown, None, None); - let result = IndexResult::new(vec![item]); + let item = CompileArtifact::new("doc-1", "Test", DocumentFormat::Markdown, None, None); + let result = CompileOutput::new(vec![item]); assert_eq!(result.doc_id(), Some("doc-1")); assert_eq!(result.len(), 1); @@ -316,7 +316,7 @@ mod tests { #[test] fn test_index_result_empty() { - let result = IndexResult::new(vec![]); + let result = CompileOutput::new(vec![]); assert!(result.is_empty()); assert_eq!(result.doc_id(), None); } @@ -324,17 +324,17 @@ mod tests { #[test] fn test_index_result_multiple() { let items = vec![ - IndexItem::new("doc-1", "A", DocumentFormat::Markdown, None, None), - IndexItem::new("doc-2", "B", DocumentFormat::Pdf, None, None), + CompileArtifact::new("doc-1", "A", DocumentFormat::Markdown, None, None), + CompileArtifact::new("doc-2", "B", DocumentFormat::Pdf, None, None), ]; - let result = IndexResult::new(items); + let result = CompileOutput::new(items); assert_eq!(result.len(), 2); assert_eq!(result.doc_id(), None); } #[test] fn test_partial_success() { - let items = vec![IndexItem::new( + let items = vec![CompileArtifact::new( "doc-1", "A", DocumentFormat::Markdown, @@ -342,7 +342,7 @@ mod tests { None, )]; let failed = vec![FailedItem::new("missing.pdf", "File not found")]; - let result = IndexResult::with_partial(items, failed); + let result = CompileOutput::with_partial(items, failed); assert_eq!(result.len(), 1); assert!(result.has_failures()); diff --git a/vectorless-core/vectorless-py/src/document.rs b/vectorless-core/vectorless-py/src/document.rs index 1d16d42..ebe2b14 100644 --- a/vectorless-core/vectorless-py/src/document.rs +++ b/vectorless-core/vectorless-py/src/document.rs @@ -18,7 +18,7 @@ use vectorless_primitives::{ use super::error::VectorlessError; // ========================================================================= -// PyDocumentInfo (existing — returned by ingest) +// PyDocumentInfo (existing — returned by compile) // ========================================================================= /// Information about an understood document. diff --git a/vectorless-core/vectorless-py/src/engine.rs b/vectorless-core/vectorless-py/src/engine.rs index 107b5c3..31f64e9 100644 --- a/vectorless-core/vectorless-py/src/engine.rs +++ b/vectorless-core/vectorless-py/src/engine.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Engine Python wrapper — async ingest/ask/forget/list_documents. +//! Engine Python wrapper — async compile/forget/list_documents. use pyo3::prelude::*; use pyo3_async_runtimes::tokio::future_into_py; @@ -20,8 +20,8 @@ use super::metrics::PyMetricsReport; // Engine async helpers (named functions to avoid FnOnce HRTB issue) // ============================================================ -async fn run_ingest(engine: Arc, input: IngestInput) -> PyResult { - let doc = engine.ingest(input).await.map_err(to_py_err)?; +async fn run_compile(engine: Arc, input: IngestInput) -> PyResult { + let doc = engine.compile(input).await.map_err(to_py_err)?; Ok(PyDocumentInfo { inner: doc }) } @@ -87,15 +87,10 @@ fn run_metrics_report(engine: Arc) -> PyMetricsReport { /// /// engine = Engine(api_key="sk-...", model="gpt-4o") /// -/// # Understand a document -/// doc = await engine.ingest("./report.pdf") +/// # Compile a document +/// doc = await engine.compile("./report.pdf") /// print(doc.summary) /// -/// # Ask a question -/// answer = await engine.ask("What is the revenue?", doc_ids=[doc.doc_id]) -/// print(answer.content) -/// print(answer.trace) # reasoning trace — always present -/// /// # List all understood documents /// docs = await engine.list_documents() /// @@ -168,7 +163,7 @@ impl PyEngine { }) } - /// Understand a document — parse, analyze, and persist. + /// Compile a document — parse, analyze, and persist. /// /// Args: /// path: File path to the document (PDF or Markdown). @@ -177,11 +172,11 @@ impl PyEngine { /// DocumentInfo with doc_id, summary, structure, concepts. /// /// Raises: - /// VectorlessError: If ingest fails. - fn ingest<'py>(&self, py: Python<'py>, path: String) -> PyResult> { + /// VectorlessError: If compilation fails. + fn compile<'py>(&self, py: Python<'py>, path: String) -> PyResult> { let engine = Arc::clone(&self.inner); let input = IngestInput::Path(path.into()); - future_into_py(py, run_ingest(engine, input)) + future_into_py(py, run_compile(engine, input)) } /// Remove a document by ID. diff --git a/vectorless/ask/__init__.py b/vectorless/ask/__init__.py index 62b5087..2b71b9f 100644 --- a/vectorless/ask/__init__.py +++ b/vectorless/ask/__init__.py @@ -1,8 +1,10 @@ """Ask pipeline — query reasoning, multi-agent retrieval, and answer synthesis.""" from vectorless.ask.dispatcher import dispatch +from vectorless.ask.errors import AskError, BudgetExceededError, LLMFailureError, NavigationError, ParseError, VerificationError from vectorless.ask.evaluate import evaluate from vectorless.ask.orchestrator import Orchestrator +from vectorless.ask.protocols import NavigableDocument from vectorless.ask.plan import Complexity, QueryIntent, QueryPlan, SubQuery from vectorless.ask.types import ( DispatchEntry, @@ -25,6 +27,7 @@ # New modules from vectorless.ask.blackboard import Discovery, SharedBlackboard +from vectorless.ask.events import AskEvent from vectorless.ask.reasoning import ( Ambiguity, AmbiguityType, @@ -47,6 +50,13 @@ "Evidence", "Metrics", "TraceStep", + # Error types + "AskError", + "LLMFailureError", + "ParseError", + "BudgetExceededError", + "NavigationError", + "VerificationError", # Worker types "WorkerOutput", "WorkerMetrics", @@ -54,6 +64,7 @@ # Orchestrator types "Orchestrator", "OrchestratorState", + "NavigableDocument", "DispatchEntry", "DocCard", "EvalResult", @@ -82,6 +93,8 @@ # Shared blackboard "Discovery", "SharedBlackboard", + # Events + "AskEvent", # Verification "VerifyPipeline", "VerificationDimension", diff --git a/vectorless/ask/blackboard.py b/vectorless/ask/blackboard.py index df86ead..60b2b5f 100644 --- a/vectorless/ask/blackboard.py +++ b/vectorless/ask/blackboard.py @@ -7,8 +7,11 @@ from __future__ import annotations +import logging from dataclasses import dataclass, field +logger = logging.getLogger(__name__) + @dataclass class Discovery: diff --git a/vectorless/ask/dispatcher.py b/vectorless/ask/dispatcher.py index 4d8906a..94c5410 100644 --- a/vectorless/ask/dispatcher.py +++ b/vectorless/ask/dispatcher.py @@ -14,6 +14,7 @@ import logging from vectorless.ask.protocols import DocLoader, EventCallback +from vectorless.ask.errors import AskError from vectorless.ask.types import DocCard, Output, Specified, Workspace from vectorless.ask.orchestrator import Orchestrator from vectorless.ask.reasoning import QueryAnalysis, QueryAnalyzer @@ -38,7 +39,13 @@ async def dispatch( # Step 1: Query reasoning (multi-stage analysis) logger.info("dispatch: query reasoning started") analyzer = QueryAnalyzer() - query_analysis = await analyzer.analyze(query, llm) + try: + query_analysis = await analyzer.analyze(query, llm) + except AskError: + raise + except Exception as e: + from vectorless.ask.errors import LLMFailureError + raise LLMFailureError(f"Query analysis failed: {e}") from e logger.info( "dispatch: query reasoning complete (intent=%s, complexity=%s)", query_analysis.intent.value, diff --git a/vectorless/ask/events.py b/vectorless/ask/events.py new file mode 100644 index 0000000..d0e113b --- /dev/null +++ b/vectorless/ask/events.py @@ -0,0 +1,24 @@ +"""Event types for the ask pipeline. + +Defines the event protocol and enum of known event names. +The Orchestrator emits events at each stage so users can hook into +monitoring, logging, cost tracking, etc. +""" + +from __future__ import annotations + +from enum import Enum + + +class AskEvent(str, Enum): + """Events emitted during the ask pipeline lifecycle.""" + + QUERY_ANALYZED = "query_analyzed" + WORKERS_DISPATCHED = "workers_dispatched" + WORKER_COMPLETED = "worker_completed" + EVIDENCE_COLLECTED = "evidence_collected" + VERIFICATION_PASSED = "verification_passed" + VERIFICATION_FAILED = "verification_failed" + REPLAN_TRIGGERED = "replan_triggered" + COMPLETED = "completed" + ERROR = "error" diff --git a/vectorless/ask/orchestrator.py b/vectorless/ask/orchestrator.py index 8172e2f..6abc72d 100644 --- a/vectorless/ask/orchestrator.py +++ b/vectorless/ask/orchestrator.py @@ -23,6 +23,8 @@ from typing import Any from vectorless.ask.protocols import DocLoader, EventCallback +from vectorless.ask.events import AskEvent +from vectorless.ask.errors import AskError, LLMFailureError, NavigationError from vectorless.ask.utils import extract_keywords, format_evidence from vectorless.ask.types import ( DispatchEntry, @@ -113,6 +115,7 @@ def __init__( query_analysis: QueryAnalysis | None = None, max_rounds: int = 15, max_llm_calls: int = 0, + max_concurrent_workers: int = 5, event_callback: EventCallback | None = None, ) -> None: self._query = query @@ -122,6 +125,7 @@ def __init__( self._skip_analysis = skip_analysis self._max_rounds = max_rounds self._max_llm_calls = max_llm_calls + self._worker_semaphore = asyncio.Semaphore(max_concurrent_workers) self._emit = event_callback or _noop_emit # Accept both old QueryPlan and new QueryAnalysis @@ -159,11 +163,17 @@ async def run(self) -> Output: if analyze_result is None: # No results or already answered + await self._emit({"event": AskEvent.COMPLETED, "reason": "no_results"}) return state.into_output("") state.total_llm_calls += analyze_result.llm_calls initial_dispatches = analyze_result.dispatches + await self._emit({ + "event": AskEvent.QUERY_ANALYZED, + "dispatches": len(initial_dispatches), + }) + # --- Phase 2: Supervisor loop --- outcome = await self._supervisor_loop( query, initial_dispatches, cards, llm, state, @@ -182,13 +192,24 @@ async def run(self) -> Output: # --- Phase 3: Finalize — rerank + assemble Output --- if state.all_evidence: - return await self._finalize_output( + await self._emit({ + "event": AskEvent.EVIDENCE_COLLECTED, + "evidence_count": len(state.all_evidence), + }) + output = await self._finalize_output( state, self._query_analysis.intent if self._query_analysis else None, confidence, ) + await self._emit({ + "event": AskEvent.COMPLETED, + "confidence": output.confidence, + "evidence_count": len(output.evidence), + }) + return output logger.info("No evidence collected — returning empty output") + await self._emit({"event": AskEvent.COMPLETED, "reason": "no_evidence"}) return state.into_output("") # ----------------------------------------------------------------------- @@ -239,9 +260,12 @@ async def _analyze( try: analysis_output = await llm.complete(system, user) - except Exception as e: + except LLMFailureError as e: logger.error("Orchestrator analysis LLM call failed: %s", e) return None + except Exception as e: + logger.error("Orchestrator analysis unexpected error: %s", e) + return None logger.info( "Phase 1: analysis complete (response_len=%d)", len(analysis_output), @@ -323,6 +347,11 @@ async def _supervisor_loop( "Dispatching %d Workers (iteration=%d)", len(current_dispatches), iteration, ) + await self._emit({ + "event": AskEvent.WORKERS_DISPATCHED, + "worker_count": len(current_dispatches), + "iteration": iteration, + }) await self._adaptive_dispatch( query, current_dispatches, cards, llm, state, blackboard, iteration, @@ -352,8 +381,11 @@ async def _supervisor_loop( llm=llm, ) llm_calls += 1 + except AskError as e: + logger.error("Verification failed (typed): %s", e) + break except Exception as e: - logger.error("Verification failed: %s", e) + logger.error("Verification unexpected error: %s", e) break logger.info( @@ -365,9 +397,20 @@ async def _supervisor_loop( if verification_result.passed: eval_sufficient = True + await self._emit({ + "event": AskEvent.VERIFICATION_PASSED, + "confidence": verification_result.overall_confidence, + "iteration": iteration, + }) break # Verification failed — check iteration limit + await self._emit({ + "event": AskEvent.VERIFICATION_FAILED, + "confidence": verification_result.overall_confidence, + "gaps": verification_result.gaps, + "iteration": iteration, + }) if iteration >= MAX_VERIFICATION_ITERATIONS - 1: logger.info( "Max verification iterations reached — returning with current confidence" @@ -394,6 +437,12 @@ async def _supervisor_loop( "Evidence insufficient (evidence=%d, iteration=%d) — replanning", len(state.all_evidence), iteration, ) + await self._emit({ + "event": AskEvent.REPLAN_TRIGGERED, + "evidence_count": len(state.all_evidence), + "gaps": verification_result.gaps if verification_result else [], + "iteration": iteration, + }) missing_info = "; ".join(verification_result.gaps) if verification_result.gaps else "" try: @@ -500,21 +549,32 @@ async def run_worker(dispatch: DispatchEntry) -> tuple[int, WorkerOutput]: "Worker completed for doc %d (%s): evidence=%d, rounds=%d", idx, card.name, len(result.evidence), result.metrics.rounds_used, ) + await self._emit({ + "event": AskEvent.WORKER_COMPLETED, + "doc_idx": idx, + "doc_name": card.name, + "evidence_count": len(result.evidence), + "rounds_used": result.metrics.rounds_used, + }) return (idx, result) tasks = [run_worker(d) for d in dispatches] - # Use TaskGroup with per-task exception wrapping to maintain - # the same fault-tolerance as gather(return_exceptions=True) + # Use TaskGroup with per-task exception wrapping and a semaphore + # to bound concurrency (default 5 workers in parallel). task_results: list[tuple[int, WorkerOutput] | Exception] = [] + semaphore = self._worker_semaphore + emit = self._emit + async with asyncio.TaskGroup() as tg: async def _safe_run(d: DispatchEntry) -> None: - try: - result = await run_worker(d) - task_results.append(result) - except Exception as e: - task_results.append(e) - logger.warning("Worker failed: %s", e) + async with semaphore: + try: + result = await run_worker(d) + task_results.append(result) + except Exception as e: + task_results.append(e) + logger.warning("Worker failed: %s", e) for d in dispatches: tg.create_task(_safe_run(d)) @@ -578,6 +638,13 @@ async def _dispatch_sequential( "Worker completed for doc %d (%s): evidence=%d, rounds=%d", idx, card.name, len(result.evidence), result.metrics.rounds_used, ) + await self._emit({ + "event": AskEvent.WORKER_COMPLETED, + "doc_idx": idx, + "doc_name": card.name, + "evidence_count": len(result.evidence), + "rounds_used": result.metrics.rounds_used, + }) state.collect_result(idx, result) @@ -644,7 +711,12 @@ async def _finalize_output( intent: Any, # QueryIntent or None confidence: float, ) -> Output: - """Rerank evidence and assemble the final Output.""" + """Rerank evidence and assemble the final Output. + + For non-factual/non-navigational intents with sufficient confidence, + runs an optional LLM synthesis step to produce a coherent answer + from the top evidence. + """ from vectorless.ask.plan import QueryIntent as PlanIntent from vectorless.ask.reasoning.types import QueryIntent @@ -662,6 +734,7 @@ async def _finalize_output( effective_intent = _plan_intent_map.get(intent_value, PlanIntent.FACTUAL) else: effective_intent = PlanIntent.FACTUAL + intent_value = "factual" reranked = rerank_process( evidence=state.all_evidence, @@ -671,7 +744,23 @@ async def _finalize_output( state.total_llm_calls += reranked.llm_calls - output = state.into_output(reranked.answer) + answer = reranked.answer + + # Optional LLM synthesis for non-trivial intents + _synthesis_intents = {"analytical", "summary", "comparative"} + if intent_value in _synthesis_intents and confidence > 0.5 and reranked.evidence: + try: + answer = await self._synthesize_answer( + query=self._query, + evidence=reranked.evidence[:5], + intent=intent_value, + llm=self._llm, + ) + state.total_llm_calls += 1 + except Exception as e: + logger.warning("Answer synthesis failed, using reranked output: %s", e) + + output = state.into_output(answer) output.confidence = reranked.confidence logger.info( @@ -681,14 +770,40 @@ async def _finalize_output( return output + async def _synthesize_answer( + self, + query: str, + evidence: list[Evidence], + intent: str, + llm: LLMClient, + ) -> str: + """LLM-powered answer synthesis from top evidence. + + Used for analytical, summary, and comparative intents where the + raw evidence formatting is not enough — the user expects a coherent + synthesized answer. + """ + evidence_text = format_evidence(evidence) + system = ( + "You are a document analysis assistant. Synthesize a clear, well-structured " + "answer from the provided evidence. Cite the source sections. Do not " + "hallucinate information not present in the evidence." + ) + user = ( + f"Question: {query}\n\n" + f"Evidence:\n{evidence_text}\n\n" + f"Provide a comprehensive answer based on the evidence above." + ) + response = await llm.complete(system, user) + return response.strip() + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _noop_emit(event: dict) -> Any: +async def _noop_emit(event: dict) -> None: """No-op event emitter.""" - return asyncio.ensure_future(asyncio.sleep(0)) def _compute_confidence( diff --git a/vectorless/ask/protocols.py b/vectorless/ask/protocols.py index a24a60f..9447d63 100644 --- a/vectorless/ask/protocols.py +++ b/vectorless/ask/protocols.py @@ -1,17 +1,144 @@ -"""Protocol definitions for callable parameters. +"""Protocol definitions for the ask pipeline. -Replaces loose `Any` / `Callable` typing with structural typing via Protocol. +Defines structural types via Protocol so that the Worker and Orchestrator +receive properly typed parameters instead of bare `Any`. """ from __future__ import annotations -from typing import Any, Protocol +from typing import Any, Protocol, runtime_checkable +# --------------------------------------------------------------------------- +# NavigableDocument — the interface Workers use to navigate documents +# --------------------------------------------------------------------------- + +@runtime_checkable +class NavigableDocument(Protocol): + """Structural type for documents that Workers can navigate. + + Implemented by the PyO3 ``Document`` class from vectorless-py. + All methods are async and return native Python objects. + """ + + # Navigation + async def ls(self) -> list[Any]: + """List children of the current node.""" + ... + + async def cd(self, node_id: str) -> None: + """Navigate into a child node by ID.""" + ... + + async def cd_by_title(self, title: str) -> None: + """Navigate into a child node by title.""" + ... + + async def cd_up(self) -> None: + """Navigate to the parent node.""" + ... + + async def back(self) -> None: + """Navigate to the previously visited node.""" + ... + + # Content access + async def cat(self, node_id: str | None = None) -> str: + """Read the full content of a node.""" + ... + + async def head(self, node_id: str, n: int) -> str: + """Read the first N lines of a node.""" + ... + + async def pwd(self) -> str: + """Return the current navigation path as a string.""" + ... + + # Search + async def find(self, keyword: str) -> list[Any]: + """Find nodes whose titles match a keyword.""" + ... + + async def grep(self, pattern: str) -> list[Any]: + """Search node contents for a pattern.""" + ... + + async def grep_node(self, node_id: str, pattern: str) -> list[Any]: + """Search a specific node's content for a pattern.""" + ... + + async def keyword_entries(self, keyword: str) -> list[Any]: + """Look up reasoning index entries for a keyword.""" + ... + + async def find_section(self, title: str) -> Any: + """Find a section by exact or partial title match.""" + ... + + # Metadata + async def toc(self, max_depth: int = 0) -> list[Any]: + """Return table of contents entries.""" + ... + + async def stats(self, node_id: str | None = None) -> Any: + """Return stats for a node.""" + ... + + async def similar(self, node_id: str) -> list[Any]: + """Find similar nodes to the given node.""" + ... + + async def section_overview(self, node_id: str) -> str: + """Get an overview of a section.""" + ... + + async def siblings(self, node_id: str) -> list[Any]: + """Get sibling nodes of the given node.""" + ... + + async def ancestors(self, node_id: str) -> list[Any]: + """Get ancestor nodes from root to the given node.""" + ... + + async def doc_card(self) -> Any: + """Get the document card with metadata.""" + ... + + async def concepts(self) -> list[Any]: + """Get extracted key concepts.""" + ... + + # Identity + async def root_id(self) -> str: + """Return the root node ID.""" + ... + + async def current_id(self) -> str: + """Return the current node ID.""" + ... + + async def node_title(self, node_id: str) -> str: + """Return the title of a node by ID.""" + ... + + async def doc_name(self) -> str: + """Return the document name.""" + ... + + async def wc(self, node_id: str | None = None) -> Any: + """Return word count info for a node.""" + ... + + +# --------------------------------------------------------------------------- +# Callable protocols +# --------------------------------------------------------------------------- + class DocLoader(Protocol): """Async callable that loads a navigable document by ID.""" - async def __call__(self, doc_id: str) -> Any: ... + async def __call__(self, doc_id: str) -> NavigableDocument: ... class EventCallback(Protocol): diff --git a/vectorless/ask/worker.py b/vectorless/ask/worker.py deleted file mode 100644 index f249abd..0000000 --- a/vectorless/ask/worker.py +++ /dev/null @@ -1,1114 +0,0 @@ -"""Worker agent — navigates a single document to collect evidence. - -The Worker uses an LLM-driven command loop: -1. Phase 0: Initial `ls` to observe the top-level structure -2. Phase 1.5 (optional): LLM generates a navigation plan from keyword hints -3. Phase 2: Main loop — LLM picks a command → execute → record trace → repeat -""" - -from __future__ import annotations - -import re -import logging -from dataclasses import dataclass -from typing import Any - -from vectorless.ask.types import TraceStep, Evidence, WorkerOutput, WorkerState -from vectorless.ask.utils import extract_keywords -from vectorless.llm_client import LLMClient -from vectorless.ask.tools import compare_nodes, summarize_section, trace_reasoning -from vectorless.ask.prompts import ( - NavigationParams, - WorkerDispatchParams, - build_plan_prompt, - build_replan_prompt, - check_sufficiency, - parse_sufficiency_response, - worker_dispatch, - worker_navigation, -) - -logger = logging.getLogger(__name__) - -MAX_HISTORY_ENTRIES = 6 - - -# --------------------------------------------------------------------------- -# Command parsing -# --------------------------------------------------------------------------- - -@dataclass -class Command: - """Parsed command from LLM output.""" - kind: str # ls, cd, cd_up, cat, find, findtree, grep, head, wc, pwd, check, done, ... - target: str = "" - target_b: str = "" # second target for compare, pattern for grep_node - lines: int = 20 - - -def _strip_quotes(s: str) -> str: - """Strip surrounding quotes (straight and smart) from a string.""" - trimmed = s.strip() - if len(trimmed) < 2: - return trimmed - first, last = trimmed[0], trimmed[-1] - matching = ( - (first == '"' and last == '"') - or (first == "'" and last == "'") - or (first == "“" and last == "”") - or (first == "‘" and last == "’") - ) - return trimmed[1:-1] if matching else trimmed - - -def parse_command(llm_output: str) -> Command: - """Parse the first non-empty line of LLM output into a Command.""" - line = "" - for l in llm_output.splitlines(): - if l.strip(): - line = l.strip() - break - - # Remove common wrapping - line = line.strip().strip("`").strip() - - parts = line.split() - if not parts: - return Command(kind="ls") - - cmd = parts[0].lower() - - if cmd == "ls": - return Command(kind="ls") - elif cmd == "cd": - if len(parts) >= 2 and parts[1] == "..": - return Command(kind="cd_up") - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="cd", target=target) - elif cmd == "cat": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "." - return Command(kind="cat", target=target) - elif cmd == "find": - keyword = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="find", target=keyword) - elif cmd == "findtree": - pattern = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="findtree", target=pattern) - elif cmd == "grep": - pattern = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="grep", target=pattern) - elif cmd == "head": - if len(parts) >= 4 and parts[1] == "-n": - target = _strip_quotes(" ".join(parts[3:])) - try: - n = int(parts[2]) - except ValueError: - n = 20 - return Command(kind="head", target=target, lines=n) - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="head", target=target) - elif cmd == "wc": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="wc", target=target) - elif cmd == "pwd": - return Command(kind="pwd") - elif cmd == "check": - return Command(kind="check") - elif cmd == "done": - return Command(kind="done") - elif cmd == "back": - return Command(kind="back") - elif cmd == "toc": - if len(parts) > 1: - try: - return Command(kind="toc", lines=int(parts[1])) - except ValueError: - pass - return Command(kind="toc", lines=0) # 0 = no depth limit - elif cmd == "stats": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="stats", target=target) - elif cmd == "grep_node": - # grep_node - if len(parts) >= 3: - return Command(kind="grep_node", target=parts[1], target_b=_strip_quotes(" ".join(parts[2:]))) - elif len(parts) == 2: - return Command(kind="grep_node", target=parts[1]) - return Command(kind="grep_node") - elif cmd == "similar": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="similar", target=target) - elif cmd in ("section_overview", "overview"): - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="section_overview", target=target) - elif cmd == "compare": - # compare — use node IDs for reliability - if len(parts) >= 3: - return Command(kind="compare", target=parts[1], target_b=parts[2]) - elif len(parts) == 2: - return Command(kind="compare", target=parts[1]) - return Command(kind="compare") - elif cmd == "trace": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="trace", target=target) - elif cmd == "summarize": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="summarize", target=target) - elif cmd == "siblings": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="siblings", target=target) - elif cmd == "ancestors": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="ancestors", target=target) - elif cmd in ("doc_card", "card"): - return Command(kind="doc_card") - elif cmd == "concepts": - return Command(kind="concepts") - elif cmd == "find_section": - target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" - return Command(kind="find_section", target=target) - else: - return Command(kind="ls") # fallback: re-observe - - -def _is_parse_failure(command: Command, raw_output: str) -> bool: - """Detect if the parsed command is a fallback (unrecognized input).""" - trimmed = raw_output.strip() - return command.kind == "ls" and not trimmed.startswith("ls") and trimmed != "" - - -# --------------------------------------------------------------------------- -# Step result -# --------------------------------------------------------------------------- - -@dataclass -class Step: - """Result of a single command execution.""" - kind: str # continue, done, force_done - reason: str = "" - - -# --------------------------------------------------------------------------- -# Worker helpers -# --------------------------------------------------------------------------- - -async def _visited_titles(state: WorkerState, doc: Any) -> str: - """Format visited node titles for prompt context.""" - titles = [] - for node_id in state.visited: - try: - title = await doc.node_title(node_id) - if title: - titles.append(title) - except Exception: - pass - return ", ".join(titles) if titles else "(none)" - - -async def _resolve_target(doc: Any, target: str, state: WorkerState) -> str | None: - """Resolve a command target (node ID, child title, or empty for current) to a node ID.""" - if not target or target == ".": - return await doc.current_id() - if re.match(r"^n\d+$", target): - return target - children = await doc.ls() - for child in children: - if child.title.lower() == target.lower(): - return child.id - for child in children: - if target.lower() in child.title.lower(): - return child.id - return None - - -# --------------------------------------------------------------------------- -# Command execution -# --------------------------------------------------------------------------- - -async def _execute_command( - command: Command, - doc: Any, - state: WorkerState, - query: str, - llm: LLMClient, -) -> Step: - """Execute a parsed command against the PyDocument. Returns Step result.""" - kind = command.kind - - if kind == "ls": - children = await doc.ls() - if not children: - state.last_feedback = "(no navigation data)" - else: - lines = [] - for i, child in enumerate(children, 1): - hints = getattr(child, "question_hints", []) - tags = getattr(child, "topic_tags", []) - annotations = [] - if hints: - for h in hints[:2]: - annotations.append(f'question "{h}"') - if tags: - for t in tags[:2]: - annotations.append(f'topic "{t}"') - ann_str = f", {', '.join(annotations)}" if annotations else "" - lines.append( - f"[{i}] {child.title} — " - f"(depth {child.depth}, {child.leaf_count} leaves{ann_str})" - ) - state.last_feedback = "\n".join(lines) - state.visited.add(await doc.current_id()) - return Step(kind="continue") - - elif kind == "cd": - target = command.target - if not target: - state.last_feedback = "Usage: cd " - return Step(kind="continue") - - # Try cd by node id first (if target looks like n42) - if re.match(r"^n\d+$", target): - try: - await doc.cd(target) - title = await doc.node_title(target) - state.breadcrumb.append(title) - current = await doc.current_id() - state.visited.add(current) - state.last_feedback = f"Entered '{title}'" - return Step(kind="continue") - except Exception: - pass - - # Try cd_by_title - try: - await doc.cd_by_title(target) - current = await doc.current_id() - title = await doc.node_title(current) - state.breadcrumb.append(title) - state.visited.add(current) - state.last_feedback = f"Entered '{title}'" - return Step(kind="continue") - except Exception: - state.last_feedback = f"Node '{target}' not found. Use ls to list children." - return Step(kind="continue") - - elif kind == "cd_up": - try: - await doc.cd_up() - if len(state.breadcrumb) > 1: - state.breadcrumb.pop() - state.last_feedback = f"Current position: /{state.path_str()}" - except Exception as e: - state.last_feedback = f"Cannot go up: {e}" - return Step(kind="continue") - - elif kind == "cat": - target = command.target - node_id = None - - if target == "." or target == "": - node_id = await doc.current_id() - elif re.match(r"^n\d+$", target): - node_id = target - else: - # Try to find by title among children - children = await doc.ls() - for child in children: - if child.title.lower() == target.lower(): - node_id = child.id - break - if target.lower() in child.title.lower(): - node_id = child.id - break - if node_id is None: - # Try find - results = await doc.find(target) - if results: - node_id = results[0].node_id - - if node_id is None: - state.last_feedback = f"Node '{target}' not found." - return Step(kind="continue") - - if node_id in state.collected_nodes: - state.last_feedback = f"Already collected evidence from '{target}'. Use done if sufficient." - return Step(kind="continue") - - try: - content = await doc.cat(node_id) - title = await doc.node_title(node_id) - pwd = await doc.pwd() - - evidence = Evidence( - source_path=pwd, - node_title=title, - content=content, - ) - state.evidence.append(evidence) - state.collected_nodes.add(node_id) - state.visited.add(node_id) - - preview = content[:500] + "..." if len(content) > 500 else content - state.last_feedback = f"[{title}] collected as evidence:\n{preview}" - return Step(kind="continue") - except Exception as e: - state.last_feedback = f"Error reading node: {e}" - return Step(kind="continue") - - elif kind == "find": - keyword = command.target - if not keyword: - state.last_feedback = "Usage: find " - return Step(kind="continue") - - try: - results = await doc.find(keyword) - except Exception: - results = [] - - if not results: - # Fallback: try keyword_entries for reasoning index hits - try: - entries = await doc.keyword_entries(keyword) - if entries: - lines = [f"Results for '{keyword}':"] - for entry in entries: - title = await doc.node_title(entry.node_id) - lines.append( - f" - {title} (depth {entry.depth}, weight {entry.weight:.2f})" - ) - state.last_feedback = "\n".join(lines) - return Step(kind="continue") - except Exception: - pass - state.last_feedback = f"No results for '{keyword}'." - return Step(kind="continue") - - lines = [f"Results for '{keyword}':"] - for r in results[:10]: - lines.append(f" - {r.title} (depth {r.depth}, {r.leaf_count} leaves)") - state.last_feedback = "\n".join(lines) - return Step(kind="continue") - - elif kind == "findtree": - pattern = command.target - if not pattern: - state.last_feedback = "Usage: findtree " - return Step(kind="continue") - - try: - results = await doc.find(pattern) - except Exception: - results = [] - - if not results: - state.last_feedback = f"No nodes matching '{pattern}' in titles." - return Step(kind="continue") - - lines = [f"Nodes matching '{pattern}':"] - for r in results[:15]: - lines.append(f" - {r.title} (depth {r.depth})") - state.last_feedback = "\n".join(lines) - return Step(kind="continue") - - elif kind == "grep": - pattern = command.target - if not pattern: - state.last_feedback = "Usage: grep " - return Step(kind="continue") - try: - matches = await doc.grep(pattern) - except Exception as e: - state.last_feedback = f"grep error: {e}" - return Step(kind="continue") - - if not matches: - state.last_feedback = f"No matches for /{pattern}/." - return Step(kind="continue") - - lines = [f"Matches for /{pattern}/:"] - for m in matches[:15]: - lines.append(f" - {m.title} (line {m.line_number}): {m.snippet[:100]}") - state.last_feedback = "\n".join(lines) - return Step(kind="continue") - - elif kind == "head": - target = command.target - n = command.lines - node_id = None - - if re.match(r"^n\d+$", target): - node_id = target - else: - children = await doc.ls() - for child in children: - if child.title.lower() == target.lower(): - node_id = child.id - break - - if node_id is None: - state.last_feedback = f"Node '{target}' not found." - return Step(kind="continue") - - try: - content = await doc.head(node_id, n) - state.last_feedback = content - except Exception as e: - state.last_feedback = f"head error: {e}" - return Step(kind="continue") - - elif kind == "wc": - target = command.target - node_id = None - - if not target: - node_id = await doc.current_id() - elif re.match(r"^n\d+$", target): - node_id = target - else: - children = await doc.ls() - for child in children: - if child.title.lower() == target.lower(): - node_id = child.id - break - - if node_id is None: - state.last_feedback = f"Node '{target}' not found." - return Step(kind="continue") - - try: - wc = await doc.wc(node_id) - state.last_feedback = f"{wc.lines} lines, {wc.words} words, {wc.chars} chars" - except Exception as e: - state.last_feedback = f"wc error: {e}" - return Step(kind="continue") - - elif kind == "pwd": - try: - pwd = await doc.pwd() - state.last_feedback = f"/{pwd}" - except Exception as e: - state.last_feedback = f"pwd error: {e}" - return Step(kind="continue") - - elif kind == "back": - try: - await doc.back() - pwd = await doc.pwd() - state.breadcrumb = [p for p in pwd.split("/") if p] - state.last_feedback = f"Current position: /{state.path_str()}" - except Exception as e: - state.last_feedback = f"Cannot go back: {e}" - return Step(kind="continue") - - elif kind == "toc": - try: - if command.lines > 0: - entries = await doc.toc(command.lines) - else: - entries = await doc.toc() - if not entries: - state.last_feedback = "(empty table of contents)" - else: - lines = ["Table of contents:"] - for entry in entries: - indent = " " * entry.depth - children = f" ({entry.child_count} children)" if entry.child_count > 0 else "" - lines.append(f"{indent}- {entry.title}{children}") - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"toc error: {e}" - return Step(kind="continue") - - elif kind == "stats": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - s = await doc.stats(node_id) - leaf = " (leaf)" if s.is_leaf else "" - state.last_feedback = ( - f"[{s.title}] depth={s.depth}, children={s.child_count}, " - f"leaves={s.leaf_count}, chars={s.char_count}, words={s.word_count}{leaf}" - ) - except Exception as e: - state.last_feedback = f"stats error: {e}" - return Step(kind="continue") - - elif kind == "grep_node": - target = command.target - pattern = command.target_b - if not target or not pattern: - state.last_feedback = "Usage: grep_node " - return Step(kind="continue") - node_id = await _resolve_target(doc, target, state) - if node_id is None: - state.last_feedback = f"Node '{target}' not found." - return Step(kind="continue") - try: - matches = await doc.grep_node(node_id, pattern) - if not matches: - state.last_feedback = f"No matches for /{pattern}/ in this node." - else: - lines = [f"Matches for /{pattern}/:"] - for m in matches[:15]: - lines.append(f" - line {m.line_number}: {m.snippet[:100]}") - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"grep_node error: {e}" - return Step(kind="continue") - - elif kind == "similar": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - results = await doc.similar(node_id) - if not results: - state.last_feedback = "No similar nodes found." - else: - lines = ["Similar nodes:"] - for r in results[:10]: - kw = ", ".join(r.shared_keywords[:3]) - lines.append(f" - {r.title} (relevance: {r.relevance:.2f}, shared: {kw})") - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"similar error: {e}" - return Step(kind="continue") - - elif kind == "section_overview": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - overview = await doc.section_overview(node_id) - state.last_feedback = overview if overview else "(no overview available)" - except Exception as e: - state.last_feedback = f"overview error: {e}" - return Step(kind="continue") - - elif kind == "compare": - target_a = command.target - target_b = command.target_b - if not target_a or not target_b: - state.last_feedback = "Usage: compare " - return Step(kind="continue") - node_a = await _resolve_target(doc, target_a, state) - node_b = await _resolve_target(doc, target_b, state) - if node_a is None: - state.last_feedback = f"Node '{target_a}' not found." - return Step(kind="continue") - if node_b is None: - state.last_feedback = f"Node '{target_b}' not found." - return Step(kind="continue") - try: - content_a = await doc.cat(node_a) - title_a = await doc.node_title(node_a) - content_b = await doc.cat(node_b) - title_b = await doc.node_title(node_b) - if node_a not in state.collected_nodes: - pwd_a = await doc.pwd() - state.evidence.append(Evidence( - source_path=pwd_a, node_title=title_a, content=content_a, - )) - state.collected_nodes.add(node_a) - if node_b not in state.collected_nodes: - pwd_b = await doc.pwd() - state.evidence.append(Evidence( - source_path=pwd_b, node_title=title_b, content=content_b, - )) - state.collected_nodes.add(node_b) - result = await compare_nodes(title_a, content_a, title_b, content_b, llm, query=query) - state.llm_calls += 1 - state.last_feedback = f"Comparison of [{title_a}] vs [{title_b}]:\n{result}" - except Exception as e: - state.last_feedback = f"compare error: {e}" - return Step(kind="continue") - - elif kind == "trace": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - content = await doc.cat(node_id) - title = await doc.node_title(node_id) - if node_id not in state.collected_nodes: - pwd = await doc.pwd() - state.evidence.append(Evidence( - source_path=pwd, node_title=title, content=content, - )) - state.collected_nodes.add(node_id) - related_context = "" - try: - similar = await doc.similar(node_id) - if similar: - related_lines = [f" - {s.title} (relevance: {s.relevance:.2f})" for s in similar[:5]] - related_context = "\nRelated sections:\n" + "\n".join(related_lines) - except Exception: - pass - result = await trace_reasoning(title, content, related_context, llm, query=query) - state.llm_calls += 1 - state.last_feedback = f"Reasoning trace for [{title}]:\n{result}" - except Exception as e: - state.last_feedback = f"trace error: {e}" - return Step(kind="continue") - - elif kind == "summarize": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - content = await doc.cat(node_id) - title = await doc.node_title(node_id) - if node_id not in state.collected_nodes: - pwd = await doc.pwd() - state.evidence.append(Evidence( - source_path=pwd, node_title=title, content=content, - )) - state.collected_nodes.add(node_id) - result = await summarize_section(title, content, llm, query=query) - state.llm_calls += 1 - state.last_feedback = f"Summary of [{title}]:\n{result}" - except Exception as e: - state.last_feedback = f"summarize error: {e}" - return Step(kind="continue") - - elif kind == "siblings": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - siblings = await doc.siblings(node_id) - if not siblings: - state.last_feedback = "(no sibling nodes)" - else: - lines = ["Sibling nodes:"] - for s in siblings: - lines.append( - f" - {s.title} (depth {s.depth}, {s.leaf_count} leaves)" - ) - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"siblings error: {e}" - return Step(kind="continue") - - elif kind == "ancestors": - node_id = await _resolve_target(doc, command.target, state) - if node_id is None: - state.last_feedback = f"Node '{command.target}' not found." - return Step(kind="continue") - try: - ancestors = await doc.ancestors(node_id) - if not ancestors: - state.last_feedback = "(at root, no ancestors)" - else: - lines = ["Path from root:"] - for a in ancestors: - lines.append( - f" {' ' * a.depth}→ {a.title} (depth {a.depth}, {a.child_count} children)" - ) - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"ancestors error: {e}" - return Step(kind="continue") - - elif kind == "doc_card": - try: - card = await doc.doc_card() - if card is None: - state.last_feedback = "(no document card available)" - else: - lines = [ - f"Document: {card.title}", - f"Overview: {card.overview}", - f"Total leaves: {card.total_leaves}", - ] - if card.question_hints: - lines.append(f"Can answer: {', '.join(card.question_hints[:5])}") - if card.topic_tags: - lines.append(f"Topics: {', '.join(card.topic_tags[:5])}") - if card.sections: - lines.append("Top-level sections:") - for s in card.sections: - lines.append(f" - {s.title}: {s.description} ({s.leaf_count} leaves)") - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"doc_card error: {e}" - return Step(kind="continue") - - elif kind == "concepts": - try: - concepts = await doc.concepts() - if not concepts: - state.last_feedback = "(no concepts extracted)" - else: - lines = ["Key concepts:"] - for c in concepts: - sections = ", ".join(c.sections[:3]) - lines.append(f" - {c.name}: {c.summary} (in: {sections})") - state.last_feedback = "\n".join(lines) - except Exception as e: - state.last_feedback = f"concepts error: {e}" - return Step(kind="continue") - - elif kind == "find_section": - title = command.target - if not title: - state.last_feedback = "Usage: find_section " - return Step(kind="continue") - try: - result = await doc.find_section(title) - if result is None: - state.last_feedback = f"No section with title '{title}'." - else: - state.last_feedback = ( - f"Found: {result.title} (id={result.node_id}, " - f"depth {result.depth}, {result.leaf_count} leaves)" - ) - except Exception as e: - state.last_feedback = f"find_section error: {e}" - return Step(kind="continue") - - elif kind == "check": - evidence_text = state.evidence_for_check() - system, user = check_sufficiency(query, evidence_text) - - try: - response = await llm.complete(system, user) - except Exception as e: - logger.warning("Check LLM call failed: %s", e) - state.last_feedback = "Could not evaluate sufficiency." - return Step(kind="continue") - - state.llm_calls += 1 - state.check_count += 1 - sufficient = parse_sufficiency_response(response) - - if sufficient: - state.last_feedback = "Evidence is sufficient. Use done to finish." - return Step(kind="done") - else: - # Extract missing info - reason = response.strip() - for prefix in ("INSUFFICIENT", "Insufficient"): - if reason.startswith(prefix): - reason = reason[len(prefix):] - break - reason = reason.lstrip("-: ") - if reason: - state.missing_info = reason - state.last_feedback = f"Evidence not yet sufficient: {response.strip()}" - return Step(kind="continue") - - elif kind == "done": - state.last_feedback = "Navigation complete." - return Step(kind="done") - - else: - state.last_feedback = f"Unknown command: {kind}" - return Step(kind="continue") - - -# --------------------------------------------------------------------------- -# Worker -# --------------------------------------------------------------------------- - -class Worker: - """Navigates a single document to collect evidence for a query. - - Usage:: - - worker = Worker(document=doc, query="What is the revenue?", llm_client=llm) - result = await worker.run() - """ - - def __init__( - self, - document: Any, - query: str, - llm_client: LLMClient, - *, - max_rounds: int = 15, - max_llm_calls: int = 0, - task: str | None = None, - intent_context: str = "", - shared_context: str = "", - ) -> None: - self._doc = document - self._query = query - self._llm = llm_client - self._max_rounds = max_rounds - self._max_llm_calls = max_llm_calls - self._task = task - self._intent_context = intent_context - self._shared_context = shared_context - - async def run(self) -> WorkerOutput: - """Execute the Worker navigation loop and return collected evidence.""" - doc = self._doc - query = self._query - llm = self._llm - task = self._task - max_rounds = self._max_rounds - max_llm = self._max_llm_calls - intent_context = self._intent_context - shared_context = self._shared_context - - state = WorkerState(remaining=max_rounds, max_rounds=max_rounds) - - # Phase 0: initial ls to observe environment - root_id = await doc.root_id() - state.visited.add(root_id) - - try: - children = await doc.ls() - if children: - lines = [] - for i, child in enumerate(children, 1): - lines.append( - f"[{i}] {child.title} — " - f"(depth {child.depth}, {child.leaf_count} leaves)" - ) - state.last_feedback = "\n".join(lines) - else: - state.last_feedback = "(no children at root)" - except Exception as e: - state.last_feedback = f"Initial ls failed: {e}" - - # Phase 1.5: optional navigation planning - keyword_hints = "" - try: - keyword_hints = await self._build_keyword_hints(doc, query) - except Exception: - pass - - if keyword_hints: - await self._generate_plan(doc, query, task, state, keyword_hints, llm) - - # Phase 2: main navigation loop - use_dispatch = task is not None - - while state.remaining > 0: - if max_llm > 0 and state.llm_calls >= max_llm: - logger.info("LLM call budget exhausted (%d/%d)", state.llm_calls, max_llm) - break - - # Build prompt - if use_dispatch and state.remaining == max_rounds: - system, user = worker_dispatch(WorkerDispatchParams( - original_query=query, - task=task or query, - doc_name=await doc.doc_name(), - breadcrumb=state.path_str(), - shared_context=shared_context, - )) - else: - visited_titles = await _visited_titles(state, doc) - system, user = worker_navigation(NavigationParams( - query=query, - task=task, - breadcrumb=state.path_str(), - evidence_summary=state.evidence_summary(), - missing_info=state.missing_info, - last_feedback=state.last_feedback, - remaining=state.remaining, - max_rounds=state.max_rounds, - history=state.history_text(), - visited_titles=visited_titles, - plan=state.plan, - intent_context=intent_context, - keyword_hints=keyword_hints, - shared_context=shared_context, - )) - - # LLM decision - round_num = max_rounds - state.remaining + 1 - try: - llm_output = await llm.complete(system, user) - except Exception as e: - logger.error("LLM call failed at round %d: %s", round_num, e) - break - state.llm_calls += 1 - - # Parse command - command = parse_command(llm_output) - is_failure = _is_parse_failure(command, llm_output) - - if is_failure: - raw_preview = llm_output.strip()[:200] - if len(llm_output.strip()) > 200: - raw_preview += "..." - state.last_feedback = ( - f"Your output was not recognized as a valid command:\n" - f'"{raw_preview}"\n\n' - f"Please output exactly one command " - f"(ls, cd, cat, head, find, grep, toc, stats, similar, overview, " - f"siblings, ancestors, doc_card, concepts, find_section, " - f"compare, trace, summarize, wc, pwd, check, or done)." - ) - state.push_history("(unrecognized) → parse failure") - continue - - is_check = command.kind == "check" - - # Execute - step = await _execute_command(command, doc, state, query, llm) - - # Re-plan after insufficient check - if is_check: - await self._handle_replan(query, task, doc, state, llm, max_llm) - - # Record history and trace - cmd_str = command.kind - if command.target: - cmd_str += f" {command.target}" - - feedback_preview = state.last_feedback - if len(feedback_preview) > 120: - feedback_preview = feedback_preview[:120] + "..." - state.push_history(f"{cmd_str} → {feedback_preview}") - - round_num_done = max_rounds - state.remaining - state.trace_steps.append(TraceStep( - action=cmd_str, - observation=state.last_feedback[:200], - round=round_num_done, - )) - - # Check termination - if step.kind == "done": - break - elif step.kind == "force_done": - break - else: - if not is_check: - state.remaining -= 1 - - doc_name = "" - try: - doc_name = await doc.doc_name() - except Exception: - pass - - return state.into_worker_output(doc_name) - - async def _build_keyword_hints(self, doc: Any, query: str) -> str: - """Build keyword hints from the document's reasoning index.""" - keywords = extract_keywords(query) - - if not keywords: - return "" - - hints = [] - for kw in keywords[:5]: # limit keywords - try: - entries = await doc.keyword_entries(kw) - for entry in entries[:3]: - title = await doc.node_title(entry.node_id) - hints.append( - f" - '{kw}' → {title} (weight {entry.weight:.2f})" - ) - except Exception: - pass - - if not hints: - return "" - - return "Keyword matches (use find <keyword> to jump directly):\n" + "\n".join(hints) + "\n" - - async def _generate_plan( - self, - doc: Any, - query: str, - task: str | None, - state: WorkerState, - keyword_hints: str, - llm: LLMClient, - ) -> None: - """Phase 1.5: generate a navigation plan from keyword hints.""" - ls_output = state.last_feedback - doc_name = await doc.doc_name() - - system, user = build_plan_prompt( - query=query, - ls_output=ls_output, - doc_name=doc_name, - keyword_hints_section=f"\n{keyword_hints}" if keyword_hints else "", - task=task, - ) - - try: - plan = await llm.complete(system, user) - state.llm_calls += 1 - plan_text = plan.strip() - if plan_text: - state.plan = plan_text - state.plan_generated = True - except Exception as e: - logger.warning("Plan generation failed: %s", e) - - async def _handle_replan( - self, - query: str, - task: str | None, - doc: Any, - state: WorkerState, - llm: LLMClient, - max_llm: int, - ) -> None: - """Dynamic re-planning after an insufficient check.""" - if not state.missing_info: - return - - if state.remaining < 3: - state.plan = "" - state.missing_info = "" - return - - if max_llm > 0 and state.llm_calls >= max_llm: - state.plan = "" - state.missing_info = "" - return - - # Build sibling hints - sibling_hints = "" - current_children = "Current position is a leaf node — consider cd .. to go back.\n" - - try: - children = await doc.ls() - if children: - items = [f" - {c.title} ({c.leaf_count} leaves)" for c in children] - current_children = f"Children at current position:\n" + "\n".join(items) + "\n" - except Exception: - pass - - system, user = build_replan_prompt( - query=query, - task=task, - path_str=state.path_str(), - evidence_summary=state.evidence_summary(), - missing_info=state.missing_info, - visited_titles=await _visited_titles(state, doc), - current_children=current_children, - sibling_hints=sibling_hints, - remaining=state.remaining, - max_rounds=state.max_rounds, - ) - - try: - new_plan = await llm.complete(system, user) - state.llm_calls += 1 - plan_text = new_plan.strip() - if plan_text: - logger.info("Re-plan generated: %s", plan_text[:200]) - state.plan = plan_text - except Exception as e: - logger.warning("Re-plan LLM call failed: %s", e) - - state.missing_info = "" diff --git a/vectorless/ask/worker/__init__.py b/vectorless/ask/worker/__init__.py new file mode 100644 index 0000000..02750be --- /dev/null +++ b/vectorless/ask/worker/__init__.py @@ -0,0 +1,6 @@ +"""Worker agent package — command-based document navigation.""" + +from vectorless.ask.worker.agent import Worker +from vectorless.ask.worker.parse import Command, parse_command, _is_parse_failure + +__all__ = ["Worker", "Command", "parse_command", "_is_parse_failure"] diff --git a/vectorless/ask/worker/agent.py b/vectorless/ask/worker/agent.py new file mode 100644 index 0000000..4910fd7 --- /dev/null +++ b/vectorless/ask/worker/agent.py @@ -0,0 +1,327 @@ +"""Worker agent — navigates a single document to collect evidence. + +The Worker uses an LLM-driven command loop: +1. Phase 0: Initial `ls` to observe the top-level structure +2. Phase 1.5 (optional): LLM generates a navigation plan from keyword hints +3. Phase 2: Main loop — LLM picks a command → execute → record trace → repeat +""" + +from __future__ import annotations + +import logging + +from vectorless.ask.protocols import NavigableDocument +from vectorless.ask.errors import LLMFailureError +from vectorless.ask.types import TraceStep, WorkerOutput, WorkerState +from vectorless.ask.utils import extract_keywords +from vectorless.llm_client import LLMClient +from vectorless.ask.worker.parse import Command, parse_command, _is_parse_failure +from vectorless.ask.worker.commands import execute_command, _visited_titles, Step +from vectorless.ask.prompts import ( + NavigationParams, + WorkerDispatchParams, + build_plan_prompt, + build_replan_prompt, + worker_dispatch, + worker_navigation, +) + +logger = logging.getLogger(__name__) + + +class Worker: + """Navigates a single document to collect evidence for a query. + + Usage:: + + worker = Worker(document=doc, query="What is the revenue?", llm_client=llm) + result = await worker.run() + """ + + def __init__( + self, + document: NavigableDocument, + query: str, + llm_client: LLMClient, + *, + max_rounds: int = 15, + max_llm_calls: int = 0, + task: str | None = None, + intent_context: str = "", + shared_context: str = "", + ) -> None: + self._doc = document + self._query = query + self._llm = llm_client + self._max_rounds = max_rounds + self._max_llm_calls = max_llm_calls + self._task = task + self._intent_context = intent_context + self._shared_context = shared_context + + async def run(self) -> WorkerOutput: + """Execute the Worker navigation loop and return collected evidence.""" + doc = self._doc + query = self._query + llm = self._llm + task = self._task + max_rounds = self._max_rounds + max_llm = self._max_llm_calls + intent_context = self._intent_context + shared_context = self._shared_context + + state = WorkerState(remaining=max_rounds, max_rounds=max_rounds) + + # Phase 0: initial ls to observe environment + root_id = await doc.root_id() + state.visited.add(root_id) + + try: + children = await doc.ls() + if children: + lines = [] + for i, child in enumerate(children, 1): + lines.append( + f"[{i}] {child.title} — " + f"(depth {child.depth}, {child.leaf_count} leaves)" + ) + state.last_feedback = "\n".join(lines) + else: + state.last_feedback = "(no children at root)" + except Exception as e: + state.last_feedback = f"Initial ls failed: {e}" + + # Phase 1.5: optional navigation planning + keyword_hints = "" + try: + keyword_hints = await self._build_keyword_hints(doc, query) + except Exception: + pass + + if keyword_hints: + await self._generate_plan(doc, query, task, state, keyword_hints, llm) + + # Phase 2: main navigation loop + use_dispatch = task is not None + + while state.remaining > 0: + if max_llm > 0 and state.llm_calls >= max_llm: + logger.info("LLM call budget exhausted (%d/%d)", state.llm_calls, max_llm) + break + + # Build prompt + if use_dispatch and state.remaining == max_rounds: + system, user = worker_dispatch(WorkerDispatchParams( + original_query=query, + task=task or query, + doc_name=await doc.doc_name(), + breadcrumb=state.path_str(), + shared_context=shared_context, + )) + else: + visited_titles = await _visited_titles(state, doc) + system, user = worker_navigation(NavigationParams( + query=query, + task=task, + breadcrumb=state.path_str(), + evidence_summary=state.evidence_summary(), + missing_info=state.missing_info, + last_feedback=state.last_feedback, + remaining=state.remaining, + max_rounds=state.max_rounds, + history=state.history_text(), + visited_titles=visited_titles, + plan=state.plan, + intent_context=intent_context, + keyword_hints=keyword_hints, + shared_context=shared_context, + )) + + # LLM decision + round_num = max_rounds - state.remaining + 1 + try: + llm_output = await llm.complete(system, user) + except LLMFailureError as e: + logger.error("LLM call failed at round %d: %s", round_num, e) + break + except Exception as e: + logger.error("Unexpected error at round %d: %s", round_num, e) + break + state.llm_calls += 1 + + # Parse command + command = parse_command(llm_output) + is_failure = _is_parse_failure(command, llm_output) + + if is_failure: + raw_preview = llm_output.strip()[:200] + if len(llm_output.strip()) > 200: + raw_preview += "..." + state.last_feedback = ( + f"Your output was not recognized as a valid command:\n" + f'"{raw_preview}"\n\n' + f"Please output exactly one command " + f"(ls, cd, cat, head, find, grep, toc, stats, similar, overview, " + f"siblings, ancestors, doc_card, concepts, find_section, " + f"compare, trace, summarize, wc, pwd, check, or done)." + ) + state.push_history("(unrecognized) \u2192 parse failure") + continue + + is_check = command.kind == "check" + + # Execute via command registry + step = await execute_command(command, doc, state, query, llm) + + # Re-plan after insufficient check + if is_check: + await self._handle_replan(query, task, doc, state, llm, max_llm) + + # Record history and trace + cmd_str = command.kind + if command.target: + cmd_str += f" {command.target}" + + feedback_preview = state.last_feedback + if len(feedback_preview) > 120: + feedback_preview = feedback_preview[:120] + "..." + state.push_history(f"{cmd_str} \u2192 {feedback_preview}") + + round_num_done = max_rounds - state.remaining + state.trace_steps.append(TraceStep( + action=cmd_str, + observation=state.last_feedback[:200], + round=round_num_done, + )) + + # Check termination + if step.kind == "done": + break + elif step.kind == "force_done": + break + else: + if not is_check: + state.remaining -= 1 + + doc_name = "" + try: + doc_name = await doc.doc_name() + except Exception: + pass + + return state.into_worker_output(doc_name) + + async def _build_keyword_hints(self, doc: NavigableDocument, query: str) -> str: + """Build keyword hints from the document's reasoning index.""" + keywords = extract_keywords(query) + + if not keywords: + return "" + + hints = [] + for kw in keywords[:5]: # limit keywords + try: + entries = await doc.keyword_entries(kw) + for entry in entries[:3]: + title = await doc.node_title(entry.node_id) + hints.append( + f" - '{kw}' \u2192 {title} (weight {entry.weight:.2f})" + ) + except Exception: + pass + + if not hints: + return "" + + return "Keyword matches (use find <keyword> to jump directly):\n" + "\n".join(hints) + "\n" + + async def _generate_plan( + self, + doc: NavigableDocument, + query: str, + task: str | None, + state: WorkerState, + keyword_hints: str, + llm: LLMClient, + ) -> None: + """Phase 1.5: generate a navigation plan from keyword hints.""" + ls_output = state.last_feedback + doc_name = await doc.doc_name() + + system, user = build_plan_prompt( + query=query, + ls_output=ls_output, + doc_name=doc_name, + keyword_hints_section=f"\n{keyword_hints}" if keyword_hints else "", + task=task, + ) + + try: + plan = await llm.complete(system, user) + state.llm_calls += 1 + plan_text = plan.strip() + if plan_text: + state.plan = plan_text + state.plan_generated = True + except Exception as e: + logger.warning("Plan generation failed: %s", e) + + async def _handle_replan( + self, + query: str, + task: str | None, + doc: NavigableDocument, + state: WorkerState, + llm: LLMClient, + max_llm: int, + ) -> None: + """Dynamic re-planning after an insufficient check.""" + if not state.missing_info: + return + + if state.remaining < 3: + state.plan = "" + state.missing_info = "" + return + + if max_llm > 0 and state.llm_calls >= max_llm: + state.plan = "" + state.missing_info = "" + return + + # Build sibling hints + sibling_hints = "" + current_children = "Current position is a leaf node \u2014 consider cd .. to go back.\n" + + try: + children = await doc.ls() + if children: + items = [f" - {c.title} ({c.leaf_count} leaves)" for c in children] + current_children = f"Children at current position:\n" + "\n".join(items) + "\n" + except Exception: + pass + + system, user = build_replan_prompt( + query=query, + task=task, + path_str=state.path_str(), + evidence_summary=state.evidence_summary(), + missing_info=state.missing_info, + visited_titles=await _visited_titles(state, doc), + current_children=current_children, + sibling_hints=sibling_hints, + remaining=state.remaining, + max_rounds=state.max_rounds, + ) + + try: + new_plan = await llm.complete(system, user) + state.llm_calls += 1 + plan_text = new_plan.strip() + if plan_text: + logger.info("Re-plan generated: %s", plan_text[:200]) + state.plan = plan_text + except Exception as e: + logger.warning("Re-plan LLM call failed: %s", e) + + state.missing_info = "" diff --git a/vectorless/ask/worker/commands.py b/vectorless/ask/worker/commands.py new file mode 100644 index 0000000..374937d --- /dev/null +++ b/vectorless/ask/worker/commands.py @@ -0,0 +1,779 @@ +"""Command execution registry — each command is a standalone async function. + +To add a new command: +1. Write an async handler: ``async def handle_<cmd>(command, doc, state, query, llm) -> Step`` +2. Register it: ``_REGISTRY["<cmd>"] = handle_<cmd>`` + +The main Worker loop looks up the handler in ``_REGISTRY`` and calls it. +No modification to the main loop is needed. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Any, Callable, Awaitable + +from vectorless.ask.protocols import NavigableDocument +from vectorless.ask.types import Evidence, WorkerState +from vectorless.llm_client import LLMClient +from vectorless.ask.tools import compare_nodes, summarize_section, trace_reasoning +from vectorless.ask.prompts import check_sufficiency, parse_sufficiency_response + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Step result — shared across all commands +# --------------------------------------------------------------------------- + +@dataclass +class Step: + """Result of a single command execution.""" + kind: str # continue, done, force_done + reason: str = "" + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + +async def _resolve_target(doc: NavigableDocument, target: str, state: WorkerState) -> str | None: + """Resolve a command target (node ID, child title, or empty for current) to a node ID.""" + if not target or target == ".": + return await doc.current_id() + if re.match(r"^n\d+$", target): + return target + children = await doc.ls() + for child in children: + if child.title.lower() == target.lower(): + return child.id + for child in children: + if target.lower() in child.title.lower(): + return child.id + return None + + +async def _visited_titles(state: WorkerState, doc: NavigableDocument) -> str: + """Format visited node titles for prompt context.""" + titles = [] + for node_id in state.visited: + try: + title = await doc.node_title(node_id) + if title: + titles.append(title) + except Exception: + pass + return ", ".join(titles) if titles else "(none)" + + +# Type alias for the handler signature +CommandHandler = Callable[..., Awaitable[Step]] + + +# --------------------------------------------------------------------------- +# Individual command handlers +# --------------------------------------------------------------------------- + +async def handle_ls( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + children = await doc.ls() + if not children: + state.last_feedback = "(no navigation data)" + else: + lines = [] + for i, child in enumerate(children, 1): + hints = getattr(child, "question_hints", []) + tags = getattr(child, "topic_tags", []) + annotations = [] + if hints: + for h in hints[:2]: + annotations.append(f'question "{h}"') + if tags: + for t in tags[:2]: + annotations.append(f'topic "{t}"') + ann_str = f", {', '.join(annotations)}" if annotations else "" + lines.append( + f"[{i}] {child.title} — " + f"(depth {child.depth}, {child.leaf_count} leaves{ann_str})" + ) + state.last_feedback = "\n".join(lines) + state.visited.add(await doc.current_id()) + return Step(kind="continue") + + +async def handle_cd( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target = command.target + if not target: + state.last_feedback = "Usage: cd <name>" + return Step(kind="continue") + + # Try cd by node id first (if target looks like n42) + if re.match(r"^n\d+$", target): + try: + await doc.cd(target) + title = await doc.node_title(target) + state.breadcrumb.append(title) + current = await doc.current_id() + state.visited.add(current) + state.last_feedback = f"Entered '{title}'" + return Step(kind="continue") + except Exception: + pass + + # Try cd_by_title + try: + await doc.cd_by_title(target) + current = await doc.current_id() + title = await doc.node_title(current) + state.breadcrumb.append(title) + state.visited.add(current) + state.last_feedback = f"Entered '{title}'" + return Step(kind="continue") + except Exception: + state.last_feedback = f"Node '{target}' not found. Use ls to list children." + return Step(kind="continue") + + +async def handle_cd_up( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + await doc.cd_up() + if len(state.breadcrumb) > 1: + state.breadcrumb.pop() + state.last_feedback = f"Current position: /{state.path_str()}" + except Exception as e: + state.last_feedback = f"Cannot go up: {e}" + return Step(kind="continue") + + +async def handle_cat( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target = command.target + node_id = None + + if target == "." or target == "": + node_id = await doc.current_id() + elif re.match(r"^n\d+$", target): + node_id = target + else: + # Try to find by title among children + children = await doc.ls() + for child in children: + if child.title.lower() == target.lower(): + node_id = child.id + break + if target.lower() in child.title.lower(): + node_id = child.id + break + if node_id is None: + # Try find + results = await doc.find(target) + if results: + node_id = results[0].node_id + + if node_id is None: + state.last_feedback = f"Node '{target}' not found." + return Step(kind="continue") + + if node_id in state.collected_nodes: + state.last_feedback = f"Already collected evidence from '{target}'. Use done if sufficient." + return Step(kind="continue") + + try: + content = await doc.cat(node_id) + title = await doc.node_title(node_id) + pwd = await doc.pwd() + + evidence = Evidence( + source_path=pwd, + node_title=title, + content=content, + ) + state.evidence.append(evidence) + state.collected_nodes.add(node_id) + state.visited.add(node_id) + + preview = content[:500] + "..." if len(content) > 500 else content + state.last_feedback = f"[{title}] collected as evidence:\n{preview}" + return Step(kind="continue") + except Exception as e: + state.last_feedback = f"Error reading node: {e}" + return Step(kind="continue") + + +async def handle_find( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + keyword = command.target + if not keyword: + state.last_feedback = "Usage: find <keyword>" + return Step(kind="continue") + + try: + results = await doc.find(keyword) + except Exception: + results = [] + + if not results: + # Fallback: try keyword_entries for reasoning index hits + try: + entries = await doc.keyword_entries(keyword) + if entries: + lines = [f"Results for '{keyword}':"] + for entry in entries: + title = await doc.node_title(entry.node_id) + lines.append( + f" - {title} (depth {entry.depth}, weight {entry.weight:.2f})" + ) + state.last_feedback = "\n".join(lines) + return Step(kind="continue") + except Exception: + pass + state.last_feedback = f"No results for '{keyword}'." + return Step(kind="continue") + + lines = [f"Results for '{keyword}':"] + for r in results[:10]: + lines.append(f" - {r.title} (depth {r.depth}, {r.leaf_count} leaves)") + state.last_feedback = "\n".join(lines) + return Step(kind="continue") + + +async def handle_findtree( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + pattern = command.target + if not pattern: + state.last_feedback = "Usage: findtree <pattern>" + return Step(kind="continue") + + try: + results = await doc.find(pattern) + except Exception: + results = [] + + if not results: + state.last_feedback = f"No nodes matching '{pattern}' in titles." + return Step(kind="continue") + + lines = [f"Nodes matching '{pattern}':"] + for r in results[:15]: + lines.append(f" - {r.title} (depth {r.depth})") + state.last_feedback = "\n".join(lines) + return Step(kind="continue") + + +async def handle_grep( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + pattern = command.target + if not pattern: + state.last_feedback = "Usage: grep <pattern>" + return Step(kind="continue") + try: + matches = await doc.grep(pattern) + except Exception as e: + state.last_feedback = f"grep error: {e}" + return Step(kind="continue") + + if not matches: + state.last_feedback = f"No matches for /{pattern}/." + return Step(kind="continue") + + lines = [f"Matches for /{pattern}/:"] + for m in matches[:15]: + lines.append(f" - {m.title} (line {m.line_number}): {m.snippet[:100]}") + state.last_feedback = "\n".join(lines) + return Step(kind="continue") + + +async def handle_head( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target = command.target + n = command.lines + node_id = None + + if re.match(r"^n\d+$", target): + node_id = target + else: + children = await doc.ls() + for child in children: + if child.title.lower() == target.lower(): + node_id = child.id + break + + if node_id is None: + state.last_feedback = f"Node '{target}' not found." + return Step(kind="continue") + + try: + content = await doc.head(node_id, n) + state.last_feedback = content + except Exception as e: + state.last_feedback = f"head error: {e}" + return Step(kind="continue") + + +async def handle_wc( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target = command.target + node_id = None + + if not target: + node_id = await doc.current_id() + elif re.match(r"^n\d+$", target): + node_id = target + else: + children = await doc.ls() + for child in children: + if child.title.lower() == target.lower(): + node_id = child.id + break + + if node_id is None: + state.last_feedback = f"Node '{target}' not found." + return Step(kind="continue") + + try: + wc = await doc.wc(node_id) + state.last_feedback = f"{wc.lines} lines, {wc.words} words, {wc.chars} chars" + except Exception as e: + state.last_feedback = f"wc error: {e}" + return Step(kind="continue") + + +async def handle_pwd( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + pwd = await doc.pwd() + state.last_feedback = f"/{pwd}" + except Exception as e: + state.last_feedback = f"pwd error: {e}" + return Step(kind="continue") + + +async def handle_back( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + await doc.back() + pwd = await doc.pwd() + state.breadcrumb = [p for p in pwd.split("/") if p] + state.last_feedback = f"Current position: /{state.path_str()}" + except Exception as e: + state.last_feedback = f"Cannot go back: {e}" + return Step(kind="continue") + + +async def handle_toc( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + if command.lines > 0: + entries = await doc.toc(command.lines) + else: + entries = await doc.toc() + if not entries: + state.last_feedback = "(empty table of contents)" + else: + lines = ["Table of contents:"] + for entry in entries: + indent = " " * entry.depth + children = f" ({entry.child_count} children)" if entry.child_count > 0 else "" + lines.append(f"{indent}- {entry.title}{children}") + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"toc error: {e}" + return Step(kind="continue") + + +async def handle_stats( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + s = await doc.stats(node_id) + leaf = " (leaf)" if s.is_leaf else "" + state.last_feedback = ( + f"[{s.title}] depth={s.depth}, children={s.child_count}, " + f"leaves={s.leaf_count}, chars={s.char_count}, words={s.word_count}{leaf}" + ) + except Exception as e: + state.last_feedback = f"stats error: {e}" + return Step(kind="continue") + + +async def handle_grep_node( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target = command.target + pattern = command.target_b + if not target or not pattern: + state.last_feedback = "Usage: grep_node <node> <pattern>" + return Step(kind="continue") + node_id = await _resolve_target(doc, target, state) + if node_id is None: + state.last_feedback = f"Node '{target}' not found." + return Step(kind="continue") + try: + matches = await doc.grep_node(node_id, pattern) + if not matches: + state.last_feedback = f"No matches for /{pattern}/ in this node." + else: + lines = [f"Matches for /{pattern}/:"] + for m in matches[:15]: + lines.append(f" - line {m.line_number}: {m.snippet[:100]}") + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"grep_node error: {e}" + return Step(kind="continue") + + +async def handle_similar( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + results = await doc.similar(node_id) + if not results: + state.last_feedback = "No similar nodes found." + else: + lines = ["Similar nodes:"] + for r in results[:10]: + kw = ", ".join(r.shared_keywords[:3]) + lines.append(f" - {r.title} (relevance: {r.relevance:.2f}, shared: {kw})") + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"similar error: {e}" + return Step(kind="continue") + + +async def handle_section_overview( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + overview = await doc.section_overview(node_id) + state.last_feedback = overview if overview else "(no overview available)" + except Exception as e: + state.last_feedback = f"overview error: {e}" + return Step(kind="continue") + + +async def handle_compare( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + target_a = command.target + target_b = command.target_b + if not target_a or not target_b: + state.last_feedback = "Usage: compare <node_a> <node_b>" + return Step(kind="continue") + node_a = await _resolve_target(doc, target_a, state) + node_b = await _resolve_target(doc, target_b, state) + if node_a is None: + state.last_feedback = f"Node '{target_a}' not found." + return Step(kind="continue") + if node_b is None: + state.last_feedback = f"Node '{target_b}' not found." + return Step(kind="continue") + try: + content_a = await doc.cat(node_a) + title_a = await doc.node_title(node_a) + content_b = await doc.cat(node_b) + title_b = await doc.node_title(node_b) + if node_a not in state.collected_nodes: + pwd_a = await doc.pwd() + state.evidence.append(Evidence( + source_path=pwd_a, node_title=title_a, content=content_a, + )) + state.collected_nodes.add(node_a) + if node_b not in state.collected_nodes: + pwd_b = await doc.pwd() + state.evidence.append(Evidence( + source_path=pwd_b, node_title=title_b, content=content_b, + )) + state.collected_nodes.add(node_b) + result = await compare_nodes(title_a, content_a, title_b, content_b, llm, query=query) + state.llm_calls += 1 + state.last_feedback = f"Comparison of [{title_a}] vs [{title_b}]:\n{result}" + except Exception as e: + state.last_feedback = f"compare error: {e}" + return Step(kind="continue") + + +async def handle_trace( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + content = await doc.cat(node_id) + title = await doc.node_title(node_id) + if node_id not in state.collected_nodes: + pwd = await doc.pwd() + state.evidence.append(Evidence( + source_path=pwd, node_title=title, content=content, + )) + state.collected_nodes.add(node_id) + related_context = "" + try: + similar = await doc.similar(node_id) + if similar: + related_lines = [f" - {s.title} (relevance: {s.relevance:.2f})" for s in similar[:5]] + related_context = "\nRelated sections:\n" + "\n".join(related_lines) + except Exception: + pass + result = await trace_reasoning(title, content, related_context, llm, query=query) + state.llm_calls += 1 + state.last_feedback = f"Reasoning trace for [{title}]:\n{result}" + except Exception as e: + state.last_feedback = f"trace error: {e}" + return Step(kind="continue") + + +async def handle_summarize( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + content = await doc.cat(node_id) + title = await doc.node_title(node_id) + if node_id not in state.collected_nodes: + pwd = await doc.pwd() + state.evidence.append(Evidence( + source_path=pwd, node_title=title, content=content, + )) + state.collected_nodes.add(node_id) + result = await summarize_section(title, content, llm, query=query) + state.llm_calls += 1 + state.last_feedback = f"Summary of [{title}]:\n{result}" + except Exception as e: + state.last_feedback = f"summarize error: {e}" + return Step(kind="continue") + + +async def handle_siblings( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + siblings = await doc.siblings(node_id) + if not siblings: + state.last_feedback = "(no sibling nodes)" + else: + lines = ["Sibling nodes:"] + for s in siblings: + lines.append( + f" - {s.title} (depth {s.depth}, {s.leaf_count} leaves)" + ) + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"siblings error: {e}" + return Step(kind="continue") + + +async def handle_ancestors( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + node_id = await _resolve_target(doc, command.target, state) + if node_id is None: + state.last_feedback = f"Node '{command.target}' not found." + return Step(kind="continue") + try: + ancestors = await doc.ancestors(node_id) + if not ancestors: + state.last_feedback = "(at root, no ancestors)" + else: + lines = ["Path from root:"] + for a in ancestors: + lines.append( + f" {' ' * a.depth}\u2192 {a.title} (depth {a.depth}, {a.child_count} children)" + ) + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"ancestors error: {e}" + return Step(kind="continue") + + +async def handle_doc_card( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + card = await doc.doc_card() + if card is None: + state.last_feedback = "(no document card available)" + else: + lines = [ + f"Document: {card.title}", + f"Overview: {card.overview}", + f"Total leaves: {card.total_leaves}", + ] + if card.question_hints: + lines.append(f"Can answer: {', '.join(card.question_hints[:5])}") + if card.topic_tags: + lines.append(f"Topics: {', '.join(card.topic_tags[:5])}") + if card.sections: + lines.append("Top-level sections:") + for s in card.sections: + lines.append(f" - {s.title}: {s.description} ({s.leaf_count} leaves)") + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"doc_card error: {e}" + return Step(kind="continue") + + +async def handle_concepts( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + try: + concepts = await doc.concepts() + if not concepts: + state.last_feedback = "(no concepts extracted)" + else: + lines = ["Key concepts:"] + for c in concepts: + sections = ", ".join(c.sections[:3]) + lines.append(f" - {c.name}: {c.summary} (in: {sections})") + state.last_feedback = "\n".join(lines) + except Exception as e: + state.last_feedback = f"concepts error: {e}" + return Step(kind="continue") + + +async def handle_find_section( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + title = command.target + if not title: + state.last_feedback = "Usage: find_section <title>" + return Step(kind="continue") + try: + result = await doc.find_section(title) + if result is None: + state.last_feedback = f"No section with title '{title}'." + else: + state.last_feedback = ( + f"Found: {result.title} (id={result.node_id}, " + f"depth {result.depth}, {result.leaf_count} leaves)" + ) + except Exception as e: + state.last_feedback = f"find_section error: {e}" + return Step(kind="continue") + + +async def handle_check( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + evidence_text = state.evidence_for_check() + system, user = check_sufficiency(query, evidence_text) + + try: + response = await llm.complete(system, user) + except Exception as e: + logger.warning("Check LLM call failed: %s", e) + state.last_feedback = "Could not evaluate sufficiency." + return Step(kind="continue") + + state.llm_calls += 1 + state.check_count += 1 + sufficient = parse_sufficiency_response(response) + + if sufficient: + state.last_feedback = "Evidence is sufficient. Use done to finish." + return Step(kind="done") + else: + # Extract missing info + reason = response.strip() + for prefix in ("INSUFFICIENT", "Insufficient"): + if reason.startswith(prefix): + reason = reason[len(prefix):] + break + reason = reason.lstrip("-: ") + if reason: + state.missing_info = reason + state.last_feedback = f"Evidence not yet sufficient: {response.strip()}" + return Step(kind="continue") + + +async def handle_done( + command: Any, doc: NavigableDocument, state: WorkerState, query: str, llm: LLMClient, +) -> Step: + state.last_feedback = "Navigation complete." + return Step(kind="done") + + +# --------------------------------------------------------------------------- +# Registry — maps command kind string to handler function +# --------------------------------------------------------------------------- + +_REGISTRY: dict[str, CommandHandler] = { + "ls": handle_ls, + "cd": handle_cd, + "cd_up": handle_cd_up, + "cat": handle_cat, + "find": handle_find, + "findtree": handle_findtree, + "grep": handle_grep, + "head": handle_head, + "wc": handle_wc, + "pwd": handle_pwd, + "back": handle_back, + "toc": handle_toc, + "stats": handle_stats, + "grep_node": handle_grep_node, + "similar": handle_similar, + "section_overview": handle_section_overview, + "compare": handle_compare, + "trace": handle_trace, + "summarize": handle_summarize, + "siblings": handle_siblings, + "ancestors": handle_ancestors, + "doc_card": handle_doc_card, + "concepts": handle_concepts, + "find_section": handle_find_section, + "check": handle_check, + "done": handle_done, +} + + +async def execute_command( + command: Any, + doc: NavigableDocument, + state: WorkerState, + query: str, + llm: LLMClient, +) -> Step: + """Execute a parsed command via the registry. + + Looks up ``command.kind`` in ``_REGISTRY`` and dispatches to the handler. + Falls back to ``handle_ls`` for unknown commands. + """ + handler = _REGISTRY.get(command.kind, handle_ls) + return await handler(command, doc, state, query, llm) diff --git a/vectorless/ask/worker/parse.py b/vectorless/ask/worker/parse.py new file mode 100644 index 0000000..0134c9a --- /dev/null +++ b/vectorless/ask/worker/parse.py @@ -0,0 +1,148 @@ +"""Command parsing — converts LLM text output into structured Command objects. + +Separated from execution for testability: parse logic has no side effects. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class Command: + """Parsed command from LLM output.""" + kind: str # ls, cd, cd_up, cat, find, findtree, grep, head, wc, pwd, check, done, ... + target: str = "" + target_b: str = "" # second target for compare, pattern for grep_node + lines: int = 20 + + +def _strip_quotes(s: str) -> str: + """Strip surrounding quotes (straight and smart) from a string.""" + trimmed = s.strip() + if len(trimmed) < 2: + return trimmed + first, last = trimmed[0], trimmed[-1] + matching = ( + (first == '"' and last == '"') + or (first == "'" and last == "'") + or (first == "\u201c" and last == "\u201d") + or (first == "\u2018" and last == "\u2019") + ) + return trimmed[1:-1] if matching else trimmed + + +def parse_command(llm_output: str) -> Command: + """Parse the first non-empty line of LLM output into a Command.""" + line = "" + for l in llm_output.splitlines(): + if l.strip(): + line = l.strip() + break + + # Remove common wrapping + line = line.strip().strip("`").strip() + + parts = line.split() + if not parts: + return Command(kind="ls") + + cmd = parts[0].lower() + + if cmd == "ls": + return Command(kind="ls") + elif cmd == "cd": + if len(parts) >= 2 and parts[1] == "..": + return Command(kind="cd_up") + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="cd", target=target) + elif cmd == "cat": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "." + return Command(kind="cat", target=target) + elif cmd == "find": + keyword = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="find", target=keyword) + elif cmd == "findtree": + pattern = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="findtree", target=pattern) + elif cmd == "grep": + pattern = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="grep", target=pattern) + elif cmd == "head": + if len(parts) >= 4 and parts[1] == "-n": + target = _strip_quotes(" ".join(parts[3:])) + try: + n = int(parts[2]) + except ValueError: + n = 20 + return Command(kind="head", target=target, lines=n) + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="head", target=target) + elif cmd == "wc": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="wc", target=target) + elif cmd == "pwd": + return Command(kind="pwd") + elif cmd == "check": + return Command(kind="check") + elif cmd == "done": + return Command(kind="done") + elif cmd == "back": + return Command(kind="back") + elif cmd == "toc": + if len(parts) > 1: + try: + return Command(kind="toc", lines=int(parts[1])) + except ValueError: + pass + return Command(kind="toc", lines=0) # 0 = no depth limit + elif cmd == "stats": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="stats", target=target) + elif cmd == "grep_node": + # grep_node <target> <pattern> + if len(parts) >= 3: + return Command(kind="grep_node", target=parts[1], target_b=_strip_quotes(" ".join(parts[2:]))) + elif len(parts) == 2: + return Command(kind="grep_node", target=parts[1]) + return Command(kind="grep_node") + elif cmd == "similar": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="similar", target=target) + elif cmd in ("section_overview", "overview"): + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="section_overview", target=target) + elif cmd == "compare": + # compare <node_a> <node_b> — use node IDs for reliability + if len(parts) >= 3: + return Command(kind="compare", target=parts[1], target_b=parts[2]) + elif len(parts) == 2: + return Command(kind="compare", target=parts[1]) + return Command(kind="compare") + elif cmd == "trace": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="trace", target=target) + elif cmd == "summarize": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="summarize", target=target) + elif cmd == "siblings": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="siblings", target=target) + elif cmd == "ancestors": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="ancestors", target=target) + elif cmd in ("doc_card", "card"): + return Command(kind="doc_card") + elif cmd == "concepts": + return Command(kind="concepts") + elif cmd == "find_section": + target = _strip_quotes(" ".join(parts[1:])) if len(parts) > 1 else "" + return Command(kind="find_section", target=target) + else: + return Command(kind="ls") # fallback: re-observe + + +def _is_parse_failure(command: Command, raw_output: str) -> bool: + """Detect if the parsed command is a fallback (unrecognized input).""" + trimmed = raw_output.strip() + return command.kind == "ls" and not trimmed.startswith("ls") and trimmed != "" diff --git a/vectorless/engine.py b/vectorless/engine.py index 31d91a0..1ce4b58 100644 --- a/vectorless/engine.py +++ b/vectorless/engine.py @@ -27,7 +27,7 @@ from vectorless.streaming import StreamingQueryResult from vectorless.types.graph import DocumentGraphWrapper from vectorless.types.results import ( - IndexResultWrapper, + CompileOutput, ) logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class Engine: """High-level Vectorless engine. - compile (ingest) runs in Rust; ask (retrieval) runs in Python. + compile runs in Rust; ask (retrieval) runs in Python. Configuration precedence: constructor args > env vars > config file > defaults. @@ -140,7 +140,7 @@ async def compile( name: str | None = None, mode: str = "default", force: bool = False, - ) -> IndexResultWrapper: + ) -> CompileOutput: """Compile a document from various sources. Exactly one source must be provided: path, paths, directory, @@ -154,13 +154,13 @@ async def compile( "Provide exactly one source: path, paths, directory, content, or bytes_data" ) - # For single file, delegate to Rust ingest + # For single file, delegate to Rust compile if path is not None: source_desc = str(path) self._events.emit_index( IndexEventData(event_type=IndexEventType.STARTED, path=source_desc) ) - doc_info = await self._rust.ingest(str(path)) + doc_info = await self._rust.compile(str(path)) self._events.emit_index( IndexEventData( event_type=IndexEventType.COMPLETE, @@ -168,7 +168,7 @@ async def compile( message=f"Indexed {doc_info.doc_id}", ) ) - return IndexResultWrapper.from_doc_info(doc_info) + return CompileOutput.from_doc_info(doc_info) # For multiple files, index them sequentially if paths is not None: @@ -189,15 +189,15 @@ async def compile( return await self.compile_batch(file_paths, mode="force" if force else mode) if content is not None: - # Write content to a temp file and ingest + # Write content to a temp file and compile import tempfile suffix = ".md" if format == "markdown" else f".{format}" with tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False) as f: f.write(content) tmp_path = f.name try: - doc_info = await self._rust.ingest(tmp_path) - return IndexResultWrapper.from_doc_info(doc_info) + doc_info = await self._rust.compile(tmp_path) + return CompileOutput.from_doc_info(doc_info) finally: import os os.unlink(tmp_path) @@ -209,8 +209,8 @@ async def compile( f.write(bytes_data) tmp_path = f.name try: - doc_info = await self._rust.ingest(tmp_path) - return IndexResultWrapper.from_doc_info(doc_info) + doc_info = await self._rust.compile(tmp_path) + return CompileOutput.from_doc_info(doc_info) finally: import os os.unlink(tmp_path) @@ -225,7 +225,7 @@ async def compile_batch( jobs: int = 1, force: bool = False, progress: bool = True, - ) -> IndexResultWrapper: + ) -> CompileOutput: """Compile multiple files with optional concurrency. Args: @@ -242,7 +242,7 @@ async def _index_one(p: str | Path) -> object: self._events.emit_index( IndexEventData(event_type=IndexEventType.STARTED, path=str(p)) ) - doc_info = await self._rust.ingest(str(p)) + doc_info = await self._rust.compile(str(p)) if progress: self._events.emit_index( IndexEventData( @@ -256,7 +256,7 @@ async def _index_one(p: str | Path) -> object: async with asyncio.TaskGroup() as tg: tasks = [tg.create_task(_index_one(p)) for p in paths] results = [t.result() for t in tasks] - return IndexResultWrapper.from_doc_infos(list(results)) + return CompileOutput.from_doc_infos(list(results)) # ── Querying (Python strategy layer) ──────────────────────── diff --git a/vectorless/types/__init__.py b/vectorless/types/__init__.py index f8d8947..9f0dc7e 100644 --- a/vectorless/types/__init__.py +++ b/vectorless/types/__init__.py @@ -8,11 +8,11 @@ WeightedKeyword, ) from vectorless.types.results import ( + CompileArtifact, + CompileOutput, Evidence, FailedItem, - IndexItemWrapper, IndexMetrics, - IndexResultWrapper, QueryMetrics, QueryResponse, QueryResult, @@ -20,11 +20,11 @@ __all__ = [ # Results + "CompileArtifact", + "CompileOutput", "Evidence", "FailedItem", - "IndexItemWrapper", "IndexMetrics", - "IndexResultWrapper", "QueryMetrics", "QueryResponse", "QueryResult", diff --git a/vectorless/types/results.py b/vectorless/types/results.py index f725dda..97e03de 100644 --- a/vectorless/types/results.py +++ b/vectorless/types/results.py @@ -184,8 +184,8 @@ def from_rust(cls, item: object) -> IndexMetrics: @dataclass(frozen=True) -class IndexItemWrapper: - """A single indexed document item.""" +class CompileArtifact: + """A single compiled document artifact.""" doc_id: str name: str @@ -196,7 +196,7 @@ class IndexItemWrapper: metrics: Optional[IndexMetrics] = None @classmethod - def from_rust(cls, item: object) -> IndexItemWrapper: + def from_rust(cls, item: object) -> CompileArtifact: metrics = IndexMetrics.from_rust(item.metrics) if item.metrics else None return cls( doc_id=item.doc_id, @@ -210,17 +210,17 @@ def from_rust(cls, item: object) -> IndexItemWrapper: @dataclass(frozen=True) -class IndexResultWrapper: - """Result of a document indexing operation.""" +class CompileOutput: + """Result of a document compilation operation.""" doc_id: Optional[str] = None - items: List[IndexItemWrapper] = field(default_factory=list) + items: List[CompileArtifact] = field(default_factory=list) failed: List[FailedItem] = field(default_factory=list) @classmethod - def from_doc_info(cls, doc_info: object) -> IndexResultWrapper: - """Create from a single Rust PyDocumentInfo (returned by ingest).""" - item = IndexItemWrapper( + def from_doc_info(cls, doc_info: object) -> CompileOutput: + """Create from a single Rust PyDocumentInfo (returned by compile).""" + item = CompileArtifact( doc_id=doc_info.doc_id, name=doc_info.name, format=doc_info.format, @@ -231,14 +231,14 @@ def from_doc_info(cls, doc_info: object) -> IndexResultWrapper: return cls(doc_id=doc_info.doc_id, items=[item]) @classmethod - def from_doc_infos(cls, doc_infos: list) -> IndexResultWrapper: + def from_doc_infos(cls, doc_infos: list) -> CompileOutput: """Create from a list of Rust PyDocumentInfo objects.""" items = [] first_doc_id = None for info in doc_infos: if first_doc_id is None: first_doc_id = info.doc_id - items.append(IndexItemWrapper( + items.append(CompileArtifact( doc_id=info.doc_id, name=info.name, format=info.format, @@ -249,8 +249,8 @@ def from_doc_infos(cls, doc_infos: list) -> IndexResultWrapper: return cls(doc_id=first_doc_id, items=items) @classmethod - def from_rust(cls, result: object) -> IndexResultWrapper: - items = [IndexItemWrapper.from_rust(i) for i in result.items] + def from_rust(cls, result: object) -> CompileOutput: + items = [CompileArtifact.from_rust(i) for i in result.items] failed = [FailedItem.from_rust(f) for f in result.failed] return cls( doc_id=result.doc_id, From 1c49ca81262976a6ca001f5bbd2181fcbeceddeb Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 19:49:35 +0800 Subject: [PATCH 17/30] feat(ask): add LLM-powered cross-document insight extraction - Introduce extract_llm_insights function in blackboard module to identify findings relevant to other documents using LLM analysis - Add new import and export for extract_llm_insights in ask module - Integrate LLM insight extraction in orchestrator for multi-document scenarios with additional error handling - Include comprehensive docstring explaining the functionality and cost implications --- vectorless/ask/__init__.py | 3 +- vectorless/ask/blackboard.py | 76 +++++++++++++++++++++++++++++++++- vectorless/ask/orchestrator.py | 13 +++++- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/vectorless/ask/__init__.py b/vectorless/ask/__init__.py index 2b71b9f..23b84fc 100644 --- a/vectorless/ask/__init__.py +++ b/vectorless/ask/__init__.py @@ -26,7 +26,7 @@ from vectorless.ask.worker import Worker # New modules -from vectorless.ask.blackboard import Discovery, SharedBlackboard +from vectorless.ask.blackboard import Discovery, SharedBlackboard, extract_llm_insights from vectorless.ask.events import AskEvent from vectorless.ask.reasoning import ( Ambiguity, @@ -93,6 +93,7 @@ # Shared blackboard "Discovery", "SharedBlackboard", + "extract_llm_insights", # Events "AskEvent", # Verification diff --git a/vectorless/ask/blackboard.py b/vectorless/ask/blackboard.py index 60b2b5f..7bf081e 100644 --- a/vectorless/ask/blackboard.py +++ b/vectorless/ask/blackboard.py @@ -9,6 +9,7 @@ import logging from dataclasses import dataclass, field +from typing import Any logger = logging.getLogger(__name__) @@ -120,8 +121,6 @@ def extract_discoveries(worker_output, doc_name: str) -> list[Discovery]: # Check if evidence content references other documents referenced_docs: list[str] = [] if evidence.content: - # Simple heuristic: look for document-like references - # (file.md, file.txt, "document X", etc.) import re doc_refs = re.findall( r'(?:see|refer to|in|from|document(?:ed)? in)\s+["\']?([\w\-\.]+\.(?:md|txt|pdf|doc))["\']?', @@ -155,3 +154,76 @@ def extract_discoveries(worker_output, doc_name: str) -> list[Discovery]: )) return discoveries + + +async def extract_llm_insights( + worker_output, + doc_name: str, + query: str, + llm: Any, +) -> list[Discovery]: + """Use LLM to extract cross-document insights from Worker output. + + More sophisticated than regex-based ``extract_discoveries``: asks the LLM + to identify findings that might be relevant to other documents being + searched in the same query. + + Cost: 1 additional LLM call per Worker. + """ + from vectorless.ask.types import WorkerOutput as WO + + if not worker_output.evidence: + return [] + + # Build a condensed summary of what the Worker found + evidence_parts: list[str] = [] + for ev in worker_output.evidence[:8]: + preview = ev.content[:300] + "..." if len(ev.content) > 300 else ev.content + evidence_parts.append(f"[{ev.node_title}]: {preview}") + evidence_summary = "\n\n".join(evidence_parts) + + system = ( + "You analyze search results from a document and identify findings that might be " + "relevant when searching OTHER documents for the same question. " + "Focus on cross-references, shared concepts, and information gaps.\n\n" + "Respond with a JSON array of objects, each with:\n" + '- "summary": brief description of the finding (one sentence)\n' + '- "finding_type": one of "lead", "cross_ref", "evidence"\n' + '- "relevance_to": list of document names or topics this relates to\n\n' + "If nothing is relevant across documents, respond with an empty array: []" + ) + user = ( + f"Query: {query}\n" + f"Document: {doc_name}\n\n" + f"Evidence collected:\n{evidence_summary}" + ) + + try: + from vectorless.ask.utils import parse_json_response + response = await llm.complete(system, user) + items = parse_json_response(response) + if not isinstance(items, list): + return [] + except Exception as e: + logger.warning("LLM insight extraction failed for %s: %s", doc_name, e) + return [] + + discoveries: list[Discovery] = [] + for item in items[:5]: + if not isinstance(item, dict): + continue + summary = str(item.get("summary", "")).strip() + if not summary: + continue + finding_type = str(item.get("finding_type", "lead")) + relevance_to = [str(r) for r in item.get("relevance_to", []) if r] + discoveries.append(Discovery( + worker_id=doc_name, + doc_name=doc_name, + node_title="llm_insight", + finding_type=finding_type, + summary=summary, + relevance_to=relevance_to, + )) + + return discoveries diff --git a/vectorless/ask/orchestrator.py b/vectorless/ask/orchestrator.py index 6abc72d..4d86000 100644 --- a/vectorless/ask/orchestrator.py +++ b/vectorless/ask/orchestrator.py @@ -42,7 +42,7 @@ from vectorless.ask.reasoning.types import QueryAnalysis from vectorless.ask.reasoning.analyzer import QueryAnalyzer from vectorless.ask.verify import VerifyPipeline, VerificationResult -from vectorless.ask.blackboard import SharedBlackboard, extract_discoveries +from vectorless.ask.blackboard import SharedBlackboard, extract_discoveries, extract_llm_insights from vectorless.ask.prompts import ( OrchestratorAnalysisParams, orchestrator_analysis, @@ -653,6 +653,17 @@ async def _dispatch_sequential( for d in discoveries: blackboard.add_discovery(d) + # LLM-powered cross-document insight extraction (only for multi-doc) + if len(cards) > 1 and result.evidence: + try: + llm_insights = await extract_llm_insights( + result, card.name, query, llm, + ) + for d in llm_insights: + blackboard.add_discovery(d) + except Exception as e: + logger.warning("LLM insight extraction failed for %s: %s", card.name, e) + # ----------------------------------------------------------------------- # Replan — mirrors Rust orchestrator/replan.rs # ----------------------------------------------------------------------- From cc7b66fb50858a2e5ac22b538f8b4deff60a61fd Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 20:47:53 +0800 Subject: [PATCH 18/30] refactor(compiler): rename stages to passes and update module structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename `stages` module to `passes` across the compiler - Update all stage-related structs to use pass terminology: - `EnhanceStage` → `EnhancePass` - `ValidateStage` → `ValidatePass` - `ConceptExtractionStage` → `ConceptPass` - `NavigationCompileStage` → `NavigationPass` - `OptimizeStage` → `OptimizePass` - `ReasoningCompileStage` → `ReasoningPass` - `VerifyStage` → `VerifyPass` - `BuildStage` → `BuildPass` - `ParseStage` → `ParsePass` - Update trait implementations from `CompileStage` to `CompilePass` - Change result types from `StageResult` to `PassResult` - Restructure module organization with frontend, analysis, and backend passes - Update import paths to use new `crate::passes` module structure - Fix all test references to use new pass naming convention --- .../vectorless-compiler/src/lib.rs | 2 +- .../{stages => passes/analysis}/enhance.rs | 30 ++-- .../src/passes/analysis/mod.rs | 10 ++ .../{stages => passes/analysis}/validate.rs | 22 +-- .../src/{stages => passes/backend}/concept.rs | 14 +- .../src/passes/backend/mod.rs | 16 ++ .../{stages => passes/backend}/navigation.rs | 58 +++---- .../{stages => passes/backend}/optimize.rs | 40 ++--- .../{stages => passes/backend}/reasoning.rs | 42 +++--- .../backend/verify.rs} | 14 +- .../src/{stages => passes/frontend}/build.rs | 18 +-- .../src/passes/frontend/mod.rs | 10 ++ .../src/{stages => passes/frontend}/parse.rs | 16 +- .../vectorless-compiler/src/passes/mod.rs | 80 ++++++++++ .../{stages => passes/transform}/enrich.rs | 18 +-- .../src/passes/transform/mod.rs | 10 ++ .../src/{stages => passes/transform}/split.rs | 26 ++-- .../src/pipeline/context.rs | 28 ++-- .../src/pipeline/executor.rs | 66 ++++---- .../vectorless-compiler/src/pipeline/mod.rs | 12 +- .../src/pipeline/orchestrator.rs | 64 ++++---- .../vectorless-compiler/src/stages/mod.rs | 141 ------------------ 22 files changed, 363 insertions(+), 374 deletions(-) rename vectorless-core/vectorless-compiler/src/{stages => passes/analysis}/enhance.rs (94%) create mode 100644 vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs rename vectorless-core/vectorless-compiler/src/{stages => passes/analysis}/validate.rs (96%) rename vectorless-core/vectorless-compiler/src/{stages => passes/backend}/concept.rs (96%) create mode 100644 vectorless-core/vectorless-compiler/src/passes/backend/mod.rs rename vectorless-core/vectorless-compiler/src/{stages => passes/backend}/navigation.rs (90%) rename vectorless-core/vectorless-compiler/src/{stages => passes/backend}/optimize.rs (92%) rename vectorless-core/vectorless-compiler/src/{stages => passes/backend}/reasoning.rs (94%) rename vectorless-core/vectorless-compiler/src/{stages/verify_ingest.rs => passes/backend/verify.rs} (88%) rename vectorless-core/vectorless-compiler/src/{stages => passes/frontend}/build.rs (97%) create mode 100644 vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs rename vectorless-core/vectorless-compiler/src/{stages => passes/frontend}/parse.rs (95%) create mode 100644 vectorless-core/vectorless-compiler/src/passes/mod.rs rename vectorless-core/vectorless-compiler/src/{stages => passes/transform}/enrich.rs (95%) create mode 100644 vectorless-core/vectorless-compiler/src/passes/transform/mod.rs rename vectorless-core/vectorless-compiler/src/{stages => passes/transform}/split.rs (94%) delete mode 100644 vectorless-core/vectorless-compiler/src/stages/mod.rs diff --git a/vectorless-core/vectorless-compiler/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs index 41ce84d..f36731e 100644 --- a/vectorless-core/vectorless-compiler/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -44,7 +44,7 @@ pub mod config; pub mod incremental; pub mod parse; pub mod pipeline; -pub mod stages; +pub mod passes; pub mod summary; // Re-export main types from pipeline diff --git a/vectorless-core/vectorless-compiler/src/stages/enhance.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs similarity index 94% rename from vectorless-core/vectorless-compiler/src/stages/enhance.rs rename to vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs index 9e8bf73..beae745 100644 --- a/vectorless-core/vectorless-compiler/src/stages/enhance.rs +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs @@ -3,7 +3,7 @@ //! Enhance stage - Generate summaries using LLM. -use super::async_trait; +use crate::passes::async_trait; use futures::StreamExt; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -16,7 +16,7 @@ use vectorless_llm::LlmClient; use vectorless_llm::memo::{MemoKey, MemoStore}; use vectorless_utils::fingerprint::Fingerprint; -use super::{CompileStage, StageResult}; +use crate::passes::{CompilePass, PassResult}; use crate::pipeline::{FailurePolicy, CompileContext, StageRetryConfig}; use crate::summary::{LlmSummaryGenerator, SummaryGenerator, SummaryStrategy}; @@ -29,14 +29,14 @@ struct PendingNode { } /// Enhance stage - generates summaries using LLM. -pub struct EnhanceStage { +pub struct EnhancePass { /// LLM client for summary generation. llm_client: Option<Arc<LlmClient>>, /// Memo store for caching LLM results. memo_store: Option<Arc<MemoStore>>, } -impl EnhanceStage { +impl EnhancePass { /// Create a new enhance stage. pub fn new() -> Self { Self { @@ -120,14 +120,14 @@ impl EnhanceStage { } } -impl Default for EnhanceStage { +impl Default for EnhancePass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for EnhanceStage { +impl CompilePass for EnhancePass { fn name(&self) -> &'static str { "enhance" } @@ -149,7 +149,7 @@ impl CompileStage for EnhanceStage { ) } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); info!( @@ -164,7 +164,7 @@ impl CompileStage for EnhanceStage { "[enhance] Skipped: strategy={:?}", ctx.options.summary_strategy ); - return Ok(StageResult::success("enhance")); + return Ok(PassResult::success("enhance")); } // Get LLM client @@ -172,7 +172,7 @@ impl CompileStage for EnhanceStage { Some(client) => client, None => { warn!("[enhance] No LLM client, skipping summary generation"); - return Ok(StageResult::success("enhance")); + return Ok(PassResult::success("enhance")); } }; @@ -181,7 +181,7 @@ impl CompileStage for EnhanceStage { Some(t) => t, None => { warn!("[enhance] No tree built, skipping"); - return Ok(StageResult::success("enhance")); + return Ok(PassResult::success("enhance")); } }; @@ -383,7 +383,7 @@ impl CompileStage for EnhanceStage { generated, shortcut_used, failed, skipped_no_content, skipped_tokens, duration ); - let mut stage_result = StageResult::success("enhance"); + let mut stage_result = PassResult::success("enhance"); stage_result.duration_ms = duration; stage_result.metadata.insert( "summaries_generated".to_string(), @@ -408,7 +408,7 @@ OVERVIEW: This section covers payment integration and billing configuration. QUESTIONS: How to set up payments?, What currencies are supported?, How to configure invoices? TAGS: payments, billing, invoices, currency"; - let (overview, questions, tags) = EnhanceStage::parse_structured_nav_response(response); + let (overview, questions, tags) = EnhancePass::parse_structured_nav_response(response); assert!(overview.contains("payment integration")); assert_eq!(questions.len(), 3); @@ -421,7 +421,7 @@ TAGS: payments, billing, invoices, currency"; fn test_parse_structured_nav_response_partial() { // Only overview, no questions or tags let response = "OVERVIEW: A general introduction to the system."; - let (overview, questions, tags) = EnhanceStage::parse_structured_nav_response(response); + let (overview, questions, tags) = EnhancePass::parse_structured_nav_response(response); assert!(overview.contains("general introduction")); assert!(questions.is_empty()); @@ -432,7 +432,7 @@ TAGS: payments, billing, invoices, currency"; fn test_parse_structured_nav_response_fallback() { // No markers at all — fallback to entire response as overview let response = "This is just a plain summary without any markers."; - let (overview, questions, tags) = EnhanceStage::parse_structured_nav_response(response); + let (overview, questions, tags) = EnhancePass::parse_structured_nav_response(response); assert_eq!(overview, response.trim()); assert!(questions.is_empty()); @@ -441,7 +441,7 @@ TAGS: payments, billing, invoices, currency"; #[test] fn test_parse_structured_nav_response_empty() { - let (overview, questions, tags) = EnhanceStage::parse_structured_nav_response(""); + let (overview, questions, tags) = EnhancePass::parse_structured_nav_response(""); assert!(overview.is_empty()); assert!(questions.is_empty()); assert!(tags.is_empty()); diff --git a/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs new file mode 100644 index 0000000..4a4ea0f --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Analysis passes — semantic validation and LLM enhancement. + +mod validate; +mod enhance; + +pub use validate::ValidatePass; +pub use enhance::EnhancePass; diff --git a/vectorless-core/vectorless-compiler/src/stages/validate.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs similarity index 96% rename from vectorless-core/vectorless-compiler/src/stages/validate.rs rename to vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs index 9855504..1dc284e 100644 --- a/vectorless-core/vectorless-compiler/src/stages/validate.rs +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs @@ -9,7 +9,7 @@ use tracing::{debug, info, warn}; use vectorless_error::Result; -use super::{AccessPattern, CompileStage, StageResult, async_trait}; +use crate::passes::{AccessPattern, CompilePass, PassResult, async_trait}; use crate::pipeline::CompileContext; /// Maximum allowed tree depth. @@ -49,9 +49,9 @@ struct ValidationIssue { /// 3. Empty title detection on leaf nodes /// 4. Token count consistency (parent >= sum of children) /// 5. Content duplication detection -pub struct ValidateStage; +pub struct ValidatePass; -impl ValidateStage { +impl ValidatePass { /// Create a new validate stage. pub fn new() -> Self { Self @@ -215,14 +215,14 @@ impl ValidateStage { } } -impl Default for ValidateStage { +impl Default for ValidatePass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for ValidateStage { +impl CompilePass for ValidatePass { fn name(&self) -> &'static str { "validate" } @@ -246,7 +246,7 @@ impl CompileStage for ValidateStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); let node_count = ctx.tree.as_ref().map(|t| t.node_count()).unwrap_or(0); @@ -283,7 +283,7 @@ impl CompileStage for ValidateStage { warnings, errors, duration ); - let mut stage_result = StageResult::success("validate"); + let mut stage_result = PassResult::success("validate"); stage_result.duration_ms = duration; stage_result .metadata @@ -314,7 +314,7 @@ mod tests { let tree = DocumentTree::new("Root", ""); let ctx = make_context_with_tree(tree); - let stage = ValidateStage::new(); + let stage = ValidatePass::new(); let issues = stage.validate_tree(&ctx); // Single root node is valid — no issues expected @@ -329,7 +329,7 @@ mod tests { let ctx = make_context_with_tree(tree); - let stage = ValidateStage::new(); + let stage = ValidatePass::new(); let issues = stage.validate_tree(&ctx); assert!(issues.is_empty()); @@ -343,7 +343,7 @@ mod tests { let ctx = make_context_with_tree(tree); - let stage = ValidateStage::new(); + let stage = ValidatePass::new(); let issues = stage.validate_tree(&ctx); let warning_count = issues @@ -359,7 +359,7 @@ mod tests { let options = crate::config::PipelineOptions::default(); let ctx = CompileContext::new(input, options); - let stage = ValidateStage::new(); + let stage = ValidatePass::new(); let issues = stage.validate_tree(&ctx); assert_eq!(issues.len(), 1); diff --git a/vectorless-core/vectorless-compiler/src/stages/concept.rs b/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs similarity index 96% rename from vectorless-core/vectorless-compiler/src/stages/concept.rs rename to vectorless-core/vectorless-compiler/src/passes/backend/concept.rs index 13cc301..b138bdb 100644 --- a/vectorless-core/vectorless-compiler/src/stages/concept.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs @@ -12,8 +12,8 @@ use vectorless_document::Concept; use vectorless_error::Result; use vectorless_llm::LlmClient; -use super::async_trait; -use super::{AccessPattern, CompileStage, StageResult}; +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; use crate::pipeline::CompileContext; /// Maximum number of top keywords to send to the LLM for concept extraction. @@ -27,11 +27,11 @@ const MAX_CONCEPTS: usize = 15; /// Takes the reasoning index's topic entries and tree summaries, then uses /// a single LLM call to extract structured [`Concept`] values. /// Falls back to basic keyword-based concepts when no LLM is available. -pub struct ConceptExtractionStage { +pub struct ConceptPass { llm_client: Option<LlmClient>, } -impl ConceptExtractionStage { +impl ConceptPass { /// Create a new stage without LLM support (keyword-based fallback). pub fn new() -> Self { Self { llm_client: None } @@ -46,7 +46,7 @@ impl ConceptExtractionStage { } #[async_trait] -impl CompileStage for ConceptExtractionStage { +impl CompilePass for ConceptPass { fn name(&self) -> &str { "concept_extraction" } @@ -67,7 +67,7 @@ impl CompileStage for ConceptExtractionStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let concepts = if let Some(ref client) = self.llm_client { extract_with_llm(ctx, client).await } else { @@ -78,7 +78,7 @@ impl CompileStage for ConceptExtractionStage { ctx.concepts = concepts; info!("[concept_extraction] Extracted {} concepts", count); - Ok(StageResult::success("concept_extraction")) + Ok(PassResult::success("concept_extraction")) } } diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs new file mode 100644 index 0000000..e170957 --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Backend passes — code generation (indexes), verification, and optimization. + +mod reasoning; +mod concept; +mod navigation; +mod verify; +mod optimize; + +pub use reasoning::ReasoningPass; +pub use concept::ConceptPass; +pub use navigation::NavigationPass; +pub use verify::VerifyPass; +pub use optimize::OptimizePass; diff --git a/vectorless-core/vectorless-compiler/src/stages/navigation.rs b/vectorless-core/vectorless-compiler/src/passes/backend/navigation.rs similarity index 90% rename from vectorless-core/vectorless-compiler/src/stages/navigation.rs rename to vectorless-core/vectorless-compiler/src/passes/backend/navigation.rs index 6602fce..f104cff 100644 --- a/vectorless-core/vectorless-compiler/src/stages/navigation.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/navigation.rs @@ -3,7 +3,7 @@ //! Navigation Index Stage — Build the Agent navigation index from the document tree. //! -//! This stage runs after EnrichStage and ReasoningCompileStage. It reads the +//! This stage runs after EnrichPass and ReasoningPass. It reads the //! enhanced TreeNode fields (summary, description, routing_keywords, leaf_count) //! and builds a [`NavigationIndex`] containing compact [`NavEntry`] and //! [`ChildRoute`] records for every non-leaf node. @@ -20,8 +20,8 @@ use tracing::{debug, info, warn}; use vectorless_document::{ChildRoute, DocumentTree, NavEntry, NavigationIndex, NodeId}; use vectorless_error::Result; -use super::async_trait; -use super::{AccessPattern, CompileStage, StageResult}; +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; use crate::pipeline::CompileContext; /// Navigation Index Stage — builds the Agent navigation index. @@ -32,9 +32,9 @@ use crate::pipeline::CompileContext; /// /// The resulting [`NavigationIndex`] is stored in `ctx.navigation_index` and /// serialized as part of [`PersistedDocument`](vectorless_storage::persistence::PersistedDocument). -pub struct NavigationCompileStage; +pub struct NavigationPass; -impl NavigationCompileStage { +impl NavigationPass { /// Create a new navigation index stage. pub fn new() -> Self { Self @@ -118,14 +118,14 @@ impl NavigationCompileStage { } } -impl Default for NavigationCompileStage { +impl Default for NavigationPass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for NavigationCompileStage { +impl CompilePass for NavigationPass { fn name(&self) -> &'static str { "navigation_index" } @@ -146,14 +146,14 @@ impl CompileStage for NavigationCompileStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); let tree = match ctx.tree.as_ref() { Some(t) => t, None => { warn!("[navigation_index] No tree, cannot build index"); - return Ok(StageResult::failure("navigation_index", "Tree not built")); + return Ok(PassResult::failure("navigation_index", "Tree not built")); } }; @@ -275,7 +275,7 @@ impl CompileStage for NavigationCompileStage { ctx.navigation_index = Some(nav_index); - let mut stage_result = StageResult::success("navigation_index"); + let mut stage_result = PassResult::success("navigation_index"); stage_result.duration_ms = duration; stage_result.metadata.insert( "nav_entries".to_string(), @@ -320,7 +320,7 @@ mod tests { let root = tree.root(); // Root has 3 leaves: 1.1, 1.2, 2.1 - assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 3); + assert_eq!(NavigationPass::count_leaves(&tree, root), 3); } #[test] @@ -328,7 +328,7 @@ mod tests { let tree = DocumentTree::new("Root", "content"); let root = tree.root(); - assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 1); + assert_eq!(NavigationPass::count_leaves(&tree, root), 1); } #[test] @@ -336,7 +336,7 @@ mod tests { let tree = build_test_tree(); let root = tree.root(); - let entry = NavigationCompileStage::build_nav_entry(&tree, root, 3); + let entry = NavigationPass::build_nav_entry(&tree, root, 3); assert_eq!(entry.overview, "A comprehensive guide"); assert_eq!(entry.leaf_count, 3); assert_eq!(entry.level, 0); @@ -347,7 +347,7 @@ mod tests { let tree = DocumentTree::new("Root", "content"); let root = tree.root(); - let entry = NavigationCompileStage::build_nav_entry(&tree, root, 1); + let entry = NavigationPass::build_nav_entry(&tree, root, 1); assert_eq!(entry.overview, "Root"); } @@ -357,14 +357,14 @@ mod tests { let root = tree.root(); let children: Vec<_> = tree.children_iter(root).collect(); - let route = NavigationCompileStage::build_child_route(&tree, children[0], 2); + let route = NavigationPass::build_child_route(&tree, children[0], 2); assert_eq!(route.title, "Section 1"); assert_eq!(route.leaf_count, 2); } #[test] fn test_stage_config() { - let stage = NavigationCompileStage::new(); + let stage = NavigationPass::new(); assert_eq!(stage.name(), "navigation_index"); assert!(stage.is_optional()); assert_eq!(stage.depends_on(), vec!["enrich"]); @@ -398,7 +398,7 @@ mod tests { ctx.tree = Some(tree); // Execute the stage - let mut stage = NavigationCompileStage::new(); + let mut stage = NavigationPass::new(); let result = stage.execute(&mut ctx).await; assert!(result.is_ok()); @@ -445,7 +445,7 @@ mod tests { ); ctx.tree = Some(tree); - let mut stage = NavigationCompileStage::new(); + let mut stage = NavigationPass::new(); let result = stage.execute(&mut ctx).await; assert!(result.is_ok()); @@ -464,7 +464,7 @@ mod tests { ); // ctx.tree is None - let mut stage = NavigationCompileStage::new(); + let mut stage = NavigationPass::new(); // Can't move ctx since tree is None, construct manually let mut ctx = ctx; ctx.tree = None; @@ -481,7 +481,7 @@ mod tests { let root = tree.root(); let child = tree.add_child(root, "Child", "this is a long content string that exceeds 100 characters and should be truncated when used as a fallback description for the child route"); - let route = NavigationCompileStage::build_child_route(&tree, child, 1); + let route = NavigationPass::build_child_route(&tree, child, 1); assert_eq!(route.title, "Child"); // description should be truncated content, not the full string assert!(route.description.len() <= 100); @@ -497,7 +497,7 @@ mod tests { // Clear any auto-generated content tree.set_summary(child, ""); - let route = NavigationCompileStage::build_child_route(&tree, child, 1); + let route = NavigationPass::build_child_route(&tree, child, 1); assert_eq!(route.title, "Orphan Section"); // Fallback: description = title when no summary and no content assert_eq!(route.description, "Orphan Section"); @@ -510,7 +510,7 @@ mod tests { let child = tree.add_child(root, "Child", "some content"); tree.set_summary(child, "A concise summary"); - let route = NavigationCompileStage::build_child_route(&tree, child, 1); + let route = NavigationPass::build_child_route(&tree, child, 1); assert_eq!(route.description, "A concise summary"); } @@ -524,14 +524,14 @@ mod tests { tree.set_summary(root, "Root overview"); tree.set_summary(sec1, "Section overview"); - let root_entry = NavigationCompileStage::build_nav_entry(&tree, root, 3); + let root_entry = NavigationPass::build_nav_entry(&tree, root, 3); assert_eq!(root_entry.level, 0); - let sec1_entry = NavigationCompileStage::build_nav_entry(&tree, sec1, 1); + let sec1_entry = NavigationPass::build_nav_entry(&tree, sec1, 1); assert_eq!(sec1_entry.level, 1); // Leaf node should still return valid NavEntry if called - let leaf_entry = NavigationCompileStage::build_nav_entry(&tree, sec1_1, 1); + let leaf_entry = NavigationPass::build_nav_entry(&tree, sec1_1, 1); assert_eq!(leaf_entry.level, 2); assert_eq!(leaf_entry.overview, "S1.1"); // no summary → fallback to title } @@ -549,15 +549,15 @@ mod tests { let _s2a = tree.add_child(sec2, "S2.A", "leaf"); // sec1 subtree has 3 leaves - assert_eq!(NavigationCompileStage::count_leaves(&tree, sec1), 3); + assert_eq!(NavigationPass::count_leaves(&tree, sec1), 3); // sec2 subtree has 1 leaf - assert_eq!(NavigationCompileStage::count_leaves(&tree, sec2), 1); + assert_eq!(NavigationPass::count_leaves(&tree, sec2), 1); // root has 4 leaves total - assert_eq!(NavigationCompileStage::count_leaves(&tree, root), 4); + assert_eq!(NavigationPass::count_leaves(&tree, root), 4); } /// Helper to check success without destructuring. - fn stage_result_is_success(result: &Result<StageResult>) -> bool { + fn stage_result_is_success(result: &Result<PassResult>) -> bool { result.as_ref().map(|r| r.success).unwrap_or(false) } } diff --git a/vectorless-core/vectorless-compiler/src/stages/optimize.rs b/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs similarity index 92% rename from vectorless-core/vectorless-compiler/src/stages/optimize.rs rename to vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs index 4654024..80761ab 100644 --- a/vectorless-core/vectorless-compiler/src/stages/optimize.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs @@ -3,7 +3,7 @@ //! Optimize stage - Optimize tree structure. -use super::{AccessPattern, async_trait}; +use crate::passes::{AccessPattern, async_trait}; use std::time::Instant; use tracing::{debug, info}; @@ -11,12 +11,12 @@ use crate::pipeline::CompileContext; use vectorless_document::NodeId; use vectorless_error::Result; -use super::{CompileStage, StageResult}; +use crate::passes::{CompilePass, PassResult}; /// Optimize stage - optimizes tree structure. -pub struct OptimizeStage; +pub struct OptimizePass; -impl OptimizeStage { +impl OptimizePass { /// Create a new optimize stage. pub fn new() -> Self { Self @@ -149,14 +149,14 @@ impl OptimizeStage { } } -impl Default for OptimizeStage { +impl Default for OptimizePass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for OptimizeStage { +impl CompilePass for OptimizePass { fn name(&self) -> &'static str { "optimize" } @@ -177,13 +177,13 @@ impl CompileStage for OptimizeStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); let config = &ctx.options.optimization; if !config.enabled { debug!("[optimize] Disabled, skipping"); - return Ok(StageResult::success("optimize")); + return Ok(PassResult::success("optimize")); } let tree = ctx @@ -225,7 +225,7 @@ impl CompileStage for OptimizeStage { merged_count, removed_count, duration ); - let mut stage_result = StageResult::success("optimize"); + let mut stage_result = PassResult::success("optimize"); stage_result.duration_ms = duration; stage_result .metadata @@ -289,7 +289,7 @@ mod tests { let mut metrics = crate::pipeline::IndexMetrics::new(); // Threshold 100: Leaf A (50) and Leaf B (30) should merge - let merged = OptimizeStage::merge_small_leaves(&mut tree, 100, &mut metrics); + let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); assert_eq!(merged, 1); assert_eq!(metrics.nodes_merged, 1); @@ -310,7 +310,7 @@ mod tests { let mut metrics = crate::pipeline::IndexMetrics::new(); // Threshold 10: all leaves are above this, nothing merges - let merged = OptimizeStage::merge_small_leaves(&mut tree, 10, &mut metrics); + let merged = OptimizePass::merge_small_leaves(&mut tree, 10, &mut metrics); assert_eq!(merged, 0); } @@ -328,7 +328,7 @@ mod tests { } let mut metrics = crate::pipeline::IndexMetrics::new(); - let _ = OptimizeStage::merge_small_leaves(&mut tree, 100, &mut metrics); + let _ = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); // Leaf A should now contain both contents with heading prefix let a_node = tree.get(a).unwrap(); @@ -356,7 +356,7 @@ mod tests { } let mut metrics = crate::pipeline::IndexMetrics::new(); - let merged = OptimizeStage::merge_small_leaves(&mut tree, 100, &mut metrics); + let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); // Section is non-leaf, only Leaf is a leaf — no adjacent pair of leaves assert_eq!(merged, 0); @@ -371,7 +371,7 @@ mod tests { let section = tree.add_child(root, "Section", ""); let _leaf = tree.add_child(section, "Leaf", "content"); - let removed = OptimizeStage::remove_empty_nodes(&mut tree); + let removed = OptimizePass::remove_empty_nodes(&mut tree); assert_eq!(removed, 1); let section_node = tree.get(section).unwrap(); @@ -383,7 +383,7 @@ mod tests { let mut tree = DocumentTree::new("Root", ""); let _child = tree.add_child(tree.root(), "Child", "content"); - let removed = OptimizeStage::remove_empty_nodes(&mut tree); + let removed = OptimizePass::remove_empty_nodes(&mut tree); assert_eq!(removed, 0); } @@ -393,7 +393,7 @@ mod tests { let root = tree.root(); let leaf = tree.add_child(root, "Leaf", ""); - let removed = OptimizeStage::remove_empty_nodes(&mut tree); + let removed = OptimizePass::remove_empty_nodes(&mut tree); assert_eq!(removed, 0, "Leaves should not be removed"); // Verify the leaf is indeed a leaf @@ -408,7 +408,7 @@ mod tests { let _c1 = tree.add_child(section, "C1", "a"); let _c2 = tree.add_child(section, "C2", "b"); - let removed = OptimizeStage::remove_empty_nodes(&mut tree); + let removed = OptimizePass::remove_empty_nodes(&mut tree); assert_eq!( removed, 0, "Nodes with multiple children should not be removed" @@ -422,13 +422,13 @@ mod tests { let section = tree.add_child(root, "Section", "has content"); let _leaf = tree.add_child(section, "Leaf", "content"); - let removed = OptimizeStage::remove_empty_nodes(&mut tree); + let removed = OptimizePass::remove_empty_nodes(&mut tree); assert_eq!(removed, 0); } #[tokio::test] async fn test_optimize_disabled_skips() { - let mut stage = OptimizeStage::new(); + let mut stage = OptimizePass::new(); assert_eq!(stage.name(), "optimize"); assert!(stage.is_optional()); assert_eq!(stage.depends_on(), vec!["enrich", "navigation_index"]); @@ -449,7 +449,7 @@ mod tests { let mut tree = DocumentTree::new("Root", ""); let mut metrics = crate::pipeline::IndexMetrics::new(); - let merged = OptimizeStage::merge_small_leaves(&mut tree, 100, &mut metrics); + let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); assert_eq!(merged, 0, "Root with no children should merge nothing"); } } diff --git a/vectorless-core/vectorless-compiler/src/stages/reasoning.rs b/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs similarity index 94% rename from vectorless-core/vectorless-compiler/src/stages/reasoning.rs rename to vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs index 228a4cb..5ee8067 100644 --- a/vectorless-core/vectorless-compiler/src/stages/reasoning.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs @@ -3,8 +3,8 @@ //! Reasoning Index Stage - Build pre-computed reasoning index. //! -//! This stage runs after EnrichStage (which generates descriptions and -//! calculates metadata) and before OptimizeStage. It builds a +//! This stage runs after EnrichPass (which generates descriptions and +//! calculates metadata) and before OptimizePass. It builds a //! [`ReasoningIndex`] from the document tree's TOC, summaries, and keywords. use std::collections::HashMap; @@ -19,8 +19,8 @@ use vectorless_error::Result; use vectorless_llm::LlmClient; use vectorless_scoring::extract_keywords; -use super::async_trait; -use super::{AccessPattern, CompileStage, StageResult}; +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; use crate::pipeline::CompileContext; /// Reasoning Index Stage - builds a pre-computed reasoning index from the document tree. @@ -29,11 +29,11 @@ use crate::pipeline::CompileContext; /// - Topic-to-path mappings from titles and summaries /// - Summary shortcuts for high-frequency "overview" queries /// - Section map for fast ToC lookup -pub struct ReasoningCompileStage { +pub struct ReasoningPass { config: ReasoningIndexConfig, } -impl ReasoningCompileStage { +impl ReasoningPass { /// Create a new reasoning index stage with default config. pub fn new() -> Self { Self { @@ -294,14 +294,14 @@ impl ReasoningCompileStage { } } -impl Default for ReasoningCompileStage { +impl Default for ReasoningPass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for ReasoningCompileStage { +impl CompilePass for ReasoningPass { fn name(&self) -> &'static str { "reasoning_index" } @@ -322,13 +322,13 @@ impl CompileStage for ReasoningCompileStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); // Check if enabled via pipeline options if !ctx.options.reasoning_index.enabled { info!("[reasoning_index] Disabled, skipping"); - return Ok(StageResult::success("reasoning_index")); + return Ok(PassResult::success("reasoning_index")); } // Use stage config, overridden by pipeline options @@ -338,7 +338,7 @@ impl CompileStage for ReasoningCompileStage { Some(t) => t, None => { warn!("[reasoning_index] No tree, cannot build index"); - return Ok(StageResult::failure("reasoning_index", "Tree not built")); + return Ok(PassResult::failure("reasoning_index", "Tree not built")); } }; @@ -427,7 +427,7 @@ impl CompileStage for ReasoningCompileStage { ctx.reasoning_index = Some(reasoning_index); - let mut stage_result = StageResult::success("reasoning_index"); + let mut stage_result = PassResult::success("reasoning_index"); stage_result.duration_ms = duration; stage_result.metadata.insert( "keywords_indexed".to_string(), @@ -452,7 +452,7 @@ mod tests { #[test] fn test_extract_node_keywords() { let keywords = - ReasoningCompileStage::extract_node_keywords("Introduction to Machine Learning", 2); + ReasoningPass::extract_node_keywords("Introduction to Machine Learning", 2); assert!(keywords.contains(&"introduction".to_string())); assert!(keywords.contains(&"machine".to_string())); assert!(keywords.contains(&"learning".to_string())); @@ -460,7 +460,7 @@ mod tests { #[test] fn test_extract_node_keywords_min_length() { - let keywords = ReasoningCompileStage::extract_node_keywords("A B CD", 2); + let keywords = ReasoningPass::extract_node_keywords("A B CD", 2); assert!(!keywords.contains(&"a".to_string())); assert!(!keywords.contains(&"b".to_string())); assert!(keywords.contains(&"cd".to_string())); @@ -468,7 +468,7 @@ mod tests { #[test] fn test_stage_config_default() { - let stage = ReasoningCompileStage::new(); + let stage = ReasoningPass::new(); assert!(stage.config.enabled); assert_eq!(stage.name(), "reasoning_index"); assert!(stage.is_optional()); @@ -493,7 +493,7 @@ mod tests { } let config = ReasoningIndexConfig::default(); - let (topic_paths, keyword_count) = ReasoningCompileStage::build_topic_paths(&tree, &config); + let (topic_paths, keyword_count) = ReasoningPass::build_topic_paths(&tree, &config); assert!( keyword_count > 0, @@ -518,7 +518,7 @@ mod tests { let _c1 = tree.add_child(root, "rust ownership", "rust borrowing rules"); let config = ReasoningIndexConfig::default(); - let (topic_paths, _) = ReasoningCompileStage::build_topic_paths(&tree, &config); + let (topic_paths, _) = ReasoningPass::build_topic_paths(&tree, &config); // All weights should be in 0.0-1.0 range for entries in topic_paths.values() { @@ -549,7 +549,7 @@ mod tests { let mut config = ReasoningIndexConfig::default(); config.max_keyword_entries = 5; - let (topic_paths, keyword_count) = ReasoningCompileStage::build_topic_paths(&tree, &config); + let (topic_paths, keyword_count) = ReasoningPass::build_topic_paths(&tree, &config); assert!( keyword_count <= 5, @@ -574,7 +574,7 @@ mod tests { n.structure = "2".to_string(); } - let section_map = ReasoningCompileStage::build_section_map(&tree); + let section_map = ReasoningPass::build_section_map(&tree); // Should index by title (lowercase) and structure index assert!(section_map.contains_key("introduction")); @@ -602,7 +602,7 @@ mod tests { n.summary = "second section summary".to_string(); } - let shortcut = ReasoningCompileStage::build_summary_shortcut(&tree); + let shortcut = ReasoningPass::build_summary_shortcut(&tree); assert!(shortcut.is_some()); let sc = shortcut.unwrap(); @@ -626,7 +626,7 @@ mod tests { n.summary = "child summary 2".to_string(); } - let shortcut = ReasoningCompileStage::build_summary_shortcut(&tree); + let shortcut = ReasoningPass::build_summary_shortcut(&tree); assert!(shortcut.is_some()); let sc = shortcut.unwrap(); diff --git a/vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs b/vectorless-core/vectorless-compiler/src/passes/backend/verify.rs similarity index 88% rename from vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs rename to vectorless-core/vectorless-compiler/src/passes/backend/verify.rs index 6f50252..01202bf 100644 --- a/vectorless-core/vectorless-compiler/src/stages/verify_ingest.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/verify.rs @@ -5,9 +5,9 @@ use tracing::{info, warn}; -use super::async_trait; -use super::{AccessPattern, CompileStage}; -use crate::pipeline::{CompileContext, StageResult}; +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass}; +use crate::pipeline::{CompileContext, PassResult}; use vectorless_error::{Error, Result}; /// Verification stage — ensures ingest produced reliable output. @@ -18,10 +18,10 @@ use vectorless_error::{Error, Result}; /// - At least one concept was extracted /// /// Any check failure produces an error — no silent degradation. -pub struct VerifyStage; +pub struct VerifyPass; #[async_trait] -impl CompileStage for VerifyStage { +impl CompilePass for VerifyPass { fn name(&self) -> &str { "verify" } @@ -41,7 +41,7 @@ impl CompileStage for VerifyStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { // Tree must exist and have nodes let tree = ctx .tree @@ -73,6 +73,6 @@ impl CompileStage for VerifyStage { ctx.concepts.len() ); - Ok(StageResult::success("verify")) + Ok(PassResult::success("verify")) } } diff --git a/vectorless-core/vectorless-compiler/src/stages/build.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs similarity index 97% rename from vectorless-core/vectorless-compiler/src/stages/build.rs rename to vectorless-core/vectorless-compiler/src/passes/frontend/build.rs index 283dfb5..2391b98 100644 --- a/vectorless-core/vectorless-compiler/src/stages/build.rs +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs @@ -3,7 +3,7 @@ //! Build stage - Build tree from raw nodes. -use super::async_trait; +use crate::passes::async_trait; use std::time::Instant; use tracing::{debug, info}; @@ -12,14 +12,14 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use super::{CompileStage, StageResult}; +use crate::passes::{CompilePass, PassResult}; use crate::ThinningConfig; use crate::pipeline::CompileContext; /// Build stage - constructs a tree from raw nodes. -pub struct BuildStage; +pub struct BuildPass; -impl BuildStage { +impl BuildPass { /// Create a new build stage. pub fn new() -> Self { Self @@ -236,14 +236,14 @@ impl BuildStage { } } -impl Default for BuildStage { +impl Default for BuildPass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for BuildStage { +impl CompilePass for BuildPass { fn name(&self) -> &'static str { "build" } @@ -252,7 +252,7 @@ impl CompileStage for BuildStage { vec!["parse"] } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); // Take raw nodes from context @@ -260,7 +260,7 @@ impl CompileStage for BuildStage { if raw_nodes.is_empty() { info!("[build] No raw nodes, skipping"); - return Ok(StageResult::success("build")); + return Ok(PassResult::success("build")); } info!( @@ -319,7 +319,7 @@ impl CompileStage for BuildStage { node_count, skipped, duration ); - let mut stage_result = StageResult::success("build"); + let mut stage_result = PassResult::success("build"); stage_result.duration_ms = duration; stage_result.metadata.insert( "node_count".to_string(), diff --git a/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs new file mode 100644 index 0000000..dbf893f --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Frontend passes — parse document into AST. + +mod parse; +mod build; + +pub use parse::ParsePass; +pub use build::BuildPass; diff --git a/vectorless-core/vectorless-compiler/src/stages/parse.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs similarity index 95% rename from vectorless-core/vectorless-compiler/src/stages/parse.rs rename to vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs index 7c90572..81ad688 100644 --- a/vectorless-core/vectorless-compiler/src/stages/parse.rs +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs @@ -3,24 +3,24 @@ //! Parse stage - Parse documents into raw nodes. -use super::async_trait; +use crate::passes::async_trait; use std::time::Instant; use tracing::{debug, info}; use vectorless_document::DocumentFormat; use vectorless_error::Result; -use super::{CompileStage, StageResult}; +use crate::passes::{CompilePass, PassResult}; use crate::SourceFormat; use crate::pipeline::{CompileContext, CompilerInput}; /// Parse stage - extracts raw nodes from documents. -pub struct ParseStage { +pub struct ParsePass { /// Optional LLM client for PDF structure extraction. llm_client: Option<vectorless_llm::LlmClient>, } -impl ParseStage { +impl ParsePass { /// Create a new parse stage. pub fn new() -> Self { Self { llm_client: None } @@ -52,19 +52,19 @@ impl ParseStage { } } -impl Default for ParseStage { +impl Default for ParsePass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for ParseStage { +impl CompilePass for ParsePass { fn name(&self) -> &'static str { "parse" } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); // Detect format @@ -151,7 +151,7 @@ impl CompileStage for ParseStage { duration ); - let mut stage_result = StageResult::success("parse"); + let mut stage_result = PassResult::success("parse"); stage_result.duration_ms = duration; stage_result.metadata.insert( "node_count".to_string(), diff --git a/vectorless-core/vectorless-compiler/src/passes/mod.rs b/vectorless-core/vectorless-compiler/src/passes/mod.rs new file mode 100644 index 0000000..33a79a5 --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/mod.rs @@ -0,0 +1,80 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Compiler passes — each pass is a discrete step in the document compilation pipeline. +//! +//! Passes are organized into four phases: +//! - **Frontend** — Parse document into AST (`parse`, `build`) +//! - **Analysis** — Semantic validation and LLM enhancement (`validate`, `enhance`) +//! - **Transform** — IR-level tree restructuring (`split`, `enrich`) +//! - **Backend** — Index generation, verification, and optimization + +pub mod frontend; +pub mod analysis; +pub mod transform; +pub mod backend; + +// Re-export all passes from submodules +pub use frontend::{ParsePass, BuildPass}; +pub use analysis::{ValidatePass, EnhancePass}; +pub use transform::{SplitPass, EnrichPass}; +pub use backend::{ReasoningPass, ConceptPass, NavigationPass, VerifyPass, OptimizePass}; + +use super::pipeline::{FailurePolicy, CompileContext, PassResult}; +pub use async_trait::async_trait; +use vectorless_error::Result; + +/// Declares which context fields a pass reads/writes. +/// Used by the orchestrator to determine safe parallel execution. +#[derive(Debug, Clone, Default)] +pub struct AccessPattern { + /// Whether this pass reads the tree. + pub reads_tree: bool, + /// Whether this pass mutates the tree (summaries, structure, etc.). + pub writes_tree: bool, + /// Whether this pass writes to `reasoning_index`. + pub writes_reasoning_index: bool, + /// Whether this pass writes to `navigation_index`. + pub writes_navigation_index: bool, + /// Whether this pass writes to `description`. + pub writes_description: bool, + /// Whether this pass writes to `concepts`. + pub writes_concepts: bool, +} + +/// Compiler pass trait. +/// +/// Each pass represents a discrete step in the document compilation pipeline. +/// Passes are executed in dependency order by the [`PipelineOrchestrator`]. +#[async_trait] +pub trait CompilePass: Send + Sync { + /// Pass name (must be unique within pipeline). + fn name(&self) -> &str; + + /// Execute the pass. + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult>; + + /// Whether this pass is optional (can be skipped on failure). + fn is_optional(&self) -> bool { + false + } + + /// Names of passes this pass depends on. + fn depends_on(&self) -> Vec<&'static str> { + Vec::new() + } + + /// Failure policy for this pass. + fn failure_policy(&self) -> FailurePolicy { + if self.is_optional() { + FailurePolicy::skip() + } else { + FailurePolicy::fail() + } + } + + /// Declare which context fields this pass accesses. + fn access_pattern(&self) -> AccessPattern { + AccessPattern::default() + } +} diff --git a/vectorless-core/vectorless-compiler/src/stages/enrich.rs b/vectorless-core/vectorless-compiler/src/passes/transform/enrich.rs similarity index 95% rename from vectorless-core/vectorless-compiler/src/stages/enrich.rs rename to vectorless-core/vectorless-compiler/src/passes/transform/enrich.rs index 2490e16..2d61794 100644 --- a/vectorless-core/vectorless-compiler/src/stages/enrich.rs +++ b/vectorless-core/vectorless-compiler/src/passes/transform/enrich.rs @@ -3,20 +3,20 @@ //! Enrich stage - Add metadata to the tree. -use super::async_trait; +use crate::passes::async_trait; use std::time::Instant; use tracing::{debug, info}; use vectorless_document::{DocumentTree, NodeId, ReferenceExtractor, TocView}; use vectorless_error::Result; -use super::{AccessPattern, CompileStage, StageResult}; +use crate::passes::{AccessPattern, CompilePass, PassResult}; use crate::pipeline::CompileContext; /// Enrich stage - adds metadata to the tree. -pub struct EnrichStage; +pub struct EnrichPass; -impl EnrichStage { +impl EnrichPass { /// Create a new enrich stage. pub fn new() -> Self { Self @@ -138,14 +138,14 @@ impl EnrichStage { } } -impl Default for EnrichStage { +impl Default for EnrichPass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for EnrichStage { +impl CompilePass for EnrichPass { fn name(&self) -> &'static str { "enrich" } @@ -163,7 +163,7 @@ impl CompileStage for EnrichStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); let tree = ctx @@ -208,7 +208,7 @@ impl CompileStage for EnrichStage { total_tokens, resolved_refs, duration ); - let mut stage_result = StageResult::success("enrich"); + let mut stage_result = PassResult::success("enrich"); stage_result.duration_ms = duration; stage_result .metadata @@ -234,7 +234,7 @@ mod tests { let mut tree = DocumentTree::new("Root", "root content"); tree.add_child(tree.root(), "Section 1", "No references here."); - let resolved = EnrichStage::resolve_references(&mut tree); + let resolved = EnrichPass::resolve_references(&mut tree); assert_eq!(resolved, 0); } } diff --git a/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs b/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs new file mode 100644 index 0000000..7355888 --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Transform passes — IR-level tree restructuring and enrichment. + +mod split; +mod enrich; + +pub use split::SplitPass; +pub use enrich::EnrichPass; diff --git a/vectorless-core/vectorless-compiler/src/stages/split.rs b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs similarity index 94% rename from vectorless-core/vectorless-compiler/src/stages/split.rs rename to vectorless-core/vectorless-compiler/src/passes/transform/split.rs index 01f6737..ff30fb3 100644 --- a/vectorless-core/vectorless-compiler/src/stages/split.rs +++ b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs @@ -10,7 +10,7 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use super::{AccessPattern, CompileStage, StageResult, async_trait}; +use crate::passes::{AccessPattern, CompilePass, PassResult, async_trait}; use crate::config::SplitConfig; use crate::pipeline::CompileContext; @@ -21,9 +21,9 @@ use crate::pipeline::CompileContext; /// child nodes from the resulting chunks. /// /// This stage runs after validate (priority 22) at priority 25. -pub struct SplitStage; +pub struct SplitPass; -impl SplitStage { +impl SplitPass { /// Create a new split stage. pub fn new() -> Self { Self @@ -201,14 +201,14 @@ impl SplitStage { } } -impl Default for SplitStage { +impl Default for SplitPass { fn default() -> Self { Self::new() } } #[async_trait] -impl CompileStage for SplitStage { +impl CompilePass for SplitPass { fn name(&self) -> &'static str { "split" } @@ -232,21 +232,21 @@ impl CompileStage for SplitStage { } } - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { let start = Instant::now(); let tree = match ctx.tree.as_mut() { Some(t) => t, None => { info!("[split] No tree, skipping"); - return Ok(StageResult::success("split")); + return Ok(PassResult::success("split")); } }; let config = &ctx.options.split; if !config.enabled { debug!("[split] Disabled, skipping"); - return Ok(StageResult::success("split")); + return Ok(PassResult::success("split")); } info!( @@ -267,7 +267,7 @@ impl CompileStage for SplitStage { split_count, node_count_before, node_count_after, duration ); - let mut stage_result = StageResult::success("split"); + let mut stage_result = PassResult::success("split"); stage_result.duration_ms = duration; stage_result .metadata @@ -292,7 +292,7 @@ mod tests { #[test] fn test_find_split_points_small_content() { let content = "Hello world"; - let points = SplitStage::find_split_points(content, 8000); + let points = SplitPass::find_split_points(content, 8000); assert!(points.is_empty()); } @@ -312,7 +312,7 @@ mod tests { content.push_str("Final content. "); } - let points = SplitStage::find_split_points(&content, 200); + let points = SplitPass::find_split_points(&content, 200); assert!(!points.is_empty()); } @@ -326,7 +326,7 @@ mod tests { content.push_str("\n\n"); } - let points = SplitStage::find_split_points(&content, 200); + let points = SplitPass::find_split_points(&content, 200); assert!(!points.is_empty()); } @@ -341,7 +341,7 @@ mod tests { tree.set_token_count(child, 15000); let config = SplitConfig::disabled(); - let count = SplitStage::split_tree(&mut tree, &config); + let count = SplitPass::split_tree(&mut tree, &config); assert_eq!(count, 0); } } diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index 2ee4ad5..8e3b0dc 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Index context for passing data between stages. +//! Compile context for passing data between passes. use std::collections::HashMap; use std::path::PathBuf; @@ -115,10 +115,10 @@ impl CompilerInput { } } -/// Result from a single stage execution. +/// Result from a single pass execution. #[derive(Debug, Clone)] -pub struct StageResult { - /// Whether the stage succeeded. +pub struct PassResult { + /// Whether the pass succeeded. pub success: bool, /// Duration in milliseconds. @@ -128,10 +128,10 @@ pub struct StageResult { pub metadata: HashMap<String, serde_json::Value>, } -impl StageResult { +impl PassResult { /// Create a successful result. pub fn success(name: &str) -> Self { - println!("Stage '{}' completed successfully", name); + println!("Pass '{}' completed successfully", name); Self { success: true, @@ -142,7 +142,7 @@ impl StageResult { /// Create a failed result. pub fn failure(name: &str, error: &str) -> Self { - println!("Stage '{}' failed: {}", name, error); + println!("Pass '{}' failed: {}", name, error); let mut metadata = HashMap::new(); metadata.insert( @@ -169,6 +169,10 @@ impl StageResult { } } +/// Backward-compatible alias. +#[deprecated(since = "0.2.0", note = "Use `PassResult` instead")] +pub type StageResult = PassResult; + /// Summary cache for lazy generation. #[derive(Debug, Clone, Default)] pub struct SummaryCache { @@ -209,7 +213,7 @@ impl SummaryCache { } } -/// Index context passed between stages. +/// Compile context passed between passes. #[derive(Debug)] pub struct CompileContext { /// Document ID. @@ -258,8 +262,8 @@ pub struct CompileContext { /// When set, the enhance and reasoning stages can reuse data from unchanged nodes. pub existing_tree: Option<DocumentTree>, - /// Stage execution results. - pub stage_results: HashMap<String, StageResult>, + /// Pass execution results. + pub stage_results: HashMap<String, PassResult>, /// Performance metrics. pub metrics: IndexMetrics, @@ -361,8 +365,8 @@ impl CompileContext { } } - /// Record a stage result. - pub fn record_stage(&mut self, name: &str, result: StageResult) { + /// Record a pass result. + pub fn record_stage(&mut self, name: &str, result: PassResult) { self.stage_results.insert(name.to_string(), result); } diff --git a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs index c60d0f0..ba6ffd2 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -12,10 +12,10 @@ use vectorless_error::Result; use vectorless_llm::LlmClient; use super::super::PipelineOptions; -use super::super::stages::{ - BuildStage, ConceptExtractionStage, EnhanceStage, EnrichStage, CompileStage, - NavigationCompileStage, OptimizeStage, ParseStage, ReasoningCompileStage, SplitStage, - ValidateStage, VerifyStage, +use super::super::passes::{ + BuildPass, ConceptPass, EnhancePass, EnrichPass, CompilePass, + NavigationPass, OptimizePass, ParsePass, ReasoningPass, SplitPass, + ValidatePass, VerifyPass, }; use super::context::{CompilerInput, CompileResult}; use super::orchestrator::PipelineOrchestrator; @@ -37,9 +37,9 @@ use super::orchestrator::PipelineOrchestrator; /// /// // Custom pipeline using orchestrator /// let orchestrator = PipelineOrchestrator::new() -/// .stage(ParseStage::new()) +/// .stage(ParsePass::new()) /// .stage_with_priority(MyCustomStage::new(), 50) -/// .stage(BuildStage::new()); +/// .stage(BuildPass::new()); /// let executor = PipelineExecutor::from_orchestrator(orchestrator); /// ``` pub struct PipelineExecutor { @@ -62,16 +62,16 @@ impl PipelineExecutor { /// 10. `optimize` - Optimize tree structure pub fn new() -> Self { let orchestrator = PipelineOrchestrator::new() - .stage_with_priority(ParseStage::new(), 10) - .stage_with_priority(BuildStage::new(), 20) - .stage_with_priority(ValidateStage::new(), 22) - .stage_with_priority(SplitStage::new(), 25) - .stage_with_priority(EnrichStage::new(), 40) - .stage_with_priority(ReasoningCompileStage::new(), 45) - .stage_with_priority(ConceptExtractionStage::new(), 47) - .stage_with_priority(NavigationCompileStage::new(), 50) - .stage_with_priority(VerifyStage, 55) - .stage_with_priority(OptimizeStage::new(), 60); + .stage_with_priority(ParsePass::new(), 10) + .stage_with_priority(BuildPass::new(), 20) + .stage_with_priority(ValidatePass::new(), 22) + .stage_with_priority(SplitPass::new(), 25) + .stage_with_priority(EnrichPass::new(), 40) + .stage_with_priority(ReasoningPass::new(), 45) + .stage_with_priority(ConceptPass::new(), 47) + .stage_with_priority(NavigationPass::new(), 50) + .stage_with_priority(VerifyPass, 55) + .stage_with_priority(OptimizePass::new(), 60); Self { orchestrator } } @@ -92,21 +92,21 @@ impl PipelineExecutor { /// 11. `optimize` - Optimize tree pub fn with_llm(client: LlmClient) -> Self { tracing::info!( - "PipelineExecutor::with_llm — cloning client to ParseStage + EnhanceStage + context" + "PipelineExecutor::with_llm — cloning client to ParsePass + EnhancePass + context" ); let orchestrator = PipelineOrchestrator::new() .with_llm_client(client.clone()) - .stage_with_priority(ParseStage::with_llm_client(client.clone()), 10) - .stage_with_priority(BuildStage::new(), 20) - .stage_with_priority(ValidateStage::new(), 22) - .stage_with_priority(SplitStage::new(), 25) - .stage_with_priority(EnhanceStage::with_llm_client(client.clone()), 30) - .stage_with_priority(EnrichStage::new(), 40) - .stage_with_priority(ReasoningCompileStage::new(), 45) - .stage_with_priority(ConceptExtractionStage::with_llm_client(client), 47) - .stage_with_priority(NavigationCompileStage::new(), 50) - .stage_with_priority(VerifyStage, 55) - .stage_with_priority(OptimizeStage::new(), 60); + .stage_with_priority(ParsePass::with_llm_client(client.clone()), 10) + .stage_with_priority(BuildPass::new(), 20) + .stage_with_priority(ValidatePass::new(), 22) + .stage_with_priority(SplitPass::new(), 25) + .stage_with_priority(EnhancePass::with_llm_client(client.clone()), 30) + .stage_with_priority(EnrichPass::new(), 40) + .stage_with_priority(ReasoningPass::new(), 45) + .stage_with_priority(ConceptPass::with_llm_client(client), 47) + .stage_with_priority(NavigationPass::new(), 50) + .stage_with_priority(VerifyPass, 55) + .stage_with_priority(OptimizePass::new(), 60); Self { orchestrator } } @@ -119,9 +119,9 @@ impl PipelineExecutor { /// /// ```rust,ignore /// let orchestrator = PipelineOrchestrator::new() - /// .stage_with_priority(ParseStage::new(), 10) + /// .stage_with_priority(ParsePass::new(), 10) /// .stage_with_priority(MyAnalysisStage::new(), 25) - /// .stage_with_priority(BuildStage::new(), 20) + /// .stage_with_priority(BuildPass::new(), 20) /// .stage_with_deps(MyValidationStage::new(), 50, &["build"]); /// /// let executor = PipelineExecutor::from_orchestrator(orchestrator); @@ -133,7 +133,7 @@ impl PipelineExecutor { /// Add a stage with default priority. /// /// The stage will be added after existing stages with the same priority. - pub fn add_stage(mut self, stage: impl CompileStage + 'static) -> Self { + pub fn add_stage(mut self, stage: impl CompilePass + 'static) -> Self { self.orchestrator = self.orchestrator.stage(stage); self } @@ -143,7 +143,7 @@ impl PipelineExecutor { /// Lower priority = earlier execution. pub fn add_stage_with_priority( mut self, - stage: impl CompileStage + 'static, + stage: impl CompilePass + 'static, priority: i32, ) -> Self { self.orchestrator = self.orchestrator.stage_with_priority(stage, priority); @@ -155,7 +155,7 @@ impl PipelineExecutor { /// The stage will run after all specified dependencies. pub fn add_stage_with_deps( mut self, - stage: impl CompileStage + 'static, + stage: impl CompilePass + 'static, priority: i32, depends_on: &[&str], ) -> Self { diff --git a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs index 31b7035..88c8e9f 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs @@ -4,12 +4,12 @@ //! Pipeline execution module. //! //! This module provides the core pipeline infrastructure: -//! - [`CompileContext`] - Context passed between stages -//! - [`PipelineExecutor`] - Executes the indexing pipeline -//! - [`PipelineOrchestrator`] - Flexible stage orchestration with dependencies +//! - [`CompileContext`] - Context passed between passes +//! - [`PipelineExecutor`] - Executes the compilation pipeline +//! - [`PipelineOrchestrator`] - Flexible pass orchestration with dependencies //! - [`IndexMetrics`] - Performance metrics collection -//! - [`FailurePolicy`] - Configurable failure handling for stages -//! - [`StageRetryConfig`] - Retry configuration for stages +//! - [`FailurePolicy`] - Configurable failure handling for passes +//! - [`StageRetryConfig`] - Retry configuration for passes mod checkpoint; mod context; @@ -18,7 +18,7 @@ mod metrics; mod orchestrator; mod policy; -pub use context::{CompileContext, CompilerInput, CompileResult, StageResult}; +pub use context::{CompileContext, CompilerInput, CompileResult, PassResult}; pub use executor::PipelineExecutor; pub use metrics::IndexMetrics; pub use policy::{FailurePolicy, StageRetryConfig}; diff --git a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs index c3c607d..f6d6283 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs @@ -30,24 +30,24 @@ use tracing::{debug, error, info, warn}; use vectorless_error::Result; use super::super::PipelineOptions; -use super::super::stages::CompileStage; +use super::super::passes::CompilePass; use super::checkpoint::{CheckpointContextData, CheckpointManager, PipelineCheckpoint}; -use super::context::{CompileContext, CompilerInput, CompileResult, StageResult}; +use super::context::{CompileContext, CompilerInput, CompileResult, PassResult}; use super::policy::FailurePolicy; /// Stage entry with metadata for orchestration. -struct StageEntry { +struct PassEntry { /// The stage implementation. - stage: Box<dyn CompileStage>, + stage: Box<dyn CompilePass>, /// Priority (lower = earlier execution). priority: i32, /// Names of stages this depends on. depends_on: Vec<String>, } -impl std::fmt::Debug for StageEntry { +impl std::fmt::Debug for PassEntry { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("StageEntry") + f.debug_struct("PassEntry") .field("name", &self.stage.name()) .field("priority", &self.priority) .field("depends_on", &self.depends_on) @@ -93,7 +93,7 @@ pub struct ExecutionGroup { /// ``` pub struct PipelineOrchestrator { /// Registered stages with metadata. - stages: Vec<StageEntry>, + stages: Vec<PassEntry>, /// Shared LLM client injected into pipeline context. llm_client: Option<vectorless_llm::LlmClient>, } @@ -124,10 +124,10 @@ impl PipelineOrchestrator { /// Dependencies are automatically read from the stage's `depends_on()` method. pub fn stage<S>(mut self, stage: S) -> Self where - S: CompileStage + 'static, + S: CompilePass + 'static, { let deps = stage.depends_on(); - self.stages.push(StageEntry { + self.stages.push(PassEntry { stage: Box::new(stage), priority: 100, depends_on: deps.into_iter().map(|s| s.to_string()).collect(), @@ -142,10 +142,10 @@ impl PipelineOrchestrator { /// Default priority is 100. pub fn stage_with_priority<S>(mut self, stage: S, priority: i32) -> Self where - S: CompileStage + 'static, + S: CompilePass + 'static, { let deps = stage.depends_on(); - self.stages.push(StageEntry { + self.stages.push(PassEntry { stage: Box::new(stage), priority, depends_on: deps.into_iter().map(|s| s.to_string()).collect(), @@ -164,7 +164,7 @@ impl PipelineOrchestrator { explicit_depends_on: &[&str], ) -> Self where - S: CompileStage + 'static, + S: CompilePass + 'static, { let trait_deps = stage.depends_on(); let mut all_deps: Vec<String> = trait_deps.into_iter().map(|s| s.to_string()).collect(); @@ -176,7 +176,7 @@ impl PipelineOrchestrator { } } - self.stages.push(StageEntry { + self.stages.push(PassEntry { stage: Box::new(stage), priority, depends_on: all_deps, @@ -346,9 +346,9 @@ impl PipelineOrchestrator { /// Execute a stage with its failure policy applied. async fn execute_stage_with_policy( - stage: &mut Box<dyn CompileStage>, + stage: &mut Box<dyn CompilePass>, ctx: &mut CompileContext, - ) -> Result<StageResult> { + ) -> Result<PassResult> { let policy = stage.failure_policy(); let stage_name = stage.name().to_string(); @@ -364,7 +364,7 @@ impl PipelineOrchestrator { Ok(result) => Ok(result), Err(e) => { warn!("Stage {} failed, skipping: {}", stage_name, e); - Ok(StageResult::failure(&stage_name, &e.to_string())) + Ok(PassResult::failure(&stage_name, &e.to_string())) } } } @@ -403,7 +403,7 @@ impl PipelineOrchestrator { /// Handle the result of a stage execution (shared between sequential and parallel paths). fn handle_stage_result( - result: Result<StageResult>, + result: Result<PassResult>, stage_name: &str, policy: &FailurePolicy, ctx: &mut CompileContext, @@ -421,7 +421,7 @@ impl PipelineOrchestrator { ); ctx.stage_results.insert( stage_name.to_string(), - StageResult::failure(stage_name, &e.to_string()), + PassResult::failure(stage_name, &e.to_string()), ); Ok(()) } else { @@ -506,7 +506,7 @@ impl PipelineOrchestrator { // Mark completed stages as done for stage_name in &checkpoint.completed_stages { ctx.stage_results - .insert(stage_name.clone(), StageResult::success(stage_name)); + .insert(stage_name.clone(), PassResult::success(stage_name)); } } else { info!("Checkpoint exists but invalid, starting fresh"); @@ -556,7 +556,7 @@ impl PipelineOrchestrator { .copied(); // For each stage, prepare (stage, context) pair. - // Swap out stages from self.stages to get owned Box<dyn CompileStage>. + // Swap out stages from self.stages to get owned Box<dyn CompilePass>. let mut entries: Vec<ParallelEntry> = Vec::with_capacity(group.stage_indices.len()); for &idx in &group.stage_indices { @@ -610,7 +610,7 @@ impl PipelineOrchestrator { // Execute writer on main ctx concurrently with readers. // Move each reader's stage+ctx into an owned async block. - // All futures are !Send (Box<dyn CompileStage>), but join_all + // All futures are !Send (Box<dyn CompilePass>), but join_all // works fine on the same thread. let reader_futs: Vec< @@ -619,7 +619,7 @@ impl PipelineOrchestrator { dyn std::future::Future< Output = ( ParallelEntry, - std::result::Result<StageResult, vectorless_error::Error>, + std::result::Result<PassResult, vectorless_error::Error>, ), > + Send, >, @@ -699,7 +699,7 @@ impl PipelineOrchestrator { ); ctx.stage_results.insert( stage_name.clone(), - StageResult::failure(&stage_name, &e.to_string()), + PassResult::failure(&stage_name, &e.to_string()), ); } else { error!("Stage {} failed, stopping pipeline: {}", stage_name, e); @@ -841,13 +841,13 @@ impl PipelineOrchestrator { struct NopStage; #[async_trait::async_trait] -impl CompileStage for NopStage { +impl CompilePass for NopStage { fn name(&self) -> &'static str { "_nop" } - async fn execute(&mut self, _ctx: &mut CompileContext) -> Result<StageResult> { - Ok(StageResult::success("_nop")) + async fn execute(&mut self, _ctx: &mut CompileContext) -> Result<PassResult> { + Ok(PassResult::success("_nop")) } } @@ -860,7 +860,7 @@ struct ParallelEntry { /// Index into orchestrator's stages vec (for swapping back). idx: usize, /// The owned stage implementation. - stage: Box<dyn CompileStage>, + stage: Box<dyn CompilePass>, /// Cloned context for reader stages; None for the tree writer /// (which uses the main ctx directly). ctx: Option<CompileContext>, @@ -869,7 +869,7 @@ struct ParallelEntry { /// Failure policy (captured before swap). policy: FailurePolicy, /// Access pattern (captured before swap). - access: crate::stages::AccessPattern, + access: crate::passes::AccessPattern, } /// Builder for creating custom stage configurations. @@ -935,7 +935,7 @@ impl CustomStageBuilder { #[cfg(test)] mod tests { - use super::super::context::StageResult; + use super::super::context::PassResult; use super::*; #[test] @@ -1016,13 +1016,13 @@ mod tests { } #[async_trait::async_trait] - impl CompileStage for MockStage { + impl CompilePass for MockStage { fn name(&self) -> &str { &self.name } - async fn execute(&mut self, _ctx: &mut CompileContext) -> Result<StageResult> { - Ok(StageResult::success(&self.name)) + async fn execute(&mut self, _ctx: &mut CompileContext) -> Result<PassResult> { + Ok(PassResult::success(&self.name)) } } } diff --git a/vectorless-core/vectorless-compiler/src/stages/mod.rs b/vectorless-core/vectorless-compiler/src/stages/mod.rs deleted file mode 100644 index c8b5360..0000000 --- a/vectorless-core/vectorless-compiler/src/stages/mod.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Index pipeline stages. - -mod build; -mod concept; -mod enhance; -mod enrich; -mod navigation; -mod optimize; -mod parse; -mod reasoning; -mod split; -mod validate; -mod verify_ingest; - -pub use build::BuildStage; -pub use concept::ConceptExtractionStage; -pub use enhance::EnhanceStage; -pub use enrich::EnrichStage; -pub use navigation::NavigationCompileStage; -pub use optimize::OptimizeStage; -pub use parse::ParseStage; -pub use reasoning::ReasoningCompileStage; -pub use split::SplitStage; -pub use validate::ValidateStage; -pub use verify_ingest::VerifyStage; - -use super::pipeline::{FailurePolicy, CompileContext, StageResult}; -pub use async_trait::async_trait; -use vectorless_error::Result; - -/// Declares which context fields a stage reads/writes. -/// Used by the orchestrator to determine safe parallel execution. -#[derive(Debug, Clone, Default)] -pub struct AccessPattern { - /// Whether this stage reads the tree. - pub reads_tree: bool, - /// Whether this stage mutates the tree (summaries, structure, etc.). - pub writes_tree: bool, - /// Whether this stage writes to `reasoning_index`. - pub writes_reasoning_index: bool, - /// Whether this stage writes to `navigation_index`. - pub writes_navigation_index: bool, - /// Whether this stage writes to `description`. - pub writes_description: bool, - /// Whether this stage writes to `concepts`. - pub writes_concepts: bool, -} - -/// Index pipeline stage. -/// -/// Each stage represents a discrete step in the document indexing process. -/// Stages are executed in dependency order by the [`PipelineOrchestrator`]. -/// -/// # Stage Lifecycle -/// -/// 1. Stage is registered with the orchestrator -/// 2. Dependencies are resolved and execution order is determined -/// 3. `execute()` is called with the shared context -/// 4. Results are stored in `ctx.stage_results` -/// -/// # Example -/// -/// ```rust,ignore -/// struct MyStage; -/// -/// #[async_trait] -/// impl CompileStage for MyStage { -/// fn name(&self) -> &str { "my_stage" } -/// -/// fn depends_on(&self) -> Vec<&'static str> { -/// vec!["parse", "build"] -/// } -/// -/// async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult> { -/// // Process the context... -/// Ok(StageResult::success("my_stage")) -/// } -/// } -/// ``` -#[async_trait] -pub trait CompileStage: Send + Sync { - /// Stage name (must be unique within pipeline). - fn name(&self) -> &str; - - /// Execute the stage. - /// - /// This method receives a mutable reference to the shared context, - /// allowing stages to read from and write to it. - async fn execute(&mut self, ctx: &mut CompileContext) -> Result<StageResult>; - - /// Whether this stage is optional (can be skipped on failure). - /// - /// Optional stages that fail will not stop the pipeline. - /// Default: `false` - fn is_optional(&self) -> bool { - false - } - - /// Names of stages this stage depends on. - /// - /// Dependencies are validated during pipeline construction. - /// A stage will only execute after all its dependencies have completed. - /// - /// # Example - /// - /// ```rust,ignore - /// fn depends_on(&self) -> Vec<&'static str> { - /// vec!["parse", "build"] - /// } - /// ``` - fn depends_on(&self) -> Vec<&'static str> { - Vec::new() - } - - /// Failure policy for this stage. - /// - /// Determines how the pipeline handles failures in this stage: - /// - `Fail`: Stop the entire pipeline (default for required stages) - /// - `Skip`: Skip this stage, continue pipeline - /// - `Retry`: Retry with exponential backoff - /// - /// Default behavior: - /// - If `is_optional()` returns true, defaults to `FailurePolicy::Skip` - /// - Otherwise, defaults to `FailurePolicy::Fail` - fn failure_policy(&self) -> FailurePolicy { - if self.is_optional() { - FailurePolicy::skip() - } else { - FailurePolicy::fail() - } - } - - /// Declare which context fields this stage accesses. - /// Used by the orchestrator for safe parallel execution. - fn access_pattern(&self) -> AccessPattern { - AccessPattern::default() - } -} From 5355ed098a1da67822e8208bb87c25052dd43a1c Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 20:52:57 +0800 Subject: [PATCH 19/30] docs(vectorless-compiler): update pipeline documentation with phase categories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace "Priority" labels with semantic phase categories (Frontend, Analysis, Transform, Backend) to better reflect the compilation pipeline stages - Update descriptions for clarity: change "Tree integrity checks (optional)" to "Tree integrity checks", remove "(optional)" from various stages as they are conditionally executed based on pipeline configuration rather than being truly optional - Add new "Concept" stage at priority 47 between "Reasoning Idx" and "Navigation Idx" phases - Rename "Symbol table (keyword→path mapping)" for "Reasoning Idx" stage and "Debug info for runtime navigation" for "Navigation Idx" stage - Add "Output validation" stage at priority 55 between "Reasoning Idx" and "Optimize" phases - Update checkpointing description from "stage group" to "pass group" for more accurate terminology --- .../vectorless-compiler/src/lib.rs | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/vectorless-core/vectorless-compiler/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs index f36731e..ba8e980 100644 --- a/vectorless-core/vectorless-compiler/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -6,39 +6,44 @@ //! This module provides a modular document compilation pipeline that transforms //! documents (Markdown, PDF) into agent-friendly intermediate artifacts. //! -//! //! ```text -//! Priority 10: ┌──────────┐ +//! Frontend 10: ┌──────────┐ //! │ Parse │ Parse document into raw nodes //! └────┬─────┘ -//! Priority 20: ┌────▼─────┐ -//! │ Build │ Construct tree + thinning (with content merge) +//! Frontend 20: ┌────▼─────┐ +//! │ Build │ Construct tree + thinning //! └────┬─────┘ -//! Priority 22: ┌────▼─────┐ +//! Analysis 22: ┌────▼─────┐ //! │ Validate │ Tree integrity checks (optional) //! └────┬─────┘ -//! Priority 25: ┌────▼─────┐ +//! Transform 25: ┌────▼─────┐ //! │ Split │ Split oversized leaf nodes (optional) //! └────┬─────┘ -//! Priority 30: ┌────▼─────┐ -//! │ Enhance │ LLM summaries (when client available) +//! Analysis 30: ┌────▼─────┐ +//! │ Enhance │ LLM summaries //! └────┬─────┘ -//! Priority 40: ┌────▼─────┐ +//! Transform 40: ┌────▼─────┐ //! │ Enrich │ Metadata + cross-references //! └────┬─────┘ -//! Priority 45: ┌────▼──────────┐ -//! │ Reasoning Idx │ Pre-computed reasoning index +//! Backend 45: ┌────▼──────────┐ +//! │ Reasoning Idx │ Symbol table (keyword→path mapping) +//! └────┬──────────┘ +//! Backend 47: ┌────▼──────────┐ +//! │ Concept │ Concept extraction //! └────┬──────────┘ -//! Priority 50: ┌────▼──────────┐ -//! │ Navigation Idx│ Agent navigation index +//! Backend 50: ┌────▼──────────┐ +//! │ Navigation Idx│ Debug info for runtime navigation //! └────┬──────────┘ -//! Priority 60: ┌────▼──────┐ +//! Backend 55: ┌────▼──────┐ +//! │ Verify │ Output validation +//! └────┬──────┘ +//! Backend 60: ┌────▼──────┐ //! │ Optimize │ Final tree optimization //! └───────────┘ //! ``` //! //! Checkpointing is available when `PipelineOptions::checkpoint_dir` is set. -//! State is saved after each stage group and resumed on restart. +//! State is saved after each pass group and resumed on restart. pub mod config; pub mod incremental; From 9fde833c2bccc999713a52afdb0d9565ab29d168 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 21:33:07 +0800 Subject: [PATCH 20/30] docs(compiler): add comprehensive documentation for compilation pipeline - Add overview documentation explaining the compiler architecture and phase breakdown - Document pipeline infrastructure including CompilePass trait, PipelineExecutor, and PipelineOrchestrator components - Detail all compilation passes with their priorities, dependencies, and functionality - Provide configuration guide for PipelineOptions and related types - Explain incremental compilation mechanism with change detection - Document checkpoint and resume functionality for pipeline recovery feat(compiler): implement RoutePass for query routing table generation - Build intent routes from nodes with question hints for Agent acceleration - Create concept routes from topic tags to enable semantic navigation - Calculate relevance scores based on content richness and hint count - Limit route targets to improve performance and reduce memory usage refactor(compiler): extend CompileContext with agent acceleration data - Add query_routes field for pre-computed routing table storage - Include chain_index for reasoning chain navigation - Add content_overlap map to prevent duplicate content visits - Introduce evidence_scores for per-node quality assessment - Update context cloning and result extraction methods accordingly --- docs/docs/compiler/checkpoint.mdx | 117 ++++++++++ docs/docs/compiler/configuration.mdx | 163 +++++++++++++ docs/docs/compiler/incremental.mdx | 125 ++++++++++ docs/docs/compiler/overview.mdx | 81 +++++++ docs/docs/compiler/passes.mdx | 218 ++++++++++++++++++ docs/docs/compiler/pipeline.mdx | 150 ++++++++++++ docs/sidebars.ts | 12 + .../src/passes/backend/route.rs | 208 +++++++++++++++++ .../vectorless-compiler/src/passes/mod.rs | 8 + .../src/pipeline/context.rs | 39 +++- .../vectorless-document/src/chain.rs | 127 ++++++++++ .../vectorless-document/src/evidence.rs | 107 +++++++++ .../vectorless-document/src/lib.rs | 12 + .../vectorless-document/src/overlap.rs | 97 ++++++++ .../vectorless-document/src/query_route.rs | 117 ++++++++++ 15 files changed, 1580 insertions(+), 1 deletion(-) create mode 100644 docs/docs/compiler/checkpoint.mdx create mode 100644 docs/docs/compiler/configuration.mdx create mode 100644 docs/docs/compiler/incremental.mdx create mode 100644 docs/docs/compiler/overview.mdx create mode 100644 docs/docs/compiler/passes.mdx create mode 100644 docs/docs/compiler/pipeline.mdx create mode 100644 vectorless-core/vectorless-compiler/src/passes/backend/route.rs create mode 100644 vectorless-core/vectorless-document/src/chain.rs create mode 100644 vectorless-core/vectorless-document/src/evidence.rs create mode 100644 vectorless-core/vectorless-document/src/overlap.rs create mode 100644 vectorless-core/vectorless-document/src/query_route.rs diff --git a/docs/docs/compiler/checkpoint.mdx b/docs/docs/compiler/checkpoint.mdx new file mode 100644 index 0000000..35b3480 --- /dev/null +++ b/docs/docs/compiler/checkpoint.mdx @@ -0,0 +1,117 @@ +--- +sidebar_position: 6 +--- + +# Checkpoint and Resume + +Checkpointing allows the pipeline to resume from where it left off after an interruption (crash, timeout, process kill). This is critical for large documents where LLM-enhanced compilation can take minutes. + +## How It Works + +When `PipelineOptions::checkpoint_dir` is set, the orchestrator saves state to disk after each execution group completes: + +```text +Group 0: [ParsePass] → save checkpoint +Group 1: [BuildPass] → save checkpoint +Group 2: [ValidatePass, SplitPass] → save checkpoint +Group 3: [EnhancePass] → save checkpoint ← expensive LLM calls +... +``` + +On restart, the orchestrator loads the checkpoint and skips already-completed passes. + +## What's Stored + +Each checkpoint contains: + +```rust +pub struct PipelineCheckpoint { + pub doc_id: String, + pub source_hash: String, // SHA-256 of source content + pub processing_version: u32, // Algorithm version + pub config_fingerprint: String, // Hash of PipelineOptions + pub completed_stages: Vec<String>, // Names of completed passes + pub context_data: CheckpointContextData, + pub timestamp: DateTime<Utc>, +} + +pub struct CheckpointContextData { + pub raw_nodes: Vec<RawNode>, // From ParsePass + pub tree: Option<DocumentTree>, // From BuildPass + pub metrics: IndexMetrics, // Cumulative metrics + pub page_count: Option<usize>, + pub line_count: Option<usize>, + pub description: Option<String>, +} +``` + +## Validation + +Before resuming, the checkpoint is validated against the current input: + +| Check | Purpose | +|---|---| +| `source_hash` matches | Source content hasn't changed | +| `processing_version` matches | Algorithm hasn't been upgraded | +| `config_fingerprint` matches | Pipeline options haven't changed | + +If any check fails, the checkpoint is discarded and the pipeline starts fresh. + +## Lifecycle + +```text +┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Start │────▶│ Load │────▶│ Valid? │ +│ Pipeline │ │ Checkpoint │ │ │ +└──────────────┘ └──────────────┘ └──┬───────┬───┘ + │ │ + Yes │ No │ + │ │ + ┌─────────▼──┐ ┌─▼──────────┐ + │ Resume from │ │ Start fresh │ + │ completed │ │ │ + │ stages │ │ │ + └──────┬──────┘ └────────────┘ + │ + ┌────────────▼─────────────┐ + │ Execute remaining passes │ + │ Save after each group │ + └────────────┬─────────────┘ + │ + ┌────────────▼─────────────┐ + │ All complete? │ + │ → Clear checkpoint file │ + └──────────────────────────┘ +``` + +## Configuration + +```rust +let options = PipelineOptions::default() + .with_checkpoint_dir("./workspace/checkpoints"); +``` + +Checkpoints are stored as individual JSON files in the checkpoint directory, one per document (keyed by `doc_id`). On successful completion, the checkpoint file is deleted. + +## CheckpointManager API + +```rust +let manager = CheckpointManager::new("./checkpoints"); + +// Save checkpoint +manager.save(&doc_id, &checkpoint)?; + +// Load checkpoint +let checkpoint = manager.load(&doc_id); + +// Check if valid for resume +let valid = CheckpointManager::is_valid_for_resume( + &checkpoint, + &source_hash, + processing_version, + &config_fingerprint, +); + +// Clear after successful completion +manager.clear(&doc_id)?; +``` diff --git a/docs/docs/compiler/configuration.mdx b/docs/docs/compiler/configuration.mdx new file mode 100644 index 0000000..34fea8e --- /dev/null +++ b/docs/docs/compiler/configuration.mdx @@ -0,0 +1,163 @@ +--- +sidebar_position: 4 +--- + +# Configuration + +`PipelineOptions` controls every aspect of the compilation pipeline. All fields have sensible defaults and can be overridden using the builder pattern. + +## PipelineOptions + +```rust +let options = PipelineOptions::default() + .with_mode(SourceFormat::Pdf) + .with_generate_ids(true) + .with_summary_strategy(SummaryStrategy::full()) + .with_thinning(ThinningConfig::enabled(300)) + .with_optimization(OptimizationConfig::new()) + .with_split(SplitConfig::with_max_tokens(2000)) + .with_generate_description(true) + .with_checkpoint_dir("./checkpoints"); +``` + +| Field | Type | Default | Description | +|---|---|---|---| +| `mode` | `SourceFormat` | `Auto` | Document format | +| `generate_ids` | `bool` | `true` | Assign unique IDs to tree nodes | +| `summary_strategy` | `SummaryStrategy` | `Full` | How to generate LLM summaries | +| `thinning` | `ThinningConfig` | disabled | Merge small nodes into parents | +| `optimization` | `OptimizationConfig` | enabled | Final tree optimization | +| `split` | `SplitConfig` | enabled (4000 tokens) | Split oversized leaf nodes | +| `generate_description` | `bool` | `true` | Generate a document-level description | +| `concurrency` | `ConcurrencyConfig` | from LLM config | Max concurrent LLM requests | +| `reasoning_index` | `ReasoningIndexConfig` | default | Symbol table configuration | +| `existing_tree` | `Option<DocumentTree>` | `None` | Previous tree for incremental updates | +| `processing_version` | `u32` | `1` | Algorithm version (forces reprocessing on change) | +| `checkpoint_dir` | `Option<PathBuf>` | `None` | Directory for pipeline checkpoints | + +## SourceFormat + +```rust +pub enum SourceFormat { + Auto, // Detect from file extension + Markdown, // Force Markdown parsing + Pdf, // Force PDF parsing +} +``` + +When set to `Auto`, the engine detects format from the file extension before calling the compiler. The compiler itself always receives a concrete format. + +## SummaryStrategy + +Controls how the EnhancePass generates LLM summaries: + +### None + +Skip summary generation entirely. Nodes retain their raw content only. + +```rust +SummaryStrategy::none() +``` + +### Full (default) + +Generate summaries for every node in the tree. + +```rust +SummaryStrategy::full() +// With custom config: +SummaryStrategy::full_with_config(SummaryStrategyConfig { + max_tokens: 200, + shortcut_threshold: 50, + ..Default::default() +}) +``` + +- Non-leaf nodes: structured output (`OVERVIEW`, `QUESTIONS`, `TAGS`) +- Leaf nodes: concise content summaries +- Nodes below `shortcut_threshold` tokens use original content (saves LLM cost) + +### Selective + +Generate summaries only for qualifying nodes. + +```rust +SummaryStrategy::selective(500, true) // min 500 tokens, branch nodes only +``` + +Parameters: +- `min_tokens`: Only generate summaries for nodes with at least this many tokens +- `branch_only`: If `true`, skip leaf nodes entirely + +### Lazy + +Generate summaries on-demand at query time instead of during compilation. + +```rust +SummaryStrategy::lazy(true) // persist generated summaries +``` + +Summaries are cached in a `SummaryCache` and optionally persisted. This is useful when many documents are compiled but only a fraction will be queried. + +## ThinningConfig + +Controls how small nodes are merged into their parents during the BuildPass: + +```rust +// Disabled (default) +ThinningConfig::disabled() + +// Enabled with 500-token threshold +ThinningConfig::enabled(500) + .with_merge_content(true) +``` + +| Field | Default | Description | +|---|---|---| +| `enabled` | `false` | Whether thinning is active | +| `threshold` | `500` | Nodes below this token count are candidates for merging | +| `merge_content` | `true` | Whether to merge child content into the parent | + +Thinning reduces tree depth by absorbing small sections (e.g., single-paragraph subsections) into their parent node. Each parent keeps at least one child. + +## SplitConfig + +Controls how oversized leaf nodes are split: + +```rust +SplitConfig::default() // enabled, 4000 tokens, pattern split on +SplitConfig::disabled() // no splitting +SplitConfig::with_max_tokens(2000) // custom threshold + .with_pattern_split(true) +``` + +| Field | Default | Description | +|---|---|---| +| `enabled` | `true` | Whether splitting is active | +| `max_tokens_per_node` | `4000` | Nodes exceeding this are split | +| `pattern_split` | `true` | Use natural break points (headings, paragraphs) | + +## OptimizationConfig + +Controls final tree optimization in the OptimizePass: + +```rust +OptimizationConfig::new() + .with_max_depth(15) + .with_max_children(20) +``` + +| Field | Default | Description | +|---|---|---| +| `enabled` | `true` | Whether optimization is active | +| `max_depth` | `None` | Flatten tree if depth exceeds this | +| `max_children` | `None` | Group children if count exceeds this | +| `merge_leaf_threshold` | `0` | Merge adjacent leaf siblings below this token count | + +## Logic Fingerprint + +`PipelineOptions::logic_fingerprint()` computes a hash of the entire configuration. This is used for: + +- **Incremental compilation**: detect when pipeline configuration has changed +- **Checkpoint validation**: reject stale checkpoints after config changes +- **Content fingerprinting**: stored alongside documents for change detection diff --git a/docs/docs/compiler/incremental.mdx b/docs/docs/compiler/incremental.mdx new file mode 100644 index 0000000..de2cd84 --- /dev/null +++ b/docs/docs/compiler/incremental.mdx @@ -0,0 +1,125 @@ +--- +sidebar_position: 5 +--- + +# Incremental Compilation + +Incremental compilation avoids reprocessing documents that haven't changed. This is analogous to incremental builds in traditional compilers — only the modified files are recompiled. + +## How It Works + +When a document is compiled, the engine stores two fingerprints alongside the persisted document: + +- **Content fingerprint**: SHA-256 hash of the source file bytes +- **Logic fingerprint**: Hash of `PipelineOptions` configuration + +On subsequent compile calls, the engine compares these fingerprints to decide whether to recompile. + +## Action Resolution + +The `resolve_action` function determines what to do with each source: + +```rust +pub enum IndexAction { + /// Content unchanged — skip entirely. + Skip(SkipInfo), + + /// New file or logic changed — full recompilation. + FullIndex { existing_id: Option<String> }, + + /// Content changed but logic unchanged — incremental update. + IncrementalUpdate { old_tree: DocumentTree, existing_id: String }, +} +``` + +The decision flow: + +```text + ┌─────────────────────┐ + │ Has the source file │ + │ changed? │ + └──────┬──────────────┘ + │ + ┌──── No ───┤─── Yes ────┐ + │ │ │ + ┌────────▼─────┐ │ ┌────────▼─────────┐ + │ Logic changed?│ │ │ Content FP match?│ + └────┬────┬────┘ │ └────┬────┬────────┘ + │ │ │ │ │ + No │ Yes │ Yes │ No + │ │ │ │ │ + Skip FullIndex │ Incremental FullIndex + │ Update + │ + (force mode: always FullIndex) +``` + +## Content Fingerprinting + +Content fingerprints are computed at multiple granularities: + +- **Document level**: SHA-256 of the entire source file +- **Node level**: Fingerprint of each tree node's content + +Node-level fingerprints enable fine-grained change detection — the system can identify exactly which sections of a document changed and only reprocess those sections. + +## Reusable Summaries + +When a document is recompiled and only some sections changed, the EnhancePass can reuse summaries from unchanged nodes: + +```rust +use vectorless_compiler::incremental; + +// Find summaries from unchanged nodes in old vs new tree +let reusable = incremental::compute_reusable_summaries(&old_tree, &new_tree); + +// Apply them to the new tree (saves LLM calls) +let count = incremental::apply_reusable_summaries(&mut new_tree, &reusable); +``` + +This dramatically reduces LLM cost for documents that change incrementally (e.g., living documents that are updated in-place). + +## ChangeDetector + +`ChangeDetector` tracks document state across compilations: + +```rust +let mut detector = ChangeDetector::new() + .with_processing_version(2); + +// Record state after compilation +detector.record_with_tree("doc-123", &content, Some(&tree), Some(&path)); + +// Check if recompilation is needed +if detector.needs_reindex_by_hash("doc-123", &new_content) { + // Content changed — recompile +} + +// Detect which nodes changed +let changeset = detector.detect_changes(&old_tree, &new_tree); +``` + +### ChangeSet + +```rust +pub struct ChangeSet { + pub added: Vec<NodeChange>, + pub removed: Vec<NodeChange>, + pub modified: Vec<NodeChange>, + pub restructured: Vec<NodeChange>, +} +``` + +Each `NodeChange` records the node title, change type, and fingerprint. + +## Processing Version + +The `processing_version` field in `PipelineOptions` acts like a compiler version — when it increments, all documents are forced to recompile even if their content hasn't changed. This is used when the compilation algorithm itself changes and existing artifacts are stale. + +## Logic Fingerprint + +`PipelineOptions::logic_fingerprint()` hashes the entire pipeline configuration into a single fingerprint. This is stored with each compiled document and compared on subsequent runs: + +- If the logic fingerprint matches and content fingerprint matches → **Skip** +- If the logic fingerprint changed → **FullIndex** (regardless of content) +- If only content changed → **IncrementalUpdate** diff --git a/docs/docs/compiler/overview.mdx b/docs/docs/compiler/overview.mdx new file mode 100644 index 0000000..32bb80f --- /dev/null +++ b/docs/docs/compiler/overview.mdx @@ -0,0 +1,81 @@ +--- +sidebar_position: 1 +--- + +# Overview + +`vectorless-compiler` is a Rust crate that compiles documents (Markdown, PDF) into agent-friendly intermediate artifacts. It follows the traditional compiler architecture — but instead of compiling source code into machine code, it compiles documents into structured trees, symbol tables, and navigation indexes. + +## Compiler Analogy + +Every concept in a traditional compiler maps directly to what this crate does: + +| Compiler Concept | Vectorless Equivalent | What It Does | +|---|---|---| +| Source code | PDF / Markdown / bytes | Raw input | +| Lexer | Markdown / PDF parser | Breaks document into nodes | +| AST | `DocumentTree` | Hierarchical data structure | +| Semantic analysis | Validate + Enhance (LLM) | Enriches semantic information | +| IR generation | Split + Enrich | Optimizes intermediate representation | +| Code generation | Reasoning / Navigation index | Generates lookup indexes | +| Symbol table | `ReasoningIndex` | name → location mapping | +| Debug info | `NavigationIndex` | Runtime navigation data | +| Object file | `PersistedDocument` | Serialized to disk | +| Incremental compilation | Fingerprint + incremental | Only recompiles changed parts | + +## Architecture + +The pipeline is organized into four phases, each containing one or more passes: + +```text + Frontend 10: Parse → Break document into raw nodes + Frontend 20: Build → Construct tree + apply thinning + Analysis 22: Validate → Tree integrity checks (optional) + Transform 25: Split → Break oversized leaf nodes (optional) + Analysis 30: Enhance → LLM summaries (optional) + Transform 40: Enrich → Metadata + cross-references + Backend 45: Reasoning → Keyword→path symbol table + Backend 47: Concept → Key concept extraction (optional) + Backend 50: Navigation→ Runtime navigation index + Backend 55: Verify → Output validation + Backend 60: Optimize → Final tree optimization +``` + +Each pass is an independent unit that declares its dependencies and access patterns. The orchestrator resolves the dependency graph, groups independent passes for parallel execution, and handles failures with configurable policies. + +## Module Structure + +```text +vectorless-compiler/src/ +├── config.rs PipelineOptions, SourceFormat, ThinningConfig +├── parse/ Document parsers (Markdown, PDF) +├── pipeline/ Executor, orchestrator, context, checkpoint +├── passes/ +│ ├── frontend/ ParsePass, BuildPass +│ ├── analysis/ ValidatePass, EnhancePass +│ ├── transform/ SplitPass, EnrichPass +│ └── backend/ ReasoningPass, ConceptPass, NavigationPass, +│ VerifyPass, OptimizePass +├── summary/ Summary strategies (Full, Selective, Lazy) +└── incremental/ Change detection, action resolution, tree update +``` + +## Quick Example + +```rust +use vectorless_compiler::{PipelineExecutor, PipelineOptions}; +use vectorless_compiler::pipeline::CompilerInput; + +// Create executor with LLM enhancement +let executor = PipelineExecutor::with_llm(llm_client); + +// Compile a document +let input = CompilerInput::file("./report.pdf"); +let options = PipelineOptions::default(); +let result = executor.execute(input, options).await?; + +// Access outputs +let tree = result.tree.expect("tree must exist"); +let reasoning = result.reasoning_index; +let navigation = result.navigation_index; +``` diff --git a/docs/docs/compiler/passes.mdx b/docs/docs/compiler/passes.mdx new file mode 100644 index 0000000..2c24f9d --- /dev/null +++ b/docs/docs/compiler/passes.mdx @@ -0,0 +1,218 @@ +--- +sidebar_position: 3 +--- + +# Compilation Passes + +The pipeline consists of 11 passes organized into four phases. Each pass is a self-contained unit with clear inputs, outputs, and dependencies. + +## Pass Overview + +| Phase | Pass | Priority | Required | Dependencies | +|---|---|---|---|---| +| Frontend | ParsePass | 10 | Yes | — | +| Frontend | BuildPass | 20 | Yes | `parse` | +| Analysis | ValidatePass | 22 | No | `build` | +| Transform | SplitPass | 25 | No | `build` | +| Analysis | EnhancePass | 30 | No | `build` | +| Transform | EnrichPass | 40 | Yes | `build` | +| Backend | ReasoningPass | 45 | No | `enrich` | +| Backend | ConceptPass | 47 | No | `reasoning_index` | +| Backend | NavigationPass | 50 | No | `enrich` | +| Backend | VerifyPass | 55 | Yes | `concept_extraction` | +| Backend | OptimizePass | 60 | No | `enrich`, `navigation_index` | + +## Frontend Phase + +Frontend passes transform raw document bytes into a structured tree — analogous to lexing and parsing in a traditional compiler. + +### ParsePass (Priority 10) + +Parses the source document into a flat list of `RawNode` values. + +- **Input**: `CompilerInput` (file path, content string, or bytes) +- **Output**: `ctx.raw_nodes: Vec<RawNode>`, `ctx.format`, `ctx.page_count` +- **No dependencies** +- Supports Markdown and PDF formats +- PDF parsing can optionally use an LLM client for better structure extraction +- Each `RawNode` contains: title, content, hierarchy level, line range, page number, token count + +```text +Source bytes → ParsePass → [RawNode, RawNode, RawNode, ...] +``` + +### BuildPass (Priority 20) + +Constructs a hierarchical `DocumentTree` from the flat raw nodes. + +- **Input**: `ctx.raw_nodes` +- **Output**: `ctx.tree: DocumentTree` +- **Depends on**: `parse` +- Applies thinning (merges nodes below token threshold into parents) +- Calculates recursive total token counts +- Assigns unique node IDs if `generate_ids` is enabled + +```text +[RawNode, ...] → BuildPass → DocumentTree (arena-based hierarchical structure) +``` + +## Analysis Phase + +Analysis passes validate and enrich the tree's semantic content. + +### ValidatePass (Priority 22) + +Checks tree integrity and reports warnings. + +- **Reads**: `ctx.tree` +- **Checks**: + - Maximum tree depth (20 levels) + - Empty titles on leaf nodes + - Token count consistency (parent ≥ sum of children) + - Content duplication detection +- **Optional** — failures are skipped, not fatal + +### EnhancePass (Priority 30) + +Generates LLM summaries for tree nodes. Only runs when an LLM client is available. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.tree` (summaries added to nodes) +- **Depends on**: `build` +- **Failure policy**: Retry with backoff +- Non-leaf nodes get structured summaries: `OVERVIEW`, `QUESTIONS`, `TAGS` +- Leaf nodes get content summaries +- Short nodes below the shortcut threshold use original content as summary (saves LLM cost) +- Uses memoization cache to avoid regenerating summaries for unchanged content + +## Transform Phase + +Transform passes restructure the tree at the IR level. + +### SplitPass (Priority 25) + +Splits oversized leaf nodes into smaller children. + +- **Reads/writes**: `ctx.tree` +- **Depends on**: `build` +- **Optional** — controlled by `SplitConfig` +- Default max tokens per node: 4000 +- Uses natural split points (headings, paragraphs) +- Pattern-based splitting enabled by default + +### EnrichPass (Priority 40) + +Adds metadata and resolves cross-references. + +- **Reads/writes**: `ctx.tree`, `ctx.description` +- **Depends on**: `build` +- **Required** +- Calculates page ranges (propagated from children up) +- Generates Table of Contents view +- Extracts and resolves in-document references (`"see Section 2.1"` → `NodeId`) +- Generates document description from root summary + +## Backend Phase + +Backend passes generate the final indexes — analogous to code generation in a traditional compiler. + +### ReasoningPass (Priority 45) + +Builds the symbol table: keyword → node path mappings. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.reasoning_index` +- **Depends on**: `enrich` +- **Optional** +- Extracts keywords with weight normalization: title (2.0×), summary (1.5×), content (1.0×) +- Builds section map for fast ToC lookup (depth-1 nodes) +- Creates summary shortcut for overview queries +- Optional LLM synonym expansion + +### ConceptPass (Priority 47) + +Extracts key concepts from topics and summaries. + +- **Reads**: `ctx.tree`, `ctx.reasoning_index` +- **Writes**: `ctx.concepts` +- **Depends on**: `reasoning_index` +- **Optional** +- Uses LLM for structured concept extraction (max 15 concepts) +- Falls back to keyword-based extraction without LLM + +### NavigationPass (Priority 50) + +Builds the runtime navigation index for agent-based traversal. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.navigation_index` +- **Depends on**: `enrich` +- **Optional** +- Creates `NavEntry` for each non-leaf node (overview, hints, tags, leaf count) +- Creates `ChildRoute` entries for children (title, description, leaf count) +- Builds `DocCard` for document-level overview + +### VerifyPass (Priority 55) + +Validates the final output. + +- **Reads**: `ctx.tree` +- **Depends on**: `concept_extraction` +- **Required** +- Checks that tree exists and has nodes +- Verifies document summary is non-empty +- Warns if no concepts were extracted + +### OptimizePass (Priority 60) + +Performs final tree structure optimization. + +- **Reads/writes**: `ctx.tree` +- **Depends on**: `enrich`, `navigation_index` +- **Optional** +- Merges adjacent small leaf nodes that are siblings +- Removes empty intermediate nodes + +## Data Flow + +The following diagram shows how data flows through the passes and which `CompileContext` fields each pass reads and writes: + +```text + ┌────────────┐ + │ CompilerInput │ + └──────┬─────┘ + │ + ┌──────────▼──────────┐ + │ ParsePass │ writes: raw_nodes, format, page_count + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ BuildPass │ reads: raw_nodes → writes: tree + └──────────┬──────────┘ + │ + ┌──────────────┼──────────────┐ + │ │ │ + ┌────────▼──────┐ ┌─────▼──────┐ ┌────▼────────┐ + │ ValidatePass │ │ SplitPass │ │ EnhancePass │ reads: tree → writes: tree + │ (read-only) │ │ │ │ │ + └───────────────┘ └────────────┘ └──────┬──────┘ + │ + ┌──────────▼──────────┐ + │ EnrichPass │ reads: tree → writes: tree, description + └──────────┬──────────┘ + │ + ┌────────────────────┼──────────────────┐ + │ │ │ + ┌─────────▼─────────┐ ┌───────▼──────────┐ ┌───▼────────┐ + │ ReasoningPass │ │ NavigationPass │ │ OptimizePass│ + │ writes: reasoning │ │ writes: navigation│ │ │ + └─────────┬─────────┘ └──────────────────┘ └────────────┘ + │ + ┌─────────▼─────────┐ + │ ConceptPass │ writes: concepts + └─────────┬─────────┘ + │ + ┌─────────▼─────────┐ + │ VerifyPass │ reads: tree (validation only) + └───────────────────┘ +``` diff --git a/docs/docs/compiler/pipeline.mdx b/docs/docs/compiler/pipeline.mdx new file mode 100644 index 0000000..86e0046 --- /dev/null +++ b/docs/docs/compiler/pipeline.mdx @@ -0,0 +1,150 @@ +--- +sidebar_position: 2 +--- + +# Pipeline Infrastructure + +The pipeline is the execution engine that runs passes in the correct order. It consists of three layers: the **trait** (`CompilePass`), the **executor** (`PipelineExecutor`), and the **orchestrator** (`PipelineOrchestrator`). + +## CompilePass Trait + +Every pass implements the `CompilePass` trait: + +```rust +#[async_trait] +pub trait CompilePass: Send + Sync { + /// Unique pass name (used for dependencies and checkpointing). + fn name(&self) -> &str; + + /// Execute the pass, reading from and writing to the shared context. + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult>; + + /// Whether the pipeline can continue if this pass fails. + fn is_optional(&self) -> bool { false } + + /// Names of passes that must complete before this one runs. + fn depends_on(&self) -> Vec<&'static str> { Vec::new() } + + /// How to handle failures: Fail, Skip, or Retry. + fn failure_policy(&self) -> FailurePolicy { ... } + + /// Which context fields this pass reads/writes (for parallel safety). + fn access_pattern(&self) -> AccessPattern { AccessPattern::default() } +} +``` + +### AccessPattern + +Declares which context fields a pass accesses, enabling safe parallel execution: + +```rust +pub struct AccessPattern { + pub reads_tree: bool, + pub writes_tree: bool, + pub writes_reasoning_index: bool, + pub writes_navigation_index: bool, + pub writes_description: bool, + pub writes_concepts: bool, +} +``` + +Within a parallel execution group, at most one pass may write to the tree. All other passes receive cloned contexts with tree snapshots. After all passes complete, outputs are merged back into the main context. + +### FailurePolicy + +| Policy | Behavior | +|---|---| +| `Fail` | Stop the entire pipeline (default for required passes) | +| `Skip` | Log the failure, mark as failed, continue pipeline | +| `Retry(config)` | Retry with exponential backoff up to `max_attempts` | + +Optional passes default to `Skip`. The `Retry` policy accepts configurable delay and max attempts. + +## PipelineExecutor + +The executor is the main entry point. It provides two preset configurations: + +```rust +// Without LLM — skips EnhancePass and ConceptPass +let executor = PipelineExecutor::new(); + +// With LLM — includes summary generation and concept extraction +let executor = PipelineExecutor::with_llm(llm_client); +``` + +Custom pipelines can be built using the orchestrator directly: + +```rust +let orchestrator = PipelineOrchestrator::new() + .stage_with_priority(ParsePass::new(), 10) + .stage_with_priority(BuildPass::new(), 20) + .stage_with_priority(MyCustomPass::new(), 35); + +let executor = PipelineExecutor::from_orchestrator(orchestrator); +``` + +You can also add passes to an existing executor: + +```rust +let executor = PipelineExecutor::with_llm(client) + .add_stage_with_priority(MyPass::new(), 55) + .add_stage_with_deps(MyValidationPass::new(), 56, &["my_pass"]); +``` + +## PipelineOrchestrator + +The orchestrator handles the complex parts of pipeline execution: + +### Dependency Resolution + +Passes declare dependencies by name. The orchestrator performs a topological sort with priority-based ordering (Kahn's algorithm): + +1. Build a dependency graph from `depends_on()` declarations +2. Validate all dependencies refer to existing passes +3. Sort by: dependencies first, then priority (lower = earlier), then registration order +4. Detect circular dependencies and report an error + +### Execution Groups + +Passes at the same dependency level with no inter-dependencies are grouped for parallel execution: + +```text +Group 0 (parallel): [ParsePass] — no deps +Group 1 (parallel): [BuildPass] — depends on "parse" +Group 2 (parallel): [ValidatePass, SplitPass] — both depend on "build" +Group 3: [EnhancePass] — depends on "build" +Group 4: [EnrichPass] — depends on "build" +Group 5 (parallel): [ReasoningPass, NavigationPass] — depend on "enrich" +... +``` + +### Parallel Execution + +When a group has multiple passes: + +1. Identify the tree writer (if any) — it gets the main context +2. All other passes receive cloned contexts with tree snapshots +3. All passes run concurrently via `tokio::join!` +4. Results are merged back by inspecting each pass's `AccessPattern` +5. Additive metrics (LLM calls, tokens) are summed across passes + +## CompileContext + +The shared context passed between passes: + +```text +CompileContext +├── doc_id, name, format, source_path # Document identity +├── input: CompilerInput # Source (File/Content/Bytes) +├── source_hash: String # SHA-256 for checkpoint validation +├── raw_nodes: Vec<RawNode> # ← ParsePass writes +├── tree: Option<DocumentTree> # ← BuildPass writes +├── reasoning_index: Option<ReasoningIndex> # ← ReasoningPass writes +├── navigation_index: Option<NavigationIndex> # ← NavigationPass writes +├── concepts: Vec<Concept> # ← ConceptPass writes +├── description: Option<String> # ← EnrichPass writes +├── summary_cache: SummaryCache # Summary memoization +├── metrics: IndexMetrics # Performance tracking +├── stage_results: HashMap<String, PassResult> # Per-pass results +└── options: PipelineOptions # Configuration +``` diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 1c2d1db..72f3e5e 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -11,6 +11,18 @@ const sidebars: SidebarsConfig = { ], }, 'architecture', + { + type: 'category', + label: 'Vectorless Compiler', + items: [ + 'compiler/overview', + 'compiler/pipeline', + 'compiler/passes', + 'compiler/configuration', + 'compiler/incremental', + 'compiler/checkpoint', + ], + }, { type: 'category', label: 'RFC', diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/route.rs b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs new file mode 100644 index 0000000..ff622bf --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs @@ -0,0 +1,208 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Route Pass — builds the query routing table from question hints and topic tags. +//! +//! This pass runs after NavigationPass. It reads the tree's question_hints and +//! routing_keywords fields (populated by EnhancePass) and generates a +//! [`QueryRoutingTable`] that lets Agents skip root-level exploration. + +use std::collections::HashMap; +use std::time::Instant; +use tracing::{debug, info, warn}; + +use vectorless_document::{DocumentTree, NodeId, QueryRoutingTable, RouteTarget, ConceptRoute}; +use vectorless_error::Result; + +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; +use crate::pipeline::CompileContext; + +/// Route Pass — builds the query routing table. +/// +/// For each non-leaf node with question_hints, creates RouteTargets from its +/// children. For each unique topic tag, creates a ConceptRoute entry. +pub struct RoutePass; + +impl RoutePass { + /// Create a new route pass. + pub fn new() -> Self { + Self + } + + /// Build route targets from a node's children. + fn build_child_routes( + tree: &DocumentTree, + parent_id: NodeId, + ) -> Vec<RouteTarget> { + let children: Vec<_> = tree.children_iter(parent_id).collect(); + let mut targets = Vec::with_capacity(children.len()); + + for child_id in children { + let node = match tree.get(child_id) { + Some(n) => n, + None => continue, + }; + + // Relevance based on question hints count and content richness + let hint_count = node.question_hints.len(); + let has_content = !node.content.is_empty(); + let relevance = if hint_count > 0 && has_content { + 0.7 + (hint_count as f64).min(3.0) * 0.1 + } else if has_content { + 0.5 + } else { + 0.3 + }; + + let reason = if hint_count > 0 { + format!("Can answer: {}", node.question_hints.first().unwrap_or(&String::new())) + } else { + format!("Section: {}", node.title) + }; + + targets.push(RouteTarget { + node_id: child_id, + relevance, + reason, + }); + } + + // Sort by relevance descending + targets.sort_by(|a, b| b.relevance.partial_cmp(&a.relevance).unwrap_or(std::cmp::Ordering::Equal)); + targets + } + + /// Build concept routes from all topic tags in the tree. + fn build_concept_routes(tree: &DocumentTree) -> Vec<ConceptRoute> { + let all_nodes = tree.traverse(); + let mut concept_map: HashMap<String, Vec<RouteTarget>> = HashMap::new(); + + for node_id in &all_nodes { + let node = match tree.get(*node_id) { + Some(n) => n, + None => continue, + }; + + for tag in &node.routing_keywords { + let relevance = if tree.is_leaf(*node_id) { 0.9 } else { 0.7 }; + concept_map + .entry(tag.to_lowercase()) + .or_default() + .push(RouteTarget { + node_id: *node_id, + relevance, + reason: format!("Tagged with: {}", tag), + }); + } + } + + // Convert to ConceptRoute, sort targets by relevance + let mut routes: Vec<ConceptRoute> = concept_map + .into_iter() + .map(|(concept, mut targets)| { + targets.sort_by(|a, b| b.relevance.partial_cmp(&a.relevance).unwrap_or(std::cmp::Ordering::Equal)); + targets.truncate(10); // limit per concept + ConceptRoute { concept, targets } + }) + .collect(); + + routes.sort_by(|a, b| b.targets.len().cmp(&a.targets.len())); + routes.truncate(50); // limit total concepts + routes + } +} + +impl Default for RoutePass { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl CompilePass for RoutePass { + fn name(&self) -> &'static str { + "route" + } + + fn depends_on(&self) -> Vec<&'static str> { + vec!["navigation_index"] + } + + fn is_optional(&self) -> bool { + true + } + + fn access_pattern(&self) -> AccessPattern { + AccessPattern { + reads_tree: true, + writes_query_routes: true, + ..Default::default() + } + } + + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { + let start = Instant::now(); + + let tree = match ctx.tree.as_ref() { + Some(t) => t, + None => { + warn!("[route] No tree, cannot build routing table"); + return Ok(PassResult::failure("route", "Tree not built")); + } + }; + + let all_nodes = tree.traverse(); + info!("[route] Building routing table for {} nodes", all_nodes.len()); + + let mut table = QueryRoutingTable::new(); + + // Phase 1: Build intent routes from nodes with question hints + let mut intent_count = 0; + for &node_id in &all_nodes { + let node = match tree.get(node_id) { + Some(n) => n, + None => continue, + }; + + if node.question_hints.is_empty() { + continue; + } + + let targets = Self::build_child_routes(tree, node_id); + if !targets.is_empty() { + table.add_intent_route(node_id, targets); + intent_count += 1; + } + } + + // Phase 2: Build concept routes from topic tags + let concept_routes = Self::build_concept_routes(tree); + let concept_count = concept_routes.len(); + for route in concept_routes { + table.add_concept_route(route); + } + + let duration = start.elapsed().as_millis() as u64; + + info!( + "[route] Complete: {} intent routes, {} concept routes in {}ms", + intent_count, concept_count, duration, + ); + + ctx.query_routes = Some(table); + + let mut result = PassResult::success("route"); + result.duration_ms = duration; + result.metadata.insert( + "intent_routes".to_string(), + serde_json::json!(intent_count), + ); + result.metadata.insert( + "concept_routes".to_string(), + serde_json::json!(concept_count), + ); + + Ok(result) + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/mod.rs b/vectorless-core/vectorless-compiler/src/passes/mod.rs index 33a79a5..5adf7b3 100644 --- a/vectorless-core/vectorless-compiler/src/passes/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/mod.rs @@ -40,6 +40,14 @@ pub struct AccessPattern { pub writes_description: bool, /// Whether this pass writes to `concepts`. pub writes_concepts: bool, + /// Whether this pass writes to `query_routes`. + pub writes_query_routes: bool, + /// Whether this pass writes to `chain_index`. + pub writes_chain_index: bool, + /// Whether this pass writes to `content_overlap`. + pub writes_content_overlap: bool, + /// Whether this pass writes to `evidence_scores`. + pub writes_evidence_scores: bool, } /// Compiler pass trait. diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index 8e3b0dc..b21a288 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -7,7 +7,10 @@ use std::collections::HashMap; use std::path::PathBuf; use crate::parse::{DocumentFormat, RawNode}; -use vectorless_document::{Concept, DocumentTree, NavigationIndex, NodeId, ReasoningIndex}; +use vectorless_document::{ + Concept, DocumentTree, NavigationIndex, NodeId, ReasoningIndex, + QueryRoutingTable, ChainIndex, ContentOverlapMap, EvidenceScoreMap, +}; use vectorless_llm::LlmClient; use super::super::{PipelineOptions, SummaryStrategy}; @@ -258,6 +261,20 @@ pub struct CompileContext { /// Key concepts extracted from the document (built by ConceptExtractionStage). pub concepts: Vec<Concept>, + // ── Agent acceleration data (built by backend passes) ── + + /// Pre-computed query routing table (built by RoutePass). + pub query_routes: Option<QueryRoutingTable>, + + /// Reasoning chain index (built by ChainPass). + pub chain_index: Option<ChainIndex>, + + /// Content overlap map (built by OverlapPass). + pub content_overlap: Option<ContentOverlapMap>, + + /// Per-node evidence quality scores (built by ScorePass). + pub evidence_scores: Option<EvidenceScoreMap>, + /// Existing tree from previous indexing (for incremental updates). /// When set, the enhance and reasoning stages can reuse data from unchanged nodes. pub existing_tree: Option<DocumentTree>, @@ -297,6 +314,10 @@ impl CompileContext { reasoning_index: None, navigation_index: None, concepts: Vec::new(), + query_routes: None, + chain_index: None, + content_overlap: None, + evidence_scores: None, existing_tree: None, stage_results: HashMap::new(), metrics: IndexMetrics::default(), @@ -396,6 +417,10 @@ impl CompileContext { reasoning_index: self.reasoning_index, navigation_index: self.navigation_index, concepts: self.concepts, + query_routes: self.query_routes, + chain_index: self.chain_index, + content_overlap: self.content_overlap, + evidence_scores: self.evidence_scores, } } } @@ -441,6 +466,18 @@ pub struct CompileResult { /// Key concepts extracted from the document. pub concepts: Vec<Concept>, + + /// Pre-computed query routing table for Agent acceleration. + pub query_routes: Option<QueryRoutingTable>, + + /// Reasoning chain index for cross-section navigation. + pub chain_index: Option<ChainIndex>, + + /// Content overlap map to prevent duplicate visits. + pub content_overlap: Option<ContentOverlapMap>, + + /// Per-node evidence quality scores. + pub evidence_scores: Option<EvidenceScoreMap>, } impl CompileResult { diff --git a/vectorless-core/vectorless-document/src/chain.rs b/vectorless-core/vectorless-document/src/chain.rs new file mode 100644 index 0000000..0533fe1 --- /dev/null +++ b/vectorless-core/vectorless-document/src/chain.rs @@ -0,0 +1,127 @@ +// Copyright (c) vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Reasoning chains — logical connections between document sections. +//! +//! Represents premise→conclusion relationships extracted from in-document +//! references and content structure. The Agent can follow chains to collect +//! supporting evidence across non-adjacent sections. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::node::NodeId; + +/// Type of logical connection between sections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ChainType { + /// A causes or leads to B. + Causal, + /// A provides evidence or support for B. + Supporting, + /// A contradicts or refutes B. + Contradicting, + /// B is a detailed expansion of A. + Elaboration, + /// A is a prerequisite step before B. + Sequence, +} + +impl ChainType { + /// Convert to a static string label. + pub fn as_str(&self) -> &'static str { + match self { + Self::Causal => "causal", + Self::Supporting => "supporting", + Self::Contradicting => "contradicting", + Self::Elaboration => "elaboration", + Self::Sequence => "sequence", + } + } +} + +/// A single reasoning chain connecting document sections. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReasoningChain { + /// Nodes that establish the premise. + pub premises: Vec<NodeId>, + /// Nodes that draw conclusions. + pub conclusions: Vec<NodeId>, + /// Type of logical connection. + pub chain_type: ChainType, + /// Human-readable summary of this chain. + pub summary: String, +} + +/// Index of reasoning chains with bidirectional node lookup. +/// +/// Allows the Agent to find all chains involving a specific node, +/// enabling "follow the reasoning" navigation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChainIndex { + /// All reasoning chains. + pub chains: Vec<ReasoningChain>, + /// Node → indices of chains that involve this node. + #[serde(with = "super::serde_helpers")] + node_chains: HashMap<NodeId, Vec<usize>>, +} + +impl ChainIndex { + /// Create a new empty chain index. + pub fn new() -> Self { + Self { + chains: Vec::new(), + node_chains: HashMap::new(), + } + } + + /// Add a chain and update the node index. + pub fn add_chain(&mut self, chain: ReasoningChain) { + let idx = self.chains.len(); + for node_id in chain.premises.iter().chain(chain.conclusions.iter()) { + self.node_chains.entry(*node_id).or_default().push(idx); + } + self.chains.push(chain); + } + + /// Get all chains involving a specific node. + pub fn chains_for(&self, node_id: NodeId) -> Vec<&ReasoningChain> { + match self.node_chains.get(&node_id) { + Some(indices) => indices.iter().filter_map(|&i| self.chains.get(i)).collect(), + None => Vec::new(), + } + } + + /// Get chains where the given node is a premise. + pub fn premises_from(&self, node_id: NodeId) -> Vec<&ReasoningChain> { + self.chains_for(node_id) + .into_iter() + .filter(|c| c.premises.contains(&node_id)) + .collect() + } + + /// Get chains where the given node is a conclusion. + pub fn conclusions_from(&self, node_id: NodeId) -> Vec<&ReasoningChain> { + self.chains_for(node_id) + .into_iter() + .filter(|c| c.conclusions.contains(&node_id)) + .collect() + } + + /// Total number of chains. + pub fn chain_count(&self) -> usize { + self.chains.len() + } + + /// Total number of nodes involved in chains. + pub fn node_count(&self) -> usize { + self.node_chains.len() + } +} + +impl Default for ChainIndex { + fn default() -> Self { + Self::new() + } +} diff --git a/vectorless-core/vectorless-document/src/evidence.rs b/vectorless-core/vectorless-document/src/evidence.rs new file mode 100644 index 0000000..ca7051f --- /dev/null +++ b/vectorless-core/vectorless-document/src/evidence.rs @@ -0,0 +1,107 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Evidence score map — per-node quality metrics computed at compile time. +//! +//! Helps the Agent prioritize high-value nodes and skip low-information content. +//! All metrics are computed from content analysis, no LLM calls required. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::node::NodeId; + +/// Per-node evidence quality scores. +/// +/// Each metric ranges from 0.0 to 1.0. Higher is better. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvidenceScore { + /// Information density: ratio of unique meaningful tokens to total tokens. + /// High density = content is packed with facts rather than filler. + pub density: f64, + /// Data richness: presence of numbers, tables, code blocks, lists. + /// High richness = content contains structured data. + pub data_richness: f64, + /// Specificity: how narrowly focused the content is on a specific topic. + /// High specificity = content is targeted, not generic prose. + pub specificity: f64, +} + +impl EvidenceScore { + /// Composite score (weighted average of all metrics). + pub fn composite(&self) -> f64 { + self.density * 0.4 + self.data_richness * 0.3 + self.specificity * 0.3 + } +} + +/// Map of evidence scores for all leaf nodes. +/// +/// Built by the `ScorePass` compiler pass. The Agent can use these scores +/// to decide which nodes to visit first when multiple candidates exist. +/// +/// # Construction +/// +/// Pure compute — statistical analysis of node content. No LLM calls. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvidenceScoreMap { + /// Scores for each scored node (typically leaf nodes). + #[serde(with = "super::serde_helpers")] + scores: HashMap<NodeId, EvidenceScore>, +} + +impl EvidenceScoreMap { + /// Create a new empty score map. + pub fn new() -> Self { + Self { + scores: HashMap::new(), + } + } + + /// Add a score for a node. + pub fn insert(&mut self, node_id: NodeId, score: EvidenceScore) { + self.scores.insert(node_id, score); + } + + /// Get the score for a specific node. + pub fn get(&self, node_id: NodeId) -> Option<&EvidenceScore> { + self.scores.get(&node_id) + } + + /// Get the composite score for a node, defaulting to 0.0. + pub fn composite_for(&self, node_id: NodeId) -> f64 { + self.scores.get(&node_id).map(|s| s.composite()).unwrap_or(0.0) + } + + /// Get nodes sorted by composite score (highest first). + pub fn ranked_nodes(&self) -> Vec<(NodeId, f64)> { + let mut nodes: Vec<_> = self + .scores + .iter() + .map(|(id, s)| (*id, s.composite())) + .collect(); + nodes.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + nodes + } + + /// Number of scored nodes. + pub fn len(&self) -> usize { + self.scores.len() + } + + /// Whether the map is empty. + pub fn is_empty(&self) -> bool { + self.scores.is_empty() + } + + /// Access the underlying scores map. + pub fn scores(&self) -> &HashMap<NodeId, EvidenceScore> { + &self.scores + } +} + +impl Default for EvidenceScoreMap { + fn default() -> Self { + Self::new() + } +} diff --git a/vectorless-core/vectorless-document/src/lib.rs b/vectorless-core/vectorless-document/src/lib.rs index 956dfde..39c8d16 100644 --- a/vectorless-core/vectorless-document/src/lib.rs +++ b/vectorless-core/vectorless-document/src/lib.rs @@ -27,6 +27,12 @@ mod toc; mod tree; pub mod understanding; +// New: Agent acceleration types +mod chain; +mod evidence; +mod overlap; +mod query_route; + pub use format::DocumentFormat; pub use navigation::{ChildRoute, DocCard, NavEntry, NavigationIndex, SectionCard}; pub use node::{NodeId, TreeNode}; @@ -39,3 +45,9 @@ pub use structure::{DocumentStructure, StructureNode}; pub use toc::{TocConfig, TocEntry, TocNode, TocView}; pub use tree::{DocumentTree, RetrievalIndex}; pub use understanding::{Concept, Document, DocumentInfo, IngestInput}; + +// Re-export agent acceleration types +pub use chain::{ChainIndex, ChainType, ReasoningChain}; +pub use evidence::{EvidenceScore, EvidenceScoreMap}; +pub use overlap::{ContentOverlapMap, OverlapEntry, OverlapType}; +pub use query_route::{ConceptRoute, QueryRoutingTable, RouteTarget}; diff --git a/vectorless-core/vectorless-document/src/overlap.rs b/vectorless-core/vectorless-document/src/overlap.rs new file mode 100644 index 0000000..9f47eb2 --- /dev/null +++ b/vectorless-core/vectorless-document/src/overlap.rs @@ -0,0 +1,97 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Content overlap map — marks overlapping regions to prevent duplicate visits. +//! +//! Built at compile time by comparing leaf node content pairwise with Jaccard +//! similarity. The Agent can skip nodes marked as overlapping, saving +//! navigation rounds. + +use serde::{Deserialize, Serialize}; + +use super::node::NodeId; + +/// Type of content overlap between two nodes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum OverlapType { + /// Content is nearly identical (Jaccard ≥ 0.9). + Duplicate, + /// One node's content is a subset of another's. + Subset, + /// One node is a summary of another. + Summary, +} + +/// A single overlap entry between two leaf nodes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OverlapEntry { + /// First node. + pub node_a: NodeId, + /// Second node. + pub node_b: NodeId, + /// Jaccard similarity score (0.0–1.0). + pub similarity: f64, + /// Type of overlap detected. + pub overlap_type: OverlapType, +} + +/// Map of content overlaps across leaf nodes. +/// +/// Built by the `OverlapPass` compiler pass. The Agent checks this map +/// when deciding whether to visit a node — if it's marked as overlapping +/// with an already-visited node, it can skip it. +/// +/// # Construction +/// +/// Pure compute — pairwise Jaccard on leaf node content. No LLM calls. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContentOverlapMap { + /// All detected overlaps. + pub overlaps: Vec<OverlapEntry>, +} + +impl ContentOverlapMap { + /// Create a new empty overlap map. + pub fn new() -> Self { + Self { overlaps: Vec::new() } + } + + /// Add an overlap entry. + pub fn add(&mut self, entry: OverlapEntry) { + self.overlaps.push(entry); + } + + /// Check if a node has any overlaps with other nodes. + pub fn has_overlap(&self, node_id: NodeId) -> bool { + self.overlaps + .iter() + .any(|o| o.node_a == node_id || o.node_b == node_id) + } + + /// Get all nodes that overlap with the given node. + pub fn overlapping_nodes(&self, node_id: NodeId) -> Vec<(NodeId, f64, OverlapType)> { + self.overlaps + .iter() + .filter_map(|o| { + if o.node_a == node_id { + Some((o.node_b, o.similarity, o.overlap_type)) + } else if o.node_b == node_id { + Some((o.node_a, o.similarity, o.overlap_type)) + } else { + None + } + }) + .collect() + } + + /// Total number of overlap pairs. + pub fn overlap_count(&self) -> usize { + self.overlaps.len() + } +} + +impl Default for ContentOverlapMap { + fn default() -> Self { + Self::new() + } +} diff --git a/vectorless-core/vectorless-document/src/query_route.rs b/vectorless-core/vectorless-document/src/query_route.rs new file mode 100644 index 0000000..c95eec9 --- /dev/null +++ b/vectorless-core/vectorless-document/src/query_route.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Pre-computed query routing table for Agent acceleration. +//! +//! Built at compile time from question hints and topic tags, the routing table +//! maps query intents and concepts to optimal entry nodes in the document tree. +//! The Agent can skip root-level exploration and navigate directly to the most +//! relevant subtree. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::node::NodeId; + +/// A scored target node for routing. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RouteTarget { + /// Target node to navigate to. + pub node_id: NodeId, + /// Relevance score (0.0–1.0). + pub relevance: f64, + /// Human-readable reason for this route (e.g., "Contains Q3 revenue data"). + pub reason: String, +} + +/// A concept-to-nodes mapping for semantic routing. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConceptRoute { + /// Concept name (e.g., "revenue", "authentication"). + pub concept: String, + /// Scored target nodes for this concept. + pub targets: Vec<RouteTarget>, +} + +/// Pre-computed routing table mapping intents and concepts to entry nodes. +/// +/// The Agent receives a query analysis (intent + concepts) and looks up +/// pre-computed routes to find the best navigation starting point, +/// bypassing the typical root → ls → explore cycle. +/// +/// # Construction +/// +/// Built by the `RoutePass` compiler pass. No LLM calls — uses existing +/// `question_hints` and `routing_keywords` from the enhance stage. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryRoutingTable { + /// Routes derived from question hints: node → scored alternatives. + /// Key nodes are those with `question_hints` populated by the enhance stage. + #[serde(with = "super::serde_helpers")] + intent_routes: HashMap<NodeId, Vec<RouteTarget>>, + + /// Routes derived from topic tags: concept → scored entry nodes. + concept_routes: Vec<ConceptRoute>, +} + +impl QueryRoutingTable { + /// Create a new empty routing table. + pub fn new() -> Self { + Self { + intent_routes: HashMap::new(), + concept_routes: Vec::new(), + } + } + + /// Add an intent-based route. + pub fn add_intent_route(&mut self, entry_node: NodeId, targets: Vec<RouteTarget>) { + self.intent_routes.insert(entry_node, targets); + } + + /// Add a concept-based route. + pub fn add_concept_route(&mut self, route: ConceptRoute) { + self.concept_routes.push(route); + } + + /// Get intent-based routes for a specific entry node. + pub fn intent_routes_for(&self, node_id: NodeId) -> Option<&[RouteTarget]> { + self.intent_routes.get(&node_id).map(Vec::as_slice) + } + + /// Get all intent route entries. + pub fn intent_routes(&self) -> &HashMap<NodeId, Vec<RouteTarget>> { + &self.intent_routes + } + + /// Get all concept routes. + pub fn concept_routes(&self) -> &[ConceptRoute] { + &self.concept_routes + } + + /// Look up concept routes matching a keyword. + pub fn routes_for_concept(&self, keyword: &str) -> Vec<&RouteTarget> { + let kw = keyword.to_lowercase(); + self.concept_routes + .iter() + .filter(|cr| cr.concept.to_lowercase().contains(&kw)) + .flat_map(|cr| cr.targets.iter()) + .collect() + } + + /// Total number of intent routes. + pub fn intent_route_count(&self) -> usize { + self.intent_routes.len() + } + + /// Total number of concept routes. + pub fn concept_route_count(&self) -> usize { + self.concept_routes.len() + } +} + +impl Default for QueryRoutingTable { + fn default() -> Self { + Self::new() + } +} From 8339fc045db7d830f7193fea8d889439369bafbb Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 21:38:29 +0800 Subject: [PATCH 21/30] docs(compiler): add documentation for custom passes, parsers, and standalone usage - Add documentation page for writing custom passes with implementation examples - Document the parsers module and RawNode structure for document parsing - Create standalone usage guide for vectorless-compiler crate - Update sidebar configuration to include new documentation pages feat(compiler): implement backend analysis passes for chain, overlap, and scoring - Add ChainPass to build reasoning chain index from document references - Implement OverlapPass to detect content overlap between leaf nodes using Jaccard similarity - Create ScorePass to compute evidence quality scores based on density, data richness, and specificity - Register new passes in the pipeline executor with appropriate priorities - Update module exports to make new passes available --- docs/docs/compiler/custom-pass.mdx | 53 +++++ docs/docs/compiler/parsers.mdx | 38 ++++ docs/docs/compiler/standalone-usage.mdx | 42 ++++ docs/sidebars.ts | 3 + .../src/passes/backend/chain.rs | 161 ++++++++++++++ .../src/passes/backend/mod.rs | 8 + .../src/passes/backend/overlap.rs | 189 +++++++++++++++++ .../src/passes/backend/score.rs | 199 ++++++++++++++++++ .../vectorless-compiler/src/passes/mod.rs | 2 +- .../src/pipeline/executor.rs | 10 +- 10 files changed, 701 insertions(+), 4 deletions(-) create mode 100644 docs/docs/compiler/custom-pass.mdx create mode 100644 docs/docs/compiler/parsers.mdx create mode 100644 docs/docs/compiler/standalone-usage.mdx create mode 100644 vectorless-core/vectorless-compiler/src/passes/backend/chain.rs create mode 100644 vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs create mode 100644 vectorless-core/vectorless-compiler/src/passes/backend/score.rs diff --git a/docs/docs/compiler/custom-pass.mdx b/docs/docs/compiler/custom-pass.mdx new file mode 100644 index 0000000..0be5138 --- /dev/null +++ b/docs/docs/compiler/custom-pass.mdx @@ -0,0 +1,53 @@ +--- +sidebar_position: 7 +--- + +# Writing a Custom Pass + +*🚧 This page is a work in progress. Content will be added soon.* + +## Overview + +The `CompilePass` trait is designed for extensibility. You can create custom passes and inject them into the pipeline using `PipelineExecutor::add_stage()` or by building a custom `PipelineOrchestrator`. + +## Topics to Cover + +- Implementing the `CompilePass` trait +- Declaring dependencies with `depends_on()` +- Setting the correct `AccessPattern` for parallel safety +- Choosing a failure policy +- Registering custom passes with the executor +- Reading from and writing to `CompileContext` +- Testing custom passes in isolation + +## Quick Start + +```rust +use vectorless_compiler::passes::{CompilePass, AccessPattern}; +use vectorless_compiler::pipeline::{CompileContext, PassResult}; + +struct MyCustomPass; + +#[async_trait::async_trait] +impl CompilePass for MyCustomPass { + fn name(&self) -> &str { "my_custom_pass" } + + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { + // Your logic here + Ok(PassResult::success("my_custom_pass")) + } + + fn depends_on(&self) -> Vec<&'static str> { + vec!["build"] + } + + fn access_pattern(&self) -> AccessPattern { + AccessPattern { + reads_tree: true, + ..Default::default() + } + } +} +``` + +{/* TODO: Full tutorial content */} diff --git a/docs/docs/compiler/parsers.mdx b/docs/docs/compiler/parsers.mdx new file mode 100644 index 0000000..d3a766b --- /dev/null +++ b/docs/docs/compiler/parsers.mdx @@ -0,0 +1,38 @@ +--- +sidebar_position: 8 +--- + +# Document Parsers + +*🚧 This page is a work in progress. Content will be added soon.* + +## Overview + +The `parse` module handles format-specific document parsing. It converts raw source bytes into a flat list of `RawNode` values that the BuildPass then assembles into a hierarchical tree. + +## Topics to Cover + +- `RawNode` structure and fields +- `DocumentMeta` metadata +- `DocumentFormat` enum and format detection +- Markdown parser: heading hierarchy, code blocks, tables +- PDF parser: page extraction, heading detection, LLM-assisted structure +- Extending with new formats (DOCX, HTML, etc.) + +## RawNode + +```rust +pub struct RawNode { + pub title: String, + pub content: String, + pub level: usize, // Hierarchy level (0 = root) + pub line_start: usize, + pub line_end: usize, + pub page: Option<usize>, // PDF only + pub token_count: Option<usize>, +} +``` + +{/* TODO: Markdown parser details */} +{/* TODO: PDF parser details */} +{/* TODO: Adding new format support */} diff --git a/docs/docs/compiler/standalone-usage.mdx b/docs/docs/compiler/standalone-usage.mdx new file mode 100644 index 0000000..0a5cd9f --- /dev/null +++ b/docs/docs/compiler/standalone-usage.mdx @@ -0,0 +1,42 @@ +--- +sidebar_position: 9 +--- + +# Standalone Crate Usage + +*🚧 This page is a work in progress. Content will be added soon.* + +## Overview + +`vectorless-compiler` is designed as a reusable crate that can be published to [crates.io](https://crates.io) and used independently of the full Vectorless engine. This page covers using it as a standalone document compiler. + +## Topics to Cover + +- Adding `vectorless-compiler` to `Cargo.toml` +- Running the pipeline without the Engine facade +- Using `PipelineExecutor::new()` (no LLM) vs `with_llm()` +- Accessing compilation outputs: tree, indexes, concepts +- Integrating with custom storage backends +- Integration testing patterns + +## Basic Usage + +```toml +# Cargo.toml +[dependencies] +vectorless-compiler = "0.1" +``` + +```rust +use vectorless_compiler::{PipelineExecutor, PipelineOptions}; +use vectorless_compiler::pipeline::CompilerInput; + +let mut executor = PipelineExecutor::new(); +let input = CompilerInput::file("./document.md"); +let result = executor.execute(input, PipelineOptions::default()).await?; + +println!("Tree nodes: {}", result.tree.unwrap().node_count()); +``` + +{/* TODO: Full standalone usage guide */} +{/* TODO: crates.io publishing details */} diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 72f3e5e..8226e8d 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -21,6 +21,9 @@ const sidebars: SidebarsConfig = { 'compiler/configuration', 'compiler/incremental', 'compiler/checkpoint', + 'compiler/custom-pass', + 'compiler/parsers', + 'compiler/standalone-usage', ], }, { diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs new file mode 100644 index 0000000..3137a73 --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs @@ -0,0 +1,161 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Chain Pass — builds reasoning chain index from in-document references. +//! +//! Analyzes TreeNode.references to find premise→conclusion relationships +//! between sections. No LLM calls — uses reference types and tree structure. + +use std::time::Instant; +use tracing::{debug, info, warn}; + +use vectorless_document::{ChainIndex, ChainType, ReasoningChain, NodeId}; +use vectorless_error::Result; + +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; +use crate::pipeline::CompileContext; + +/// Chain Pass — builds reasoning chain index. +/// +/// Uses the references field on TreeNode (populated by EnrichPass) to +/// identify logical connections between sections. +pub struct ChainPass; + +impl ChainPass { + /// Create a new chain pass. + pub fn new() -> Self { + Self + } + + /// Determine chain type from the reference structure. + fn classify_chain( + ref_type: &str, + source_depth: usize, + target_depth: usize, + ) -> ChainType { + match ref_type { + "Section" | "section" => { + if target_depth > source_depth { + ChainType::Elaboration + } else { + ChainType::Supporting + } + } + "Appendix" | "appendix" => ChainType::Supporting, + "Table" | "Figure" | "Equation" | "table" | "figure" | "equation" => { + ChainType::Supporting + } + "Footnote" | "footnote" => ChainType::Elaboration, + _ => ChainType::Supporting, + } + } +} + +impl Default for ChainPass { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl CompilePass for ChainPass { + fn name(&self) -> &'static str { + "chain" + } + + fn depends_on(&self) -> Vec<&'static str> { + vec!["enrich"] + } + + fn is_optional(&self) -> bool { + true + } + + fn access_pattern(&self) -> AccessPattern { + AccessPattern { + reads_tree: true, + writes_chain_index: true, + ..Default::default() + } + } + + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { + let start = Instant::now(); + + let tree = match ctx.tree.as_ref() { + Some(t) => t, + None => { + warn!("[chain] No tree, cannot build chain index"); + return Ok(PassResult::failure("chain", "Tree not built")); + } + }; + + let all_nodes = tree.traverse(); + let mut index = ChainIndex::new(); + + for &node_id in &all_nodes { + let node = match tree.get(node_id) { + Some(n) => n, + None => continue, + }; + + if node.references.is_empty() { + continue; + } + + for reference in &node.references { + // Only process references with resolved targets + let target_id = match reference.target_node { + Some(id) => id, + None => continue, + }; + + let chain_type = Self::classify_chain( + &reference.ref_text, + node.depth, + tree.get(target_id).map(|n| n.depth).unwrap_or(0), + ); + + let target_title = tree + .get(target_id) + .map(|n| n.title.as_str()) + .unwrap_or("unknown"); + + index.add_chain(ReasoningChain { + premises: vec![node_id], + conclusions: vec![target_id], + chain_type, + summary: format!( + "{} references {} ({})", + node.title, target_title, reference.ref_text + ), + }); + } + } + + let chain_count = index.chain_count(); + let node_count = index.node_count(); + let duration = start.elapsed().as_millis() as u64; + + info!( + "[chain] Complete: {} chains involving {} nodes in {}ms", + chain_count, node_count, duration, + ); + + ctx.chain_index = Some(index); + + let mut result = PassResult::success("chain"); + result.duration_ms = duration; + result.metadata.insert( + "chains".to_string(), + serde_json::json!(chain_count), + ); + result.metadata.insert( + "nodes".to_string(), + serde_json::json!(node_count), + ); + + Ok(result) + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs index e170957..1591936 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs @@ -8,9 +8,17 @@ mod concept; mod navigation; mod verify; mod optimize; +mod route; +mod chain; +mod overlap; +mod score; pub use reasoning::ReasoningPass; pub use concept::ConceptPass; pub use navigation::NavigationPass; pub use verify::VerifyPass; pub use optimize::OptimizePass; +pub use route::RoutePass; +pub use chain::ChainPass; +pub use overlap::OverlapPass; +pub use score::ScorePass; diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs new file mode 100644 index 0000000..961907a --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs @@ -0,0 +1,189 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Overlap Pass — detects content overlap between leaf nodes. +//! +//! Computes pairwise Jaccard similarity on leaf node content to identify +//! duplicate or near-duplicate sections. The Agent can skip overlapping nodes. + +use std::collections::HashSet; +use std::time::Instant; +use tracing::{debug, info, warn}; + +use vectorless_document::{ContentOverlapMap, OverlapEntry, OverlapType, NodeId}; +use vectorless_error::Result; + +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; +use crate::pipeline::CompileContext; + +/// Jaccard similarity threshold for overlap detection. +const SIMILARITY_THRESHOLD: f64 = 0.8; + +/// Overlap Pass — builds content overlap map. +pub struct OverlapPass; + +impl OverlapPass { + /// Create a new overlap pass. + pub fn new() -> Self { + Self + } + + /// Compute Jaccard similarity between two strings (word-level). + fn jaccard(a: &str, b: &str) -> f64 { + let words_a: HashSet<&str> = a.to_lowercase().split_whitespace().collect(); + let words_b: HashSet<&str> = b.to_lowercase().split_whitespace().collect(); + + if words_a.is_empty() && words_b.is_empty() { + return 1.0; + } + if words_a.is_empty() || words_b.is_empty() { + return 0.0; + } + + let intersection = words_a.intersection(&words_b).count() as f64; + let union = words_a.union(&words_b).count() as f64; + intersection / union + } + + /// Classify overlap type based on similarity and content length ratio. + fn classify_overlap(similarity: f64, len_a: usize, len_b: usize) -> OverlapType { + if similarity >= 0.9 { + OverlapType::Duplicate + } else { + let ratio = if len_a > 0 && len_b > 0 { + (len_a.min(len_b) as f64) / (len_a.max(len_b) as f64) + } else { + 0.0 + }; + if ratio < 0.5 { + OverlapType::Summary + } else { + OverlapType::Subset + } + } + } +} + +impl Default for OverlapPass { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl CompilePass for OverlapPass { + fn name(&self) -> &'static str { + "overlap" + } + + fn depends_on(&self) -> Vec<&'static str> { + vec!["build"] + } + + fn is_optional(&self) -> bool { + true + } + + fn access_pattern(&self) -> AccessPattern { + AccessPattern { + reads_tree: true, + writes_content_overlap: true, + ..Default::default() + } + } + + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { + let start = Instant::now(); + + let tree = match ctx.tree.as_ref() { + Some(t) => t, + None => { + warn!("[overlap] No tree, cannot compute overlaps"); + return Ok(PassResult::failure("overlap", "Tree not built")); + } + }; + + let leaves: Vec<NodeId> = tree.leaves(); + let leaf_count = leaves.len(); + + info!("[overlap] Computing overlaps for {} leaf nodes", leaf_count); + + // Skip if too few leaves (no pairs to compare) + if leaf_count < 2 { + debug!("[overlap] Fewer than 2 leaves, no overlaps possible"); + ctx.content_overlap = Some(ContentOverlapMap::new()); + let mut result = PassResult::success("overlap"); + result.duration_ms = start.elapsed().as_millis() as u64; + return Ok(result); + } + + let mut overlap_map = ContentOverlapMap::new(); + let mut comparisons = 0usize; + + // Pairwise Jaccard on leaf content + for i in 0..leaf_count { + let node_a = leaves[i]; + let content_a = match tree.get(node_a) { + Some(n) => &n.content, + None => continue, + }; + + // Skip very short content (< 50 chars) + if content_a.len() < 50 { + continue; + } + + for j in (i + 1)..leaf_count { + let node_b = leaves[j]; + let content_b = match tree.get(node_b) { + Some(n) => &n.content, + None => continue, + }; + + if content_b.len() < 50 { + continue; + } + + comparisons += 1; + let similarity = Self::jaccard(content_a, content_b); + + if similarity >= SIMILARITY_THRESHOLD { + overlap_map.add(OverlapEntry { + node_a, + node_b, + similarity, + overlap_type: Self::classify_overlap( + similarity, + content_a.len(), + content_b.len(), + ), + }); + } + } + } + + let overlap_count = overlap_map.overlap_count(); + let duration = start.elapsed().as_millis() as u64; + + info!( + "[overlap] Complete: {} overlaps from {} comparisons in {}ms", + overlap_count, comparisons, duration, + ); + + ctx.content_overlap = Some(overlap_map); + + let mut result = PassResult::success("overlap"); + result.duration_ms = duration; + result.metadata.insert( + "overlaps".to_string(), + serde_json::json!(overlap_count), + ); + result.metadata.insert( + "comparisons".to_string(), + serde_json::json!(comparisons), + ); + + Ok(result) + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/score.rs b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs new file mode 100644 index 0000000..2cedd53 --- /dev/null +++ b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs @@ -0,0 +1,199 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Score Pass — computes per-node evidence quality scores. +//! +//! Analyzes leaf node content for information density, data richness, +//! and topic specificity. All metrics are pure compute, no LLM calls. + +use std::collections::HashSet; +use std::time::Instant; +use tracing::{debug, info, warn}; + +use vectorless_document::{EvidenceScore, EvidenceScoreMap, NodeId}; +use vectorless_error::Result; + +use crate::passes::async_trait; +use crate::passes::{AccessPattern, CompilePass, PassResult}; +use crate::pipeline::CompileContext; + +/// Score Pass — builds evidence quality score map. +pub struct ScorePass; + +impl ScorePass { + /// Create a new score pass. + pub fn new() -> Self { + Self + } + + /// Compute information density: unique meaningful tokens / total tokens. + fn compute_density(content: &str) -> f64 { + let words: Vec<&str> = content.split_whitespace().collect(); + if words.is_empty() { + return 0.0; + } + let unique: HashSet<&str> = words.iter().copied().collect(); + unique.len() as f64 / words.len() as f64 + } + + /// Compute data richness: presence of numbers, tables, code, lists. + fn compute_data_richness(content: &str) -> f64 { + let mut score = 0.0f64; + let len = content.len() as f64; + + if len == 0.0 { + return 0.0; + } + + // Numbers (digits, percentages, currencies) + let digit_count = content.chars().filter(|c| c.is_ascii_digit()).count() as f64; + if digit_count > 0.0 { + score += 0.3 * (digit_count / len).min(0.1) / 0.1; + } + + // Table markers (|, tabs, CSV patterns) + let pipe_count = content.matches('|').count() as f64; + let tab_count = content.matches('\t').count() as f64; + if pipe_count > 3.0 || tab_count > 3.0 { + score += 0.3; + } + + // Code blocks (``` or indented lines) + if content.contains("```") || content.contains(" ") { + score += 0.2; + } + + // Lists (-, *, numbered) + let list_markers = content.lines().filter(|l| { + let trimmed = l.trim(); + trimmed.starts_with("- ") + || trimmed.starts_with("* ") + || trimmed.starts_with("+ ") + || (trimmed.len() > 2 && trimmed.as_bytes().first().map(|b| b.is_ascii_digit()).unwrap_or(false) && trimmed.contains('.')) + }).count(); + if list_markers > 0 { + score += 0.2 * (list_markers as f64).min(5.0) / 5.0; + } + + score.min(1.0) + } + + /// Compute specificity: how focused the content is (vs generic filler). + fn compute_specificity(content: &str) -> f64 { + let words: Vec<&str> = content.split_whitespace().collect(); + if words.is_empty() { + return 0.0; + } + + // Generic filler words that indicate low specificity + let filler = [ + "the", "is", "a", "an", "and", "or", "but", "in", "on", "at", + "to", "for", "of", "with", "this", "that", "it", "from", "by", + "was", "were", "be", "have", "has", "had", "are", "will", "would", + ]; + let filler_set: HashSet<&str> = filler.iter().copied().collect(); + + let filler_count = words.iter().filter(|w| filler_set.contains(w.to_lowercase().as_str())).count() as f64; + let total = words.len() as f64; + + // Higher ratio of non-filler = higher specificity + 1.0 - (filler_count / total).min(0.8) + } +} + +impl Default for ScorePass { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl CompilePass for ScorePass { + fn name(&self) -> &'static str { + "score" + } + + fn depends_on(&self) -> Vec<&'static str> { + vec!["enrich"] + } + + fn is_optional(&self) -> bool { + true + } + + fn access_pattern(&self) -> AccessPattern { + AccessPattern { + reads_tree: true, + writes_evidence_scores: true, + ..Default::default() + } + } + + async fn execute(&mut self, ctx: &mut CompileContext) -> Result<PassResult> { + let start = Instant::now(); + + let tree = match ctx.tree.as_ref() { + Some(t) => t, + None => { + warn!("[score] No tree, cannot compute scores"); + return Ok(PassResult::failure("score", "Tree not built")); + } + }; + + let leaves = tree.leaves(); + let mut score_map = EvidenceScoreMap::new(); + + info!("[score] Computing evidence scores for {} leaf nodes", leaves.len()); + + for &node_id in &leaves { + let node = match tree.get(node_id) { + Some(n) => n, + None => continue, + }; + + let content = &node.content; + if content.is_empty() { + continue; + } + + let density = Self::compute_density(content); + let data_richness = Self::compute_data_richness(content); + let specificity = Self::compute_specificity(content); + + score_map.insert(node_id, EvidenceScore { + density, + data_richness, + specificity, + }); + } + + let scored_count = score_map.len(); + let avg_density = if scored_count > 0 { + score_map.scores().values().map(|s| s.density).sum::<f64>() / scored_count as f64 + } else { + 0.0 + }; + + let duration = start.elapsed().as_millis() as u64; + + info!( + "[score] Complete: {} nodes scored (avg density: {:.2}) in {}ms", + scored_count, avg_density, duration, + ); + + ctx.evidence_scores = Some(score_map); + + let mut result = PassResult::success("score"); + result.duration_ms = duration; + result.metadata.insert( + "scored_nodes".to_string(), + serde_json::json!(scored_count), + ); + result.metadata.insert( + "avg_density".to_string(), + serde_json::json!(format!("{:.3}", avg_density)), + ); + + Ok(result) + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/mod.rs b/vectorless-core/vectorless-compiler/src/passes/mod.rs index 5adf7b3..3f1b833 100644 --- a/vectorless-core/vectorless-compiler/src/passes/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/mod.rs @@ -18,7 +18,7 @@ pub mod backend; pub use frontend::{ParsePass, BuildPass}; pub use analysis::{ValidatePass, EnhancePass}; pub use transform::{SplitPass, EnrichPass}; -pub use backend::{ReasoningPass, ConceptPass, NavigationPass, VerifyPass, OptimizePass}; +pub use backend::{ReasoningPass, ConceptPass, NavigationPass, VerifyPass, OptimizePass, RoutePass, ChainPass, OverlapPass, ScorePass}; use super::pipeline::{FailurePolicy, CompileContext, PassResult}; pub use async_trait::async_trait; diff --git a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs index ba6ffd2..64586be 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -13,9 +13,9 @@ use vectorless_llm::LlmClient; use super::super::PipelineOptions; use super::super::passes::{ - BuildPass, ConceptPass, EnhancePass, EnrichPass, CompilePass, - NavigationPass, OptimizePass, ParsePass, ReasoningPass, SplitPass, - ValidatePass, VerifyPass, + BuildPass, ChainPass, ConceptPass, EnhancePass, EnrichPass, CompilePass, + NavigationPass, OptimizePass, OverlapPass, ParsePass, ReasoningPass, + RoutePass, ScorePass, SplitPass, ValidatePass, VerifyPass, }; use super::context::{CompilerInput, CompileResult}; use super::orchestrator::PipelineOrchestrator; @@ -70,6 +70,10 @@ impl PipelineExecutor { .stage_with_priority(ReasoningPass::new(), 45) .stage_with_priority(ConceptPass::new(), 47) .stage_with_priority(NavigationPass::new(), 50) + .stage_with_priority(RoutePass::new(), 52) + .stage_with_priority(ChainPass::new(), 54) + .stage_with_priority(OverlapPass::new(), 56) + .stage_with_priority(ScorePass::new(), 58) .stage_with_priority(VerifyPass, 55) .stage_with_priority(OptimizePass::new(), 60); From 666d51a6efb6773d558796dc49caefb4e169d08e Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 21:53:44 +0800 Subject: [PATCH 22/30] feat(compiler): add new backend passes for query routing, reasoning chains, overlap detection and scoring - Add RoutePass for building query routing tables with intent and concept routes - Add ChainPass for creating reasoning chain indexes from document references - Add OverlapPass for detecting content overlap with Jaccard similarity algorithm - Add ScorePass for evidence quality scoring using density, data richness and specificity - Update pipeline executor to include new backend stages at priorities 52-58 - Add comprehensive unit tests for each new pass covering edge cases and end-to-end scenarios - Update documentation diagrams to show new backend components (Route, Chain, Overlap, Score) - Add metrics recording for each new pass including timing and count statistics - Update validation pass to track new output flags - Export NodeReference type for external usage --- .../vectorless-compiler/src/lib.rs | 12 ++ .../src/passes/analysis/validate.rs | 4 + .../src/passes/backend/chain.rs | 139 +++++++++++++- .../src/passes/backend/concept.rs | 2 - .../src/passes/backend/overlap.rs | 160 ++++++++++++++++- .../src/passes/backend/route.rs | 151 +++++++++++++++- .../src/passes/backend/score.rs | 170 +++++++++++++++++- .../src/passes/transform/split.rs | 4 + .../src/pipeline/context.rs | 4 + .../src/pipeline/executor.rs | 4 + .../vectorless-document/src/lib.rs | 2 +- .../vectorless-metrics/src/index.rs | 67 +++++++ 12 files changed, 709 insertions(+), 10 deletions(-) diff --git a/vectorless-core/vectorless-compiler/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs index ba8e980..162d16d 100644 --- a/vectorless-core/vectorless-compiler/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -34,6 +34,18 @@ //! Backend 50: ┌────▼──────────┐ //! │ Navigation Idx│ Debug info for runtime navigation //! └────┬──────────┘ +//! Backend 52: ┌────▼──────────┐ +//! │ Route │ Query routing table +//! └────┬──────────┘ +//! Backend 54: ┌────▼──────────┐ +//! │ Chain │ Reasoning chain index +//! └────┬──────────┘ +//! Backend 56: ┌────▼──────────┐ +//! │ Overlap │ Content overlap detection +//! └────┬──────────┘ +//! Backend 58: ┌────▼──────────┐ +//! │ Score │ Evidence quality scoring +//! └────┬──────────┘ //! Backend 55: ┌────▼──────┐ //! │ Verify │ Output validation //! └────┬──────┘ diff --git a/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs index 1dc284e..88f9e04 100644 --- a/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/validate.rs @@ -243,6 +243,10 @@ impl CompilePass for ValidatePass { writes_navigation_index: false, writes_description: false, writes_concepts: false, + writes_query_routes: false, + writes_chain_index: false, + writes_content_overlap: false, + writes_evidence_scores: false, } } diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs index 3137a73..e95db72 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs @@ -7,9 +7,9 @@ //! between sections. No LLM calls — uses reference types and tree structure. use std::time::Instant; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; -use vectorless_document::{ChainIndex, ChainType, ReasoningChain, NodeId}; +use vectorless_document::{ChainIndex, ChainType, ReasoningChain}; use vectorless_error::Result; use crate::passes::async_trait; @@ -143,6 +143,8 @@ impl CompilePass for ChainPass { chain_count, node_count, duration, ); + ctx.metrics.record_chain(duration, chain_count); + ctx.chain_index = Some(index); let mut result = PassResult::success("chain"); @@ -159,3 +161,136 @@ impl CompilePass for ChainPass { Ok(result) } } + +#[cfg(test)] +mod tests { + use super::*; + use vectorless_document::NodeReference; + + fn build_test_tree_with_refs() -> vectorless_document::DocumentTree { + let mut tree = vectorless_document::DocumentTree::new("Root", "root content"); + let root = tree.root(); + + let sec1 = tree.add_child(root, "Introduction", "See Section 2 for details"); + let sec2 = tree.add_child(root, "Methods", "As shown in Table 1"); + let appendix = tree.add_child(root, "Appendix A", "Supporting data"); + + // Add references: sec1 → sec2 (Section ref), sec2 → appendix (Appendix ref) + if let Some(n) = tree.get_mut(sec1) { + n.references = vec![NodeReference { + ref_text: "Section".to_string(), + target_node: Some(sec2), + }]; + } + if let Some(n) = tree.get_mut(sec2) { + n.references = vec![NodeReference { + ref_text: "Appendix".to_string(), + target_node: Some(appendix), + }]; + } + + tree + } + + #[test] + fn test_stage_config() { + let pass = ChainPass::new(); + assert_eq!(pass.name(), "chain"); + assert!(pass.is_optional()); + assert_eq!(pass.depends_on(), vec!["enrich"]); + + let ap = pass.access_pattern(); + assert!(ap.reads_tree); + assert!(ap.writes_chain_index); + assert!(!ap.writes_tree); + } + + #[test] + fn test_classify_chain_section_elaboration() { + // Section ref to deeper node = Elaboration + assert_eq!(ChainPass::classify_chain("Section", 0, 1), ChainType::Elaboration); + } + + #[test] + fn test_classify_chain_section_supporting() { + // Section ref to same or shallower depth = Supporting + assert_eq!(ChainPass::classify_chain("Section", 1, 0), ChainType::Supporting); + } + + #[test] + fn test_classify_chain_appendix() { + assert_eq!(ChainPass::classify_chain("Appendix", 0, 1), ChainType::Supporting); + } + + #[test] + fn test_classify_chain_table_figure() { + assert_eq!(ChainPass::classify_chain("Table", 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain("Figure", 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain("Equation", 0, 1), ChainType::Supporting); + } + + #[test] + fn test_classify_chain_footnote() { + assert_eq!(ChainPass::classify_chain("Footnote", 0, 2), ChainType::Elaboration); + } + + #[test] + fn test_classify_chain_unknown() { + assert_eq!(ChainPass::classify_chain("custom", 0, 0), ChainType::Supporting); + } + + #[tokio::test] + async fn test_execute_end_to_end() { + let tree = build_test_tree_with_refs(); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = ChainPass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + let pass_result = result.unwrap(); + assert!(pass_result.success); + + let index = ctx.chain_index.unwrap(); + assert_eq!(index.chain_count(), 2); // sec1→sec2, sec2→appendix + assert!(index.node_count() >= 2); + } + + #[tokio::test] + async fn test_execute_no_tree() { + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = None; + + let mut pass = ChainPass::new(); + let result = pass.execute(&mut ctx).await.unwrap(); + assert!(!result.success); + assert!(ctx.chain_index.is_none()); + } + + #[tokio::test] + async fn test_execute_no_references() { + let tree = vectorless_document::DocumentTree::new("Root", "no references"); + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = ChainPass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + + let index = ctx.chain_index.unwrap(); + assert_eq!(index.chain_count(), 0); + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs b/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs index b138bdb..f5a1a34 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/concept.rs @@ -3,8 +3,6 @@ //! Concept extraction stage — extracts key concepts from topics and summaries. -use std::collections::HashMap; - use serde::Deserialize; use tracing::{info, warn}; diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs index 961907a..2110f39 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs @@ -31,8 +31,10 @@ impl OverlapPass { /// Compute Jaccard similarity between two strings (word-level). fn jaccard(a: &str, b: &str) -> f64 { - let words_a: HashSet<&str> = a.to_lowercase().split_whitespace().collect(); - let words_b: HashSet<&str> = b.to_lowercase().split_whitespace().collect(); + let a_lower = a.to_lowercase(); + let b_lower = b.to_lowercase(); + let words_a: HashSet<&str> = a_lower.split_whitespace().collect(); + let words_b: HashSet<&str> = b_lower.split_whitespace().collect(); if words_a.is_empty() && words_b.is_empty() { return 1.0; @@ -171,6 +173,8 @@ impl CompilePass for OverlapPass { overlap_count, comparisons, duration, ); + ctx.metrics.record_overlap(duration, overlap_count); + ctx.content_overlap = Some(overlap_map); let mut result = PassResult::success("overlap"); @@ -187,3 +191,155 @@ impl CompilePass for OverlapPass { Ok(result) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_jaccard_identical() { + let sim = OverlapPass::jaccard("hello world foo bar", "hello world foo bar"); + assert!((sim - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_jaccard_no_overlap() { + let sim = OverlapPass::jaccard("alpha beta gamma", "delta epsilon zeta"); + assert!((sim - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_jaccard_partial() { + let sim = OverlapPass::jaccard("alpha beta gamma", "beta gamma delta"); + // intersection: beta, gamma (2), union: alpha, beta, gamma, delta (4) = 0.5 + assert!((sim - 0.5).abs() < 1e-10); + } + + #[test] + fn test_jaccard_empty() { + assert!((OverlapPass::jaccard("", "") - 1.0).abs() < f64::EPSILON); + assert!((OverlapPass::jaccard("content", "") - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_jaccard_case_insensitive() { + let sim = OverlapPass::jaccard("Hello World", "hello world"); + assert!((sim - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_classify_overlap_duplicate() { + assert_eq!(OverlapPass::classify_overlap(0.95, 100, 100), OverlapType::Duplicate); + } + + #[test] + fn test_classify_overlap_subset() { + // 0.85 similarity with similar lengths → Subset + assert_eq!(OverlapPass::classify_overlap(0.85, 100, 90), OverlapType::Subset); + } + + #[test] + fn test_classify_overlap_summary() { + // 0.85 similarity with very different lengths → Summary + assert_eq!(OverlapPass::classify_overlap(0.85, 100, 30), OverlapType::Summary); + } + + #[test] + fn test_stage_config() { + let pass = OverlapPass::new(); + assert_eq!(pass.name(), "overlap"); + assert!(pass.is_optional()); + assert_eq!(pass.depends_on(), vec!["build"]); + + let ap = pass.access_pattern(); + assert!(ap.reads_tree); + assert!(ap.writes_content_overlap); + assert!(!ap.writes_tree); + } + + #[tokio::test] + async fn test_execute_no_tree() { + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = None; + + let mut pass = OverlapPass::new(); + let result = pass.execute(&mut ctx).await.unwrap(); + assert!(!result.success); + assert!(ctx.content_overlap.is_none()); + } + + #[tokio::test] + async fn test_execute_single_leaf() { + let tree = vectorless_document::DocumentTree::new("Root", "single leaf content"); + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = OverlapPass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + let map = ctx.content_overlap.unwrap(); + assert_eq!(map.overlap_count(), 0); + } + + #[tokio::test] + async fn test_execute_with_duplicates() { + let mut tree = vectorless_document::DocumentTree::new("Root", ""); + let root = tree.root(); + + // Two leaf nodes with identical content (long enough to pass the 50-char threshold) + let long_content = "This is a sufficiently long piece of content that should pass the minimum length threshold for overlap detection in the system.".to_string(); + tree.add_child(root, "Section A", &long_content); + tree.add_child(root, "Section B", &long_content); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = OverlapPass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + + let map = ctx.content_overlap.unwrap(); + assert_eq!(map.overlap_count(), 1); + assert_eq!(map.overlaps[0].overlap_type, OverlapType::Duplicate); + } + + #[tokio::test] + async fn test_execute_no_overlap() { + let mut tree = vectorless_document::DocumentTree::new("Root", ""); + let root = tree.root(); + + // Two leaf nodes with completely different content + tree.add_child(root, "Section A", + "Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega"); + tree.add_child(root, "Section B", + "Apple banana cherry date elderberry fig grape honeydew kiwi lemon mango nectarine orange papaya quince raspberry strawberry tangerine"); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = OverlapPass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + + let map = ctx.content_overlap.unwrap(); + assert_eq!(map.overlap_count(), 0); + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/route.rs b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs index ff622bf..9abb3db 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/route.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::time::Instant; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; use vectorless_document::{DocumentTree, NodeId, QueryRoutingTable, RouteTarget, ConceptRoute}; use vectorless_error::Result; @@ -190,6 +190,8 @@ impl CompilePass for RoutePass { intent_count, concept_count, duration, ); + ctx.metrics.record_route(duration, intent_count, concept_count); + ctx.query_routes = Some(table); let mut result = PassResult::success("route"); @@ -206,3 +208,150 @@ impl CompilePass for RoutePass { Ok(result) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn build_test_tree_with_hints() -> DocumentTree { + let mut tree = DocumentTree::new("Root", "root content"); + let root = tree.root(); + + let sec1 = tree.add_child(root, "Revenue Q3", "Q3 revenue was $4.2B"); + let sec2 = tree.add_child(root, "Revenue Q4", "Q4 revenue was $5.1B"); + + // Add question hints + if let Some(n) = tree.get_mut(root) { + n.question_hints = vec!["What was the revenue?".to_string()]; + n.routing_keywords = vec!["revenue".to_string(), "finance".to_string()]; + } + if let Some(n) = tree.get_mut(sec1) { + n.routing_keywords = vec!["revenue".to_string(), "Q3".to_string()]; + } + if let Some(n) = tree.get_mut(sec2) { + n.question_hints = vec!["What was Q4 revenue?".to_string()]; + n.routing_keywords = vec!["revenue".to_string(), "Q4".to_string()]; + } + + tree + } + + #[test] + fn test_stage_config() { + let pass = RoutePass::new(); + assert_eq!(pass.name(), "route"); + assert!(pass.is_optional()); + assert_eq!(pass.depends_on(), vec!["navigation_index"]); + + let ap = pass.access_pattern(); + assert!(ap.reads_tree); + assert!(ap.writes_query_routes); + assert!(!ap.writes_tree); + } + + #[test] + fn test_build_child_routes_basic() { + let tree = build_test_tree_with_hints(); + let root = tree.root(); + let targets = RoutePass::build_child_routes(&tree, root); + + assert_eq!(targets.len(), 2); + // Should be sorted by relevance descending + assert!(targets[0].relevance >= targets[1].relevance); + } + + #[test] + fn test_build_child_routes_with_hints() { + let tree = build_test_tree_with_hints(); + let root = tree.root(); + + // Root has question hints, so build routes from its children + let targets = RoutePass::build_child_routes(&tree, root); + assert!(!targets.is_empty()); + + // At least one child should have content-based reason + let has_section_reason = targets.iter().any(|t| t.reason.starts_with("Section:")); + assert!(has_section_reason); + } + + #[test] + fn test_build_concept_routes() { + let tree = build_test_tree_with_hints(); + let routes = RoutePass::build_concept_routes(&tree); + + assert!(!routes.is_empty()); + + // "revenue" appears on all 3 nodes + let revenue_route = routes.iter().find(|r| r.concept == "revenue"); + assert!(revenue_route.is_some()); + assert!(revenue_route.unwrap().targets.len() >= 2); + } + + #[test] + fn test_build_concept_routes_empty() { + let tree = DocumentTree::new("Root", "no keywords"); + let routes = RoutePass::build_concept_routes(&tree); + assert!(routes.is_empty()); + } + + #[tokio::test] + async fn test_execute_end_to_end() { + let tree = build_test_tree_with_hints(); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = RoutePass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + let pass_result = result.unwrap(); + assert!(pass_result.success); + + // Verify routing table + let table = ctx.query_routes.unwrap(); + assert!(table.intent_route_count() > 0); + assert!(table.concept_route_count() > 0); + + // Verify metrics recorded + assert!(ctx.metrics.route_time_ms > 0 || pass_result.duration_ms >= 0); + } + + #[tokio::test] + async fn test_execute_no_tree() { + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = None; + + let mut pass = RoutePass::new(); + let result = pass.execute(&mut ctx).await.unwrap(); + assert!(!result.success); + assert!(ctx.query_routes.is_none()); + } + + #[tokio::test] + async fn test_execute_no_hints_no_keywords() { + let tree = DocumentTree::new("Root", "plain content"); + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = RoutePass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + let pass_result = result.unwrap(); + assert!(pass_result.success); + + let table = ctx.query_routes.unwrap(); + assert_eq!(table.intent_route_count(), 0); + assert_eq!(table.concept_route_count(), 0); + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/score.rs b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs index 2cedd53..b93a521 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/score.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs @@ -8,9 +8,9 @@ use std::collections::HashSet; use std::time::Instant; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; -use vectorless_document::{EvidenceScore, EvidenceScoreMap, NodeId}; +use vectorless_document::{EvidenceScore, EvidenceScoreMap}; use vectorless_error::Result; use crate::passes::async_trait; @@ -181,6 +181,8 @@ impl CompilePass for ScorePass { scored_count, avg_density, duration, ); + ctx.metrics.record_score(duration, scored_count); + ctx.evidence_scores = Some(score_map); let mut result = PassResult::success("score"); @@ -197,3 +199,167 @@ impl CompilePass for ScorePass { Ok(result) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compute_density_unique() { + // All unique words → density = 1.0 + assert!((ScorePass::compute_density("alpha beta gamma delta") - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_compute_density_repeated() { + // "word" appears 3 times out of 3 → density = 1/3 + let d = ScorePass::compute_density("word word word"); + assert!((d - (1.0 / 3.0)).abs() < 1e-10); + } + + #[test] + fn test_compute_density_empty() { + assert!((ScorePass::compute_density("") - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_compute_data_richness_numbers() { + let score = ScorePass::compute_data_richness("Revenue was $4.2B in Q3 2024, up 12.5%"); + assert!(score > 0.0, "Should detect numbers"); + } + + #[test] + fn test_compute_data_richness_table() { + let score = ScorePass::compute_data_richness("| A | B | C |\n|---|---|---|\n| 1 | 2 | 3 |"); + assert!(score > 0.0, "Should detect table markers"); + } + + #[test] + fn test_compute_data_richness_code() { + let score = ScorePass::compute_data_richness("```rust\nfn main() {}\n```"); + assert!(score > 0.0, "Should detect code blocks"); + } + + #[test] + fn test_compute_data_richness_list() { + let score = ScorePass::compute_data_richness("- item one\n- item two\n- item three"); + assert!(score > 0.0, "Should detect lists"); + } + + #[test] + fn test_compute_data_richness_plain() { + let score = ScorePass::compute_data_richness("just some plain text without any structured data"); + assert!((score - 0.0).abs() < f64::EPSILON, "Plain text should have low richness"); + } + + #[test] + fn test_compute_data_richness_empty() { + assert!((ScorePass::compute_data_richness("") - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_compute_specificity_high() { + // Lots of technical terms, few filler words + let s = ScorePass::compute_specificity("HashMap NodeId DocumentTree CompileContext PipelineExecutor"); + assert!(s > 0.8, "Technical content should have high specificity"); + } + + #[test] + fn test_compute_specificity_low() { + // All filler words + let s = ScorePass::compute_specificity("the is a an and or but in on at to for of with this that"); + assert!(s < 0.3, "Filler content should have low specificity"); + } + + #[test] + fn test_compute_specificity_empty() { + assert!((ScorePass::compute_specificity("") - 0.0).abs() < f64::EPSILON); + } + + #[test] + fn test_evidence_score_composite() { + let score = EvidenceScore { + density: 1.0, + data_richness: 1.0, + specificity: 1.0, + }; + assert!((score.composite() - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_stage_config() { + let pass = ScorePass::new(); + assert_eq!(pass.name(), "score"); + assert!(pass.is_optional()); + assert_eq!(pass.depends_on(), vec!["enrich"]); + + let ap = pass.access_pattern(); + assert!(ap.reads_tree); + assert!(ap.writes_evidence_scores); + assert!(!ap.writes_tree); + } + + #[tokio::test] + async fn test_execute_end_to_end() { + let mut tree = vectorless_document::DocumentTree::new("Root", ""); + let root = tree.root(); + tree.add_child(root, "Section 1", "Revenue was $4.2B in Q3 2024"); + tree.add_child(root, "Section 2", "HashMap implementation details"); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = ScorePass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + + let scores = ctx.evidence_scores.unwrap(); + assert_eq!(scores.len(), 2); + + // All scored nodes should have positive composite + for (_, composite) in scores.ranked_nodes() { + assert!(composite > 0.0); + } + } + + #[tokio::test] + async fn test_execute_no_tree() { + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = None; + + let mut pass = ScorePass::new(); + let result = pass.execute(&mut ctx).await.unwrap(); + assert!(!result.success); + assert!(ctx.evidence_scores.is_none()); + } + + #[tokio::test] + async fn test_execute_empty_content() { + let mut tree = vectorless_document::DocumentTree::new("Root", ""); + let root = tree.root(); + tree.add_child(root, "Empty Section", ""); + + let mut ctx = CompileContext::new( + crate::pipeline::CompilerInput::content("test"), + crate::config::PipelineOptions::default(), + ); + ctx.tree = Some(tree); + + let mut pass = ScorePass::new(); + let result = pass.execute(&mut ctx).await; + + assert!(result.is_ok()); + assert!(result.unwrap().success); + + let scores = ctx.evidence_scores.unwrap(); + assert_eq!(scores.len(), 0); // Empty content nodes are skipped + } +} diff --git a/vectorless-core/vectorless-compiler/src/passes/transform/split.rs b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs index ff30fb3..ce013e9 100644 --- a/vectorless-core/vectorless-compiler/src/passes/transform/split.rs +++ b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs @@ -229,6 +229,10 @@ impl CompilePass for SplitPass { writes_navigation_index: false, writes_description: false, writes_concepts: false, + writes_query_routes: false, + writes_chain_index: false, + writes_content_overlap: false, + writes_evidence_scores: false, } } diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index b21a288..8b8b188 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -501,6 +501,10 @@ impl CompileResult { + self.metrics.enrich_time_ms + self.metrics.reasoning_index_time_ms + self.metrics.navigation_index_time_ms + + self.metrics.route_time_ms + + self.metrics.chain_time_ms + + self.metrics.overlap_time_ms + + self.metrics.score_time_ms + self.metrics.optimize_time_ms } } diff --git a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs index 64586be..53884b9 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -109,6 +109,10 @@ impl PipelineExecutor { .stage_with_priority(ReasoningPass::new(), 45) .stage_with_priority(ConceptPass::with_llm_client(client), 47) .stage_with_priority(NavigationPass::new(), 50) + .stage_with_priority(RoutePass::new(), 52) + .stage_with_priority(ChainPass::new(), 54) + .stage_with_priority(OverlapPass::new(), 56) + .stage_with_priority(ScorePass::new(), 58) .stage_with_priority(VerifyPass, 55) .stage_with_priority(OptimizePass::new(), 60); diff --git a/vectorless-core/vectorless-document/src/lib.rs b/vectorless-core/vectorless-document/src/lib.rs index 39c8d16..eee8f79 100644 --- a/vectorless-core/vectorless-document/src/lib.rs +++ b/vectorless-core/vectorless-document/src/lib.rs @@ -40,7 +40,7 @@ pub use reasoning::{ ReasoningIndex, ReasoningIndexBuilder, ReasoningIndexConfig, SectionSummary, SummaryShortcut, TopicEntry, }; -pub use reference::ReferenceExtractor; +pub use reference::{NodeReference, ReferenceExtractor}; pub use structure::{DocumentStructure, StructureNode}; pub use toc::{TocConfig, TocEntry, TocNode, TocView}; pub use tree::{DocumentTree, RetrievalIndex}; diff --git a/vectorless-core/vectorless-metrics/src/index.rs b/vectorless-core/vectorless-metrics/src/index.rs index 3d1e556..af5d9d5 100644 --- a/vectorless-core/vectorless-metrics/src/index.rs +++ b/vectorless-core/vectorless-metrics/src/index.rs @@ -87,6 +87,44 @@ pub struct IndexMetrics { /// Number of nodes merged. #[serde(default)] pub nodes_merged: usize, + + // ── Agent acceleration pass metrics ── + + /// Route pass duration (ms). + #[serde(default)] + pub route_time_ms: u64, + + /// Number of intent routes built. + #[serde(default)] + pub intent_routes: usize, + + /// Number of concept routes built. + #[serde(default)] + pub concept_routes: usize, + + /// Chain pass duration (ms). + #[serde(default)] + pub chain_time_ms: u64, + + /// Number of reasoning chains indexed. + #[serde(default)] + pub chains_indexed: usize, + + /// Overlap pass duration (ms). + #[serde(default)] + pub overlap_time_ms: u64, + + /// Number of content overlap pairs detected. + #[serde(default)] + pub overlaps_detected: usize, + + /// Score pass duration (ms). + #[serde(default)] + pub score_time_ms: u64, + + /// Number of nodes scored for evidence quality. + #[serde(default)] + pub nodes_scored: usize, } impl IndexMetrics { @@ -184,6 +222,31 @@ impl IndexMetrics { self.nodes_merged += 1; } + /// Record route pass time. + pub fn record_route(&mut self, duration_ms: u64, intent_routes: usize, concept_routes: usize) { + self.route_time_ms = duration_ms; + self.intent_routes = intent_routes; + self.concept_routes = concept_routes; + } + + /// Record chain pass time. + pub fn record_chain(&mut self, duration_ms: u64, chains: usize) { + self.chain_time_ms = duration_ms; + self.chains_indexed = chains; + } + + /// Record overlap pass time. + pub fn record_overlap(&mut self, duration_ms: u64, overlaps: usize) { + self.overlap_time_ms = duration_ms; + self.overlaps_detected = overlaps; + } + + /// Record score pass time. + pub fn record_score(&mut self, duration_ms: u64, scored_nodes: usize) { + self.score_time_ms = duration_ms; + self.nodes_scored = scored_nodes; + } + /// Get total time. pub fn total_time_ms(&self) -> u64 { self.parse_time_ms @@ -194,6 +257,10 @@ impl IndexMetrics { + self.enrich_time_ms + self.reasoning_index_time_ms + self.navigation_index_time_ms + + self.route_time_ms + + self.chain_time_ms + + self.overlap_time_ms + + self.score_time_ms + self.optimize_time_ms } } From 1c824f455e11d0b24c69879298e91ee01ee2546c Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:05:08 +0800 Subject: [PATCH 23/30] feat(compiler): add new compilation passes for enhanced functionality - Add RoutePass for pre-computed query routing table to accelerate agent-based queries - Add ChainPass for building reasoning chain index from in-document cross-references - Add OverlapPass for detecting content overlap between leaf nodes using Jaccard similarity - Add ScorePass for computing per-node evidence quality scores based on density, richness, and specificity metrics Update documentation to reflect 15 passes instead of 1 in the pipeline, including detailed descriptions of new passes, their dependencies, and data flow diagrams. Modify ChainPass implementation to use proper RefType enum instead of string matching for reference classification. --- docs/docs/compiler/overview.mdx | 12 +++ docs/docs/compiler/passes.mdx | 88 +++++++++++++++---- .../src/passes/backend/chain.rs | 57 ++++++------ .../vectorless-document/src/lib.rs | 2 +- 4 files changed, 117 insertions(+), 42 deletions(-) diff --git a/docs/docs/compiler/overview.mdx b/docs/docs/compiler/overview.mdx index 32bb80f..daab1fe 100644 --- a/docs/docs/compiler/overview.mdx +++ b/docs/docs/compiler/overview.mdx @@ -20,6 +20,9 @@ Every concept in a traditional compiler maps directly to what this crate does: | Code generation | Reasoning / Navigation index | Generates lookup indexes | | Symbol table | `ReasoningIndex` | name → location mapping | | Debug info | `NavigationIndex` | Runtime navigation data | +| Linker | RoutePass / ChainPass | Pre-computed routing + reasoning chains | +| Dead code elimination | OverlapPass | Detects duplicate content regions | +| Optimization hints | ScorePass | Evidence quality scoring per node | | Object file | `PersistedDocument` | Serialized to disk | | Incremental compilation | Fingerprint + incremental | Only recompiles changed parts | @@ -37,6 +40,10 @@ The pipeline is organized into four phases, each containing one or more passes: Backend 45: Reasoning → Keyword→path symbol table Backend 47: Concept → Key concept extraction (optional) Backend 50: Navigation→ Runtime navigation index + Backend 52: Route → Query routing table (optional) + Backend 54: Chain → Reasoning chain index (optional) + Backend 56: Overlap → Content overlap detection (optional) + Backend 58: Score → Evidence quality scoring (optional) Backend 55: Verify → Output validation Backend 60: Optimize → Final tree optimization ``` @@ -55,6 +62,7 @@ vectorless-compiler/src/ │ ├── analysis/ ValidatePass, EnhancePass │ ├── transform/ SplitPass, EnrichPass │ └── backend/ ReasoningPass, ConceptPass, NavigationPass, +│ RoutePass, ChainPass, OverlapPass, ScorePass, │ VerifyPass, OptimizePass ├── summary/ Summary strategies (Full, Selective, Lazy) └── incremental/ Change detection, action resolution, tree update @@ -78,4 +86,8 @@ let result = executor.execute(input, options).await?; let tree = result.tree.expect("tree must exist"); let reasoning = result.reasoning_index; let navigation = result.navigation_index; +let routes = result.query_routes; // Agent acceleration +let chains = result.chain_index; // Cross-section reasoning +let overlaps = result.content_overlap; // Dedup hints +let scores = result.evidence_scores; // Priority scoring ``` diff --git a/docs/docs/compiler/passes.mdx b/docs/docs/compiler/passes.mdx index 2c24f9d..394fc29 100644 --- a/docs/docs/compiler/passes.mdx +++ b/docs/docs/compiler/passes.mdx @@ -4,7 +4,7 @@ sidebar_position: 3 # Compilation Passes -The pipeline consists of 11 passes organized into four phases. Each pass is a self-contained unit with clear inputs, outputs, and dependencies. +The pipeline consists of 15 passes organized into four phases. Each pass is a self-contained unit with clear inputs, outputs, and dependencies. ## Pass Overview @@ -19,6 +19,10 @@ The pipeline consists of 11 passes organized into four phases. Each pass is a se | Backend | ReasoningPass | 45 | No | `enrich` | | Backend | ConceptPass | 47 | No | `reasoning_index` | | Backend | NavigationPass | 50 | No | `enrich` | +| Backend | RoutePass | 52 | No | `navigation_index` | +| Backend | ChainPass | 54 | No | `enrich` | +| Backend | OverlapPass | 56 | No | `build` | +| Backend | ScorePass | 58 | No | `enrich` | | Backend | VerifyPass | 55 | Yes | `concept_extraction` | | Backend | OptimizePass | 60 | No | `enrich`, `navigation_index` | @@ -152,6 +156,59 @@ Builds the runtime navigation index for agent-based traversal. - Creates `ChildRoute` entries for children (title, description, leaf count) - Builds `DocCard` for document-level overview +### RoutePass (Priority 52) + +Builds the pre-computed query routing table for Agent acceleration. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.query_routes` +- **Depends on**: `navigation_index` +- **Optional** +- Phase 1: Builds intent routes from nodes with `question_hints` +- Phase 2: Builds concept routes from `routing_keywords` tags +- No LLM calls — uses existing tree metadata + +### ChainPass (Priority 54) + +Builds reasoning chain index from in-document cross-references. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.chain_index` +- **Depends on**: `enrich` +- **Optional** +- Analyzes `TreeNode.references` to find premise→conclusion relationships +- Classifies chains by type: Causal, Supporting, Contradicting, Elaboration, Sequence +- Provides bidirectional node lookup (node → chains involving that node) +- No LLM calls — uses reference types and tree structure + +### OverlapPass (Priority 56) + +Detects content overlap between leaf nodes using Jaccard similarity. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.content_overlap` +- **Depends on**: `build` +- **Optional** +- Pairwise Jaccard similarity on leaf node content (word-level) +- Classifies overlaps: Duplicate (≥0.9), Subset, Summary +- Skips nodes with content shorter than 50 characters +- No LLM calls — pure statistical comparison + +### ScorePass (Priority 58) + +Computes per-node evidence quality scores. + +- **Reads**: `ctx.tree` +- **Writes**: `ctx.evidence_scores` +- **Depends on**: `enrich` +- **Optional** +- Three metrics per leaf node: + - **Density**: unique tokens / total tokens (information density) + - **Data richness**: presence of numbers, tables, code, lists + - **Specificity**: ratio of domain terms to filler words +- Composite score: density×0.4 + richness×0.3 + specificity×0.3 +- No LLM calls — pure content analysis + ### VerifyPass (Priority 55) Validates the final output. @@ -201,18 +258,19 @@ The following diagram shows how data flows through the passes and which `Compile │ EnrichPass │ reads: tree → writes: tree, description └──────────┬──────────┘ │ - ┌────────────────────┼──────────────────┐ - │ │ │ - ┌─────────▼─────────┐ ┌───────▼──────────┐ ┌───▼────────┐ - │ ReasoningPass │ │ NavigationPass │ │ OptimizePass│ - │ writes: reasoning │ │ writes: navigation│ │ │ - └─────────┬─────────┘ └──────────────────┘ └────────────┘ - │ - ┌─────────▼─────────┐ - │ ConceptPass │ writes: concepts - └─────────┬─────────┘ - │ - ┌─────────▼─────────┐ - │ VerifyPass │ reads: tree (validation only) - └───────────────────┘ + ┌─────────────────────────────┼────────────────────────────┐ + │ │ │ │ │ + ┌─────────▼─────┐ ┌─────▼────────┐ ┌───▼───────┐ ┌──▼──────────┐ ┌▼────────────┐ + │ ReasoningPass │ │NavigationPass│ │ Optimize │ │ ChainPass │ │ ScorePass │ + │writes:reasoning│ │writes:nav_idx│ │ │ │writes:chains│ │writes:scores│ + └─────────┬─────┘ └──────┬───────┘ └───────────┘ └─────────────┘ └─────────────┘ + │ │ + ┌─────────▼─────┐ ┌──────▼───────┐ ┌─────────────┐ + │ ConceptPass │ │ RoutePass │ │ OverlapPass │ + │writes:concepts│ │writes:routes │ │writes:overlap│ + └─────────┬─────┘ └──────────────┘ └─────────────┘ + │ + ┌─────────▼─────┐ + │ VerifyPass │ reads: tree (validation only) + └───────────────┘ ``` diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs index e95db72..3a36577 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs @@ -9,7 +9,7 @@ use std::time::Instant; use tracing::{info, warn}; -use vectorless_document::{ChainIndex, ChainType, ReasoningChain}; +use vectorless_document::{ChainIndex, ChainType, ReasoningChain, RefType}; use vectorless_error::Result; use crate::passes::async_trait; @@ -30,23 +30,22 @@ impl ChainPass { /// Determine chain type from the reference structure. fn classify_chain( - ref_type: &str, + ref_type: RefType, source_depth: usize, target_depth: usize, ) -> ChainType { match ref_type { - "Section" | "section" => { + RefType::Section => { if target_depth > source_depth { ChainType::Elaboration } else { ChainType::Supporting } } - "Appendix" | "appendix" => ChainType::Supporting, - "Table" | "Figure" | "Equation" | "table" | "figure" | "equation" => { + RefType::Appendix | RefType::Table | RefType::Figure | RefType::Equation => { ChainType::Supporting } - "Footnote" | "footnote" => ChainType::Elaboration, + RefType::Footnote => ChainType::Elaboration, _ => ChainType::Supporting, } } @@ -112,7 +111,7 @@ impl CompilePass for ChainPass { }; let chain_type = Self::classify_chain( - &reference.ref_text, + reference.ref_type, node.depth, tree.get(target_id).map(|n| n.depth).unwrap_or(0), ); @@ -165,7 +164,7 @@ impl CompilePass for ChainPass { #[cfg(test)] mod tests { use super::*; - use vectorless_document::NodeReference; + use vectorless_document::{NodeReference, RefType}; fn build_test_tree_with_refs() -> vectorless_document::DocumentTree { let mut tree = vectorless_document::DocumentTree::new("Root", "root content"); @@ -177,16 +176,24 @@ mod tests { // Add references: sec1 → sec2 (Section ref), sec2 → appendix (Appendix ref) if let Some(n) = tree.get_mut(sec1) { - n.references = vec![NodeReference { - ref_text: "Section".to_string(), - target_node: Some(sec2), - }]; + n.references = vec![NodeReference::resolved( + "see Section 2".to_string(), + "2".to_string(), + RefType::Section, + 4, + sec2, + 1.0, + )]; } if let Some(n) = tree.get_mut(sec2) { - n.references = vec![NodeReference { - ref_text: "Appendix".to_string(), - target_node: Some(appendix), - }]; + n.references = vec![NodeReference::resolved( + "Appendix A".to_string(), + "A".to_string(), + RefType::Appendix, + 12, + appendix, + 1.0, + )]; } tree @@ -207,36 +214,34 @@ mod tests { #[test] fn test_classify_chain_section_elaboration() { - // Section ref to deeper node = Elaboration - assert_eq!(ChainPass::classify_chain("Section", 0, 1), ChainType::Elaboration); + assert_eq!(ChainPass::classify_chain(RefType::Section, 0, 1), ChainType::Elaboration); } #[test] fn test_classify_chain_section_supporting() { - // Section ref to same or shallower depth = Supporting - assert_eq!(ChainPass::classify_chain("Section", 1, 0), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Section, 1, 0), ChainType::Supporting); } #[test] fn test_classify_chain_appendix() { - assert_eq!(ChainPass::classify_chain("Appendix", 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Appendix, 0, 1), ChainType::Supporting); } #[test] fn test_classify_chain_table_figure() { - assert_eq!(ChainPass::classify_chain("Table", 0, 1), ChainType::Supporting); - assert_eq!(ChainPass::classify_chain("Figure", 0, 1), ChainType::Supporting); - assert_eq!(ChainPass::classify_chain("Equation", 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Table, 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Figure, 0, 1), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Equation, 0, 1), ChainType::Supporting); } #[test] fn test_classify_chain_footnote() { - assert_eq!(ChainPass::classify_chain("Footnote", 0, 2), ChainType::Elaboration); + assert_eq!(ChainPass::classify_chain(RefType::Footnote, 0, 2), ChainType::Elaboration); } #[test] fn test_classify_chain_unknown() { - assert_eq!(ChainPass::classify_chain("custom", 0, 0), ChainType::Supporting); + assert_eq!(ChainPass::classify_chain(RefType::Unknown, 0, 0), ChainType::Supporting); } #[tokio::test] diff --git a/vectorless-core/vectorless-document/src/lib.rs b/vectorless-core/vectorless-document/src/lib.rs index eee8f79..a0f2360 100644 --- a/vectorless-core/vectorless-document/src/lib.rs +++ b/vectorless-core/vectorless-document/src/lib.rs @@ -40,7 +40,7 @@ pub use reasoning::{ ReasoningIndex, ReasoningIndexBuilder, ReasoningIndexConfig, SectionSummary, SummaryShortcut, TopicEntry, }; -pub use reference::{NodeReference, ReferenceExtractor}; +pub use reference::{NodeReference, RefType, ReferenceExtractor}; pub use structure::{DocumentStructure, StructureNode}; pub use toc::{TocConfig, TocEntry, TocNode, TocView}; pub use tree::{DocumentTree, RetrievalIndex}; From 03fc6be7999baf72786e121b790ff2b251fe685b Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:19:32 +0800 Subject: [PATCH 24/30] refactor(compiler): rename index pipeline to compile pipeline BREAKING CHANGE: Renamed IndexMetrics to CompileMetrics and IndexedDocument to CompiledDocument throughout the codebase. - Updated documentation to reflect compile pipeline terminology - Changed metric type from IndexMetrics to CompileMetrics - Renamed internal document type from IndexedDocument to CompiledDocument - Added new agent acceleration data fields to compiled document - Updated schema version from 1 to 2 due to structural changes - Modified persistence layer to include new index types --- docs/docs/compiler/standalone-usage.mdx | 7 --- .../vectorless-compiler/src/config.rs | 4 +- .../vectorless-compiler/src/lib.rs | 2 +- .../vectorless-compiler/src/parse/mod.rs | 2 +- .../src/parse/pdf/parser.rs | 4 +- .../src/passes/backend/optimize.rs | 12 ++--- .../src/pipeline/checkpoint.rs | 8 +-- .../src/pipeline/context.rs | 12 ++--- .../src/pipeline/executor.rs | 10 ++-- .../src/pipeline/metrics.rs | 4 +- .../vectorless-compiler/src/pipeline/mod.rs | 4 +- .../src/pipeline/orchestrator.rs | 2 +- .../src/pipeline/policy.rs | 10 ++-- ...dexed_document.rs => compiled_document.rs} | 46 +++++++++++------ .../vectorless-engine/src/engine.rs | 22 ++++---- .../vectorless-engine/src/indexer.rs | 50 +++++++++++-------- vectorless-core/vectorless-engine/src/lib.rs | 2 +- .../vectorless-engine/src/types.rs | 8 +-- .../vectorless-metrics/src/index.rs | 8 +-- vectorless-core/vectorless-metrics/src/lib.rs | 2 +- .../vectorless-storage/src/persistence.rs | 33 ++++++++++-- 21 files changed, 148 insertions(+), 104 deletions(-) rename vectorless-core/vectorless-engine/src/{indexed_document.rs => compiled_document.rs} (68%) diff --git a/docs/docs/compiler/standalone-usage.mdx b/docs/docs/compiler/standalone-usage.mdx index 0a5cd9f..1bf21b9 100644 --- a/docs/docs/compiler/standalone-usage.mdx +++ b/docs/docs/compiler/standalone-usage.mdx @@ -21,12 +21,6 @@ sidebar_position: 9 ## Basic Usage -```toml -# Cargo.toml -[dependencies] -vectorless-compiler = "0.1" -``` - ```rust use vectorless_compiler::{PipelineExecutor, PipelineOptions}; use vectorless_compiler::pipeline::CompilerInput; @@ -39,4 +33,3 @@ println!("Tree nodes: {}", result.tree.unwrap().node_count()); ``` {/* TODO: Full standalone usage guide */} -{/* TODO: crates.io publishing details */} diff --git a/vectorless-core/vectorless-compiler/src/config.rs b/vectorless-core/vectorless-compiler/src/config.rs index 3bce23a..b129ddc 100644 --- a/vectorless-core/vectorless-compiler/src/config.rs +++ b/vectorless-core/vectorless-compiler/src/config.rs @@ -1,9 +1,9 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Configuration types for the index pipeline. +//! Configuration types for the compile pipeline. //! -//! This module contains all configuration types used by the indexing pipeline: +//! This module contains all configuration types used by the compilation pipeline: //! - [`SourceFormat`] - Document format selection //! - [`PipelineOptions`] - Full pipeline configuration //! - [`OptimizationConfig`] - Tree optimization settings diff --git a/vectorless-core/vectorless-compiler/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs index 162d16d..9425b44 100644 --- a/vectorless-core/vectorless-compiler/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -65,7 +65,7 @@ pub mod passes; pub mod summary; // Re-export main types from pipeline -pub use pipeline::{CompilerInput, IndexMetrics, PipelineExecutor, CompileResult}; +pub use pipeline::{CompilerInput, CompileMetrics, PipelineExecutor, CompileResult}; // Re-export config types pub use config::{SourceFormat, PipelineOptions, ThinningConfig}; diff --git a/vectorless-core/vectorless-compiler/src/parse/mod.rs b/vectorless-core/vectorless-compiler/src/parse/mod.rs index d9bde2b..593f654 100644 --- a/vectorless-core/vectorless-compiler/src/parse/mod.rs +++ b/vectorless-core/vectorless-compiler/src/parse/mod.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Document parsing for the index pipeline. +//! Document parsing for the compile pipeline. //! //! Supports Markdown and PDF formats. Parsing is dispatched directly //! via `match` — no trait objects or registry needed. diff --git a/vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs b/vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs index 61e0578..6bdeb22 100644 --- a/vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs +++ b/vectorless-core/vectorless-compiler/src/parse/pdf/parser.rs @@ -257,7 +257,7 @@ impl Default for PdfParser { } impl PdfParser { - /// Parse a PDF file into raw nodes for the index pipeline. + /// Parse a PDF file into raw nodes for the compile pipeline. pub async fn parse_file(&self, path: &Path) -> Result<ParseResult> { let bytes = tokio::fs::read(path) .await @@ -267,7 +267,7 @@ impl PdfParser { .await } - /// Parse PDF bytes into raw nodes for the index pipeline. + /// Parse PDF bytes into raw nodes for the compile pipeline. pub async fn parse_bytes_async( &self, bytes: &[u8], diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs b/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs index 80761ab..61a6a5e 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/optimize.rs @@ -30,7 +30,7 @@ impl OptimizePass { fn merge_small_leaves( tree: &mut vectorless_document::DocumentTree, min_tokens: usize, - metrics: &mut crate::IndexMetrics, + metrics: &mut crate::CompileMetrics, ) -> usize { let mut merged_count = 0; @@ -286,7 +286,7 @@ mod tests { fn test_merge_small_leaves_merges_adjacent_pair() { let mut tree = make_merge_test_tree(); let root = tree.root(); - let mut metrics = crate::pipeline::IndexMetrics::new(); + let mut metrics = crate::pipeline::CompileMetrics::new(); // Threshold 100: Leaf A (50) and Leaf B (30) should merge let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); @@ -307,7 +307,7 @@ mod tests { #[test] fn test_merge_small_leaves_nothing_above_threshold() { let mut tree = make_merge_test_tree(); - let mut metrics = crate::pipeline::IndexMetrics::new(); + let mut metrics = crate::pipeline::CompileMetrics::new(); // Threshold 10: all leaves are above this, nothing merges let merged = OptimizePass::merge_small_leaves(&mut tree, 10, &mut metrics); @@ -327,7 +327,7 @@ mod tests { n.token_count = Some(5); } - let mut metrics = crate::pipeline::IndexMetrics::new(); + let mut metrics = crate::pipeline::CompileMetrics::new(); let _ = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); // Leaf A should now contain both contents with heading prefix @@ -355,7 +355,7 @@ mod tests { n.token_count = Some(5); } - let mut metrics = crate::pipeline::IndexMetrics::new(); + let mut metrics = crate::pipeline::CompileMetrics::new(); let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); // Section is non-leaf, only Leaf is a leaf — no adjacent pair of leaves @@ -447,7 +447,7 @@ mod tests { #[test] fn test_merge_small_leaves_empty_tree() { let mut tree = DocumentTree::new("Root", ""); - let mut metrics = crate::pipeline::IndexMetrics::new(); + let mut metrics = crate::pipeline::CompileMetrics::new(); let merged = OptimizePass::merge_small_leaves(&mut tree, 100, &mut metrics); assert_eq!(merged, 0, "Root with no children should merge nothing"); diff --git a/vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs b/vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs index ad60721..ee5313d 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/checkpoint.rs @@ -16,7 +16,7 @@ use tracing::{info, warn}; use crate::parse::RawNode; use vectorless_document::DocumentTree; -use super::metrics::IndexMetrics; +use super::metrics::CompileMetrics; /// Serializable checkpoint capturing pipeline state at a point in time. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -53,7 +53,7 @@ pub struct CheckpointContextData { pub tree: Option<DocumentTree>, /// Metrics collected so far. - pub metrics: IndexMetrics, + pub metrics: CompileMetrics, /// Page count (for PDFs). pub page_count: Option<usize>, @@ -192,7 +192,7 @@ mod tests { context_data: CheckpointContextData { raw_nodes: Vec::new(), tree: Some(DocumentTree::new("Test", "content")), - metrics: IndexMetrics::default(), + metrics: CompileMetrics::default(), page_count: None, line_count: Some(10), description: None, @@ -309,7 +309,7 @@ mod tests { context_data: CheckpointContextData { raw_nodes: Vec::new(), tree: Some(tree), - metrics: IndexMetrics::default(), + metrics: CompileMetrics::default(), page_count: None, line_count: None, description: None, diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index 8b8b188..364dfa3 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -14,9 +14,9 @@ use vectorless_document::{ use vectorless_llm::LlmClient; use super::super::{PipelineOptions, SummaryStrategy}; -use super::metrics::IndexMetrics; +use super::metrics::CompileMetrics; -/// Input for the index pipeline. +/// Input for the compile pipeline. #[derive(Debug, Clone)] pub enum CompilerInput { /// Index from file path. @@ -283,7 +283,7 @@ pub struct CompileContext { pub stage_results: HashMap<String, PassResult>, /// Performance metrics. - pub metrics: IndexMetrics, + pub metrics: CompileMetrics, /// Document description. pub description: Option<String>, @@ -320,7 +320,7 @@ impl CompileContext { evidence_scores: None, existing_tree: None, stage_results: HashMap::new(), - metrics: IndexMetrics::default(), + metrics: CompileMetrics::default(), description: None, page_count: None, line_count: None, @@ -425,7 +425,7 @@ impl CompileContext { } } -/// Final result from the index pipeline. +/// Final result from the compile pipeline. #[derive(Debug)] pub struct CompileResult { /// Document ID. @@ -453,7 +453,7 @@ pub struct CompileResult { pub line_count: Option<usize>, /// Performance metrics. - pub metrics: IndexMetrics, + pub metrics: CompileMetrics, /// Summary cache. pub summary_cache: SummaryCache, diff --git a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs index 53884b9..2416794 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Pipeline executor for running index stages. +//! Pipeline executor for running compile passes. //! //! The executor uses [`PipelineOrchestrator`] internally for flexible //! stage management with priority-based ordering and dependency resolution. @@ -20,10 +20,10 @@ use super::super::passes::{ use super::context::{CompilerInput, CompileResult}; use super::orchestrator::PipelineOrchestrator; -/// Pipeline executor for document indexing. +/// Pipeline executor for document compilation. /// -/// Uses [`PipelineOrchestrator`] internally for stage management. -/// Supports both preset configurations and custom stage pipelines. +/// Uses [`PipelineOrchestrator`] internally for pass management. +/// Supports both preset configurations and custom pass pipelines. /// /// # Example /// @@ -192,7 +192,7 @@ impl PipelineExecutor { options: PipelineOptions, ) -> Result<CompileResult> { info!( - "Starting index pipeline with {} stages", + "Starting compile pipeline with {} passes", self.orchestrator.stage_count() ); self.orchestrator.execute(input, options).await diff --git a/vectorless-core/vectorless-compiler/src/pipeline/metrics.rs b/vectorless-core/vectorless-compiler/src/pipeline/metrics.rs index 9c08d69..a8bc36e 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/metrics.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/metrics.rs @@ -1,6 +1,6 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Re-export IndexMetrics from the metrics module. +//! Re-export CompileMetrics from the metrics module. -pub use vectorless_metrics::IndexMetrics; +pub use vectorless_metrics::CompileMetrics; diff --git a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs index 88c8e9f..d48144e 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs @@ -7,7 +7,7 @@ //! - [`CompileContext`] - Context passed between passes //! - [`PipelineExecutor`] - Executes the compilation pipeline //! - [`PipelineOrchestrator`] - Flexible pass orchestration with dependencies -//! - [`IndexMetrics`] - Performance metrics collection +//! - [`CompileMetrics`] - Performance metrics collection //! - [`FailurePolicy`] - Configurable failure handling for passes //! - [`StageRetryConfig`] - Retry configuration for passes @@ -20,5 +20,5 @@ mod policy; pub use context::{CompileContext, CompilerInput, CompileResult, PassResult}; pub use executor::PipelineExecutor; -pub use metrics::IndexMetrics; +pub use metrics::CompileMetrics; pub use policy::{FailurePolicy, StageRetryConfig}; diff --git a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs index f6d6283..096cf89 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Pipeline orchestrator for managing and executing index stages. +//! Pipeline orchestrator for managing and executing compile passes. //! //! The orchestrator provides: //! - Stage registration with priority diff --git a/vectorless-core/vectorless-compiler/src/pipeline/policy.rs b/vectorless-core/vectorless-compiler/src/pipeline/policy.rs index da3c5b2..d93d842 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/policy.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/policy.rs @@ -1,15 +1,15 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Failure policies for pipeline stages. +//! Failure policies for pipeline passes. //! -//! This module provides configurable failure handling for index pipeline stages. +//! This module provides configurable failure handling for compile pipeline passes. //! //! # Policies //! -//! - **Fail** - Stop the entire pipeline on stage failure (default for required stages) -//! - **Skip** - Skip the failed stage and continue the pipeline -//! - **Retry** - Retry the stage with exponential backoff before failing +//! - **Fail** - Stop the entire pipeline on pass failure (default for required passes) +//! - **Skip** - Skip the failed pass and continue the pipeline +//! - **Retry** - Retry the pass with exponential backoff before failing //! //! # Example //! diff --git a/vectorless-core/vectorless-engine/src/indexed_document.rs b/vectorless-core/vectorless-engine/src/compiled_document.rs similarity index 68% rename from vectorless-core/vectorless-engine/src/indexed_document.rs rename to vectorless-core/vectorless-engine/src/compiled_document.rs index ee1cbbc..24f912d 100644 --- a/vectorless-core/vectorless-engine/src/indexed_document.rs +++ b/vectorless-core/vectorless-engine/src/compiled_document.rs @@ -1,9 +1,9 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Internal intermediate type produced by the indexing pipeline. +//! Internal intermediate type produced by the compile pipeline. //! -//! [`IndexedDocument`] is an internal-only type that carries data from +//! [`CompiledDocument`] is an internal-only type that carries data from //! [`IndexerClient`](super::indexer::IndexerClient) to [`Engine`](super::Engine). //! It is **not** part of the public API. @@ -11,15 +11,15 @@ use std::path::PathBuf; use vectorless_document::DocumentFormat; use vectorless_document::DocumentTree; -use vectorless_metrics::IndexMetrics; +use vectorless_metrics::CompileMetrics; use vectorless_storage::PageContent; -/// An indexed document with its tree structure and metadata. +/// A compiled document with its tree structure, indexes, and metadata. /// -/// Internal intermediate produced by the indexing pipeline and consumed +/// Internal intermediate produced by the compile pipeline and consumed /// by [`Engine`](super::Engine) to create a [`PersistedDocument`](vectorless_storage::PersistedDocument). #[derive(Debug, Clone)] -pub(crate) struct IndexedDocument { +pub(crate) struct CompiledDocument { /// Unique document identifier. pub id: String, @@ -44,8 +44,8 @@ pub(crate) struct IndexedDocument { /// Per-page content (for PDFs). pub pages: Vec<PageContent>, - /// Indexing pipeline metrics. - pub metrics: Option<IndexMetrics>, + /// Compile pipeline metrics. + pub metrics: Option<CompileMetrics>, /// Pre-computed reasoning index for retrieval acceleration. pub reasoning_index: Option<vectorless_document::ReasoningIndex>, @@ -55,10 +55,24 @@ pub(crate) struct IndexedDocument { /// Key concepts extracted from the document. pub concepts: Vec<vectorless_document::Concept>, + + // ── Agent acceleration data ── + + /// Pre-computed query routing table for Agent acceleration. + pub query_routes: Option<vectorless_document::QueryRoutingTable>, + + /// Reasoning chain index for cross-section navigation. + pub chain_index: Option<vectorless_document::ChainIndex>, + + /// Content overlap map to prevent duplicate visits. + pub content_overlap: Option<vectorless_document::ContentOverlapMap>, + + /// Per-node evidence quality scores. + pub evidence_scores: Option<vectorless_document::EvidenceScoreMap>, } -impl IndexedDocument { - /// Create a new indexed document. +impl CompiledDocument { + /// Create a new compiled document. pub fn new(id: impl Into<String>, format: DocumentFormat) -> Self { Self { id: id.into(), @@ -73,6 +87,10 @@ impl IndexedDocument { reasoning_index: None, navigation_index: None, concepts: Vec::new(), + query_routes: None, + chain_index: None, + content_overlap: None, + evidence_scores: None, } } @@ -106,8 +124,8 @@ impl IndexedDocument { self } - /// Set the indexing metrics. - pub fn with_metrics(mut self, metrics: IndexMetrics) -> Self { + /// Set the compile metrics. + pub fn with_metrics(mut self, metrics: CompileMetrics) -> Self { self.metrics = Some(metrics); self } @@ -118,8 +136,8 @@ mod tests { use super::*; #[test] - fn test_indexed_document() { - let doc = IndexedDocument::new("doc-1", DocumentFormat::Markdown) + fn test_compiled_document() { + let doc = CompiledDocument::new("doc-1", DocumentFormat::Markdown) .with_name("Test Document") .with_description("A test document"); diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index 4082c17..e6ad951 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -36,7 +36,7 @@ use super::{ /// The main Engine client. /// -/// Provides high-level operations for document indexing and retrieval. +/// Provides high-level operations for document compilation and retrieval. /// Uses interior mutability to allow sharing across async tasks. /// /// # Cloning @@ -51,7 +51,7 @@ pub struct Engine { /// Configuration (immutable, shared). config: Arc<Config>, - /// Indexer client for document indexing. + /// Indexer client for document compilation. indexer: IndexerClient, /// Workspace client for persistence. @@ -278,7 +278,7 @@ impl Engine { name: Option<&str>, pipeline_options: PipelineOptions, existing_tree: Option<&DocumentTree>, - ) -> Result<super::indexed_document::IndexedDocument> { + ) -> Result<super::compiled_document::CompiledDocument> { let retry = &self.config.llm.retry; let max_attempts = retry.max_attempts; @@ -313,13 +313,13 @@ impl Engine { unreachable!() } - /// Convert an [`IndexedDocument`] to an [`CompileArtifact`] and persist it. + /// Convert an [`CompiledDocument`] to an [`CompileArtifact`] and persist it. /// /// If `old_id` is provided, the old document is removed after a /// successful save (atomic save-first, then remove old). async fn index_and_persist( &self, - doc: super::indexed_document::IndexedDocument, + doc: super::compiled_document::CompiledDocument, pipeline_options: &PipelineOptions, source_label: &str, old_id: Option<&str>, @@ -347,8 +347,8 @@ impl Engine { (vec![item], Vec::new()) } - /// Build an [`CompileArtifact`] from an [`IndexedDocument`](super::indexed_document::IndexedDocument). - fn build_index_item(doc: &super::indexed_document::IndexedDocument) -> CompileArtifact { + /// Build an [`CompileArtifact`] from an [`CompiledDocument`](super::compiled_document::CompiledDocument). + fn build_index_item(doc: &super::compiled_document::CompiledDocument) -> CompileArtifact { CompileArtifact::new( doc.id.clone(), doc.name.clone(), @@ -772,10 +772,10 @@ mod tests { // -- build_index_item ---------------------------------------------------------------- // Build_index_item only transforms data -- no I/O. - use crate::indexed_document::IndexedDocument; + use crate::compiled_document::CompiledDocument; - fn make_doc() -> IndexedDocument { - IndexedDocument::new("test-id", vectorless_compiler::parse::DocumentFormat::Markdown) + fn make_doc() -> CompiledDocument { + CompiledDocument::new("test-id", vectorless_compiler::parse::DocumentFormat::Markdown) .with_name("test.md") .with_description("test doc") .with_source_path(std::path::PathBuf::from("/tmp/test.md")) @@ -799,7 +799,7 @@ mod tests { #[test] fn test_build_index_item_no_source_path() { - let doc = IndexedDocument::new("id", vectorless_compiler::parse::DocumentFormat::Pdf); + let doc = CompiledDocument::new("id", vectorless_compiler::parse::DocumentFormat::Pdf); let item = Engine::build_index_item(&doc); assert_eq!(item.source_path, Some(String::new())); // unwrap_or_default diff --git a/vectorless-core/vectorless-engine/src/indexer.rs b/vectorless-core/vectorless-engine/src/indexer.rs index 816dd72..0044af6 100644 --- a/vectorless-core/vectorless-engine/src/indexer.rs +++ b/vectorless-core/vectorless-engine/src/indexer.rs @@ -1,9 +1,9 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Document indexing client. +//! Document compile client. //! -//! This module provides document indexing operations including +//! This module provides document compilation operations including //! format detection, parsing, and tree building. //! //! # Example @@ -33,14 +33,14 @@ use vectorless_llm::LlmClient; use vectorless_storage::{DocumentMeta, PersistedDocument}; use super::compile_input::CompileSource; -use super::indexed_document::IndexedDocument; +use super::compiled_document::CompiledDocument; use vectorless_events::{EventEmitter, IndexEvent}; -/// Document indexing client. +/// Document compile client. /// -/// Provides operations for parsing and indexing documents. -/// Each index operation creates a fresh pipeline executor, enabling -/// true parallel document indexing without mutex contention. +/// Provides operations for compiling documents. +/// Each compile operation creates a fresh pipeline executor, enabling +/// true parallel document compilation without mutex contention. pub(crate) struct IndexerClient { /// Factory for creating pipeline executors (one per index operation). executor_factory: Arc<dyn Fn() -> PipelineExecutor + Send + Sync>, @@ -74,7 +74,7 @@ impl IndexerClient { source: &CompileSource, name: Option<&str>, pipeline_options: PipelineOptions, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { self.index_with_existing(source, name, pipeline_options, None) .await } @@ -88,7 +88,7 @@ impl IndexerClient { name: Option<&str>, mut pipeline_options: PipelineOptions, existing_tree: Option<&vectorless_document::DocumentTree>, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { pipeline_options.existing_tree = existing_tree.cloned(); match source { CompileSource::Path(path) => self.index_from_path(path, name, pipeline_options).await, @@ -111,7 +111,7 @@ impl IndexerClient { path: &Path, name: Option<&str>, pipeline_options: PipelineOptions, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { let path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); // Validate file before indexing @@ -151,7 +151,7 @@ impl IndexerClient { format: DocumentFormat, name: Option<&str>, pipeline_options: PipelineOptions, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { // Validate content before indexing let validation = vectorless_utils::validate_content(content, format); if !validation.valid { @@ -183,7 +183,7 @@ impl IndexerClient { format: DocumentFormat, name: Option<&str>, pipeline_options: PipelineOptions, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { // Validate bytes before indexing let validation = vectorless_utils::validate_bytes(bytes, format); if !validation.valid { @@ -224,7 +224,7 @@ impl IndexerClient { name: Option<&str>, path: Option<&Path>, pipeline_options: PipelineOptions, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { self.events.emit_index(IndexEvent::Started { path: source_label.to_string(), }); @@ -238,18 +238,18 @@ impl IndexerClient { let mut executor = (self.executor_factory)(); let result = executor.execute(input, pipeline_options).await?; - self.build_indexed_document(doc_id, result, format, name, path) + self.build_compiled_document(doc_id, result, format, name, path) } - /// Build indexed document from pipeline result. - fn build_indexed_document( + /// Build compiled document from pipeline result. + fn build_compiled_document( &self, doc_id: String, result: vectorless_compiler::CompileResult, format: DocumentFormat, name: Option<&str>, path: Option<&Path>, - ) -> Result<IndexedDocument> { + ) -> Result<CompiledDocument> { let tree = result .tree .ok_or_else(|| Error::Parse("Document tree not generated".to_string()))?; @@ -265,7 +265,7 @@ impl IndexerClient { }) .unwrap_or_else(|| result.name.clone()); - let mut doc = IndexedDocument::new(&doc_id, format) + let mut doc = CompiledDocument::new(&doc_id, format) .with_name(&doc_name) .with_tree(tree) .with_metrics(result.metrics); @@ -273,6 +273,10 @@ impl IndexerClient { doc.reasoning_index = result.reasoning_index; doc.navigation_index = result.navigation_index; doc.concepts = result.concepts; + doc.query_routes = result.query_routes; + doc.chain_index = result.chain_index; + doc.content_overlap = result.content_overlap; + doc.evidence_scores = result.evidence_scores; if let Some(p) = path { doc = doc.with_source_path(p); @@ -311,14 +315,14 @@ impl IndexerClient { .ok_or_else(|| Error::Parse(format!("Unsupported format: {}", ext))) } - /// Convert [`IndexedDocument`] to [`PersistedDocument`]. + /// Convert [`CompiledDocument`] to [`PersistedDocument`]. /// /// This is an associated function — it does not depend on client state. /// Stores content and logic fingerprints from the pipeline options. /// /// Uses async file I/O to avoid blocking the tokio runtime. pub async fn to_persisted( - doc: IndexedDocument, + doc: CompiledDocument, pipeline_options: &PipelineOptions, ) -> PersistedDocument { let mut meta = DocumentMeta::new(&doc.id, &doc.name, doc.format.extension()) @@ -342,7 +346,7 @@ impl IndexerClient { let logic_fp = pipeline_options.logic_fingerprint(); meta = meta.with_logic_fingerprint(logic_fp); - let tree = doc.tree.expect("IndexedDocument must have a tree"); + let tree = doc.tree.expect("CompiledDocument must have a tree"); // Extract stats from metrics let node_count = tree.node_count(); @@ -361,6 +365,10 @@ impl IndexerClient { persisted.reasoning_index = doc.reasoning_index; persisted.navigation_index = doc.navigation_index; persisted.concepts = doc.concepts; + persisted.query_routes = doc.query_routes; + persisted.chain_index = doc.chain_index; + persisted.content_overlap = doc.content_overlap; + persisted.evidence_scores = doc.evidence_scores; persisted .meta .update_processing_stats(node_count, summary_tokens, duration_ms); diff --git a/vectorless-core/vectorless-engine/src/lib.rs b/vectorless-core/vectorless-engine/src/lib.rs index b588592..b11fa5a 100644 --- a/vectorless-core/vectorless-engine/src/lib.rs +++ b/vectorless-core/vectorless-engine/src/lib.rs @@ -13,7 +13,7 @@ mod builder; mod compile_input; mod engine; -mod indexed_document; +mod compiled_document; mod indexer; mod types; mod workspace; diff --git a/vectorless-core/vectorless-engine/src/types.rs b/vectorless-core/vectorless-engine/src/types.rs index 61c44b1..f78df63 100644 --- a/vectorless-core/vectorless-engine/src/types.rs +++ b/vectorless-core/vectorless-engine/src/types.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use vectorless_document::DocumentFormat; -use vectorless_metrics::IndexMetrics; +use vectorless_metrics::CompileMetrics; // ============================================================ // Partial Success @@ -210,7 +210,7 @@ pub struct CompileArtifact { /// Page count (for PDFs). pub page_count: Option<usize>, /// Indexing pipeline metrics (timing, LLM usage, node stats). - pub metrics: Option<IndexMetrics>, + pub metrics: Option<CompileMetrics>, } impl CompileArtifact { @@ -240,13 +240,13 @@ impl CompileArtifact { } /// Set the indexing metrics. - pub fn with_metrics(mut self, metrics: IndexMetrics) -> Self { + pub fn with_metrics(mut self, metrics: CompileMetrics) -> Self { self.metrics = Some(metrics); self } /// Set the indexing metrics (optional). - pub fn with_metrics_opt(mut self, metrics: Option<IndexMetrics>) -> Self { + pub fn with_metrics_opt(mut self, metrics: Option<CompileMetrics>) -> Self { self.metrics = metrics; self } diff --git a/vectorless-core/vectorless-metrics/src/index.rs b/vectorless-core/vectorless-metrics/src/index.rs index af5d9d5..6bdb063 100644 --- a/vectorless-core/vectorless-metrics/src/index.rs +++ b/vectorless-core/vectorless-metrics/src/index.rs @@ -1,13 +1,13 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Indexing pipeline metrics. +//! Compile pipeline metrics. use serde::{Deserialize, Serialize}; -/// Performance metrics for the indexing pipeline. +/// Performance metrics for the compile pipeline. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct IndexMetrics { +pub struct CompileMetrics { /// Parse stage duration (ms). #[serde(default)] pub parse_time_ms: u64, @@ -127,7 +127,7 @@ pub struct IndexMetrics { pub nodes_scored: usize, } -impl IndexMetrics { +impl CompileMetrics { /// Create new metrics with start time. pub fn new() -> Self { Self::default() diff --git a/vectorless-core/vectorless-metrics/src/lib.rs b/vectorless-core/vectorless-metrics/src/lib.rs index 26ab641..09e9b19 100644 --- a/vectorless-core/vectorless-metrics/src/lib.rs +++ b/vectorless-core/vectorless-metrics/src/lib.rs @@ -51,6 +51,6 @@ mod llm; mod retrieval; pub use hub::{MetricsHub, MetricsReport}; -pub use index::IndexMetrics; +pub use index::CompileMetrics; pub use llm::LlmMetricsReport; pub use retrieval::RetrievalMetricsReport; diff --git a/vectorless-core/vectorless-storage/src/persistence.rs b/vectorless-core/vectorless-storage/src/persistence.rs index f7fd0a4..edaa5c5 100644 --- a/vectorless-core/vectorless-storage/src/persistence.rs +++ b/vectorless-core/vectorless-storage/src/persistence.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026 vectorless developers // SPDX-License-Identifier: Apache-2.0 -//! Persistence utilities for saving and loading document indices. +//! Persistence utilities for saving and loading compiled documents. //! //! # Features //! @@ -13,7 +13,10 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::path::PathBuf; -use vectorless_document::{DocumentTree, NavigationIndex, ReasoningIndex}; +use vectorless_document::{ + ChainIndex, ContentOverlapMap, DocumentTree, EvidenceScoreMap, NavigationIndex, + QueryRoutingTable, ReasoningIndex, +}; use vectorless_error::Error; use vectorless_error::Result; @@ -25,7 +28,7 @@ const FORMAT_VERSION: u32 = 1; /// Increment this when the document structure changes in a /// backward-incompatible way (e.g. field renames, new required fields). /// Old documents will be detected and logged as stale on load. -const SCHEMA_VERSION: u32 = 1; +const SCHEMA_VERSION: u32 = 2; /// Metadata for a persisted document. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -208,7 +211,7 @@ impl DocumentMeta { } } -/// A persisted document index containing tree and metadata. +/// A persisted compiled document containing tree, indexes, and metadata. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PersistedDocument { /// Schema version — incremented on backward-incompatible changes. @@ -237,6 +240,24 @@ pub struct PersistedDocument { /// Key concepts extracted from the document. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub concepts: Vec<vectorless_document::Concept>, + + // ── Agent acceleration data ── + + /// Pre-computed query routing table for Agent acceleration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_routes: Option<QueryRoutingTable>, + + /// Reasoning chain index for cross-section navigation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chain_index: Option<ChainIndex>, + + /// Content overlap map to prevent duplicate visits. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content_overlap: Option<ContentOverlapMap>, + + /// Per-node evidence quality scores. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub evidence_scores: Option<EvidenceScoreMap>, } impl PersistedDocument { @@ -250,6 +271,10 @@ impl PersistedDocument { reasoning_index: None, navigation_index: None, concepts: Vec::new(), + query_routes: None, + chain_index: None, + content_overlap: None, + evidence_scores: None, } } From 6f212cc42351064f894c42cf8f8a7b2aa1a16eaa Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:26:53 +0800 Subject: [PATCH 25/30] refactor(compiler): remove deprecated StageResult alias and unused CustomStageBuilder BREAKING CHANGE: Removed deprecated StageResult type alias that was marked for removal since version 0.2.0. Also removed CustomStageBuilder struct which was unused in the codebase. These changes clean up the API surface and remove dead code. --- .../src/pipeline/context.rs | 4 - .../src/pipeline/orchestrator.rs | 74 ------------------- 2 files changed, 78 deletions(-) diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index 364dfa3..a44b4b2 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -172,10 +172,6 @@ impl PassResult { } } -/// Backward-compatible alias. -#[deprecated(since = "0.2.0", note = "Use `PassResult` instead")] -pub type StageResult = PassResult; - /// Summary cache for lazy generation. #[derive(Debug, Clone, Default)] pub struct SummaryCache { diff --git a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs index 096cf89..7981b1f 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs @@ -872,67 +872,6 @@ struct ParallelEntry { access: crate::passes::AccessPattern, } -/// Builder for creating custom stage configurations. -/// -/// This is a convenience type for configuring custom stages -/// without manually calling the orchestrator methods. -pub struct CustomStageBuilder { - name: String, - priority: i32, - depends_on: Vec<String>, - optional: bool, -} - -impl CustomStageBuilder { - /// Create a new custom stage builder. - pub fn new(name: impl Into<String>) -> Self { - Self { - name: name.into(), - priority: 100, - depends_on: Vec::new(), - optional: false, - } - } - - /// Set priority (lower = earlier). - pub fn priority(mut self, priority: i32) -> Self { - self.priority = priority; - self - } - - /// Add a dependency. - pub fn depends_on(mut self, stage: impl Into<String>) -> Self { - self.depends_on.push(stage.into()); - self - } - - /// Mark as optional (failures won't stop pipeline). - pub fn optional(mut self) -> Self { - self.optional = true; - self - } - - /// Get the stage name. - pub fn name(&self) -> &str { - &self.name - } - - /// Get the priority. - pub fn get_priority(&self) -> i32 { - self.priority - } - - /// Get dependencies. - pub fn get_deps(&self) -> &[String] { - &self.depends_on - } - - /// Check if optional. - pub fn is_optional(&self) -> bool { - self.optional - } -} - #[cfg(test)] mod tests { use super::super::context::PassResult; @@ -989,19 +928,6 @@ mod tests { assert!(orchestrator.has_stage("b")); } - #[test] - fn test_custom_stage_builder() { - let builder = CustomStageBuilder::new("my_stage") - .priority(50) - .depends_on("parse") - .optional(); - - assert_eq!(builder.name(), "my_stage"); - assert_eq!(builder.get_priority(), 50); - assert_eq!(builder.get_deps(), &["parse".to_string()]); - assert!(builder.is_optional()); - } - /// Mock stage for testing. struct MockStage { name: String, From e091cef25d63780f2beb76ade22d6681fd056384 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:42:42 +0800 Subject: [PATCH 26/30] feat(document): add agent acceleration data to compiled documents Add pre-computed agent acceleration data structures to the Document type including query routing tables, reasoning chain indices, content overlap maps, and evidence quality scores. Update documentation to reflect compilation terminology instead of ingestion terminology. BREAKING CHANGE: Document understanding terminology changed from ingestion to compilation process. feat(navigator): implement agent acceleration query methods Add new methods to DocumentNavigator for querying agent acceleration data including intent routes, concept routes, reasoning chains, content overlaps, and evidence scores. Include helper method for node ID conversion. refactor(python): expose agent acceleration APIs to Python bindings Expose new agent acceleration data structures and query methods through Python bindings. Add corresponding Python wrapper classes and async methods for all new functionality. feat(agent): utilize acceleration data for improved keyword hints Enhance agent keyword hint generation by incorporating pre-computed concept routes and evidence quality scores alongside traditional keyword index matches. Provide richer context for agent decision making. --- .../vectorless-document/src/understanding.rs | 32 ++- .../vectorless-engine/src/engine.rs | 4 + .../vectorless-primitives/src/navigator.rs | 4 + .../src/navigator_search.rs | 138 ++++++++++ .../vectorless-primitives/src/types.rs | 65 +++++ vectorless-core/vectorless-py/src/document.rs | 243 +++++++++++++++++- vectorless-core/vectorless-py/src/lib.rs | 12 +- vectorless/ask/protocols.py | 25 ++ vectorless/ask/worker/agent.py | 55 +++- 9 files changed, 559 insertions(+), 19 deletions(-) diff --git a/vectorless-core/vectorless-document/src/understanding.rs b/vectorless-core/vectorless-document/src/understanding.rs index 2b97ce3..3fc0a78 100644 --- a/vectorless-core/vectorless-document/src/understanding.rs +++ b/vectorless-core/vectorless-document/src/understanding.rs @@ -4,8 +4,8 @@ //! Understanding types — the core objects that define the Document Understanding Engine. //! //! These types form the stable public contract: -//! - [`Document`] — the unified post-ingest artifact (internal first-class citizen) -//! - [`DocumentInfo`] — what `ingest()` returns to users +//! - [`Document`] — the unified post-compile artifact (internal first-class citizen) +//! - [`DocumentInfo`] — what `compile()` returns to users //! - [`Concept`] — key concept extracted from a document use serde::{Deserialize, Serialize}; @@ -13,14 +13,14 @@ use serde::{Deserialize, Serialize}; use super::toc::TocNode; // --------------------------------------------------------------------------- -// Document — unified post-ingest artifact +// Document — unified post-compile artifact // --------------------------------------------------------------------------- -/// A understood document — the core artifact of the understand phase. +/// A compiled document — the core artifact of the compile pipeline. /// -/// This is what `ingest()` produces internally. +/// This is what `compile()` produces internally. /// It unifies tree + navigation index + reasoning index + summary + concepts -/// into a single first-class type. +/// + agent acceleration data into a single first-class type. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Document { /// Unique document identifier. @@ -29,11 +29,11 @@ pub struct Document { pub name: String, /// Document format ("pdf", "markdown", "docx"). pub format: String, - /// Source file path (if indexed from a file). + /// Source file path (if compiled from a file). #[serde(default, skip_serializing_if = "Option::is_none")] pub source_path: Option<String>, - // ── Three indexes (engine internal) ── + // ── Indexes ── /// Hierarchical semantic tree. pub tree: super::tree::DocumentTree, /// Pre-computed navigation structure. @@ -41,13 +41,27 @@ pub struct Document { /// Keyword / topic / section summaries. pub reasoning_index: super::reasoning::ReasoningIndex, - // ── Understanding results (ingest stage output) ── + // ── Compile results ── /// Document-level summary. pub summary: String, /// Key concepts the engine identified. #[serde(default)] pub concepts: Vec<Concept>, + // ── Agent acceleration data ── + /// Pre-computed query routing table. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_routes: Option<super::query_route::QueryRoutingTable>, + /// Reasoning chain index. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chain_index: Option<super::chain::ChainIndex>, + /// Content overlap map. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content_overlap: Option<super::overlap::ContentOverlapMap>, + /// Per-node evidence quality scores. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub evidence_scores: Option<super::evidence::EvidenceScoreMap>, + // ── Metadata ── /// Page count (for PDFs). #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index e6ad951..0a22486 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -507,6 +507,10 @@ impl Engine { reasoning_index, summary: persisted.meta.description.unwrap_or_default(), concepts: persisted.concepts, + query_routes: persisted.query_routes, + chain_index: persisted.chain_index, + content_overlap: persisted.content_overlap, + evidence_scores: persisted.evidence_scores, page_count: persisted.meta.page_count, section_count, } diff --git a/vectorless-core/vectorless-primitives/src/navigator.rs b/vectorless-core/vectorless-primitives/src/navigator.rs index ae52433..5cdc79c 100644 --- a/vectorless-core/vectorless-primitives/src/navigator.rs +++ b/vectorless-core/vectorless-primitives/src/navigator.rs @@ -78,6 +78,10 @@ impl DocumentNavigator { usize::from(id.0) as u64 } + fn u64_to_id(&self, num: u64) -> Option<NodeId> { + self.node_id_map.get(&num).copied() + } + fn resolve_optional_id(&self, opt: Option<&str>) -> Result<NodeId> { match opt { Some(s) => self.parse_id(s), diff --git a/vectorless-core/vectorless-primitives/src/navigator_search.rs b/vectorless-core/vectorless-primitives/src/navigator_search.rs index 1aa6762..f505ea1 100644 --- a/vectorless-core/vectorless-primitives/src/navigator_search.rs +++ b/vectorless-core/vectorless-primitives/src/navigator_search.rs @@ -248,4 +248,142 @@ impl DocumentNavigator { } result } + + // ----------------------------------------------------------------------- + // Agent acceleration queries + // ----------------------------------------------------------------------- + + /// Get all intent routes from the query routing table. + pub async fn intent_routes(&self) -> Vec<RouteTargetInfo> { + self.doc + .query_routes + .as_ref() + .map(|table| { + table + .intent_routes() + .values() + .flat_map(|targets| { + targets.iter().map(|t| RouteTargetInfo { + node_id: self.id_to_u64(t.node_id), + relevance: t.relevance, + reason: t.reason.clone(), + }) + }) + .collect() + }) + .unwrap_or_default() + } + + /// Get concept routes matching a keyword. + pub async fn concept_routes(&self, keyword: &str) -> Vec<ConceptRouteInfo> { + self.doc + .query_routes + .as_ref() + .map(|table| { + table + .routes_for_concept(keyword) + .into_iter() + .map(|t| RouteTargetInfo { + node_id: self.id_to_u64(t.node_id), + relevance: t.relevance, + reason: t.reason.clone(), + }) + .collect::<Vec<_>>() + }) + .unwrap_or_default() + .into_iter() + .map(|targets| ConceptRouteInfo { + concept: keyword.to_string(), + targets, + }) + .collect() + .into_iter() + .take(1) + .collect() + } + + /// Get reasoning chains involving a specific node. + pub async fn chains_for(&self, node_id: u64) -> Vec<ChainInfo> { + let nid = match self.u64_to_id(node_id) { + Some(id) => id, + None => return Vec::new(), + }; + self.doc + .chain_index + .as_ref() + .map(|idx| { + idx.chains_for(nid) + .into_iter() + .map(|c| ChainInfo { + premises: c.premises.iter().map(|&id| self.id_to_u64(id)).collect(), + conclusions: c.conclusions.iter().map(|&id| self.id_to_u64(id)).collect(), + chain_type: c.chain_type.as_str().to_string(), + summary: c.summary.clone(), + }) + .collect() + }) + .unwrap_or_default() + } + + /// Get overlapping nodes for a specific node. + pub async fn overlaps_for(&self, node_id: u64) -> Vec<OverlapInfo> { + let nid = match self.u64_to_id(node_id) { + Some(id) => id, + None => return Vec::new(), + }; + self.doc + .content_overlap + .as_ref() + .map(|map| { + map.overlapping_nodes(nid) + .into_iter() + .map(|(id, sim, ot)| OverlapInfo { + node_a: node_id, + node_b: self.id_to_u64(id), + similarity: sim, + overlap_type: match ot { + vectorless_document::OverlapType::Duplicate => "duplicate", + vectorless_document::OverlapType::Subset => "subset", + vectorless_document::OverlapType::Summary => "summary", + } + .to_string(), + }) + .collect() + }) + .unwrap_or_default() + } + + /// Get evidence quality score for a specific node. + pub async fn evidence_score_for(&self, node_id: u64) -> Option<EvidenceScoreInfo> { + let nid = self.u64_to_id(node_id)?; + self.doc.evidence_scores.as_ref()?.get(nid).map(|s| EvidenceScoreInfo { + node_id, + density: s.density, + data_richness: s.data_richness, + specificity: s.specificity, + composite: s.composite(), + }) + } + + /// Get all evidence scores, sorted by composite descending. + pub async fn evidence_scores_ranked(&self) -> Vec<EvidenceScoreInfo> { + self.doc + .evidence_scores + .as_ref() + .map(|map| { + map.ranked_nodes() + .into_iter() + .filter_map(|(nid, composite)| { + map.get(nid).map(|s| EvidenceScoreInfo { + node_id: self.id_to_u64(nid), + density: s.density, + data_richness: s.data_richness, + specificity: s.specificity, + composite, + }) + }) + .collect() + }) + .unwrap_or_default() + } } diff --git a/vectorless-core/vectorless-primitives/src/types.rs b/vectorless-core/vectorless-primitives/src/types.rs index 4bc3712..5d2ae95 100644 --- a/vectorless-core/vectorless-primitives/src/types.rs +++ b/vectorless-core/vectorless-primitives/src/types.rs @@ -185,3 +185,68 @@ pub struct ConceptInfo { /// Which sections this concept appears in. pub sections: Vec<String>, } + +// --------------------------------------------------------------------------- +// Agent acceleration types +// --------------------------------------------------------------------------- + +/// A scored target node from the query routing table. +#[derive(Debug, Clone)] +pub struct RouteTargetInfo { + /// Target node ID. + pub node_id: u64, + /// Relevance score (0.0–1.0). + pub relevance: f64, + /// Human-readable reason for this route. + pub reason: String, +} + +/// A concept-based route from the query routing table. +#[derive(Debug, Clone)] +pub struct ConceptRouteInfo { + /// Concept keyword. + pub concept: String, + /// Scored target nodes. + pub targets: Vec<RouteTargetInfo>, +} + +/// A reasoning chain connecting document sections. +#[derive(Debug, Clone)] +pub struct ChainInfo { + /// Premise node IDs. + pub premises: Vec<u64>, + /// Conclusion node IDs. + pub conclusions: Vec<u64>, + /// Chain type label. + pub chain_type: String, + /// Human-readable summary. + pub summary: String, +} + +/// An overlap entry between two nodes. +#[derive(Debug, Clone)] +pub struct OverlapInfo { + /// First node ID. + pub node_a: u64, + /// Second node ID. + pub node_b: u64, + /// Jaccard similarity score. + pub similarity: f64, + /// Overlap type label. + pub overlap_type: String, +} + +/// Evidence quality score for a single node. +#[derive(Debug, Clone)] +pub struct EvidenceScoreInfo { + /// Node ID. + pub node_id: u64, + /// Information density (0.0–1.0). + pub density: f64, + /// Data richness (0.0–1.0). + pub data_richness: f64, + /// Topic specificity (0.0–1.0). + pub specificity: f64, + /// Weighted composite score. + pub composite: f64, +} diff --git a/vectorless-core/vectorless-py/src/document.rs b/vectorless-core/vectorless-py/src/document.rs index ebe2b14..a2b59f8 100644 --- a/vectorless-core/vectorless-py/src/document.rs +++ b/vectorless-core/vectorless-py/src/document.rs @@ -10,8 +10,9 @@ use pyo3_async_runtimes::tokio::future_into_py; use tokio::sync::Mutex; use vectorless_primitives::{ - CollectedEvidence, ConceptInfo, DocCardInfo, DocumentNavigator, FindResult, MatchResult, - NodeInfo, NodeStats, SectionCardInfo, SectionSummaryInfo, SimilarResult, TocEntry, + ChainInfo, CollectedEvidence, ConceptInfo, ConceptRouteInfo, DocCardInfo, DocumentNavigator, + EvidenceScoreInfo, FindResult, MatchResult, NodeInfo, NodeStats, OverlapInfo, + RouteTargetInfo, SectionCardInfo, SectionSummaryInfo, SimilarResult, TocEntry, TopicEntryInfo, WordCount, }; @@ -345,6 +346,121 @@ impl PyDocument { }) } + // ── Agent acceleration ──────────────────────────────────────────────── + + /// Get all intent routes from the query routing table. + fn intent_routes<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let nav = nav.lock().await; + Ok(nav + .intent_routes() + .await + .into_iter() + .map(PyRouteTarget::from) + .collect::<Vec<_>>()) + }) + } + + /// Get concept routes matching a keyword. + fn concept_routes<'py>( + &self, + py: Python<'py>, + keyword: String, + ) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let nav = nav.lock().await; + Ok(nav + .concept_routes(&keyword) + .await + .into_iter() + .map(PyConceptRoute::from) + .collect::<Vec<_>>()) + }) + } + + /// Get reasoning chains involving a specific node. + fn chains_for<'py>( + &self, + py: Python<'py>, + node_id: String, + ) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let num = node_id + .strip_prefix('n') + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .parse::<u64>() + .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + let nav = nav.lock().await; + Ok(nav + .chains_for(num) + .await + .into_iter() + .map(PyChainInfo::from) + .collect::<Vec<_>>()) + }) + } + + /// Get overlapping nodes for a specific node. + fn overlaps_for<'py>( + &self, + py: Python<'py>, + node_id: String, + ) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let num = node_id + .strip_prefix('n') + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .parse::<u64>() + .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + let nav = nav.lock().await; + Ok(nav + .overlaps_for(num) + .await + .into_iter() + .map(PyOverlapInfo::from) + .collect::<Vec<_>>()) + }) + } + + /// Get evidence quality score for a specific node. + fn evidence_score<'py>( + &self, + py: Python<'py>, + node_id: String, + ) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let num = node_id + .strip_prefix('n') + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .parse::<u64>() + .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + let nav = nav.lock().await; + Ok(nav + .evidence_score_for(num) + .await + .map(PyEvidenceScore::from)) + }) + } + + /// Get all evidence scores ranked by composite score. + fn evidence_scores_ranked<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> { + let nav = Arc::clone(&self.inner); + future_into_py(py, async move { + let nav = nav.lock().await; + Ok(nav + .evidence_scores_ranked() + .await + .into_iter() + .map(PyEvidenceScore::from) + .collect::<Vec<_>>()) + }) + } + // ── Evidence ──────────────────────────────────────────────────────── /// Explicitly collect evidence from a node. @@ -955,3 +1071,126 @@ impl From<ConceptInfo> for PyConceptInfo { } } } + +// ========================================================================= +// Agent acceleration types +// ========================================================================= + +/// A scored target node from the query routing table. +#[pyclass(name = "RouteTarget", skip_from_py_object)] +#[derive(Clone)] +pub struct PyRouteTarget { + #[pyo3(get)] + pub node_id: String, + #[pyo3(get)] + pub relevance: f64, + #[pyo3(get)] + pub reason: String, +} + +impl From<RouteTargetInfo> for PyRouteTarget { + fn from(v: RouteTargetInfo) -> Self { + Self { + node_id: format!("n{}", v.node_id), + relevance: v.relevance, + reason: v.reason, + } + } +} + +/// A concept-based route from the query routing table. +#[pyclass(name = "ConceptRoute", skip_from_py_object)] +#[derive(Clone)] +pub struct PyConceptRoute { + #[pyo3(get)] + pub concept: String, + #[pyo3(get)] + pub targets: Vec<PyRouteTarget>, +} + +impl From<ConceptRouteInfo> for PyConceptRoute { + fn from(v: ConceptRouteInfo) -> Self { + Self { + concept: v.concept, + targets: v.targets.into_iter().map(PyRouteTarget::from).collect(), + } + } +} + +/// A reasoning chain connecting document sections. +#[pyclass(name = "ChainInfo", skip_from_py_object)] +#[derive(Clone)] +pub struct PyChainInfo { + #[pyo3(get)] + pub premises: Vec<String>, + #[pyo3(get)] + pub conclusions: Vec<String>, + #[pyo3(get)] + pub chain_type: String, + #[pyo3(get)] + pub summary: String, +} + +impl From<ChainInfo> for PyChainInfo { + fn from(v: ChainInfo) -> Self { + Self { + premises: v.premises.into_iter().map(|id| format!("n{id}")).collect(), + conclusions: v.conclusions.into_iter().map(|id| format!("n{id}")).collect(), + chain_type: v.chain_type, + summary: v.summary, + } + } +} + +/// An overlap entry between two nodes. +#[pyclass(name = "OverlapInfo", skip_from_py_object)] +#[derive(Clone)] +pub struct PyOverlapInfo { + #[pyo3(get)] + pub node_a: String, + #[pyo3(get)] + pub node_b: String, + #[pyo3(get)] + pub similarity: f64, + #[pyo3(get)] + pub overlap_type: String, +} + +impl From<OverlapInfo> for PyOverlapInfo { + fn from(v: OverlapInfo) -> Self { + Self { + node_a: format!("n{}", v.node_a), + node_b: format!("n{}", v.node_b), + similarity: v.similarity, + overlap_type: v.overlap_type, + } + } +} + +/// Evidence quality score for a node. +#[pyclass(name = "EvidenceScore", skip_from_py_object)] +#[derive(Clone)] +pub struct PyEvidenceScore { + #[pyo3(get)] + pub node_id: String, + #[pyo3(get)] + pub density: f64, + #[pyo3(get)] + pub data_richness: f64, + #[pyo3(get)] + pub specificity: f64, + #[pyo3(get)] + pub composite: f64, +} + +impl From<EvidenceScoreInfo> for PyEvidenceScore { + fn from(v: EvidenceScoreInfo) -> Self { + Self { + node_id: format!("n{}", v.node_id), + density: v.density, + data_richness: v.data_richness, + specificity: v.specificity, + composite: v.composite, + } + } +} diff --git a/vectorless-core/vectorless-py/src/lib.rs b/vectorless-core/vectorless-py/src/lib.rs index 5c3c0ad..39a419b 100644 --- a/vectorless-core/vectorless-py/src/lib.rs +++ b/vectorless-core/vectorless-py/src/lib.rs @@ -14,9 +14,10 @@ mod metrics; use config::PyConfig; use document::{ - PyCollectedEvidence, PyConcept, PyConceptInfo, PyDocCard, PyDocument, PyDocumentInfo, - PyFindResult, PyMatchResult, PyNodeInfo, PyNodeStats, PySectionCard, PySectionSummary, - PySimilarResult, PyTocEntry, PyTopicEntry, PyWordCount, + PyChainInfo, PyCollectedEvidence, PyConcept, PyConceptInfo, PyConceptRoute, PyDocCard, + PyDocument, PyDocumentInfo, PyEvidenceScore, PyFindResult, PyMatchResult, PyNodeInfo, + PyNodeStats, PyOverlapInfo, PyRouteTarget, PySectionCard, PySectionSummary, PySimilarResult, + PyTocEntry, PyTopicEntry, PyWordCount, }; use engine::PyEngine; use error::VectorlessError; @@ -44,6 +45,11 @@ fn _vectorless(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::<PySectionCard>()?; m.add_class::<PyDocCard>()?; m.add_class::<PyConceptInfo>()?; + m.add_class::<PyRouteTarget>()?; + m.add_class::<PyConceptRoute>()?; + m.add_class::<PyChainInfo>()?; + m.add_class::<PyOverlapInfo>()?; + m.add_class::<PyEvidenceScore>()?; m.add_class::<PyDocumentGraphNode>()?; m.add_class::<PyDocumentGraph>()?; m.add_class::<PyGraphEdge>()?; diff --git a/vectorless/ask/protocols.py b/vectorless/ask/protocols.py index 9447d63..ed68da5 100644 --- a/vectorless/ask/protocols.py +++ b/vectorless/ask/protocols.py @@ -130,6 +130,31 @@ async def wc(self, node_id: str | None = None) -> Any: """Return word count info for a node.""" ... + # Agent acceleration (pre-computed from compile pipeline) + async def intent_routes(self) -> list[Any]: + """Get all intent routes from the query routing table.""" + ... + + async def concept_routes(self, keyword: str) -> list[Any]: + """Get concept routes matching a keyword.""" + ... + + async def chains_for(self, node_id: str) -> list[Any]: + """Get reasoning chains involving a specific node.""" + ... + + async def overlaps_for(self, node_id: str) -> list[Any]: + """Get overlapping nodes for a specific node.""" + ... + + async def evidence_score(self, node_id: str) -> Any: + """Get evidence quality score for a specific node.""" + ... + + async def evidence_scores_ranked(self) -> list[Any]: + """Get all evidence scores ranked by composite score.""" + ... + # --------------------------------------------------------------------------- # Callable protocols diff --git a/vectorless/ask/worker/agent.py b/vectorless/ask/worker/agent.py index 4910fd7..30f4921 100644 --- a/vectorless/ask/worker/agent.py +++ b/vectorless/ask/worker/agent.py @@ -212,28 +212,73 @@ async def run(self) -> WorkerOutput: return state.into_worker_output(doc_name) async def _build_keyword_hints(self, doc: NavigableDocument, query: str) -> str: - """Build keyword hints from the document's reasoning index.""" + """Build keyword hints from the document's reasoning index and acceleration data.""" keywords = extract_keywords(query) if not keywords: return "" hints = [] - for kw in keywords[:5]: # limit keywords + + # Keyword index matches + for kw in keywords[:5]: try: entries = await doc.keyword_entries(kw) for entry in entries[:3]: title = await doc.node_title(entry.node_id) hints.append( - f" - '{kw}' \u2192 {title} (weight {entry.weight:.2f})" + f" - '{kw}' → {title} (weight {entry.weight:.2f})" ) except Exception: pass - if not hints: + # Concept routes (pre-computed by RoutePass) + route_hints = [] + for kw in keywords[:5]: + try: + routes = await doc.concept_routes(kw) + for route in routes[:1]: + for target in route.targets[:3]: + title = await doc.node_title(target.node_id) + route_hints.append( + f" - [{route.concept}] {title} " + f"(relevance {target.relevance:.2f}: {target.reason})" + ) + except Exception: + pass + + # Top evidence scores (pre-computed by ScorePass) + score_hints = [] + try: + scores = await doc.evidence_scores_ranked() + for s in scores[:5]: + title = await doc.node_title(s.node_id) + score_hints.append( + f" - {title} (score {s.composite:.2f}: " + f"density={s.density:.2f} richness={s.data_richness:.2f})" + ) + except Exception: + pass + + sections = [] + if hints: + sections.append( + "Keyword matches (use find <keyword> to jump directly):\n" + + "\n".join(hints) + ) + if route_hints: + sections.append( + "Pre-computed routes:\n" + "\n".join(route_hints) + ) + if score_hints: + sections.append( + "High-value evidence nodes:\n" + "\n".join(score_hints) + ) + + if not sections: return "" - return "Keyword matches (use find <keyword> to jump directly):\n" + "\n".join(hints) + "\n" + return "\n\n".join(sections) + "\n" async def _generate_plan( self, From e4ef4626b1a42638e7ddc3a38893ba3f55c5aaa4 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:47:32 +0800 Subject: [PATCH 27/30] docs(architecture): rename index pipeline to compile pipeline with enhanced phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename "Index Pipeline" to "Compile Pipeline" to better reflect the compilation nature of the process - Replace stage-based terminology with phase-based structure (Frontend → Analysis → Transform → Backend) - Add detailed documentation for new backend passes including Route, Chain, Overlap, Score, and Verify - Document agent acceleration data and how it guides worker navigation - Update references from "indexing" to "compilation" throughout the architecture documentation feat(navigator): optimize concept routes lookup with early termination - Refactor concept_routes method to return early when no targets found - Limit results to one ConceptRouteInfo instead of collecting all matches - Improve performance by avoiding unnecessary collection operations fix(python): ensure error messages are properly converted to strings - Convert string literals to owned String objects in VectorlessError creation - Maintain consistency in error message handling across Python bindings - Prevent potential issues with string ownership in error contexts --- docs/docs/architecture.mdx | 56 +++++++++++++------ .../src/navigator_search.rs | 25 +++++---- vectorless-core/vectorless-py/src/document.rs | 12 ++-- 3 files changed, 57 insertions(+), 36 deletions(-) diff --git a/docs/docs/architecture.mdx b/docs/docs/architecture.mdx index 6986771..ed72091 100644 --- a/docs/docs/architecture.mdx +++ b/docs/docs/architecture.mdx @@ -10,7 +10,7 @@ Vectorless transforms documents into hierarchical semantic trees and uses LLM-po ```text ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ Document │────▶│ Index │────▶│ Storage │ +│ Document │────▶│ Compile │────▶│ Storage │ │ (PDF/MD) │ │ Pipeline │ │ (Disk) │ └──────────────┘ └──────────────┘ └──────┬───────┘ │ @@ -20,23 +20,42 @@ Vectorless transforms documents into hierarchical semantic trees and uses LLM-po └──────────────┘ └──────────────┘ ``` -## Index Pipeline +## Compile Pipeline -The indexing pipeline processes documents through ordered stages: +The compile pipeline processes documents through four phases (Frontend → Analysis → Transform → Backend), each containing independent passes: -| Stage | Priority | Description | -|-------|----------|-------------| -| **Parse** | 10 | Parse document into raw nodes (Markdown headings, PDF pages) | -| **Build** | 20 | Construct arena-based tree with thinning and content merge | -| **Validate** | 22 | Tree integrity checks | -| **Split** | 25 | Split oversized leaf nodes (>4000 tokens) | -| **Enhance** | 30 | Generate LLM summaries (Full, Selective, or Lazy strategy) | -| **Enrich** | 40 | Calculate metadata, page ranges, resolve cross-references | -| **Reasoning Index** | 45 | Build keyword-to-node mappings, synonym expansion, summary shortcuts | -| **Navigation Index** | 50 | Build NavEntry + ChildRoute data for agent navigation | -| **Optimize** | 60 | Final tree optimization | +| Phase | Pass | Priority | Description | +|-------|------|----------|-------------| +| **Frontend** | Parse | 10 | Parse document into raw nodes (Markdown headings, PDF pages) | +| **Frontend** | Build | 20 | Construct arena-based tree with thinning and content merge | +| **Analysis** | Validate | 22 | Tree integrity checks | +| **Transform** | Split | 25 | Split oversized leaf nodes (>4000 tokens) | +| **Transform** | Enhance | 30 | Generate LLM summaries (Full, Selective, or Lazy strategy) | +| **Transform** | Enrich | 40 | Calculate metadata, page ranges, resolve cross-references | +| **Backend** | Reasoning Index | 45 | Build keyword-to-node mappings, synonym expansion, summary shortcuts | +| **Backend** | Concept | 46 | Extract key concepts with section associations | +| **Backend** | Navigation Index | 50 | Build NavEntry + ChildRoute data for agent navigation | +| **Backend** | Route | 52 | Build query routing table (intent routes + concept routes) | +| **Backend** | Chain | 54 | Build reasoning chains from cross-references | +| **Backend** | Overlap | 56 | Detect content overlap between nodes (Jaccard similarity) | +| **Backend** | Score | 58 | Compute evidence quality scores (density, richness, specificity) | +| **Backend** | Verify | 59 | Validate compiled output integrity | +| **Backend** | Optimize | 60 | Final tree optimization | -Each stage is independently configurable. The pipeline supports incremental re-indexing via content fingerprinting. +Each pass is independently configurable. The pipeline supports incremental recompilation via content fingerprinting and checkpoint/resume for fault tolerance. + +### Agent Acceleration Data + +The backend passes produce pre-computed acceleration data used by Workers during retrieval: + +| Data | Pass | Purpose | +|------|------|---------| +| **QueryRoutingTable** | Route | Maps intents and concepts to scored target nodes | +| **ChainIndex** | Chain | Connects sections via reasoning chains (elaboration, supporting, etc.) | +| **ContentOverlapMap** | Overlap | Flags duplicate/subset/summary overlap between nodes | +| **EvidenceScoreMap** | Score | Ranks nodes by information density and data richness | + +This data is injected as Phase 1.5 hints into the Worker's navigation plan, allowing the LLM to make informed routing decisions without additional navigation steps. ## Tree Structure @@ -104,7 +123,7 @@ When the user specifies document IDs directly, the Orchestrator skips the analys Each Worker navigates a single document's tree to collect evidence through a command-based loop: 1. **Bird's-eye** — `ls` the root for an overview -2. **Plan** — LLM generates a navigation plan based on keyword index hits +2. **Plan** — LLM generates a navigation plan based on keyword index hits + acceleration data 3. **Navigate** — Loop: LLM selects command → execute → observe result → repeat 4. **Return** — Collected evidence only — no answer synthesis @@ -132,6 +151,7 @@ Workers prioritize keyword-based navigation over manual exploration: 1. When keyword index hits are available, Workers use `find` with the exact keyword to jump directly to relevant sections 2. Workers use `ls` when no keyword hints exist or when discovering unknown structure 3. Workers use `findtree` when the section title pattern is known but not the exact name +4. Pre-computed acceleration data (routes, scores, chains) is injected as Phase 1.5 hints to guide the Worker toward high-value nodes #### Dynamic Re-planning @@ -149,11 +169,11 @@ The system returns raw evidence text — no LLM synthesis or paraphrasing. This ## DocCard Catalog -When multiple documents are indexed, Vectorless maintains a lightweight `catalog.bin` containing DocCard metadata for each document. This allows the Orchestrator to analyze and select relevant documents without loading the full document trees — a significant optimization for workspaces with many documents. +When multiple documents are compiled, Vectorless maintains a lightweight `catalog.bin` containing DocCard metadata for each document. This allows the Orchestrator to analyze and select relevant documents without loading the full document trees — a significant optimization for workspaces with many documents. ## Cross-Document Graph -When multiple documents are indexed, Vectorless automatically builds a relationship graph based on shared keywords and Jaccard similarity. The graph is constructed as a background task after each indexing operation. +When multiple documents are compiled, Vectorless automatically builds a relationship graph based on shared keywords and Jaccard similarity. The graph is constructed as a background task after each compilation operation. ## Zero Infrastructure diff --git a/vectorless-core/vectorless-primitives/src/navigator_search.rs b/vectorless-core/vectorless-primitives/src/navigator_search.rs index f505ea1..4d43581 100644 --- a/vectorless-core/vectorless-primitives/src/navigator_search.rs +++ b/vectorless-core/vectorless-primitives/src/navigator_search.rs @@ -276,7 +276,8 @@ impl DocumentNavigator { /// Get concept routes matching a keyword. pub async fn concept_routes(&self, keyword: &str) -> Vec<ConceptRouteInfo> { - self.doc + let targets: Vec<RouteTargetInfo> = self + .doc .query_routes .as_ref() .map(|table| { @@ -288,18 +289,18 @@ impl DocumentNavigator { relevance: t.relevance, reason: t.reason.clone(), }) - .collect::<Vec<_>>() - }) - .unwrap_or_default() - .into_iter() - .map(|targets| ConceptRouteInfo { - concept: keyword.to_string(), - targets, + .collect() }) - .collect() - .into_iter() - .take(1) - .collect() + .unwrap_or_default(); + + if targets.is_empty() { + return Vec::new(); + } + + vec![ConceptRouteInfo { + concept: keyword.to_string(), + targets, + }] } /// Get reasoning chains involving a specific node. diff --git a/vectorless-core/vectorless-py/src/document.rs b/vectorless-core/vectorless-py/src/document.rs index a2b59f8..73a62b1 100644 --- a/vectorless-core/vectorless-py/src/document.rs +++ b/vectorless-core/vectorless-py/src/document.rs @@ -390,9 +390,9 @@ impl PyDocument { future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? .parse::<u64>() - .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; Ok(nav .chains_for(num) @@ -413,9 +413,9 @@ impl PyDocument { future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? .parse::<u64>() - .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; Ok(nav .overlaps_for(num) @@ -436,9 +436,9 @@ impl PyDocument { future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'", "navigation"))? + .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? .parse::<u64>() - .map_err(|_| VectorlessError::new("Invalid NodeId", "navigation"))?; + .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; Ok(nav .evidence_score_for(num) From 88b1b61e7d1265e9762ee6aa508879e36b75810f Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:51:44 +0800 Subject: [PATCH 28/30] docs(markdown): update example code snippets to use correct module paths - Replace old module path `vectorless::parser::markdown` with new path `vectorless_compiler::parse::markdown::config::MarkdownConfig` - Update examples to use proper crate structure and remove outdated configuration methods - Change rust code blocks to `rust,ignore` to prevent compilation errors --- .../vectorless-compiler/src/parse/markdown/config.rs | 10 ++-------- .../vectorless-compiler/src/parse/markdown/mod.rs | 6 ++---- .../vectorless-compiler/src/parse/markdown/parser.rs | 9 ++------- .../vectorless-compiler/src/parse/pdf/mod.rs | 7 ++----- .../vectorless-compiler/src/parse/toc/processor.rs | 12 ++++-------- 5 files changed, 12 insertions(+), 32 deletions(-) diff --git a/vectorless-core/vectorless-compiler/src/parse/markdown/config.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/config.rs index 7a013f5..32d2972 100644 --- a/vectorless-core/vectorless-compiler/src/parse/markdown/config.rs +++ b/vectorless-core/vectorless-compiler/src/parse/markdown/config.rs @@ -9,18 +9,12 @@ /// /// # Example /// -/// ```rust -/// use vectorless::parser::markdown::MarkdownConfig; +/// ```rust,ignore +/// use vectorless_compiler::parse::markdown::config::MarkdownConfig; /// /// // Default GFM configuration /// let config = MarkdownConfig::default(); /// -/// // Strict CommonMark -/// let config = MarkdownConfig::commonmark(); -/// -/// // Documentation-focused -/// let config = MarkdownConfig::documentation(); -/// /// // Custom configuration /// let config = MarkdownConfig { /// max_heading_level: 3, diff --git a/vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs index 168f364..e384f52 100644 --- a/vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs +++ b/vectorless-core/vectorless-compiler/src/parse/markdown/mod.rs @@ -15,12 +15,10 @@ //! //! # Example //! -//! ```rust -//! use vectorless::parser::markdown::{MarkdownParser, MarkdownConfig}; +//! ```rust,ignore +//! use vectorless_compiler::parse::markdown::MarkdownParser; //! //! let parser = MarkdownParser::new(); -//! // or with custom config: -//! // let parser = MarkdownParser::with_config(MarkdownConfig::gfm()); //! ``` mod config; diff --git a/vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs b/vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs index b8980f7..2b6e431 100644 --- a/vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs +++ b/vectorless-core/vectorless-compiler/src/parse/markdown/parser.rs @@ -26,18 +26,13 @@ use super::frontmatter; /// /// # Example /// -/// ```rust -/// use vectorless::parser::markdown::MarkdownParser; -/// use vectorless::parser::DocumentParser; +/// ```rust,ignore +/// use vectorless_compiler::parse::markdown::MarkdownParser; /// -/// # #[tokio::main] -/// # async fn main() -> vectorless::Result<()> { /// let parser = MarkdownParser::new(); /// let result = parser.parse("# Title\n\nContent").await?; /// /// println!("Found {} nodes", result.node_count()); -/// # Ok(()) -/// # } /// ``` #[derive(Debug, Clone)] pub struct MarkdownParser { diff --git a/vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs b/vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs index dc92da8..3226e44 100644 --- a/vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs +++ b/vectorless-core/vectorless-compiler/src/parse/pdf/mod.rs @@ -9,11 +9,10 @@ //! //! # Example //! -//! ```rust,no_run -//! use vectorless::parser::pdf::{PdfParser, PdfPage}; +//! ```rust,ignore +//! use vectorless_compiler::parse::pdf::{PdfParser, PdfPage}; //! use std::path::Path; //! -//! # fn main() -> vectorless::Result<()> { //! let parser = PdfParser::new(); //! let result = parser.parse_file(Path::new("document.pdf"))?; //! @@ -21,8 +20,6 @@ //! for page in &result.pages { //! println!("Page {}: {} tokens", page.number, page.token_count); //! } -//! # Ok(()) -//! # } //! ``` mod parser; diff --git a/vectorless-core/vectorless-compiler/src/parse/toc/processor.rs b/vectorless-core/vectorless-compiler/src/parse/toc/processor.rs index bc8d52a..c1cb774 100644 --- a/vectorless-core/vectorless-compiler/src/parse/toc/processor.rs +++ b/vectorless-core/vectorless-compiler/src/parse/toc/processor.rs @@ -94,14 +94,12 @@ impl Default for TocProcessorConfig { /// /// # Example /// -/// ```rust,no_run -/// use vectorless::parser::toc::TocProcessor; -/// use vectorless::parser::pdf::PdfParser; +/// ```rust,ignore +/// use vectorless_compiler::parse::toc::processor::TocProcessor; +/// use vectorless_compiler::parse::pdf::PdfParser; /// -/// # #[tokio::main] -/// # async fn main() -> vectorless::Result<()> { /// let pdf_parser = PdfParser::new(); -/// let result = pdf_parser.parse_file("document.pdf".as_ref()).await?; +/// let result = pdf_parser.parse_file("document.pdf".as_ref())?; /// /// let processor = TocProcessor::new(); /// let entries = processor.process(&result.pages).await?; @@ -109,8 +107,6 @@ impl Default for TocProcessorConfig { /// for entry in &entries { /// println!("{} - Page {:?}", entry.title, entry.physical_page); /// } -/// # Ok(()) -/// # } /// ``` pub struct TocProcessor { config: TocProcessorConfig, From d8664b6dc04e285c30acfdd6d2ac455ff2510251 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:52:19 +0800 Subject: [PATCH 29/30] refactor(compiler): reorganize module declarations and imports - Move pipeline module declaration after passes module - Reorder re-exported types from pipeline and config modules - Adjust import order for consistency across multiple files refactor(pipeline): reorder context field exports - Move ChainIndex before Concept in Document imports - Remove unnecessary blank lines in context definitions - Clean up unused imports in various pipeline modules refactor(passes): reorganize pass module structure - Move chain module before reasoning in backend - Reorder imports and module declarations consistently - Move parse module after build in frontend - Move split module after enrich in transform refactor(engine): clean up imports and declarations - Reorder compiler imports in engine module - Simplify import grouping in indexer module - Move engine module declaration position refactor(storage): remove redundant documentation comment style: format function calls and assertions with proper line breaks - Wrap long assertion statements in test cases - Format method chaining for better readability - Break down complex expressions across multiple lines --- .../vectorless-compiler/src/lib.rs | 6 +- .../src/passes/analysis/enhance.rs | 2 +- .../src/passes/analysis/mod.rs | 4 +- .../src/passes/backend/chain.rs | 60 +++++++++------ .../src/passes/backend/mod.rs | 16 ++-- .../src/passes/backend/overlap.rs | 31 +++++--- .../src/passes/backend/reasoning.rs | 3 +- .../src/passes/backend/route.rs | 39 ++++++---- .../src/passes/backend/score.rs | 75 ++++++++++++------- .../src/passes/frontend/build.rs | 2 +- .../src/passes/frontend/mod.rs | 4 +- .../src/passes/frontend/parse.rs | 2 +- .../vectorless-compiler/src/passes/mod.rs | 17 +++-- .../src/passes/transform/mod.rs | 4 +- .../src/passes/transform/split.rs | 2 +- .../src/pipeline/context.rs | 5 +- .../src/pipeline/executor.rs | 8 +- .../vectorless-compiler/src/pipeline/mod.rs | 2 +- .../src/pipeline/orchestrator.rs | 2 +- .../vectorless-document/src/evidence.rs | 5 +- .../vectorless-document/src/overlap.rs | 4 +- .../src/compiled_document.rs | 1 - .../vectorless-engine/src/engine.rs | 25 ++++--- .../vectorless-engine/src/indexer.rs | 2 +- vectorless-core/vectorless-engine/src/lib.rs | 4 +- vectorless-core/vectorless-llm/src/lib.rs | 1 - .../vectorless-metrics/src/index.rs | 1 - vectorless-core/vectorless-py/src/document.rs | 52 ++++++------- vectorless-core/vectorless-storage/src/lib.rs | 2 +- .../vectorless-storage/src/persistence.rs | 1 - 30 files changed, 218 insertions(+), 164 deletions(-) diff --git a/vectorless-core/vectorless-compiler/src/lib.rs b/vectorless-core/vectorless-compiler/src/lib.rs index 9425b44..29be8f6 100644 --- a/vectorless-core/vectorless-compiler/src/lib.rs +++ b/vectorless-core/vectorless-compiler/src/lib.rs @@ -60,15 +60,15 @@ pub mod config; pub mod incremental; pub mod parse; -pub mod pipeline; pub mod passes; +pub mod pipeline; pub mod summary; // Re-export main types from pipeline -pub use pipeline::{CompilerInput, CompileMetrics, PipelineExecutor, CompileResult}; +pub use pipeline::{CompileMetrics, CompileResult, CompilerInput, PipelineExecutor}; // Re-export config types -pub use config::{SourceFormat, PipelineOptions, ThinningConfig}; +pub use config::{PipelineOptions, SourceFormat, ThinningConfig}; pub use vectorless_document::ReasoningIndexConfig; // Re-export summary diff --git a/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs index beae745..59b87ba 100644 --- a/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/enhance.rs @@ -17,7 +17,7 @@ use vectorless_llm::memo::{MemoKey, MemoStore}; use vectorless_utils::fingerprint::Fingerprint; use crate::passes::{CompilePass, PassResult}; -use crate::pipeline::{FailurePolicy, CompileContext, StageRetryConfig}; +use crate::pipeline::{CompileContext, FailurePolicy, StageRetryConfig}; use crate::summary::{LlmSummaryGenerator, SummaryGenerator, SummaryStrategy}; /// A node that needs LLM summary generation. diff --git a/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs b/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs index 4a4ea0f..7777249 100644 --- a/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/analysis/mod.rs @@ -3,8 +3,8 @@ //! Analysis passes — semantic validation and LLM enhancement. -mod validate; mod enhance; +mod validate; -pub use validate::ValidatePass; pub use enhance::EnhancePass; +pub use validate::ValidatePass; diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs index 3a36577..3e5a3d9 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/chain.rs @@ -29,11 +29,7 @@ impl ChainPass { } /// Determine chain type from the reference structure. - fn classify_chain( - ref_type: RefType, - source_depth: usize, - target_depth: usize, - ) -> ChainType { + fn classify_chain(ref_type: RefType, source_depth: usize, target_depth: usize) -> ChainType { match ref_type { RefType::Section => { if target_depth > source_depth { @@ -148,14 +144,12 @@ impl CompilePass for ChainPass { let mut result = PassResult::success("chain"); result.duration_ms = duration; - result.metadata.insert( - "chains".to_string(), - serde_json::json!(chain_count), - ); - result.metadata.insert( - "nodes".to_string(), - serde_json::json!(node_count), - ); + result + .metadata + .insert("chains".to_string(), serde_json::json!(chain_count)); + result + .metadata + .insert("nodes".to_string(), serde_json::json!(node_count)); Ok(result) } @@ -214,34 +208,58 @@ mod tests { #[test] fn test_classify_chain_section_elaboration() { - assert_eq!(ChainPass::classify_chain(RefType::Section, 0, 1), ChainType::Elaboration); + assert_eq!( + ChainPass::classify_chain(RefType::Section, 0, 1), + ChainType::Elaboration + ); } #[test] fn test_classify_chain_section_supporting() { - assert_eq!(ChainPass::classify_chain(RefType::Section, 1, 0), ChainType::Supporting); + assert_eq!( + ChainPass::classify_chain(RefType::Section, 1, 0), + ChainType::Supporting + ); } #[test] fn test_classify_chain_appendix() { - assert_eq!(ChainPass::classify_chain(RefType::Appendix, 0, 1), ChainType::Supporting); + assert_eq!( + ChainPass::classify_chain(RefType::Appendix, 0, 1), + ChainType::Supporting + ); } #[test] fn test_classify_chain_table_figure() { - assert_eq!(ChainPass::classify_chain(RefType::Table, 0, 1), ChainType::Supporting); - assert_eq!(ChainPass::classify_chain(RefType::Figure, 0, 1), ChainType::Supporting); - assert_eq!(ChainPass::classify_chain(RefType::Equation, 0, 1), ChainType::Supporting); + assert_eq!( + ChainPass::classify_chain(RefType::Table, 0, 1), + ChainType::Supporting + ); + assert_eq!( + ChainPass::classify_chain(RefType::Figure, 0, 1), + ChainType::Supporting + ); + assert_eq!( + ChainPass::classify_chain(RefType::Equation, 0, 1), + ChainType::Supporting + ); } #[test] fn test_classify_chain_footnote() { - assert_eq!(ChainPass::classify_chain(RefType::Footnote, 0, 2), ChainType::Elaboration); + assert_eq!( + ChainPass::classify_chain(RefType::Footnote, 0, 2), + ChainType::Elaboration + ); } #[test] fn test_classify_chain_unknown() { - assert_eq!(ChainPass::classify_chain(RefType::Unknown, 0, 0), ChainType::Supporting); + assert_eq!( + ChainPass::classify_chain(RefType::Unknown, 0, 0), + ChainType::Supporting + ); } #[tokio::test] diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs index 1591936..2b2502e 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/mod.rs @@ -3,22 +3,22 @@ //! Backend passes — code generation (indexes), verification, and optimization. -mod reasoning; +mod chain; mod concept; mod navigation; -mod verify; mod optimize; -mod route; -mod chain; mod overlap; +mod reasoning; +mod route; mod score; +mod verify; -pub use reasoning::ReasoningPass; +pub use chain::ChainPass; pub use concept::ConceptPass; pub use navigation::NavigationPass; -pub use verify::VerifyPass; pub use optimize::OptimizePass; -pub use route::RoutePass; -pub use chain::ChainPass; pub use overlap::OverlapPass; +pub use reasoning::ReasoningPass; +pub use route::RoutePass; pub use score::ScorePass; +pub use verify::VerifyPass; diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs index 2110f39..c0c12c8 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/overlap.rs @@ -10,7 +10,7 @@ use std::collections::HashSet; use std::time::Instant; use tracing::{debug, info, warn}; -use vectorless_document::{ContentOverlapMap, OverlapEntry, OverlapType, NodeId}; +use vectorless_document::{ContentOverlapMap, NodeId, OverlapEntry, OverlapType}; use vectorless_error::Result; use crate::passes::async_trait; @@ -179,14 +179,12 @@ impl CompilePass for OverlapPass { let mut result = PassResult::success("overlap"); result.duration_ms = duration; - result.metadata.insert( - "overlaps".to_string(), - serde_json::json!(overlap_count), - ); - result.metadata.insert( - "comparisons".to_string(), - serde_json::json!(comparisons), - ); + result + .metadata + .insert("overlaps".to_string(), serde_json::json!(overlap_count)); + result + .metadata + .insert("comparisons".to_string(), serde_json::json!(comparisons)); Ok(result) } @@ -229,19 +227,28 @@ mod tests { #[test] fn test_classify_overlap_duplicate() { - assert_eq!(OverlapPass::classify_overlap(0.95, 100, 100), OverlapType::Duplicate); + assert_eq!( + OverlapPass::classify_overlap(0.95, 100, 100), + OverlapType::Duplicate + ); } #[test] fn test_classify_overlap_subset() { // 0.85 similarity with similar lengths → Subset - assert_eq!(OverlapPass::classify_overlap(0.85, 100, 90), OverlapType::Subset); + assert_eq!( + OverlapPass::classify_overlap(0.85, 100, 90), + OverlapType::Subset + ); } #[test] fn test_classify_overlap_summary() { // 0.85 similarity with very different lengths → Summary - assert_eq!(OverlapPass::classify_overlap(0.85, 100, 30), OverlapType::Summary); + assert_eq!( + OverlapPass::classify_overlap(0.85, 100, 30), + OverlapType::Summary + ); } #[test] diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs b/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs index 5ee8067..eca565f 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/reasoning.rs @@ -451,8 +451,7 @@ mod tests { #[test] fn test_extract_node_keywords() { - let keywords = - ReasoningPass::extract_node_keywords("Introduction to Machine Learning", 2); + let keywords = ReasoningPass::extract_node_keywords("Introduction to Machine Learning", 2); assert!(keywords.contains(&"introduction".to_string())); assert!(keywords.contains(&"machine".to_string())); assert!(keywords.contains(&"learning".to_string())); diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/route.rs b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs index 9abb3db..1c47948 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/route.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/route.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::time::Instant; use tracing::{info, warn}; -use vectorless_document::{DocumentTree, NodeId, QueryRoutingTable, RouteTarget, ConceptRoute}; +use vectorless_document::{ConceptRoute, DocumentTree, NodeId, QueryRoutingTable, RouteTarget}; use vectorless_error::Result; use crate::passes::async_trait; @@ -31,10 +31,7 @@ impl RoutePass { } /// Build route targets from a node's children. - fn build_child_routes( - tree: &DocumentTree, - parent_id: NodeId, - ) -> Vec<RouteTarget> { + fn build_child_routes(tree: &DocumentTree, parent_id: NodeId) -> Vec<RouteTarget> { let children: Vec<_> = tree.children_iter(parent_id).collect(); let mut targets = Vec::with_capacity(children.len()); @@ -56,7 +53,10 @@ impl RoutePass { }; let reason = if hint_count > 0 { - format!("Can answer: {}", node.question_hints.first().unwrap_or(&String::new())) + format!( + "Can answer: {}", + node.question_hints.first().unwrap_or(&String::new()) + ) } else { format!("Section: {}", node.title) }; @@ -69,7 +69,11 @@ impl RoutePass { } // Sort by relevance descending - targets.sort_by(|a, b| b.relevance.partial_cmp(&a.relevance).unwrap_or(std::cmp::Ordering::Equal)); + targets.sort_by(|a, b| { + b.relevance + .partial_cmp(&a.relevance) + .unwrap_or(std::cmp::Ordering::Equal) + }); targets } @@ -101,7 +105,11 @@ impl RoutePass { let mut routes: Vec<ConceptRoute> = concept_map .into_iter() .map(|(concept, mut targets)| { - targets.sort_by(|a, b| b.relevance.partial_cmp(&a.relevance).unwrap_or(std::cmp::Ordering::Equal)); + targets.sort_by(|a, b| { + b.relevance + .partial_cmp(&a.relevance) + .unwrap_or(std::cmp::Ordering::Equal) + }); targets.truncate(10); // limit per concept ConceptRoute { concept, targets } }) @@ -153,7 +161,10 @@ impl CompilePass for RoutePass { }; let all_nodes = tree.traverse(); - info!("[route] Building routing table for {} nodes", all_nodes.len()); + info!( + "[route] Building routing table for {} nodes", + all_nodes.len() + ); let mut table = QueryRoutingTable::new(); @@ -190,16 +201,16 @@ impl CompilePass for RoutePass { intent_count, concept_count, duration, ); - ctx.metrics.record_route(duration, intent_count, concept_count); + ctx.metrics + .record_route(duration, intent_count, concept_count); ctx.query_routes = Some(table); let mut result = PassResult::success("route"); result.duration_ms = duration; - result.metadata.insert( - "intent_routes".to_string(), - serde_json::json!(intent_count), - ); + result + .metadata + .insert("intent_routes".to_string(), serde_json::json!(intent_count)); result.metadata.insert( "concept_routes".to_string(), serde_json::json!(concept_count), diff --git a/vectorless-core/vectorless-compiler/src/passes/backend/score.rs b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs index b93a521..1142b9e 100644 --- a/vectorless-core/vectorless-compiler/src/passes/backend/score.rs +++ b/vectorless-core/vectorless-compiler/src/passes/backend/score.rs @@ -64,13 +64,22 @@ impl ScorePass { } // Lists (-, *, numbered) - let list_markers = content.lines().filter(|l| { - let trimmed = l.trim(); - trimmed.starts_with("- ") - || trimmed.starts_with("* ") - || trimmed.starts_with("+ ") - || (trimmed.len() > 2 && trimmed.as_bytes().first().map(|b| b.is_ascii_digit()).unwrap_or(false) && trimmed.contains('.')) - }).count(); + let list_markers = content + .lines() + .filter(|l| { + let trimmed = l.trim(); + trimmed.starts_with("- ") + || trimmed.starts_with("* ") + || trimmed.starts_with("+ ") + || (trimmed.len() > 2 + && trimmed + .as_bytes() + .first() + .map(|b| b.is_ascii_digit()) + .unwrap_or(false) + && trimmed.contains('.')) + }) + .count(); if list_markers > 0 { score += 0.2 * (list_markers as f64).min(5.0) / 5.0; } @@ -87,13 +96,16 @@ impl ScorePass { // Generic filler words that indicate low specificity let filler = [ - "the", "is", "a", "an", "and", "or", "but", "in", "on", "at", - "to", "for", "of", "with", "this", "that", "it", "from", "by", - "was", "were", "be", "have", "has", "had", "are", "will", "would", + "the", "is", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", + "with", "this", "that", "it", "from", "by", "was", "were", "be", "have", "has", "had", + "are", "will", "would", ]; let filler_set: HashSet<&str> = filler.iter().copied().collect(); - let filler_count = words.iter().filter(|w| filler_set.contains(w.to_lowercase().as_str())).count() as f64; + let filler_count = words + .iter() + .filter(|w| filler_set.contains(w.to_lowercase().as_str())) + .count() as f64; let total = words.len() as f64; // Higher ratio of non-filler = higher specificity @@ -143,7 +155,10 @@ impl CompilePass for ScorePass { let leaves = tree.leaves(); let mut score_map = EvidenceScoreMap::new(); - info!("[score] Computing evidence scores for {} leaf nodes", leaves.len()); + info!( + "[score] Computing evidence scores for {} leaf nodes", + leaves.len() + ); for &node_id in &leaves { let node = match tree.get(node_id) { @@ -160,11 +175,14 @@ impl CompilePass for ScorePass { let data_richness = Self::compute_data_richness(content); let specificity = Self::compute_specificity(content); - score_map.insert(node_id, EvidenceScore { - density, - data_richness, - specificity, - }); + score_map.insert( + node_id, + EvidenceScore { + density, + data_richness, + specificity, + }, + ); } let scored_count = score_map.len(); @@ -187,10 +205,9 @@ impl CompilePass for ScorePass { let mut result = PassResult::success("score"); result.duration_ms = duration; - result.metadata.insert( - "scored_nodes".to_string(), - serde_json::json!(scored_count), - ); + result + .metadata + .insert("scored_nodes".to_string(), serde_json::json!(scored_count)); result.metadata.insert( "avg_density".to_string(), serde_json::json!(format!("{:.3}", avg_density)), @@ -248,8 +265,12 @@ mod tests { #[test] fn test_compute_data_richness_plain() { - let score = ScorePass::compute_data_richness("just some plain text without any structured data"); - assert!((score - 0.0).abs() < f64::EPSILON, "Plain text should have low richness"); + let score = + ScorePass::compute_data_richness("just some plain text without any structured data"); + assert!( + (score - 0.0).abs() < f64::EPSILON, + "Plain text should have low richness" + ); } #[test] @@ -260,14 +281,18 @@ mod tests { #[test] fn test_compute_specificity_high() { // Lots of technical terms, few filler words - let s = ScorePass::compute_specificity("HashMap NodeId DocumentTree CompileContext PipelineExecutor"); + let s = ScorePass::compute_specificity( + "HashMap NodeId DocumentTree CompileContext PipelineExecutor", + ); assert!(s > 0.8, "Technical content should have high specificity"); } #[test] fn test_compute_specificity_low() { // All filler words - let s = ScorePass::compute_specificity("the is a an and or but in on at to for of with this that"); + let s = ScorePass::compute_specificity( + "the is a an and or but in on at to for of with this that", + ); assert!(s < 0.3, "Filler content should have low specificity"); } diff --git a/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs index 2391b98..fa5d4c8 100644 --- a/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/build.rs @@ -12,8 +12,8 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use crate::passes::{CompilePass, PassResult}; use crate::ThinningConfig; +use crate::passes::{CompilePass, PassResult}; use crate::pipeline::CompileContext; /// Build stage - constructs a tree from raw nodes. diff --git a/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs index dbf893f..75b237d 100644 --- a/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/mod.rs @@ -3,8 +3,8 @@ //! Frontend passes — parse document into AST. -mod parse; mod build; +mod parse; -pub use parse::ParsePass; pub use build::BuildPass; +pub use parse::ParsePass; diff --git a/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs b/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs index 81ad688..d757bca 100644 --- a/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs +++ b/vectorless-core/vectorless-compiler/src/passes/frontend/parse.rs @@ -10,8 +10,8 @@ use tracing::{debug, info}; use vectorless_document::DocumentFormat; use vectorless_error::Result; -use crate::passes::{CompilePass, PassResult}; use crate::SourceFormat; +use crate::passes::{CompilePass, PassResult}; use crate::pipeline::{CompileContext, CompilerInput}; /// Parse stage - extracts raw nodes from documents. diff --git a/vectorless-core/vectorless-compiler/src/passes/mod.rs b/vectorless-core/vectorless-compiler/src/passes/mod.rs index 3f1b833..991cd41 100644 --- a/vectorless-core/vectorless-compiler/src/passes/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/mod.rs @@ -9,18 +9,21 @@ //! - **Transform** — IR-level tree restructuring (`split`, `enrich`) //! - **Backend** — Index generation, verification, and optimization -pub mod frontend; pub mod analysis; -pub mod transform; pub mod backend; +pub mod frontend; +pub mod transform; // Re-export all passes from submodules -pub use frontend::{ParsePass, BuildPass}; -pub use analysis::{ValidatePass, EnhancePass}; -pub use transform::{SplitPass, EnrichPass}; -pub use backend::{ReasoningPass, ConceptPass, NavigationPass, VerifyPass, OptimizePass, RoutePass, ChainPass, OverlapPass, ScorePass}; +pub use analysis::{EnhancePass, ValidatePass}; +pub use backend::{ + ChainPass, ConceptPass, NavigationPass, OptimizePass, OverlapPass, ReasoningPass, RoutePass, + ScorePass, VerifyPass, +}; +pub use frontend::{BuildPass, ParsePass}; +pub use transform::{EnrichPass, SplitPass}; -use super::pipeline::{FailurePolicy, CompileContext, PassResult}; +use super::pipeline::{CompileContext, FailurePolicy, PassResult}; pub use async_trait::async_trait; use vectorless_error::Result; diff --git a/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs b/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs index 7355888..65abc48 100644 --- a/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs +++ b/vectorless-core/vectorless-compiler/src/passes/transform/mod.rs @@ -3,8 +3,8 @@ //! Transform passes — IR-level tree restructuring and enrichment. -mod split; mod enrich; +mod split; -pub use split::SplitPass; pub use enrich::EnrichPass; +pub use split::SplitPass; diff --git a/vectorless-core/vectorless-compiler/src/passes/transform/split.rs b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs index ce013e9..7ff4cd9 100644 --- a/vectorless-core/vectorless-compiler/src/passes/transform/split.rs +++ b/vectorless-core/vectorless-compiler/src/passes/transform/split.rs @@ -10,8 +10,8 @@ use vectorless_document::{DocumentTree, NodeId}; use vectorless_error::Result; use vectorless_utils::estimate_tokens; -use crate::passes::{AccessPattern, CompilePass, PassResult, async_trait}; use crate::config::SplitConfig; +use crate::passes::{AccessPattern, CompilePass, PassResult, async_trait}; use crate::pipeline::CompileContext; /// Split stage — breaks oversized leaf nodes into smaller children. diff --git a/vectorless-core/vectorless-compiler/src/pipeline/context.rs b/vectorless-core/vectorless-compiler/src/pipeline/context.rs index a44b4b2..7aab10e 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/context.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/context.rs @@ -8,8 +8,8 @@ use std::path::PathBuf; use crate::parse::{DocumentFormat, RawNode}; use vectorless_document::{ - Concept, DocumentTree, NavigationIndex, NodeId, ReasoningIndex, - QueryRoutingTable, ChainIndex, ContentOverlapMap, EvidenceScoreMap, + ChainIndex, Concept, ContentOverlapMap, DocumentTree, EvidenceScoreMap, NavigationIndex, + NodeId, QueryRoutingTable, ReasoningIndex, }; use vectorless_llm::LlmClient; @@ -258,7 +258,6 @@ pub struct CompileContext { pub concepts: Vec<Concept>, // ── Agent acceleration data (built by backend passes) ── - /// Pre-computed query routing table (built by RoutePass). pub query_routes: Option<QueryRoutingTable>, diff --git a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs index 2416794..d73c3bc 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/executor.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/executor.rs @@ -13,11 +13,11 @@ use vectorless_llm::LlmClient; use super::super::PipelineOptions; use super::super::passes::{ - BuildPass, ChainPass, ConceptPass, EnhancePass, EnrichPass, CompilePass, - NavigationPass, OptimizePass, OverlapPass, ParsePass, ReasoningPass, - RoutePass, ScorePass, SplitPass, ValidatePass, VerifyPass, + BuildPass, ChainPass, CompilePass, ConceptPass, EnhancePass, EnrichPass, NavigationPass, + OptimizePass, OverlapPass, ParsePass, ReasoningPass, RoutePass, ScorePass, SplitPass, + ValidatePass, VerifyPass, }; -use super::context::{CompilerInput, CompileResult}; +use super::context::{CompileResult, CompilerInput}; use super::orchestrator::PipelineOrchestrator; /// Pipeline executor for document compilation. diff --git a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs index d48144e..ec93098 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/mod.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/mod.rs @@ -18,7 +18,7 @@ mod metrics; mod orchestrator; mod policy; -pub use context::{CompileContext, CompilerInput, CompileResult, PassResult}; +pub use context::{CompileContext, CompileResult, CompilerInput, PassResult}; pub use executor::PipelineExecutor; pub use metrics::CompileMetrics; pub use policy::{FailurePolicy, StageRetryConfig}; diff --git a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs index 7981b1f..26c0860 100644 --- a/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs +++ b/vectorless-core/vectorless-compiler/src/pipeline/orchestrator.rs @@ -32,7 +32,7 @@ use vectorless_error::Result; use super::super::PipelineOptions; use super::super::passes::CompilePass; use super::checkpoint::{CheckpointContextData, CheckpointManager, PipelineCheckpoint}; -use super::context::{CompileContext, CompilerInput, CompileResult, PassResult}; +use super::context::{CompileContext, CompileResult, CompilerInput, PassResult}; use super::policy::FailurePolicy; /// Stage entry with metadata for orchestration. diff --git a/vectorless-core/vectorless-document/src/evidence.rs b/vectorless-core/vectorless-document/src/evidence.rs index ca7051f..4582479 100644 --- a/vectorless-core/vectorless-document/src/evidence.rs +++ b/vectorless-core/vectorless-document/src/evidence.rs @@ -70,7 +70,10 @@ impl EvidenceScoreMap { /// Get the composite score for a node, defaulting to 0.0. pub fn composite_for(&self, node_id: NodeId) -> f64 { - self.scores.get(&node_id).map(|s| s.composite()).unwrap_or(0.0) + self.scores + .get(&node_id) + .map(|s| s.composite()) + .unwrap_or(0.0) } /// Get nodes sorted by composite score (highest first). diff --git a/vectorless-core/vectorless-document/src/overlap.rs b/vectorless-core/vectorless-document/src/overlap.rs index 9f47eb2..5bebf3b 100644 --- a/vectorless-core/vectorless-document/src/overlap.rs +++ b/vectorless-core/vectorless-document/src/overlap.rs @@ -53,7 +53,9 @@ pub struct ContentOverlapMap { impl ContentOverlapMap { /// Create a new empty overlap map. pub fn new() -> Self { - Self { overlaps: Vec::new() } + Self { + overlaps: Vec::new(), + } } /// Add an overlap entry. diff --git a/vectorless-core/vectorless-engine/src/compiled_document.rs b/vectorless-core/vectorless-engine/src/compiled_document.rs index 24f912d..1b50f6a 100644 --- a/vectorless-core/vectorless-engine/src/compiled_document.rs +++ b/vectorless-core/vectorless-engine/src/compiled_document.rs @@ -57,7 +57,6 @@ pub(crate) struct CompiledDocument { pub concepts: Vec<vectorless_document::Concept>, // ── Agent acceleration data ── - /// Pre-computed query routing table for Agent acceleration. pub query_routes: Option<vectorless_document::QueryRoutingTable>, diff --git a/vectorless-core/vectorless-engine/src/engine.rs b/vectorless-core/vectorless-engine/src/engine.rs index 0a22486..d13beff 100644 --- a/vectorless-core/vectorless-engine/src/engine.rs +++ b/vectorless-core/vectorless-engine/src/engine.rs @@ -14,23 +14,21 @@ use std::{collections::HashMap, sync::Arc}; use futures::StreamExt; use tracing::{info, warn}; -use vectorless_config::Config; -use vectorless_document::{ - Document as UnderstandingDocument, DocumentTree, IngestInput, -}; -use vectorless_error::{Error, Result}; -use vectorless_events::EventEmitter; use vectorless_compiler::{ PipelineOptions, incremental::{self, IndexAction}, }; +use vectorless_config::Config; +use vectorless_document::{Document as UnderstandingDocument, DocumentTree, IngestInput}; +use vectorless_error::{Error, Result}; +use vectorless_events::EventEmitter; use vectorless_metrics::MetricsHub; use vectorless_storage::{PersistedDocument, Workspace}; use super::{ compile_input::{CompileInput, CompileSource}, indexer::IndexerClient, - types::{FailedItem, CompileArtifact, CompileMode, CompileOutput}, + types::{CompileArtifact, CompileMode, CompileOutput, FailedItem}, workspace::WorkspaceClient, }; @@ -545,7 +543,7 @@ impl Engine { options: &super::types::CompileOptions, source: &CompileSource, ) -> PipelineOptions { - use vectorless_compiler::{SourceFormat, ReasoningIndexConfig, SummaryStrategy}; + use vectorless_compiler::{ReasoningIndexConfig, SourceFormat, SummaryStrategy}; let format = match source { CompileSource::Path(path) => self @@ -779,10 +777,13 @@ mod tests { use crate::compiled_document::CompiledDocument; fn make_doc() -> CompiledDocument { - CompiledDocument::new("test-id", vectorless_compiler::parse::DocumentFormat::Markdown) - .with_name("test.md") - .with_description("test doc") - .with_source_path(std::path::PathBuf::from("/tmp/test.md")) + CompiledDocument::new( + "test-id", + vectorless_compiler::parse::DocumentFormat::Markdown, + ) + .with_name("test.md") + .with_description("test doc") + .with_source_path(std::path::PathBuf::from("/tmp/test.md")) } #[test] diff --git a/vectorless-core/vectorless-engine/src/indexer.rs b/vectorless-core/vectorless-engine/src/indexer.rs index 0044af6..2527dca 100644 --- a/vectorless-core/vectorless-engine/src/indexer.rs +++ b/vectorless-core/vectorless-engine/src/indexer.rs @@ -26,9 +26,9 @@ use std::sync::Arc; use tracing::info; use uuid::Uuid; +use vectorless_compiler::{CompilerInput, PipelineExecutor, PipelineOptions, SourceFormat}; use vectorless_document::DocumentFormat; use vectorless_error::{Error, Result}; -use vectorless_compiler::{CompilerInput, SourceFormat, PipelineExecutor, PipelineOptions}; use vectorless_llm::LlmClient; use vectorless_storage::{DocumentMeta, PersistedDocument}; diff --git a/vectorless-core/vectorless-engine/src/lib.rs b/vectorless-core/vectorless-engine/src/lib.rs index b11fa5a..85dae58 100644 --- a/vectorless-core/vectorless-engine/src/lib.rs +++ b/vectorless-core/vectorless-engine/src/lib.rs @@ -12,8 +12,8 @@ mod builder; mod compile_input; -mod engine; mod compiled_document; +mod engine; mod indexer; mod types; mod workspace; @@ -35,7 +35,7 @@ pub use compile_input::CompileInput; // Result & Info Types // ============================================================ -pub use types::{FailedItem, CompileArtifact, CompileMode, CompileOptions, CompileOutput}; +pub use types::{CompileArtifact, CompileMode, CompileOptions, CompileOutput, FailedItem}; // ============================================================ // Parser Types (needed for CompileInput::from_content) diff --git a/vectorless-core/vectorless-llm/src/lib.rs b/vectorless-core/vectorless-llm/src/lib.rs index 02e6cca..baa04f6 100644 --- a/vectorless-core/vectorless-llm/src/lib.rs +++ b/vectorless-core/vectorless-llm/src/lib.rs @@ -40,4 +40,3 @@ pub mod throttle; pub use client::LlmClient; pub use error::LlmResult; pub use pool::LlmPool; - diff --git a/vectorless-core/vectorless-metrics/src/index.rs b/vectorless-core/vectorless-metrics/src/index.rs index 6bdb063..dccc5ff 100644 --- a/vectorless-core/vectorless-metrics/src/index.rs +++ b/vectorless-core/vectorless-metrics/src/index.rs @@ -89,7 +89,6 @@ pub struct CompileMetrics { pub nodes_merged: usize, // ── Agent acceleration pass metrics ── - /// Route pass duration (ms). #[serde(default)] pub route_time_ms: u64, diff --git a/vectorless-core/vectorless-py/src/document.rs b/vectorless-core/vectorless-py/src/document.rs index 73a62b1..5c7d53f 100644 --- a/vectorless-core/vectorless-py/src/document.rs +++ b/vectorless-core/vectorless-py/src/document.rs @@ -11,9 +11,8 @@ use tokio::sync::Mutex; use vectorless_primitives::{ ChainInfo, CollectedEvidence, ConceptInfo, ConceptRouteInfo, DocCardInfo, DocumentNavigator, - EvidenceScoreInfo, FindResult, MatchResult, NodeInfo, NodeStats, OverlapInfo, - RouteTargetInfo, SectionCardInfo, SectionSummaryInfo, SimilarResult, TocEntry, - TopicEntryInfo, WordCount, + EvidenceScoreInfo, FindResult, MatchResult, NodeInfo, NodeStats, OverlapInfo, RouteTargetInfo, + SectionCardInfo, SectionSummaryInfo, SimilarResult, TocEntry, TopicEntryInfo, WordCount, }; use super::error::VectorlessError; @@ -363,11 +362,7 @@ impl PyDocument { } /// Get concept routes matching a keyword. - fn concept_routes<'py>( - &self, - py: Python<'py>, - keyword: String, - ) -> PyResult<Bound<'py, PyAny>> { + fn concept_routes<'py>(&self, py: Python<'py>, keyword: String) -> PyResult<Bound<'py, PyAny>> { let nav = Arc::clone(&self.inner); future_into_py(py, async move { let nav = nav.lock().await; @@ -381,16 +376,14 @@ impl PyDocument { } /// Get reasoning chains involving a specific node. - fn chains_for<'py>( - &self, - py: Python<'py>, - node_id: String, - ) -> PyResult<Bound<'py, PyAny>> { + fn chains_for<'py>(&self, py: Python<'py>, node_id: String) -> PyResult<Bound<'py, PyAny>> { let nav = Arc::clone(&self.inner); future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? + .ok_or_else(|| { + VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation") + })? .parse::<u64>() .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; @@ -404,16 +397,14 @@ impl PyDocument { } /// Get overlapping nodes for a specific node. - fn overlaps_for<'py>( - &self, - py: Python<'py>, - node_id: String, - ) -> PyResult<Bound<'py, PyAny>> { + fn overlaps_for<'py>(&self, py: Python<'py>, node_id: String) -> PyResult<Bound<'py, PyAny>> { let nav = Arc::clone(&self.inner); future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? + .ok_or_else(|| { + VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation") + })? .parse::<u64>() .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; @@ -427,23 +418,18 @@ impl PyDocument { } /// Get evidence quality score for a specific node. - fn evidence_score<'py>( - &self, - py: Python<'py>, - node_id: String, - ) -> PyResult<Bound<'py, PyAny>> { + fn evidence_score<'py>(&self, py: Python<'py>, node_id: String) -> PyResult<Bound<'py, PyAny>> { let nav = Arc::clone(&self.inner); future_into_py(py, async move { let num = node_id .strip_prefix('n') - .ok_or_else(|| VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation"))? + .ok_or_else(|| { + VectorlessError::new("NodeId must start with 'n'".to_string(), "navigation") + })? .parse::<u64>() .map_err(|_| VectorlessError::new("Invalid NodeId".to_string(), "navigation"))?; let nav = nav.lock().await; - Ok(nav - .evidence_score_for(num) - .await - .map(PyEvidenceScore::from)) + Ok(nav.evidence_score_for(num).await.map(PyEvidenceScore::from)) }) } @@ -1135,7 +1121,11 @@ impl From<ChainInfo> for PyChainInfo { fn from(v: ChainInfo) -> Self { Self { premises: v.premises.into_iter().map(|id| format!("n{id}")).collect(), - conclusions: v.conclusions.into_iter().map(|id| format!("n{id}")).collect(), + conclusions: v + .conclusions + .into_iter() + .map(|id| format!("n{id}")) + .collect(), chain_type: v.chain_type, summary: v.summary, } diff --git a/vectorless-core/vectorless-storage/src/lib.rs b/vectorless-core/vectorless-storage/src/lib.rs index 2d6a698..d13fc9f 100644 --- a/vectorless-core/vectorless-storage/src/lib.rs +++ b/vectorless-core/vectorless-storage/src/lib.rs @@ -8,7 +8,7 @@ //! - **Persistence** — Save/load document trees and metadata with atomic writes //! - **Cache** — LRU cache for loaded documents //! - **Lock** — File locking for multi-process safety -//! - **Backend** — Storage backend abstraction (file, memory, etc.) +//! - **Backend** — Storage backend abstraction (file, memory, etc.) pub mod backend; pub mod cache; diff --git a/vectorless-core/vectorless-storage/src/persistence.rs b/vectorless-core/vectorless-storage/src/persistence.rs index edaa5c5..5ea7645 100644 --- a/vectorless-core/vectorless-storage/src/persistence.rs +++ b/vectorless-core/vectorless-storage/src/persistence.rs @@ -242,7 +242,6 @@ pub struct PersistedDocument { pub concepts: Vec<vectorless_document::Concept>, // ── Agent acceleration data ── - /// Pre-computed query routing table for Agent acceleration. #[serde(default, skip_serializing_if = "Option::is_none")] pub query_routes: Option<QueryRoutingTable>, From e1d06320ec0acab107d6cd2c152bfc20dfca6f47 Mon Sep 17 00:00:00 2001 From: zTgx <747674262@qq.com> Date: Fri, 24 Apr 2026 22:53:37 +0800 Subject: [PATCH 30/30] docs(HISTORY): add release notes for version 0.1.12 - **Compile pipeline**: renamed index pipeline to compile pipeline with passes-based architecture - **Compiler refactor**: renamed stages to passes, removed deprecated `StageResult` alias and `CustomStageBuilder` - New backend compilation passes: query routing, reasoning chains, overlap detection, and scoring - Agent acceleration data added to compiled documents - LLM-powered cross-document insight extraction in ask module - Enhanced JSON parsing with proper error handling - Upgraded minimum Python version to 3.11 - Removed unused modules: agent, memory backend, validation, ReferenceResolver, SufficiencyLevel - Restructured configuration modules and removed legacy retrieval config - Simplified storage layer by removing memory backend - Documentation updates for architecture and compilation pipeline --- HISTORY.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 387ed3f..85ed76a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,19 @@ # HISTORY +## 0.1.12 (2026-04-24) + +- **Compile pipeline**: renamed index pipeline to compile pipeline with passes-based architecture +- **Compiler refactor**: renamed stages to passes, removed deprecated `StageResult` alias and `CustomStageBuilder` +- New backend compilation passes: query routing, reasoning chains, overlap detection, and scoring +- Agent acceleration data added to compiled documents +- LLM-powered cross-document insight extraction in ask module +- Enhanced JSON parsing with proper error handling +- Upgraded minimum Python version to 3.11 +- Removed unused modules: agent, memory backend, validation, ReferenceResolver, SufficiencyLevel +- Restructured configuration modules and removed legacy retrieval config +- Simplified storage layer by removing memory backend +- Documentation updates for architecture and compilation pipeline + ## 0.1.11 (2026-04-21) - Project description updated to "reasoning-based document engine"