From de11d6563f9de605b8a944f230253859acccad7a Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 14:49:33 -0400 Subject: [PATCH 1/9] Batch file embeddings during indexing --- src/db/error.rs | 6 ++ src/db/mod.rs | 124 ++++++++++++++++++++++++++++++++ src/embed/bge.rs | 39 ++++++++-- src/embed/mod.rs | 4 ++ src/index/file.rs | 71 ++++++++++++++----- src/index/mod.rs | 59 +++++++++++++++- src/index/scanner.rs | 164 +++++++++++++++++++++++++++++++++---------- 7 files changed, 404 insertions(+), 63 deletions(-) diff --git a/src/db/error.rs b/src/db/error.rs index 756e458..d80cc50 100644 --- a/src/db/error.rs +++ b/src/db/error.rs @@ -82,6 +82,12 @@ pub enum DbError { source: sqlx::Error, }, + #[error("failed to commit indexed file batch")] + CommitFileBatch { + #[source] + source: sqlx::Error, + }, + #[error("failed to replace classifications for {path}")] ReplaceDirectoryClassifications { path: String, diff --git a/src/db/mod.rs b/src/db/mod.rs index 389b06d..b3b57ca 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -240,6 +240,130 @@ impl Database { Ok(()) } + pub async fn upsert_files_with_chunks( + &self, + files: &[(&IndexedFile, &[IndexedFileChunk])], + ) -> Result<()> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| DbError::UpsertFile { + path: "".to_string(), + source, + })?; + + for (file, chunks) in files { + sqlx::query( + " + INSERT INTO indexed_files ( + path, + directory_path, + name, + extension, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + content_fingerprint, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + directory_path = excluded.directory_path, + name = excluded.name, + extension = excluded.extension, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + content_fingerprint = excluded.content_fingerprint, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + ) + .bind(&file.path) + .bind(&file.directory_path) + .bind(&file.name) + .bind(&file.extension) + .bind( + i64::try_from(file.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(file.created_unix_seconds) + .bind(file.modified_unix_seconds) + .bind(file.accessed_unix_seconds) + .bind(file.readonly) + .bind(&file.content_fingerprint) + .bind(file.indexed_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::UpsertFile { + path: file.path.clone(), + source, + })?; + + sqlx::query("DELETE FROM indexed_file_chunks WHERE file_path = ?") + .bind(&file.path) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::DeleteFileChunks { + path: file.path.clone(), + source, + })?; + + for chunk in *chunks { + sqlx::query( + " + INSERT INTO indexed_file_chunks ( + file_path, + directory_path, + chunk_index, + content, + embedding, + embedding_dim, + start_byte, + end_byte, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&chunk.file_path) + .bind(&chunk.directory_path) + .bind(i64::from(chunk.chunk_index)) + .bind(&chunk.content) + .bind(encode_embedding(&chunk.embedding)) + .bind( + i64::try_from(chunk.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind( + i64::try_from(chunk.start_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind( + i64::try_from(chunk.end_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(chunk.indexed_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::InsertFileChunk { + path: chunk.file_path.clone(), + chunk_index: chunk.chunk_index, + source, + })?; + } + } + + transaction + .commit() + .await + .map_err(|source| DbError::CommitFileBatch { source })?; + + Ok(()) + } + pub async fn replace_directory_classifications( &self, directory_path: &str, diff --git a/src/embed/bge.rs b/src/embed/bge.rs index 124dae9..3ff403b 100644 --- a/src/embed/bge.rs +++ b/src/embed/bge.rs @@ -34,17 +34,40 @@ impl BgeSmallEmbedder { } fn embed_prefixed(&self, prefix: &str, text: &str) -> Result> { - let input = format!("{prefix}: {text}"); + let mut embeddings = self.embed_prefixed_batch(prefix, &[text.to_string()])?; + + embeddings.pop().ok_or_else(|| EmbedError::Model { + message: "embedding model returned no vectors".to_string(), + }) + } + + fn embed_prefixed_batch(&self, prefix: &str, texts: &[String]) -> Result>> { + if texts.is_empty() { + return Ok(Vec::new()); + } + + let inputs = texts + .iter() + .map(|text| format!("{prefix}: {text}")) + .collect::>(); let mut model = self.model.lock().map_err(|_| EmbedError::Lock)?; - let mut embeddings = model - .embed([input], None) + let embeddings = model + .embed(inputs, None) .map_err(|source| EmbedError::Model { message: source.to_string(), })?; - embeddings.pop().ok_or_else(|| EmbedError::Model { - message: "embedding model returned no vectors".to_string(), - }) + if embeddings.len() != texts.len() { + return Err(EmbedError::Model { + message: format!( + "embedding model returned {} vectors for {} inputs", + embeddings.len(), + texts.len() + ), + }); + } + + Ok(embeddings) } } @@ -61,6 +84,10 @@ impl Embedder for BgeSmallEmbedder { self.embed_prefixed("passage", text) } + fn embed_documents(&self, texts: &[String]) -> Result>> { + self.embed_prefixed_batch("passage", texts) + } + fn embed_query(&self, text: &str) -> Result> { self.embed_prefixed("query", text) } diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 8001185..18eb92c 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -16,6 +16,10 @@ pub trait Embedder { self.embed(text) } + fn embed_documents(&self, texts: &[String]) -> Result>> { + texts.iter().map(|text| self.embed_document(text)).collect() + } + fn embed_query(&self, text: &str) -> Result> { self.embed(text) } diff --git a/src/index/file.rs b/src/index/file.rs index a305f8f..51848af 100644 --- a/src/index/file.rs +++ b/src/index/file.rs @@ -5,7 +5,6 @@ use std::time::{SystemTime, UNIX_EPOCH}; use crate::config::Settings; use crate::db::{IndexedFile, IndexedFileChunk}; -use crate::embed::Embedder; use crate::index::{IndexError, Result}; #[derive(Debug, Clone, PartialEq)] @@ -14,15 +13,60 @@ pub struct IndexedFileData { pub chunks: Vec, } -pub fn index_text_file( +#[derive(Debug, Clone, PartialEq)] +pub struct PreparedIndexedFileData { + pub file: IndexedFile, + pub chunks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreparedFileChunk { + pub file_path: String, + pub directory_path: String, + pub chunk_index: u32, + pub content: String, + pub start_byte: u64, + pub end_byte: u64, + pub indexed_unix_seconds: i64, +} + +impl PreparedIndexedFileData { + pub fn into_indexed(self, embeddings: &mut impl Iterator>) -> IndexedFileData { + let chunks = self + .chunks + .into_iter() + .map(|chunk| { + chunk.into_indexed(embeddings.next().expect("embedding count is validated")) + }) + .collect(); + + IndexedFileData { + file: self.file, + chunks, + } + } +} + +impl PreparedFileChunk { + fn into_indexed(self, embedding: Vec) -> IndexedFileChunk { + IndexedFileChunk { + file_path: self.file_path, + directory_path: self.directory_path, + chunk_index: self.chunk_index, + content: self.content, + embedding, + start_byte: self.start_byte, + end_byte: self.end_byte, + indexed_unix_seconds: self.indexed_unix_seconds, + } + } +} + +pub fn prepare_text_file( path: &Path, directory: &Path, settings: &Settings, - embedder: &E, -) -> Result> -where - E: Embedder, -{ +) -> Result> { let metadata = fs::metadata(path).map_err(|source| IndexError::StatFile { path: path.to_path_buf(), source, @@ -81,27 +125,18 @@ where .into_iter() .enumerate() { - let embedding = - embedder - .embed_document(chunk.text) - .map_err(|source| IndexError::EmbedSummary { - path: path.to_path_buf(), - source, - })?; - - chunks.push(IndexedFileChunk { + chunks.push(PreparedFileChunk { file_path: file_path.clone(), directory_path: directory_path.clone(), chunk_index: u32::try_from(chunk_index).unwrap_or(u32::MAX), content: chunk.text.to_string(), - embedding, start_byte: chunk.start_byte, end_byte: chunk.end_byte, indexed_unix_seconds, }); } - Ok(Some(IndexedFileData { file, chunks })) + Ok(Some(PreparedIndexedFileData { file, chunks })) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/index/mod.rs b/src/index/mod.rs index fecd305..c527c1b 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -122,10 +122,14 @@ fn is_excluded_index_root(settings: &Settings, root: &std::path::Path) -> bool { #[cfg(test)] mod tests { use std::fs; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use super::*; use crate::db::{Database, DocumentKind, IndexedDocument}; - use crate::embed::FakeEmbedder; + use crate::embed::{Embedder, FakeEmbedder}; #[tokio::test] async fn indexes_directory_summaries_and_skips_excluded_paths() { @@ -231,6 +235,29 @@ mod tests { ); } + #[tokio::test] + async fn batches_file_chunk_embeddings() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let app = root.join("app"); + fs::create_dir_all(&app).unwrap(); + fs::write(app.join("one.txt"), "alpha").unwrap(); + fs::write(app.join("two.txt"), "bravo").unwrap(); + fs::write(app.join("three.txt"), "charlie").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = CountingEmbedder::new(16); + let calls = Arc::clone(&embedder.document_batch_calls); + let indexer = Indexer::new(&settings, &database, &embedder); + + let report = indexer.index_roots(vec![root]).await.unwrap(); + + assert_eq!(report.text_files_indexed, 3); + assert_eq!(report.file_chunks_indexed, 3); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn skips_hidden_roots_and_prunes_stale_rows() { let temp = tempfile::tempdir().unwrap(); @@ -347,4 +374,34 @@ mod tests { None ); } + + #[derive(Debug)] + struct CountingEmbedder { + dimensions: usize, + document_batch_calls: Arc, + } + + impl CountingEmbedder { + fn new(dimensions: usize) -> Self { + Self { + dimensions, + document_batch_calls: Arc::new(AtomicUsize::new(0)), + } + } + } + + impl Embedder for CountingEmbedder { + fn dimensions(&self) -> usize { + self.dimensions + } + + fn embed(&self, _text: &str) -> crate::embed::Result> { + Ok(vec![1.0; self.dimensions]) + } + + fn embed_documents(&self, texts: &[String]) -> crate::embed::Result>> { + self.document_batch_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![vec![1.0; self.dimensions]; texts.len()]) + } + } } diff --git a/src/index/scanner.rs b/src/index/scanner.rs index 623fab0..5c97da6 100644 --- a/src/index/scanner.rs +++ b/src/index/scanner.rs @@ -2,13 +2,16 @@ use std::fs; use std::future::Future; use std::path::Path; use std::pin::Pin; +use std::sync::Mutex; use super::{classify, file, summary}; use crate::config::Settings; use crate::db::{Database, IndexedDocument}; -use crate::embed::Embedder; +use crate::embed::{EmbedError, Embedder}; use crate::index::{IndexError, IndexProgress, Result}; +const FILE_CHUNK_EMBED_BATCH_SIZE: usize = 64; + #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct IndexReport { pub roots_scanned: u64, @@ -49,16 +52,17 @@ where E: Embedder, P: IndexProgress + ?Sized, { + let batch = FileIndexBatch::new(database, embedder); scan_directory( root, settings, - database, - embedder, + &batch, report, progress, DepthPosition::IndexRoot, ) - .await + .await?; + batch.flush().await } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -81,8 +85,7 @@ impl DepthPosition { fn scan_directory<'a, E, P>( directory: &'a Path, settings: &'a Settings, - database: &'a Database, - embedder: &'a E, + batch: &'a FileIndexBatch<'a, E>, report: &'a mut IndexReport, progress: &'a mut P, depth_position: DepthPosition, @@ -101,7 +104,8 @@ where } })?; - database + batch + .database .upsert_document(&document) .await .map_err(|source| IndexError::StoreDocument { @@ -109,7 +113,8 @@ where source: Box::new(source), })?; let classifications = classify::classify_directory(directory, settings)?; - database + batch + .database .replace_directory_classifications(&document.path, &classifications) .await .map_err(|source| IndexError::StoreDirectoryClassifications { @@ -140,7 +145,8 @@ where })?; if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { - database + batch + .database .delete_path_tree(&path.to_string_lossy()) .await .map_err(|source| IndexError::PruneExcludedPath { @@ -153,25 +159,16 @@ where if file_type.is_dir() { let Some(child_position) = depth_position.child_position() else { - prune_unindexed_directory(database, report, &path).await?; + prune_unindexed_directory(batch.database, report, &path).await?; continue; }; if exceeds_max_depth(settings, child_position) { - prune_unindexed_directory(database, report, &path).await?; + prune_unindexed_directory(batch.database, report, &path).await?; continue; } - scan_directory( - &path, - settings, - database, - embedder, - report, - progress, - child_position, - ) - .await?; + scan_directory(&path, settings, batch, report, progress, child_position).await?; } else if file_type.is_file() { if settings.index.is_excluded_name(&name) { report.entries_skipped += 1; @@ -179,26 +176,11 @@ where } report.files_seen += 1; - if let Some(indexed_file) = - file::index_text_file(&path, directory, settings, embedder)? - { + if let Some(indexed_file) = file::prepare_text_file(&path, directory, settings)? { report.text_files_indexed += 1; report.file_chunks_indexed += u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); - database - .upsert_file(&indexed_file.file) - .await - .map_err(|source| IndexError::StoreFile { - path: indexed_file.file.path.clone(), - source: Box::new(source), - })?; - database - .replace_file_chunks(&indexed_file.file.path, &indexed_file.chunks) - .await - .map_err(|source| IndexError::StoreFileChunks { - path: indexed_file.file.path, - source: Box::new(source), - })?; + batch.push(indexed_file).await?; } } else { report.entries_skipped += 1; @@ -234,6 +216,112 @@ async fn prune_unindexed_directory( Ok(()) } +struct FileIndexBatch<'a, E> { + database: &'a Database, + embedder: &'a E, + state: Mutex, +} + +#[derive(Debug, Default)] +struct FileIndexBatchState { + files: Vec, + chunk_count: usize, +} + +impl<'a, E> FileIndexBatch<'a, E> +where + E: Embedder, +{ + fn new(database: &'a Database, embedder: &'a E) -> Self { + Self { + database, + embedder, + state: Mutex::new(FileIndexBatchState::default()), + } + } + + async fn push(&self, file: file::PreparedIndexedFileData) -> Result<()> { + let should_flush = { + let mut state = self + .state + .lock() + .expect("file index batch lock is poisoned"); + state.chunk_count = state.chunk_count.saturating_add(file.chunks.len()); + state.files.push(file); + state.chunk_count >= FILE_CHUNK_EMBED_BATCH_SIZE + }; + + if should_flush { + self.flush().await?; + } + + Ok(()) + } + + async fn flush(&self) -> Result<()> { + let files = { + let mut state = self + .state + .lock() + .expect("file index batch lock is poisoned"); + if state.files.is_empty() { + return Ok(()); + } + state.chunk_count = 0; + std::mem::take(&mut state.files) + }; + let first_path = files + .first() + .map(|file| file.file.path.clone()) + .unwrap_or_else(|| "".to_string()); + let texts = files + .iter() + .flat_map(|file| file.chunks.iter().map(|chunk| chunk.content.clone())) + .collect::>(); + + let embeddings = + self.embedder + .embed_documents(&texts) + .map_err(|source| IndexError::EmbedSummary { + path: first_path.clone().into(), + source, + })?; + + if embeddings.len() != texts.len() { + return Err(IndexError::EmbedSummary { + path: first_path.into(), + source: EmbedError::Model { + message: format!( + "embedding model returned {} vectors for {} file chunks", + embeddings.len(), + texts.len() + ), + }, + }); + } + + let mut embeddings = embeddings.into_iter(); + let indexed_files = files + .into_iter() + .map(|file| file.into_indexed(&mut embeddings)) + .collect::>(); + let file_refs = indexed_files + .iter() + .map(|file| (&file.file, file.chunks.as_slice())) + .collect::>(); + + self.database + .upsert_files_with_chunks(&file_refs) + .await + .map_err(|source| IndexError::StoreFileChunks { + path: first_path, + source: Box::new(source), + })?; + + Ok(()) + } +} + #[allow(dead_code)] fn _assert_document_is_send(document: IndexedDocument) -> IndexedDocument { document From 256d6bbc7bd911a59f9689de1fc528fa8bb033ec Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 14:54:19 -0400 Subject: [PATCH 2/9] Improve init progress output --- src/app/mod.rs | 49 +++++++++++++++++++++- src/main.rs | 109 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 151 insertions(+), 7 deletions(-) diff --git a/src/app/mod.rs b/src/app/mod.rs index 0637fcf..eac8a20 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -20,19 +20,64 @@ pub struct InitReport { pub index: IndexReport, } +pub trait InitProgress { + fn paths_started(&mut self) {} + fn paths_ready(&mut self, _config_dir: &Path, _data_dir: &Path, _cache_dir: &Path) {} + fn config_started(&mut self, _path: &Path) {} + fn config_ready(&mut self, _path: &Path, _created: bool) {} + fn database_started(&mut self, _path: &Path) {} + fn database_ready(&mut self, _path: &Path, _created: bool) {} + fn model_started(&mut self, _cache_dir: &Path) {} + fn model_ready(&mut self, _cache_dir: &Path) {} + fn index_started(&mut self, _roots: &[String]) {} +} + +#[derive(Debug, Default)] +pub struct NoopInitProgress; + +impl InitProgress for NoopInitProgress {} + pub async fn init() -> Result { - let mut progress = NoopProgress; - init_with_progress(&mut progress).await + let mut index_progress = NoopProgress; + let mut init_progress = NoopInitProgress; + init_with_progress_and_steps(&mut index_progress, &mut init_progress).await } pub async fn init_with_progress

(progress: &mut P) -> Result where P: IndexProgress + ?Sized, { + let mut init_progress = NoopInitProgress; + init_with_progress_and_steps(progress, &mut init_progress).await +} + +pub async fn init_with_progress_and_steps( + progress: &mut P, + init_progress: &mut I, +) -> Result +where + P: IndexProgress + ?Sized, + I: InitProgress + ?Sized, +{ + init_progress.paths_started(); let paths = AppPaths::discover()?; + init_progress.paths_ready(&paths.config_dir, &paths.data_dir, &paths.cache_dir); + + init_progress.config_started(&paths.config_file); + let config_created = !paths.config_file.exists(); let settings = Settings::load_or_create(&paths.config_file)?; + init_progress.config_ready(&paths.config_file, config_created); + + init_progress.database_started(&paths.database_file); + let database_created = !paths.database_file.exists(); let database = Database::open(&paths.database_file).await?; + init_progress.database_ready(&paths.database_file, database_created); + + init_progress.model_started(&paths.cache_dir); let embedder = RuntimeEmbedder::load(&paths)?; + init_progress.model_ready(&paths.cache_dir); + + init_progress.index_started(&settings.index.roots); let indexer = Indexer::new(&settings, &database, &embedder); let index = indexer .index_configured_roots_with_progress(progress) diff --git a/src/main.rs b/src/main.rs index a3b50a8..4ada87b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,12 +65,12 @@ async fn run(invocation: Invocation) -> error::Result<()> { } Invocation::ShellInit { shell } => write_stdout(shell_init(shell).as_bytes())?, Invocation::Init => { + let mut init_progress = TerminalInitProgress::start(); let mut progress = TerminalIndexProgress::start(); - let report = app::init_with_progress(&mut progress).await?; + let report = + app::init_with_progress_and_steps(&mut progress, &mut init_progress).await?; progress.finish(); - println!("config ready: {}", report.config_file.display()); - println!("database ready: {}", report.database_file.display()); - println!("{}", report.index.human_summary()); + init_progress.finish(&report); } Invocation::Index { roots } => { let mut progress = TerminalIndexProgress::start(); @@ -122,6 +122,98 @@ fn confirm_reset() -> error::Result { )) } +struct TerminalInitProgress { + step: usize, +} + +impl TerminalInitProgress { + const STEP_COUNT: usize = 5; + + fn start() -> Self { + println!("cds init"); + flush_stdout(); + Self { step: 0 } + } + + fn begin(&mut self, label: &str) { + self.step += 1; + println!("[{}/{}] {label}", self.step, Self::STEP_COUNT); + flush_stdout(); + } + + fn detail(label: &str, value: impl std::fmt::Display) { + println!(" {label}: {value}"); + flush_stdout(); + } + + fn status(label: &str, path: &Path, created: bool) { + let action = if created { "created" } else { "ready" }; + Self::detail(label, format_args!("{action} {}", path.display())); + } + + fn finish(&mut self, report: &app::InitReport) { + Self::detail("index", report.index.human_summary()); + Self::detail("config", report.config_file.display()); + Self::detail("database", report.database_file.display()); + println!("cds init complete"); + flush_stdout(); + } +} + +impl app::InitProgress for TerminalInitProgress { + fn paths_started(&mut self) { + self.begin("Resolve cds directories"); + } + + fn paths_ready(&mut self, config_dir: &Path, data_dir: &Path, cache_dir: &Path) { + Self::detail("config dir", config_dir.display()); + Self::detail("data dir", data_dir.display()); + Self::detail("cache dir", cache_dir.display()); + } + + fn config_started(&mut self, path: &Path) { + self.begin("Prepare config"); + Self::detail("path", path.display()); + } + + fn config_ready(&mut self, path: &Path, created: bool) { + Self::status("config", path, created); + } + + fn database_started(&mut self, path: &Path) { + self.begin("Prepare database"); + Self::detail("path", path.display()); + } + + fn database_ready(&mut self, path: &Path, created: bool) { + Self::status("database", path, created); + } + + fn model_started(&mut self, cache_dir: &Path) { + self.begin("Load embedding model"); + Self::detail("model", "BAAI/bge-small-en-v1.5"); + Self::detail("cache", cache_dir.join("models").display()); + } + + fn model_ready(&mut self, _cache_dir: &Path) { + Self::detail("model", "ready"); + } + + fn index_started(&mut self, roots: &[String]) { + self.begin("Index configured roots"); + let roots = if roots.is_empty() { + "".to_string() + } else { + roots.join(", ") + }; + Self::detail("roots", roots); + } +} + +fn flush_stdout() { + let _ = io::stdout().flush(); +} + const SEARCH_LABEL: &str = "Searching"; struct SearchAnimation { @@ -195,18 +287,21 @@ struct TerminalIndexProgress { state: Arc>, stop: Arc, worker: Option>, + enabled: bool, } impl TerminalIndexProgress { fn start() -> Self { let state = Arc::new(Mutex::new(ProgressState::default())); let stop = Arc::new(AtomicBool::new(false)); - let worker = Some(spawn_progress_worker(Arc::clone(&state), Arc::clone(&stop))); + let enabled = io::stderr().is_terminal(); + let worker = enabled.then(|| spawn_progress_worker(Arc::clone(&state), Arc::clone(&stop))); Self { state, stop, worker, + enabled, } } @@ -226,6 +321,10 @@ impl Drop for TerminalIndexProgress { impl IndexProgress for TerminalIndexProgress { fn directory_started(&mut self, directory: &Path) { + if !self.enabled { + return; + } + let Ok(mut state) = self.state.lock() else { return; }; From c63559cc26326ab32fc9dbbe4e12567c839f32f2 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 15:11:04 -0400 Subject: [PATCH 3/9] Speed up Docker cd equivalence CI --- .github/workflows/ci.yml | 2 +- AGENTS.md | 10 +++++----- Cargo.toml | 6 +++++- README.md | 19 ++++++++++++++----- src/app/mod.rs | 29 ++++++++++++++++++++++++----- src/embed/mod.rs | 2 ++ tests/docker_cd_equivalence.rs | 4 +++- tests/docker_cd_equivalence.sh | 7 +++++-- 8 files changed, 59 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa69d4c..219cd45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,6 @@ jobs: cd-equivalence: name: Docker cd Equivalence runs-on: ubuntu-latest - needs: rust steps: - name: Checkout @@ -92,4 +91,5 @@ jobs: - name: Run Docker cd equivalence test env: CDS_DOCKER_IMAGE: rust:1 + CDS_DOCKER_RANDOM_CASES: 20 run: cargo test --test docker_cd_equivalence -- --nocapture diff --git a/AGENTS.md b/AGENTS.md index 840a049..9add1c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,16 +184,16 @@ against Bash's built-in `cd` for status, stdout, stderr, `PWD`, `OLDPWD`, and ph Keep these details in mind when editing it: - CI currently uses `CDS_DOCKER_IMAGE=rust:1`. -- The Docker image must include Cargo and C++ standard library/linker support. The - FastEmbed/ONNX dependency links against `libstdc++`, so very small images such as - `rust:1-slim` can fail with `unable to find library -lstdc++`. +- The Docker image must include Cargo. The test builds `cds` with `--no-default-features` + and `CDS_EMBEDDER=fake`, so it does not need the FastEmbed/ONNX dependency stack. - The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some container invocations otherwise fail with `cargo: command not found`. - Bash includes source line numbers in `cd` diagnostics. Normalize only those line numbers before comparing stderr, while keeping the actual diagnostic text exact. -- The Docker test can skip locally when Docker is unavailable, so use +- The Docker test can skip locally when Docker is unavailable. Use `CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence -- --nocapture` - with Docker access when changing equivalence behavior. + with Docker access when changing equivalence behavior. Set `CDS_DOCKER_RANDOM_CASES` + to increase or decrease the randomized sample count. ## Repository Hygiene diff --git a/Cargo.toml b/Cargo.toml index 540b35a..0313a16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,14 @@ name = "cds" version = "0.1.0" edition = "2024" +[features] +default = ["real-embedder"] +real-embedder = ["dep:fastembed"] + [dependencies] clap = { version = "4.5.40", features = ["cargo"] } color-eyre = "0.6.5" -fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"] } +fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"], optional = true } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "sqlite", "migrate", "macros"] } diff --git a/README.md b/README.md index 98a09ad..5da4d2a 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ cargo test --all ``` The test suite includes a Docker-backed equivalence test. When Docker is available, it -mounts this project into a Rust container, builds `cds`, generates a fresh random -filesystem tree, and compares `cds` against the shell's built-in `cd` for exit status, -stdout, stderr, `PWD`, `OLDPWD`, and physical path behavior. +mounts this project into a Rust container, builds `cds` without the real embedding +runtime, generates a fresh random filesystem tree, and compares `cds` against the shell's +built-in `cd` for exit status, stdout, stderr, `PWD`, `OLDPWD`, and physical path +behavior. Use a specific container image with: @@ -72,8 +73,16 @@ Use a specific container image with: CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence ``` -The Docker image must include Cargo and the C++ standard library/linker support required by -the embedding runtime. The default image is `rust:1`. +Adjust the number of randomized cases with: + +```sh +CDS_DOCKER_RANDOM_CASES=200 cargo test --test docker_cd_equivalence +``` + +The Docker image must include Cargo. The default image is `rust:1`. + +Default builds include the real local embedding runtime. Use `--no-default-features` with +`CDS_EMBEDDER=fake` for shell-only checks that do not need `bge-small-en-v1.5`. ## Indexing diff --git a/src/app/mod.rs b/src/app/mod.rs index eac8a20..06fab66 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -6,7 +6,9 @@ use std::path::{Path, PathBuf}; use crate::config::{AppPaths, Settings, expand_tilde}; use crate::db::{Database, DirectoryTypeCount}; -use crate::embed::{BgeSmallEmbedder, Embedder, FakeEmbedder}; +#[cfg(feature = "real-embedder")] +use crate::embed::BgeSmallEmbedder; +use crate::embed::{Embedder, FakeEmbedder}; use crate::error::{Result, app_err, config_err, embed_err}; use crate::index::{IndexProgress, IndexReport, Indexer, NoopProgress}; use crate::search::{SearchResult, Searcher}; @@ -236,6 +238,7 @@ fn current_shell_directory() -> Option { } enum RuntimeEmbedder { + #[cfg(feature = "real-embedder")] Bge(Box), Fake(FakeEmbedder), } @@ -246,16 +249,14 @@ impl RuntimeEmbedder { return Ok(Self::Fake(FakeEmbedder::default())); } - BgeSmallEmbedder::new(&paths.cache_dir) - .map(Box::new) - .map(Self::Bge) - .map_err(embed_err) + load_real_embedder(paths) } } impl Embedder for RuntimeEmbedder { fn dimensions(&self) -> usize { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.dimensions(), Self::Fake(embedder) => embedder.dimensions(), } @@ -263,6 +264,7 @@ impl Embedder for RuntimeEmbedder { fn embed(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed(text), Self::Fake(embedder) => embedder.embed(text), } @@ -270,6 +272,7 @@ impl Embedder for RuntimeEmbedder { fn embed_document(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed_document(text), Self::Fake(embedder) => embedder.embed_document(text), } @@ -277,12 +280,28 @@ impl Embedder for RuntimeEmbedder { fn embed_query(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed_query(text), Self::Fake(embedder) => embedder.embed_query(text), } } } +#[cfg(feature = "real-embedder")] +fn load_real_embedder(paths: &AppPaths) -> Result { + BgeSmallEmbedder::new(&paths.cache_dir) + .map(Box::new) + .map(RuntimeEmbedder::Bge) + .map_err(embed_err) +} + +#[cfg(not(feature = "real-embedder"))] +fn load_real_embedder(_paths: &AppPaths) -> Result { + Err(embed_err(crate::embed::EmbedError::Model { + message: "real embedder support is disabled; rebuild with the real-embedder feature or set CDS_EMBEDDER=fake".to_string(), + })) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 18eb92c..4259c8a 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -1,7 +1,9 @@ +#[cfg(feature = "real-embedder")] mod bge; mod error; mod fake; +#[cfg(feature = "real-embedder")] pub use bge::{BGE_SMALL_EN_V15_DIMENSIONS, BgeSmallEmbedder}; pub use error::EmbedError; pub use fake::FakeEmbedder; diff --git a/tests/docker_cd_equivalence.rs b/tests/docker_cd_equivalence.rs index 7766f4b..97e1708 100644 --- a/tests/docker_cd_equivalence.rs +++ b/tests/docker_cd_equivalence.rs @@ -11,7 +11,7 @@ if ! command -v cargo >/dev/null 2>&1; then exit 127 fi -cargo build --quiet +CDS_EMBEDDER=fake cargo build --quiet --no-default-features --bin cds bash tests/docker_cd_equivalence.sh "#; @@ -48,6 +48,8 @@ fn docker_cd_equivalence_random_tree() { .arg("CARGO_HOME=/tmp/cargo") .arg("--env") .arg("CARGO_TARGET_DIR=/tmp/cds-target") + .arg("--env") + .arg("CDS_EMBEDDER=fake") .arg(image) .arg("bash") .arg("-lc") diff --git a/tests/docker_cd_equivalence.sh b/tests/docker_cd_equivalence.sh index 871bd5d..7459785 100644 --- a/tests/docker_cd_equivalence.sh +++ b/tests/docker_cd_equivalence.sh @@ -65,8 +65,10 @@ for dir in "${dirs[@]}"; do printf 'fixture file for %s\n' "$dir" > "$dir/file_$RANDOM.txt" done +cds_init="$(cds --shell-init bash)" + setup_cds() { - eval "$(cds --shell-init bash)" + eval "$cds_init" } normalize_stderr() { @@ -183,7 +185,8 @@ compare_case "cd dash" "$tree_root" "$space_dir" "" - compare_case "invalid directory" "$tree_root" "$space_dir" "" "$tree_root/does-not-exist" compare_case "too many args" "$tree_root" "$space_dir" "" "$space_dir" "$quote_dir" -for i in $(seq 1 60); do +random_cases="${CDS_DOCKER_RANDOM_CASES:-60}" +for i in $(seq 1 "$random_cases"); do start="${dirs[$((RANDOM % ${#dirs[@]}))]}" oldpwd="${dirs[$((RANDOM % ${#dirs[@]}))]}" target="${dirs[$((RANDOM % ${#dirs[@]}))]}" From f259afad3c30786d041c1c808e31cd9ced03c33d Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 15:36:02 -0400 Subject: [PATCH 4/9] Added pnpm style spinner for searching state --- src/main.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4ada87b..b5693c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -214,7 +214,10 @@ fn flush_stdout() { let _ = io::stdout().flush(); } -const SEARCH_LABEL: &str = "Searching"; +const SEARCH_LABEL: &str = "Searching.."; +const SEARCH_SPINNER_COLOR: &str = "\x1b[32m"; +const SEARCH_SPINNER_RESET: &str = "\x1b[0m"; +const SEARCH_SPINNER_FRAMES: [&str; 10] = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; struct SearchAnimation { stop: Arc, @@ -254,7 +257,7 @@ fn spawn_search_animation(stop: Arc) -> JoinHandle<()> { while !stop.load(Ordering::Relaxed) { render_search_frame(tick); tick = tick.wrapping_add(1); - thread::sleep(Duration::from_millis(120)); + thread::sleep(Duration::from_millis(160)); } clear_search_frame(); @@ -272,8 +275,8 @@ fn clear_search_frame() { } fn search_frame(tick: usize) -> String { - let dots = ".".repeat((tick % 3) + 1); - format!("{SEARCH_LABEL}{dots}") + let frame = SEARCH_SPINNER_FRAMES[tick % SEARCH_SPINNER_FRAMES.len()]; + format!("{SEARCH_SPINNER_COLOR}{frame}{SEARCH_SPINNER_RESET} {SEARCH_LABEL}") } #[derive(Debug, Default)] @@ -379,10 +382,10 @@ mod tests { use super::*; #[test] - fn search_frame_cycles_dots() { - assert_eq!(search_frame(0), "Searching."); - assert_eq!(search_frame(1), "Searching.."); - assert_eq!(search_frame(2), "Searching..."); - assert_eq!(search_frame(3), "Searching."); + fn search_frame_cycles_dot_spinner() { + assert_eq!(search_frame(0), "\x1b[32m⠋\x1b[0m Searching.."); + assert_eq!(search_frame(1), "\x1b[32m⠙\x1b[0m Searching.."); + assert_eq!(search_frame(9), "\x1b[32m⠏\x1b[0m Searching.."); + assert_eq!(search_frame(10), "\x1b[32m⠋\x1b[0m Searching.."); } } From d70f3b1248fbea0340904d555f2c56b7c32fb549 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 15:42:29 -0400 Subject: [PATCH 5/9] Ensure reset clears indexed data --- src/db/mod.rs | 64 +++++++++++++++++++++++++++++++++++-- tests/search_integration.rs | 34 ++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/db/mod.rs b/src/db/mod.rs index b3b57ca..28bf15a 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -533,19 +533,43 @@ impl Database { } pub async fn reset(&self) -> Result<()> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| DbError::ResetDatabase { source })?; + + sqlx::query("PRAGMA secure_delete = ON") + .execute(&mut *transaction) + .await + .map_err(|source| DbError::ResetDatabase { source })?; sqlx::query("DELETE FROM directory_classifications") - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::ResetDatabase { source })?; sqlx::query("DELETE FROM indexed_file_chunks") - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::ResetDatabase { source })?; sqlx::query("DELETE FROM indexed_files") - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::ResetDatabase { source })?; sqlx::query("DELETE FROM indexed_documents") + .execute(&mut *transaction) + .await + .map_err(|source| DbError::ResetDatabase { source })?; + + transaction + .commit() + .await + .map_err(|source| DbError::ResetDatabase { source })?; + + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)") + .execute(&self.pool) + .await + .map_err(|source| DbError::ResetDatabase { source })?; + sqlx::query("VACUUM") .execute(&self.pool) .await .map_err(|source| DbError::ResetDatabase { source })?; @@ -1139,6 +1163,32 @@ mod tests { }) .await .unwrap(); + let indexed_file = IndexedFile { + path: "/tmp/project/README.md".to_string(), + directory_path: "/tmp/project".to_string(), + name: "README.md".to_string(), + extension: Some("md".to_string()), + size_bytes: 12, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: false, + content_fingerprint: "mtime:12:len:12:hash:abc".to_string(), + indexed_unix_seconds: 34, + }; + let indexed_chunk = IndexedFileChunk { + file_path: indexed_file.path.clone(), + directory_path: indexed_file.directory_path.clone(), + chunk_index: 0, + content: "project readme cargo".to_string(), + embedding: vec![0.1, 0.2, 0.3], + start_byte: 0, + end_byte: 20, + indexed_unix_seconds: 34, + }; + db.upsert_files_with_chunks(&[(&indexed_file, &[indexed_chunk])]) + .await + .unwrap(); db.replace_directory_classifications( "/tmp/project", &[DirectoryClassification { @@ -1157,6 +1207,14 @@ mod tests { db.reset().await.unwrap(); assert_eq!(db.document_count().await.unwrap(), 0); + assert_eq!(table_count(&db, "indexed_files").await, 0); + assert_eq!(table_count(&db, "indexed_file_chunks").await, 0); + assert_eq!(table_count(&db, "directory_classifications").await, 0); assert!(db.directory_type_counts().await.unwrap().is_empty()); } + + async fn table_count(db: &Database, table: &str) -> i64 { + let sql = format!("SELECT COUNT(*) FROM {table}"); + sqlx::query_scalar(&sql).fetch_one(&db.pool).await.unwrap() + } } diff --git a/tests/search_integration.rs b/tests/search_integration.rs index f6ceba9..6b90f6c 100644 --- a/tests/search_integration.rs +++ b/tests/search_integration.rs @@ -140,6 +140,40 @@ fn reset_prompts_and_deletes_indexed_data_when_confirmed() { let counts = fixture.cds().arg("--dir-type-count").output().unwrap(); assert!(counts.status.success()); assert_eq!(String::from_utf8(counts.stdout).unwrap(), ""); + + let search = fixture + .cds() + .arg("--search") + .arg("manifest") + .output() + .unwrap(); + assert!( + search.status.success(), + "stderr: {}", + String::from_utf8_lossy(&search.stderr) + ); + assert_eq!(String::from_utf8(search.stdout).unwrap(), ""); + + let cwd = fixture.temp.path().join("cwd-after-reset"); + std::fs::create_dir_all(&cwd).unwrap(); + let emit = fixture + .cds() + .arg("--cds-emit") + .arg("--") + .arg("manifest") + .current_dir(&cwd) + .env("PWD", &cwd) + .output() + .unwrap(); + assert!( + emit.status.success(), + "stderr: {}", + String::from_utf8_lossy(&emit.stderr) + ); + assert_eq!( + String::from_utf8(emit.stdout).unwrap(), + "builtin cd 'manifest'\n" + ); } struct SearchFixture { From 5f3be41a737d9a26a34584ee7ef66e25df5ec79c Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 15:56:34 -0400 Subject: [PATCH 6/9] Optimize initial indexing --- README.md | 24 ++- src/config/settings.rs | 52 ++++++ src/db/mod.rs | 123 ++++++++++++ src/index/file.rs | 82 +++++++- src/index/mod.rs | 6 +- src/index/scanner.rs | 412 +++++++++++++++++++++++++++++------------ src/index/summary.rs | 6 + 7 files changed, 571 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 5da4d2a..250f209 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,16 @@ The initial config looks like this: "*.xcassets", "*.imageset", "*.appiconset", - "*.colorset" + "*.colorset", + "*.svg", + "*.png", + "*.jpg", + "*.jpeg", + "*.gif", + "*.webp", + "*.ico", + "*.pdf", + "*.zip" ], "max_file_bytes": 65536, "max_excerpt_bytes": 4096, @@ -145,11 +154,14 @@ Or index explicit roots: cds --index ~/Projects ~/work ``` -The current indexer stores directory metadata and text-file content separately in SQLite. -Directory rows store structured filesystem metadata: name, type, parent path, size in bytes, -created time, modified time, accessed time, readonly status, and index time. Text files are -stored as file metadata plus embedded content chunks linked back to their containing -directories. Directory metadata itself is not embedded. +The current indexer stores directory metadata and high-signal text-file content separately +in SQLite. Directory rows store structured filesystem metadata: name, type, parent path, +size in bytes, created time, modified time, accessed time, readonly status, and index time. +High-signal files such as READMEs, package manifests, project config files, and SQL schemas +are stored as file metadata plus embedded content chunks linked back to their containing +directories. Low-signal arbitrary text files are skipped by default to keep initial indexing +fast, and media/asset/archive files such as SVGs, images, PDFs, and ZIPs are ignored. +Directory metadata itself is not embedded. Hidden directories are always skipped and pruned from the local index. Each configured index root is treated as a container, and each top-level directory inside it is indexed only through `max_depth_per_top_level_directory` levels. diff --git a/src/config/settings.rs b/src/config/settings.rs index bf4c35a..1cb778d 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -108,6 +108,15 @@ impl Default for IndexSettings { "*.imageset".to_string(), "*.appiconset".to_string(), "*.colorset".to_string(), + "*.svg".to_string(), + "*.png".to_string(), + "*.jpg".to_string(), + "*.jpeg".to_string(), + "*.gif".to_string(), + "*.webp".to_string(), + "*.ico".to_string(), + "*.pdf".to_string(), + "*.zip".to_string(), ], max_file_bytes: 65_536, max_excerpt_bytes: 4_096, @@ -147,6 +156,46 @@ fn is_low_signal_name(name: &str) -> bool { || lower.ends_with(".imageset") || lower.ends_with(".appiconset") || lower.ends_with(".colorset") + || is_low_signal_asset_extension(&lower) +} + +fn is_low_signal_asset_extension(name: &str) -> bool { + matches!( + name.rsplit_once('.').map(|(_, extension)| extension), + Some( + "svg" + | "png" + | "jpg" + | "jpeg" + | "gif" + | "webp" + | "ico" + | "bmp" + | "tif" + | "tiff" + | "pdf" + | "zip" + | "tar" + | "gz" + | "tgz" + | "bz2" + | "xz" + | "7z" + | "rar" + | "dmg" + | "mp3" + | "mp4" + | "mov" + | "avi" + | "webm" + | "wav" + | "flac" + | "woff" + | "woff2" + | "ttf" + | "otf" + ) + ) } fn is_hidden_name(name: &str) -> bool { @@ -286,6 +335,9 @@ mod tests { assert!(index.is_excluded_name("private.key")); assert!(index.is_excluded_name("Assets.xcassets")); assert!(index.is_excluded_name("Custom Icon.appiconset")); + assert!(index.is_excluded_name("logo.svg")); + assert!(index.is_excluded_name("screenshot.png")); + assert!(index.is_excluded_name("archive.zip")); assert!(index.is_excluded_directory_name(".vscode")); assert!(!index.is_excluded_name(".vscode")); assert!(!index.is_excluded_name("README.md")); diff --git a/src/db/mod.rs b/src/db/mod.rs index 28bf15a..2e3952f 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -180,6 +180,129 @@ impl Database { Ok(()) } + pub async fn upsert_directories_with_classifications( + &self, + directories: &[(&IndexedDocument, &[DirectoryClassification])], + ) -> Result<()> { + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::UpsertDocument { + path: "".to_string(), + source, + })?; + + for (document, classifications) in directories { + sqlx::query( + " + INSERT INTO indexed_documents ( + path, + name, + kind, + parent_path, + searchable_text, + embedding, + embedding_dim, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + name = excluded.name, + kind = excluded.kind, + parent_path = excluded.parent_path, + searchable_text = excluded.searchable_text, + embedding = excluded.embedding, + embedding_dim = excluded.embedding_dim, + metadata_fingerprint = excluded.metadata_fingerprint, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + ) + .bind(&document.path) + .bind(&document.name) + .bind(document.kind.as_str()) + .bind(&document.parent_path) + .bind(&document.searchable_text) + .bind(encode_embedding(&document.embedding)) + .bind( + i64::try_from(document.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind(&document.metadata_fingerprint) + .bind( + i64::try_from(document.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(document.created_unix_seconds) + .bind(document.modified_unix_seconds) + .bind(document.accessed_unix_seconds) + .bind(document.readonly) + .bind(document.indexed_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; + + sqlx::query("DELETE FROM directory_classifications WHERE directory_path = ?") + .bind(&document.path) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::ReplaceDirectoryClassifications { + path: document.path.clone(), + source, + })?; + + for classification in *classifications { + sqlx::query( + " + INSERT INTO directory_classifications ( + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&classification.directory_path) + .bind(&classification.label) + .bind(f64::from(classification.confidence)) + .bind(&classification.detector) + .bind(&classification.evidence_path) + .bind(&classification.evidence_summary) + .bind(classification.detected_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::InsertDirectoryClassification { + path: classification.directory_path.clone(), + label: classification.label.clone(), + source, + })?; + } + } + + transaction + .commit() + .await + .map_err(|source| DbError::CommitFileBatch { source })?; + + Ok(()) + } + pub async fn replace_file_chunks( &self, file_path: &str, diff --git a/src/index/file.rs b/src/index/file.rs index 51848af..eb97482 100644 --- a/src/index/file.rs +++ b/src/index/file.rs @@ -76,6 +76,15 @@ pub fn prepare_text_file( return Ok(None); } + let name = path + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_string_lossy() + .into_owned(); + if !is_high_signal_content_file(&name) { + return Ok(None); + } + let bytes = fs::read(path).map_err(|source| IndexError::ReadFile { path: path.to_path_buf(), source, @@ -94,11 +103,6 @@ pub fn prepare_text_file( let modified_unix_seconds = unix_seconds(metadata.modified().unwrap_or(UNIX_EPOCH)); let file_path = path_to_string(path); let directory_path = path_to_string(directory); - let name = path - .file_name() - .unwrap_or_else(|| OsStr::new("")) - .to_string_lossy() - .into_owned(); let file = IndexedFile { path: file_path.clone(), @@ -180,6 +184,38 @@ fn normalize_whitespace(value: &str) -> String { value.split_whitespace().collect::>().join(" ") } +fn is_high_signal_content_file(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + if lower.starts_with("readme") || lower == "gemfile" || lower == "dockerfile" { + return true; + } + + matches!( + lower.as_str(), + "cargo.toml" + | "package.json" + | "manifest.json" + | "pyproject.toml" + | "requirements.txt" + | "setup.py" + | "go.mod" + | "makefile" + | "docker-compose.yml" + | "compose.yml" + | "compose.yaml" + | "tsconfig.json" + | "vite.config.js" + | "vite.config.ts" + | "next.config.js" + | "next.config.ts" + | "tailwind.config.js" + | "tailwind.config.ts" + ) || matches!( + lower.rsplit_once('.').map(|(_, extension)| extension), + Some("md" | "markdown" | "toml" | "yaml" | "yml" | "sql") + ) +} + fn unix_seconds(time: SystemTime) -> i64 { time.duration_since(UNIX_EPOCH) .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) @@ -210,4 +246,40 @@ mod tests { assert_eq!(chunks[1].text, "ef gh"); assert_eq!(chunks[2].text, "i"); } + + #[test] + fn skips_low_signal_text_files() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("notes.txt"); + fs::write(&path, "large low-signal notes").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap(); + + assert_eq!(prepared, None); + } + + #[test] + fn skips_svg_assets_even_though_they_are_text() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("logo.svg"); + fs::write(&path, "Logo").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap(); + + assert_eq!(prepared, None); + } + + #[test] + fn prepares_high_signal_text_files() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("README.md"); + fs::write(&path, "high signal project summary").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()) + .unwrap() + .expect("README.md is indexed"); + + assert_eq!(prepared.file.name, "README.md"); + assert_eq!(prepared.chunks.len(), 1); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index c527c1b..fdd18ac 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -241,9 +241,9 @@ mod tests { let root = temp.path().join("Projects"); let app = root.join("app"); fs::create_dir_all(&app).unwrap(); - fs::write(app.join("one.txt"), "alpha").unwrap(); - fs::write(app.join("two.txt"), "bravo").unwrap(); - fs::write(app.join("three.txt"), "charlie").unwrap(); + fs::write(app.join("README.md"), "alpha").unwrap(); + fs::write(app.join("Cargo.toml"), "bravo").unwrap(); + fs::write(app.join("schema.sql"), "charlie").unwrap(); let settings = Settings::default(); let database = Database::open_in_memory().await.unwrap(); diff --git a/src/index/scanner.rs b/src/index/scanner.rs index 5c97da6..ec26213 100644 --- a/src/index/scanner.rs +++ b/src/index/scanner.rs @@ -1,8 +1,8 @@ +use std::collections::VecDeque; use std::fs; -use std::future::Future; -use std::path::Path; -use std::pin::Pin; -use std::sync::Mutex; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; use super::{classify, file, summary}; use crate::config::Settings; @@ -38,6 +38,17 @@ impl IndexReport { self.roots_not_directories, ) } + + fn merge(&mut self, other: Self) { + self.roots_scanned += other.roots_scanned; + self.roots_missing += other.roots_missing; + self.roots_not_directories += other.roots_not_directories; + self.directories_indexed += other.directories_indexed; + self.files_seen += other.files_seen; + self.text_files_indexed += other.text_files_indexed; + self.file_chunks_indexed += other.file_chunks_indexed; + self.entries_skipped += other.entries_skipped; + } } pub async fn scan_root_with_progress( @@ -52,17 +63,49 @@ where E: Embedder, P: IndexProgress + ?Sized, { + let mut scan = scan_root_concurrently(root, settings)?; + scan.directories + .sort_by(|left, right| left.document.path.cmp(&right.document.path)); + scan.files + .sort_by(|left, right| left.file.path.cmp(&right.file.path)); + scan.pruned_paths.sort(); + scan.pruned_paths.dedup(); + + for path in &scan.pruned_paths { + database + .delete_path_tree(path) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: path.clone(), + source: Box::new(source), + })?; + } + + for directory in &scan.directories { + progress.directory_started(Path::new(&directory.document.path)); + } + + let directory_refs = scan + .directories + .iter() + .map(|directory| (&directory.document, directory.classifications.as_slice())) + .collect::>(); + database + .upsert_directories_with_classifications(&directory_refs) + .await + .map_err(|source| IndexError::StoreDocument { + path: root.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + let batch = FileIndexBatch::new(database, embedder); - scan_directory( - root, - settings, - &batch, - report, - progress, - DepthPosition::IndexRoot, - ) - .await?; - batch.flush().await + for file in scan.files { + batch.push(file).await?; + } + batch.flush().await?; + + report.merge(scan.report); + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -82,138 +125,267 @@ impl DepthPosition { } } -fn scan_directory<'a, E, P>( - directory: &'a Path, - settings: &'a Settings, - batch: &'a FileIndexBatch<'a, E>, - report: &'a mut IndexReport, - progress: &'a mut P, +fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { + match depth_position { + DepthPosition::IndexRoot => false, + DepthPosition::TopLevelDirectory { depth } => { + depth > settings.index.max_depth_per_top_level_directory + } + } +} + +#[derive(Debug, Clone)] +struct ScanJob { + directory: PathBuf, depth_position: DepthPosition, -) -> Pin> + 'a>> -where - E: Embedder + 'a, - P: IndexProgress + ?Sized + 'a, -{ - Box::pin(async move { - progress.directory_started(directory); +} - let document = summary::summarize_directory(directory, settings).map_err(|source| { - IndexError::SummarizeDirectory { - path: directory.to_path_buf(), - source: Box::new(source), +#[derive(Debug, Default)] +struct ConcurrentScan { + directories: Vec, + files: Vec, + pruned_paths: Vec, + report: IndexReport, +} + +#[derive(Debug)] +struct ScannedDirectory { + document: IndexedDocument, + classifications: Vec, +} + +#[derive(Debug)] +struct ScanOutput { + directory: ScannedDirectory, + files: Vec, + child_jobs: Vec, + pruned_paths: Vec, + report: IndexReport, +} + +struct ScanQueue { + state: Mutex, + changed: Condvar, +} + +#[derive(Debug)] +struct ScanQueueState { + pending: VecDeque, + active: usize, + stopped: bool, +} + +fn scan_root_concurrently(root: &Path, settings: &Settings) -> Result { + let worker_count = scan_worker_count(); + let queue = Arc::new(ScanQueue { + state: Mutex::new(ScanQueueState { + pending: VecDeque::from([ScanJob { + directory: root.to_path_buf(), + depth_position: DepthPosition::IndexRoot, + }]), + active: 0, + stopped: false, + }), + changed: Condvar::new(), + }); + let settings = Arc::new(settings.clone()); + let scan = Arc::new(Mutex::new(ConcurrentScan::default())); + let error = Arc::new(Mutex::new(None)); + + thread::scope(|scope| { + for _ in 0..worker_count { + let queue = Arc::clone(&queue); + let settings = Arc::clone(&settings); + let scan = Arc::clone(&scan); + let error = Arc::clone(&error); + + scope.spawn(move || worker_loop(queue, settings, scan, error)); + } + }); + + if let Some(error) = error.lock().expect("scan error lock is poisoned").take() { + return Err(error); + } + + let scan = Arc::try_unwrap(scan) + .expect("scan workers have stopped") + .into_inner() + .expect("scan result lock is poisoned"); + Ok(scan) +} + +fn worker_loop( + queue: Arc, + settings: Arc, + scan: Arc>, + error: Arc>>, +) { + loop { + let Some(job) = next_scan_job(&queue) else { + return; + }; + + let result = scan_directory_job(&job, &settings); + match result { + Ok(output) => finish_scan_job(&queue, &scan, output), + Err(source) => { + *error.lock().expect("scan error lock is poisoned") = Some(source); + stop_scan_workers(&queue); + return; } - })?; + } + } +} - batch - .database - .upsert_document(&document) - .await - .map_err(|source| IndexError::StoreDocument { - path: document.path.clone(), - source: Box::new(source), - })?; - let classifications = classify::classify_directory(directory, settings)?; - batch - .database - .replace_directory_classifications(&document.path, &classifications) - .await - .map_err(|source| IndexError::StoreDirectoryClassifications { - path: document.path.clone(), - source: Box::new(source), - })?; - report.directories_indexed += 1; +fn next_scan_job(queue: &ScanQueue) -> Option { + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); - let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { - path: directory.to_path_buf(), - source, - })?; + loop { + if state.stopped { + return None; + } + + if let Some(job) = state.pending.pop_front() { + state.active += 1; + return Some(job); + } + + if state.active == 0 { + state.stopped = true; + queue.changed.notify_all(); + return None; + } - for entry in entries { + state = queue + .changed + .wait(state) + .expect("scan queue lock is poisoned"); + } +} + +fn finish_scan_job(queue: &ScanQueue, scan: &Mutex, output: ScanOutput) { + { + let mut scan = scan.lock().expect("scan result lock is poisoned"); + scan.directories.push(output.directory); + scan.files.extend(output.files); + scan.pruned_paths.extend(output.pruned_paths); + scan.report.merge(output.report); + } + + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); + state.pending.extend(output.child_jobs); + state.active = state.active.saturating_sub(1); + queue.changed.notify_all(); +} + +fn stop_scan_workers(queue: &ScanQueue) { + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); + state.stopped = true; + state.pending.clear(); + queue.changed.notify_all(); +} + +fn scan_directory_job(job: &ScanJob, settings: &Settings) -> Result { + let document = summary::summarize_directory(&job.directory, settings).map_err(|source| { + IndexError::SummarizeDirectory { + path: job.directory.clone(), + source: Box::new(source), + } + })?; + let classifications = classify::classify_directory(&job.directory, settings)?; + let mut output = ScanOutput { + directory: ScannedDirectory { + document, + classifications, + }, + files: Vec::new(), + child_jobs: Vec::new(), + pruned_paths: Vec::new(), + report: IndexReport { + directories_indexed: 1, + ..IndexReport::default() + }, + }; + + let mut entries = fs::read_dir(&job.directory) + .map_err(|source| IndexError::ReadDirectory { + path: job.directory.clone(), + source, + })? + .map(|entry| { let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { - path: directory.to_path_buf(), + path: job.directory.clone(), source, })?; let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - + let name = entry.file_name().to_string_lossy().into_owned(); let file_type = entry .file_type() .map_err(|source| IndexError::ReadFileType { path: path.clone(), source, })?; + Ok((path, name, file_type)) + }) + .collect::>>()?; + entries.sort_by(|left, right| left.0.cmp(&right.0)); + + for (path, name, file_type) in entries { + if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; + continue; + } - if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { - batch - .database - .delete_path_tree(&path.to_string_lossy()) - .await - .map_err(|source| IndexError::PruneExcludedPath { - path: path.to_string_lossy().into_owned(), - source: Box::new(source), - })?; - report.entries_skipped += 1; + if file_type.is_dir() { + let Some(child_position) = job.depth_position.child_position() else { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; continue; - } + }; - if file_type.is_dir() { - let Some(child_position) = depth_position.child_position() else { - prune_unindexed_directory(batch.database, report, &path).await?; - continue; - }; - - if exceeds_max_depth(settings, child_position) { - prune_unindexed_directory(batch.database, report, &path).await?; - continue; - } - - scan_directory(&path, settings, batch, report, progress, child_position).await?; - } else if file_type.is_file() { - if settings.index.is_excluded_name(&name) { - report.entries_skipped += 1; - continue; - } - - report.files_seen += 1; - if let Some(indexed_file) = file::prepare_text_file(&path, directory, settings)? { - report.text_files_indexed += 1; - report.file_chunks_indexed += - u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); - batch.push(indexed_file).await?; - } - } else { - report.entries_skipped += 1; + if exceeds_max_depth(settings, child_position) { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; + continue; } - } - Ok(()) - }) -} + output.child_jobs.push(ScanJob { + directory: path, + depth_position: child_position, + }); + } else if file_type.is_file() { + if settings.index.is_excluded_name(&name) { + output.report.entries_skipped += 1; + continue; + } -fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { - match depth_position { - DepthPosition::IndexRoot => false, - DepthPosition::TopLevelDirectory { depth } => { - depth > settings.index.max_depth_per_top_level_directory + output.report.files_seen += 1; + if let Some(indexed_file) = file::prepare_text_file(&path, &job.directory, settings)? { + output.report.text_files_indexed += 1; + output.report.file_chunks_indexed += + u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); + output.files.push(indexed_file); + } + } else { + output.report.entries_skipped += 1; } } + + Ok(output) } -async fn prune_unindexed_directory( - database: &Database, - report: &mut IndexReport, - path: &Path, -) -> Result<()> { - database - .delete_path_tree(&path.to_string_lossy()) - .await - .map_err(|source| IndexError::PruneExcludedPath { - path: path.to_string_lossy().into_owned(), - source: Box::new(source), - })?; - report.entries_skipped += 1; - Ok(()) +fn scan_worker_count() -> usize { + thread::available_parallelism() + .map(usize::from) + .unwrap_or(4) + .clamp(2, 8) } struct FileIndexBatch<'a, E> { diff --git a/src/index/summary.rs b/src/index/summary.rs index 58421c4..08a4422 100644 --- a/src/index/summary.rs +++ b/src/index/summary.rs @@ -218,6 +218,11 @@ mod tests { fs::create_dir(temp.path().join(".vscode")).unwrap(); fs::write(temp.path().join("README.md"), "Chrome extension workspace").unwrap(); fs::write(temp.path().join(".env"), "SECRET=true").unwrap(); + fs::write( + temp.path().join("logo.svg"), + "Logo", + ) + .unwrap(); let settings = Settings::default(); let document = summarize_directory(temp.path(), &settings).unwrap(); @@ -235,6 +240,7 @@ mod tests { assert!(document.modified_unix_seconds > 0); assert!(document.searchable_text.contains("child directories: src")); assert!(!document.searchable_text.contains(".vscode")); + assert!(!document.searchable_text.contains("logo.svg")); assert!(document.searchable_text.contains("README.md")); assert!(document.searchable_text.contains("type: directory")); assert!(document.searchable_text.contains("size bytes:")); From 6b9136e590041a043bf30130f35b9f5aca6a08cf Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 19:01:50 -0400 Subject: [PATCH 7/9] Add daemon-backed indexing and search --- AGENTS.md | 2 + Cargo.lock | 104 + Cargo.toml | 1 + README.md | 76 +- install.sh | 85 +- migrations/0002_create_chunk_history.sql | 23 + ...0003_index_chunk_history_modified_time.sql | 5 + src/app/error.rs | 61 + src/app/mod.rs | 1038 ++++++++- src/cli.rs | 141 +- src/config/mod.rs | 1 + src/config/settings.rs | 44 + src/db/document.rs | 22 + src/db/error.rs | 15 + src/db/mod.rs | 472 ++++- src/index/mod.rs | 61 +- src/index/progress.rs | 2 + src/index/scanner.rs | 191 +- src/lib.rs | 18 +- src/main.rs | 121 +- src/search/mod.rs | 1863 +++++++++++++++-- start-daemon.sh | 20 + tests/install_script.rs | 11 +- tests/search_integration.rs | 36 + 24 files changed, 4208 insertions(+), 205 deletions(-) create mode 100644 migrations/0002_create_chunk_history.sql create mode 100644 migrations/0003_index_chunk_history_modified_time.sql create mode 100755 start-daemon.sh diff --git a/AGENTS.md b/AGENTS.md index 9add1c2..fca71fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,10 +105,12 @@ Current user-facing commands: ```sh cds --init +cds --daemon cds --index [PATH...] cds --search QUERY... cds --dir-type-count cds --reset +cds --restart-daemon cds --shell-init [bash|zsh] ``` diff --git a/Cargo.lock b/Cargo.lock index 677f017..3af4c3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -215,6 +224,7 @@ dependencies = [ name = "cds" version = "0.1.0" dependencies = [ + "chrono", "clap", "color-eyre", "fastembed", @@ -238,6 +248,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "clap" version = "4.6.1" @@ -369,6 +390,12 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1086,6 +1113,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -3253,12 +3304,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 0313a16..f9d5c4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ default = ["real-embedder"] real-embedder = ["dep:fastembed"] [dependencies] +chrono = { version = "0.4.38", default-features = false, features = ["clock", "std"] } clap = { version = "4.5.40", features = ["cargo"] } color-eyre = "0.6.5" fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"], optional = true } diff --git a/README.md b/README.md index 250f209..8ade3a3 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,9 @@ From the repository root: ./install.sh ``` -The installer runs `cargo install --path .` and adds an idempotent shell integration -block to your `~/.zshrc` or `~/.bashrc`. After it finishes: +The installer runs `cargo install --path .`, adds an idempotent shell integration block to +your `~/.zshrc` or `~/.bashrc`, and starts the indexing daemon in the background. The daemon +pid and log are written under `~/.cache/cds/` by default. After it finishes: ```sh source ~/.zshrc @@ -136,12 +137,49 @@ The initial config looks like this: "max_excerpt_bytes": 4096, "max_entries_per_directory": 80, "max_depth_per_top_level_directory": 3, - "max_chunk_bytes": 4096 + "max_chunk_bytes": 4096, + "generic_terms": [ + "a", + "an", + "and", + "app", + "application", + "code", + "dir", + "directory", + "for", + "from", + "game", + "in", + "last", + "me", + "my", + "of", + "on", + "program", + "project", + "repo", + "repository", + "search", + "site", + "that", + "the", + "thing", + "to", + "tool", + "web", + "website", + "with", + "work" + ] }, "detectors": [] } ``` +`generic_terms` controls low-signal query words that should not dominate path filtering or +ranking. You can add or remove words from that list without reindexing. + Run an indexing pass over configured roots: ```sh @@ -154,6 +192,27 @@ Or index explicit roots: cds --index ~/Projects ~/work ``` +Run the indexing daemon in the foreground: + +```sh +cds --daemon +``` + +Kill existing cds daemons and start a fresh background daemon: + +```sh +cds --restart-daemon +``` + +The daemon keeps the embedding model loaded, polls configured roots for filesystem changes, +and reindexes changed roots without requiring another manual scan. Generated file chunk +embeddings are also recorded in an append-only history table keyed by file fingerprint, so +search can still match earlier content versions after a file changes. On Unix platforms, the +daemon also opens a local socket under the cds cache directory; normal `cds --search` and +shell-integration searches use that socket when available so the query can reuse the +daemon's already-loaded embedding model. If the daemon is not running or the socket cannot be +created, searches fall back to the normal direct path. + The current indexer stores directory metadata and high-signal text-file content separately in SQLite. Directory rows store structured filesystem metadata: name, type, parent path, size in bytes, created time, modified time, accessed time, readonly status, and index time. @@ -309,14 +368,21 @@ List detected directory types: cds --dir-type-count ``` +Restart the background daemon: + +```sh +cds --restart-daemon +``` + Delete all indexed data from the SQLite database: ```sh cds --reset ``` -This prompts before deleting directory metadata, file metadata, content chunks, and directory -type classifications. The database file and schema are kept in place. +This prompts before deleting directory metadata, file metadata, current and historical +content chunks, and directory type classifications. The database file and schema are kept in +place. When the shell integration is installed, `cds` also tries semantic search automatically for plain directory changes that do not look like local `cd` usage. For example, `cds Projects` diff --git a/install.sh b/install.sh index a407a9b..3236027 100755 --- a/install.sh +++ b/install.sh @@ -6,7 +6,7 @@ shell_name="$(basename "${SHELL:-}")" force_install=0 usage() { - cat <&2 - usage >&2 - exit 2 - ;; - esac - shift + case "$1" in + --force) + force_install=1 + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift done if ! command -v cargo >/dev/null 2>&1; then - echo "error: cargo is required to install cds" >&2 - echo "Install Rust from https://rustup.rs, then rerun ./install.sh" >&2 - exit 1 + echo "error: cargo is required to install cds" >&2 + echo "Install Rust from https://rustup.rs, then rerun ./install.sh" >&2 + exit 1 fi case "$shell_name" in - zsh) - profile="${ZDOTDIR:-$HOME}/.zshrc" - init_shell="zsh" - ;; - bash) - profile="$HOME/.bashrc" - init_shell="bash" - ;; - *) - profile="${ZDOTDIR:-$HOME}/.zshrc" - init_shell="zsh" - echo "warning: unsupported shell '${shell_name:-unknown}', defaulting setup to zsh" >&2 - ;; +zsh) + profile="${ZDOTDIR:-$HOME}/.zshrc" + init_shell="zsh" + ;; +bash) + profile="$HOME/.bashrc" + init_shell="bash" + ;; +*) + profile="${ZDOTDIR:-$HOME}/.zshrc" + init_shell="zsh" + echo "warning: unsupported shell '${shell_name:-unknown}', defaulting setup to zsh" >&2 + ;; esac echo "Installing cds with cargo..." cargo_args=(install --path "$repo_dir") if [ "$force_install" -eq 1 ]; then - cargo_args+=(--force) + cargo_args+=(--force) fi cargo "${cargo_args[@]}" cargo_bin="${CARGO_HOME:-$HOME/.cargo}/bin" +cache_dir="${CDS_CACHE_DIR:-$HOME/.cache/cds}" mkdir -p "$(dirname "$profile")" touch "$profile" if ! grep -Fq '# >>> cds init >>>' "$profile"; then - cat >> "$profile" <>"$profile" <>> cds init >>> if [ -x "$cargo_bin/cds" ]; then @@ -75,17 +76,21 @@ if [ -x "$cargo_bin/cds" ]; then fi # <<< cds init <<< EOF - echo "Added cds shell integration to $profile" + echo "Added cds shell integration to $profile" else - echo "cds shell integration already exists in $profile" + echo "cds shell integration already exists in $profile" fi if ! command -v cds >/dev/null 2>&1; then - echo "warning: cds is installed at $cargo_bin/cds, but $cargo_bin is not on PATH" >&2 - echo "Add this to your shell profile before the cds integration block:" >&2 - echo "export PATH=\"$cargo_bin:\$PATH\"" >&2 + echo "warning: cds is installed at $cargo_bin/cds, but $cargo_bin is not on PATH" >&2 + echo "Add this to your shell profile before the cds integration block:" >&2 + echo "export PATH=\"$cargo_bin:\$PATH\"" >&2 fi +mkdir -p "$cache_dir" +echo "Restarting cds daemon..." +"$cargo_bin/cds" --restart-daemon + cat < Result { pub async fn init_with_progress

(progress: &mut P) -> Result where - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, { let mut init_progress = NoopInitProgress; init_with_progress_and_steps(progress, &mut init_progress).await @@ -58,7 +68,7 @@ pub async fn init_with_progress_and_steps( init_progress: &mut I, ) -> Result where - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, I: InitProgress + ?Sized, { init_progress.paths_started(); @@ -99,7 +109,7 @@ pub async fn index(roots: Vec) -> Result { pub async fn index_with_progress

(roots: Vec, progress: &mut P) -> Result where - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, { let paths = AppPaths::discover()?; let settings = Settings::load_or_create(&paths.config_file)?; @@ -131,11 +141,25 @@ pub async fn search(query: Vec, limit: usize) -> Result, limit: usize) -> Result { + let query = join_query(query)?; + let paths = AppPaths::discover()?; + daemon_dry_run(&paths, &query, limit)? + .ok_or_else(|| app_err(AppError::DaemonUnavailable { mode: "dry-run" })) +} + pub async fn search_text(query: &str, limit: usize) -> Result> { let paths = AppPaths::discover()?; + if let Some(results) = daemon_search(&paths, query, limit)? { + return Ok(results); + } + + eprintln!("cds: warning: daemon unavailable or busy; searching locally"); + + let settings = Settings::load_or_create(&paths.config_file)?; let database = Database::open_existing(&paths.database_file).await?; let embedder = RuntimeEmbedder::load(&paths)?; - let searcher = Searcher::new(&database, &embedder); + let searcher = Searcher::new_with_settings(&database, &embedder, &settings); Ok(searcher.search(query, limit).await?) } @@ -151,7 +175,838 @@ pub async fn reset_database() -> Result<()> { Ok(database.reset().await?) } +pub async fn daemon() -> Result<()> { + daemon_with_options(DaemonOptions::default()).await +} + +pub async fn daemon_once() -> Result<()> { + let paths = AppPaths::discover()?; + let settings = Settings::load_or_create(&paths.config_file)?; + let database = Database::open(&paths.database_file).await?; + let embedder = RuntimeEmbedder::load(&paths)?; + let indexer = Indexer::new(&settings, &database, &embedder); + indexer.index_configured_roots().await?; + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DaemonRestartReport { + pub killed_daemons: usize, + pub pid: u32, + pub log_file: PathBuf, +} + +pub fn restart_daemon() -> Result { + let paths = AppPaths::discover()?; + let killed_daemons = stop_cds_daemons(&paths)?; + let started = start_daemon_process(&paths)?; + + Ok(DaemonRestartReport { + killed_daemons, + pid: started.pid, + log_file: started.log_file, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DaemonOptions { + poll_interval: Duration, + run_once: bool, +} + +impl Default for DaemonOptions { + fn default() -> Self { + Self { + poll_interval: Duration::from_secs(60), + run_once: false, + } + } +} + +async fn daemon_with_options(options: DaemonOptions) -> Result<()> { + let paths = AppPaths::discover()?; + if !options.run_once { + stop_cds_daemons(&paths)?; + } + let search_listener = if options.run_once { + empty_daemon_listener() + } else { + prepare_daemon_socket(&paths)? + }; + if !options.run_once { + write_daemon_pid(&paths, process::id())?; + } + let database = Database::open(&paths.database_file).await?; + let embedder = RuntimeEmbedder::load(&paths)?; + let mut root_snapshots = HashMap::::new(); + let mut search_cache = None; + let mut last_poll = Instant::now() - options.poll_interval; + + loop { + handle_daemon_search_requests( + &search_listener, + &paths, + &database, + &embedder, + &mut search_cache, + ) + .await?; + + if last_poll.elapsed() >= options.poll_interval { + if daemon_index_changed_files(&paths, &database, &embedder, &mut root_snapshots).await? + { + search_cache = None; + } + last_poll = Instant::now(); + } + + if options.run_once { + return Ok(()); + } + + thread::sleep(Duration::from_millis(200)); + } +} + +async fn daemon_index_changed_files( + paths: &AppPaths, + database: &Database, + embedder: &E, + root_snapshots: &mut HashMap, +) -> Result +where + E: Embedder, +{ + let settings = Settings::load_or_create(&paths.config_file)?; + let roots = settings.expanded_roots().map_err(config_err)?; + let mut changed_files = Vec::new(); + let mut deleted_files = Vec::new(); + + for root in roots { + let snapshot = daemon_root_file_snapshot(&root, &settings)?; + if let Some(previous) = root_snapshots.insert(root, snapshot.clone()) { + let diff = previous.diff(&snapshot); + changed_files.extend(diff.changed_files); + deleted_files.extend(diff.deleted_files); + } + } + + if !changed_files.is_empty() || !deleted_files.is_empty() { + let indexer = Indexer::new(&settings, database, embedder); + indexer + .index_file_changes(changed_files, deleted_files) + .await?; + return Ok(true); + } + + Ok(false) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct StartedDaemon { + pid: u32, + log_file: PathBuf, +} + +fn daemon_pid_file(paths: &AppPaths) -> PathBuf { + paths.cache_dir.join("daemon.pid") +} + +fn daemon_log_file(paths: &AppPaths) -> PathBuf { + paths.cache_dir.join("daemon.log") +} + +fn write_daemon_pid(paths: &AppPaths, pid: u32) -> Result<()> { + fs::create_dir_all(&paths.cache_dir).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: paths.cache_dir.clone(), + source, + }) + })?; + fs::write(daemon_pid_file(paths), pid.to_string()).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: daemon_pid_file(paths), + source, + }) + }) +} + +fn start_daemon_process(paths: &AppPaths) -> Result { + fs::create_dir_all(&paths.cache_dir).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: paths.cache_dir.clone(), + source, + }) + })?; + let log_file = daemon_log_file(paths); + let log = OpenOptions::new() + .create(true) + .append(true) + .open(&log_file) + .map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: log_file.clone(), + source, + }) + })?; + let stderr = log.try_clone().map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: log_file.clone(), + source, + }) + })?; + let executable = env::current_exe() + .map_err(|source| app_err(AppError::ResolveCurrentExecutable { source }))?; + let child = Command::new(executable) + .arg("--daemon") + .stdin(Stdio::null()) + .stdout(Stdio::from(log)) + .stderr(Stdio::from(stderr)) + .spawn() + .map_err(|source| app_err(AppError::StartDaemon { source }))?; + let pid = child.id(); + write_daemon_pid(paths, pid)?; + + Ok(StartedDaemon { pid, log_file }) +} + +#[cfg(unix)] +fn stop_cds_daemons(paths: &AppPaths) -> Result { + let mut pids = cds_daemon_pids()?; + pids.sort_unstable(); + pids.dedup(); + + for pid in &pids { + terminate_process(*pid)?; + } + + let deadline = Instant::now() + Duration::from_secs(2); + loop { + let still_running = pids + .iter() + .copied() + .filter(|pid| process_is_alive(*pid)) + .collect::>(); + if still_running.is_empty() { + break; + } + if Instant::now() >= deadline { + for pid in still_running { + kill_process(pid)?; + } + break; + } + thread::sleep(Duration::from_millis(50)); + } + + let _ = fs::remove_file(daemon_pid_file(paths)); + let _ = fs::remove_file(daemon_socket_path(paths)); + Ok(pids.len()) +} + +#[cfg(not(unix))] +fn stop_cds_daemons(paths: &AppPaths) -> Result { + let _ = fs::remove_file(daemon_pid_file(paths)); + Ok(0) +} + +#[cfg(unix)] +fn cds_daemon_pids() -> Result> { + let current_pid = process::id(); + let current_uid = current_uid()?; + let output = Command::new("ps") + .args(["-axo", "pid=,uid=,command="]) + .output() + .map_err(|source| { + app_err(AppError::DaemonProcessCommand { + command: "ps", + source, + }) + })?; + if !output.status.success() { + return Err(app_err(AppError::DaemonProcessStatus { + command: "ps", + status: output.status.to_string(), + })); + } + + let process_table = String::from_utf8_lossy(&output.stdout); + Ok(process_table + .lines() + .filter_map(|line| cds_pid_from_process_line(line, current_uid, current_pid)) + .collect()) +} + +#[cfg(unix)] +fn current_uid() -> Result> { + let output = Command::new("id").arg("-u").output().map_err(|source| { + app_err(AppError::DaemonProcessCommand { + command: "id", + source, + }) + })?; + if !output.status.success() { + return Err(app_err(AppError::DaemonProcessStatus { + command: "id", + status: output.status.to_string(), + })); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .ok()) +} + +#[cfg(unix)] +fn cds_pid_from_process_line( + line: &str, + current_uid: Option, + current_pid: u32, +) -> Option { + let mut parts = line.split_whitespace(); + let pid = parts.next()?.parse::().ok()?; + let uid = parts.next()?.parse::().ok()?; + let command = parts.next()?; + + if pid == current_pid { + return None; + } + if current_uid.is_some_and(|current_uid| uid != current_uid) { + return None; + } + is_cds_process_command(command).then_some(pid) +} + +#[cfg(unix)] +fn is_cds_process_command(command: &str) -> bool { + let executable_name = Path::new(command) + .file_name() + .and_then(|name| name.to_str()); + executable_name == Some("cds") +} + +#[cfg(unix)] +fn terminate_process(pid: u32) -> Result<()> { + let status = Command::new("kill") + .arg("-TERM") + .arg(pid.to_string()) + .status() + .map_err(|source| { + app_err(AppError::DaemonProcessCommand { + command: "kill", + source, + }) + })?; + if !status.success() && process_is_alive(pid) { + return Err(app_err(AppError::DaemonProcessStatus { + command: "kill", + status: status.to_string(), + })); + } + + Ok(()) +} + +#[cfg(unix)] +fn kill_process(pid: u32) -> Result<()> { + let status = Command::new("kill") + .arg("-KILL") + .arg(pid.to_string()) + .status() + .map_err(|source| { + app_err(AppError::DaemonProcessCommand { + command: "kill", + source, + }) + })?; + if !status.success() && process_is_alive(pid) { + return Err(app_err(AppError::DaemonProcessStatus { + command: "kill", + status: status.to_string(), + })); + } + + Ok(()) +} + +#[cfg(unix)] +fn process_is_alive(pid: u32) -> bool { + Command::new("kill") + .arg("-0") + .arg(pid.to_string()) + .status() + .is_ok_and(|status| status.success()) +} + +#[derive(Debug, Serialize, Deserialize)] +struct DaemonSearchRequest { + query: String, + limit: usize, + #[serde(default)] + dry_run: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +struct DaemonSearchResponse { + results: Vec, + dry_run: Option, +} + +#[cfg(unix)] +type DaemonSearchListener = Option; + +#[cfg(not(unix))] +struct DaemonSearchListener; + +#[cfg(unix)] +fn daemon_socket_path(paths: &AppPaths) -> PathBuf { + paths.cache_dir.join("daemon.sock") +} + +#[cfg(unix)] +fn empty_daemon_listener() -> DaemonSearchListener { + None +} + +#[cfg(not(unix))] +fn empty_daemon_listener() -> DaemonSearchListener { + DaemonSearchListener +} + +#[cfg(unix)] +fn prepare_daemon_socket(paths: &AppPaths) -> Result { + use std::os::unix::net::{UnixListener, UnixStream}; + + fs::create_dir_all(&paths.cache_dir).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: paths.cache_dir.clone(), + source, + }) + })?; + let socket = daemon_socket_path(paths); + if socket.exists() { + if UnixStream::connect(&socket).is_ok() { + return Err(app_err(AppError::DaemonAlreadyRunning { + socket: socket.clone(), + })); + } + + fs::remove_file(&socket).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: socket.clone(), + source, + }) + })?; + } + + let listener = match UnixListener::bind(&socket) { + Ok(listener) => listener, + Err(source) if source.kind() == std::io::ErrorKind::PermissionDenied => return Ok(None), + Err(source) => { + return Err(app_err(AppError::InspectDaemonWatchPath { + path: socket.clone(), + source, + })); + } + }; + listener.set_nonblocking(true).map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: socket, + source, + }) + })?; + Ok(Some(listener)) +} + +#[cfg(not(unix))] +fn prepare_daemon_socket(_paths: &AppPaths) -> Result { + Ok(DaemonSearchListener) +} + +#[cfg(unix)] +fn daemon_search(paths: &AppPaths, query: &str, limit: usize) -> Result>> { + let response = daemon_request( + paths, + DaemonSearchRequest { + query: query.to_string(), + limit, + dry_run: false, + }, + DAEMON_SEARCH_TIMEOUT, + )?; + Ok(response.map(|response| response.results)) +} + +#[cfg(unix)] +fn daemon_dry_run(paths: &AppPaths, query: &str, limit: usize) -> Result> { + let Some(response) = daemon_request( + paths, + DaemonSearchRequest { + query: query.to_string(), + limit, + dry_run: true, + }, + DAEMON_DRY_RUN_TIMEOUT, + )? + else { + return Ok(None); + }; + response + .dry_run + .map(Some) + .ok_or_else(|| app_err(AppError::DaemonDryRunMissing)) +} + +#[cfg(unix)] +fn daemon_request( + paths: &AppPaths, + request: DaemonSearchRequest, + timeout: Duration, +) -> Result> { + use std::os::unix::net::UnixStream; + + let socket = daemon_socket_path(paths); + let Ok(mut stream) = UnixStream::connect(socket) else { + return Ok(None); + }; + let _ = stream.set_read_timeout(Some(timeout)); + let _ = stream.set_write_timeout(Some(timeout)); + let request = serde_json::to_vec(&request) + .map_err(|source| app_err(AppError::SerializeDaemonMessage { source }))?; + if let Err(source) = stream.write_all(&request) { + if is_daemon_timeout(&source) { + return Ok(None); + } + return Err(app_err(AppError::DaemonSocketIo { source })); + } + stream + .shutdown(std::net::Shutdown::Write) + .map_err(|source| app_err(AppError::DaemonSocketIo { source }))?; + + let mut response = Vec::new(); + if let Err(source) = stream.read_to_end(&mut response) { + if is_daemon_timeout(&source) { + return Ok(None); + } + return Err(app_err(AppError::DaemonSocketIo { source })); + } + let response = serde_json::from_slice::(&response) + .map_err(|source| app_err(AppError::ParseDaemonMessage { source }))?; + Ok(Some(response)) +} + +fn is_daemon_timeout(source: &std::io::Error) -> bool { + matches!( + source.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ) +} + +#[cfg(not(unix))] +fn daemon_search( + _paths: &AppPaths, + _query: &str, + _limit: usize, +) -> Result>> { + Ok(None) +} + +#[cfg(not(unix))] +fn daemon_dry_run(_paths: &AppPaths, _query: &str, _limit: usize) -> Result> { + Ok(None) +} + +#[cfg(unix)] +async fn handle_daemon_search_requests( + listener: &DaemonSearchListener, + paths: &AppPaths, + database: &Database, + embedder: &E, + search_cache: &mut Option, +) -> Result<()> +where + E: Embedder, +{ + let Some(listener) = listener else { + return Ok(()); + }; + + loop { + let (mut stream, _) = match listener.accept() { + Ok(connection) => connection, + Err(source) if source.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(source) => return Err(app_err(AppError::DaemonSocketIo { source })), + }; + stream + .set_nonblocking(false) + .map_err(|source| app_err(AppError::DaemonSocketIo { source }))?; + + let mut request = Vec::new(); + if let Err(source) = stream.read_to_end(&mut request) { + if is_daemon_client_disconnect(&source) { + continue; + } + return Err(app_err(AppError::DaemonSocketIo { source })); + } + let request = match serde_json::from_slice::(&request) { + Ok(request) => request, + Err(_) => continue, + }; + let settings = Settings::load_or_create(&paths.config_file)?; + let generic_terms = search_generic_terms(&settings); + let (cache, cache_status) = + search_cache_for(database, search_cache, &generic_terms).await?; + let searcher = Searcher::new_with_settings(database, embedder, &settings); + let response = if request.dry_run { + let report = searcher + .dry_run_with_cache_status(&request.query, request.limit, cache, cache_status) + .await?; + DaemonSearchResponse { + results: report.results.clone(), + dry_run: Some(report), + } + } else { + let results = searcher + .search_with_cache(&request.query, request.limit, cache) + .await?; + DaemonSearchResponse { + results, + dry_run: None, + } + }; + let response = serde_json::to_vec(&response) + .map_err(|source| app_err(AppError::SerializeDaemonMessage { source }))?; + if let Err(source) = stream.write_all(&response) { + if is_daemon_client_disconnect(&source) { + continue; + } + return Err(app_err(AppError::DaemonSocketIo { source })); + } + } +} + +async fn search_cache_for<'a>( + database: &Database, + search_cache: &'a mut Option, + generic_terms: &HashSet, +) -> Result<(&'a SearchCache, CacheDryRunStatus)> { + let status = if search_cache + .as_ref() + .is_none_or(|cache| !cache.matches_generic_terms(generic_terms)) + { + *search_cache = Some(SearchCache::load(database, generic_terms).await?); + CacheDryRunStatus::Miss + } else { + CacheDryRunStatus::Hit + }; + + Ok(( + search_cache + .as_ref() + .expect("search cache should be populated"), + status, + )) +} + +fn search_generic_terms(settings: &Settings) -> HashSet { + settings + .index + .generic_terms + .iter() + .map(|term| term.to_ascii_lowercase()) + .collect() +} + +fn is_daemon_client_disconnect(source: &std::io::Error) -> bool { + matches!( + source.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::NotConnected + ) +} + +#[cfg(not(unix))] +async fn handle_daemon_search_requests( + _listener: &DaemonSearchListener, + _paths: &AppPaths, + _database: &Database, + _embedder: &E, + _search_cache: &mut Option, +) -> Result<()> +where + E: Embedder, +{ + Ok(()) +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct RootFileSnapshot { + files: HashMap, +} + +impl RootFileSnapshot { + fn diff(&self, next: &Self) -> FileSnapshotDiff { + let mut changed_files = Vec::new(); + let mut deleted_files = Vec::new(); + + for (path, snapshot) in &next.files { + if self.files.get(path) != Some(snapshot) { + changed_files.push(path.clone()); + } + } + + for path in self.files.keys() { + if !next.files.contains_key(path) { + deleted_files.push(path.to_string_lossy().into_owned()); + } + } + + changed_files.sort(); + deleted_files.sort(); + + FileSnapshotDiff { + changed_files, + deleted_files, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct FileSnapshot { + size_bytes: u64, + modified_unix_nanos: u128, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct FileSnapshotDiff { + changed_files: Vec, + deleted_files: Vec, +} + +fn daemon_root_file_snapshot(root: &Path, settings: &Settings) -> Result { + let mut snapshot = RootFileSnapshot::default(); + collect_file_snapshots( + root, + settings, + &mut snapshot, + DaemonSnapshotDepth::IndexRoot, + )?; + Ok(snapshot) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DaemonSnapshotDepth { + IndexRoot, + TopLevelDirectory { depth: usize }, +} + +impl DaemonSnapshotDepth { + fn child_position(self) -> Option { + match self { + Self::IndexRoot => Some(Self::TopLevelDirectory { depth: 0 }), + Self::TopLevelDirectory { depth } => depth + .checked_add(1) + .map(|depth| Self::TopLevelDirectory { depth }), + } + } + + fn exceeds_max_depth(self, settings: &Settings) -> bool { + match self { + Self::IndexRoot => false, + Self::TopLevelDirectory { depth } => { + depth > settings.index.max_depth_per_top_level_directory + } + } + } +} + +fn collect_file_snapshots( + path: &Path, + settings: &Settings, + snapshot: &mut RootFileSnapshot, + depth: DaemonSnapshotDepth, +) -> Result<()> { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + return Ok(()); + } + Err(source) => { + return Err(app_err(AppError::InspectDaemonWatchPath { + path: path.to_path_buf(), + source, + })); + } + }; + + if !metadata.is_dir() { + snapshot.files.insert( + path.to_path_buf(), + FileSnapshot { + size_bytes: metadata.len(), + modified_unix_nanos: metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_nanos()) + .unwrap_or(0), + }, + ); + return Ok(()); + } + + let mut entries = fs::read_dir(path) + .map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: path.to_path_buf(), + source, + }) + })? + .filter_map(|entry| entry.ok()) + .collect::>(); + entries.sort_by_key(|entry| entry.path()); + + for entry in entries { + let name = entry.file_name().to_string_lossy().into_owned(); + let path = entry.path(); + let file_type = entry.file_type().map_err(|source| { + app_err(AppError::InspectDaemonWatchPath { + path: path.clone(), + source, + }) + })?; + + if file_type.is_dir() { + if settings.index.is_excluded_directory_name(&name) { + continue; + } + let Some(child_depth) = depth.child_position() else { + continue; + }; + if child_depth.exceeds_max_depth(settings) { + continue; + } + collect_file_snapshots(&path, settings, snapshot, child_depth)?; + } else if settings.index.is_excluded_name(&name) { + continue; + } else { + collect_file_snapshots(&path, settings, snapshot, depth)?; + } + } + + Ok(()) +} + pub async fn resolve_cd_script(args: Vec) -> Vec { + if is_explicit_cds_command(&args) { + return crate::emit_cds_command_script(&args); + } + if let Some(query) = implied_search_query(&args) && let Ok(results) = search_text(&query, 5).await && let Some(result) = results @@ -164,6 +1019,29 @@ pub async fn resolve_cd_script(args: Vec) -> Vec { crate::emit_cd_script(&args) } +fn is_explicit_cds_command(args: &[OsString]) -> bool { + let Some(first) = args.first().and_then(|arg| arg.to_str()) else { + return false; + }; + + matches!( + first, + "--daemon" + | "--dir-type-count" + | "--dry-run" + | "--help" + | "--index" + | "--init" + | "--reset" + | "--restart-daemon" + | "--search" + | "--shell-init" + | "--version" + | "-h" + | "-V" + ) +} + pub fn implied_search_query(args: &[OsString]) -> Option { semantic_query(args) } @@ -362,6 +1240,20 @@ mod tests { assert_eq!(semantic_query_in(&[os("~")], Some(temp.path())), None); } + #[test] + fn daemon_client_disconnect_errors_are_non_fatal() { + for kind in [ + std::io::ErrorKind::BrokenPipe, + std::io::ErrorKind::ConnectionAborted, + std::io::ErrorKind::ConnectionReset, + std::io::ErrorKind::TimedOut, + std::io::ErrorKind::WouldBlock, + std::io::ErrorKind::NotConnected, + ] { + assert!(is_daemon_client_disconnect(&std::io::Error::from(kind))); + } + } + #[test] fn semantic_query_joins_plain_words() { let temp = tempfile::tempdir().unwrap(); @@ -378,4 +1270,140 @@ mod tests { assert_eq!(implied_search_query(&[os("../Projects")]), None); assert_eq!(implied_search_query(&[]), None); } + + #[tokio::test] + async fn explicit_cds_commands_are_emitted_as_commands() { + assert_eq!( + resolve_cd_script(vec![os("--restart-daemon")]).await, + b"command cds '--restart-daemon'\n" + ); + assert_eq!( + resolve_cd_script(vec![os("--reset")]).await, + b"command cds '--reset'\n" + ); + assert_eq!( + resolve_cd_script(vec![os("--dry-run"), os("github"), os("clone")]).await, + b"command cds '--dry-run' 'github' 'clone'\n" + ); + } + + #[cfg(unix)] + #[test] + fn detects_cds_process_commands() { + assert!(is_cds_process_command("/Users/me/.cargo/bin/cds")); + assert!(is_cds_process_command("target/debug/cds")); + assert!(!is_cds_process_command("/Users/me/.cargo/bin/cds-helper")); + assert!(!is_cds_process_command("bash")); + + let pid = cds_pid_from_process_line( + " 123 501 /Users/me/.cargo/bin/cds --daemon", + Some(501), + 999, + ); + assert_eq!(pid, Some(123)); + let restart = cds_pid_from_process_line( + " 124 501 /Users/me/.cargo/bin/cds --restart-daemon", + Some(501), + 999, + ); + assert_eq!(restart, Some(124)); + let search = cds_pid_from_process_line( + " 125 501 /Users/me/.cargo/bin/cds --search daemon", + Some(501), + 999, + ); + assert_eq!(search, Some(125)); + let current_process = cds_pid_from_process_line( + " 999 501 /Users/me/.cargo/bin/cds --restart-daemon", + Some(501), + 999, + ); + assert_eq!(current_process, None); + let other_user = + cds_pid_from_process_line(" 123 502 /Users/me/.cargo/bin/cds --daemon", Some(501), 999); + assert_eq!(other_user, None); + } + + #[test] + fn file_snapshots_diff_changed_and_deleted_files() { + let unchanged = PathBuf::from("/tmp/project/README.md"); + let changed = PathBuf::from("/tmp/project/package.json"); + let deleted = PathBuf::from("/tmp/project/Cargo.toml"); + + let previous = RootFileSnapshot { + files: HashMap::from([ + ( + unchanged.clone(), + FileSnapshot { + size_bytes: 10, + modified_unix_nanos: 10, + }, + ), + ( + changed.clone(), + FileSnapshot { + size_bytes: 20, + modified_unix_nanos: 20, + }, + ), + ( + deleted.clone(), + FileSnapshot { + size_bytes: 30, + modified_unix_nanos: 30, + }, + ), + ]), + }; + let next = RootFileSnapshot { + files: HashMap::from([ + ( + unchanged, + FileSnapshot { + size_bytes: 10, + modified_unix_nanos: 10, + }, + ), + ( + changed.clone(), + FileSnapshot { + size_bytes: 20, + modified_unix_nanos: 21, + }, + ), + ]), + }; + + let diff = previous.diff(&next); + + assert_eq!(diff.changed_files, vec![changed]); + assert_eq!( + diff.deleted_files, + vec![deleted.to_string_lossy().into_owned()] + ); + } + + #[test] + fn daemon_file_snapshot_respects_recursive_depth_limit() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("project"); + let included = root.join("top").join("allowed").join("README.md"); + let excluded = root + .join("top") + .join("allowed") + .join("too-deep") + .join("README.md"); + std::fs::create_dir_all(included.parent().unwrap()).unwrap(); + std::fs::create_dir_all(excluded.parent().unwrap()).unwrap(); + std::fs::write(&included, "included content").unwrap(); + std::fs::write(&excluded, "excluded content").unwrap(); + + let mut settings = Settings::default(); + settings.index.max_depth_per_top_level_directory = 1; + + let snapshot = daemon_root_file_snapshot(&root, &settings).unwrap(); + + assert!(snapshot.files.contains_key(&included)); + assert!(!snapshot.files.contains_key(&excluded)); + } } diff --git a/src/cli.rs b/src/cli.rs index b922327..040b14c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,15 +2,18 @@ use std::env; use std::ffi::{OsStr, OsString}; use clap::builder::OsStringValueParser; -use clap::{Arg, ArgAction, Command, error::ErrorKind}; +use clap::{Arg, ArgAction, ArgGroup, Command, error::ErrorKind}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Invocation { + Daemon { run_once: bool }, DirectoryTypeCount, + DryRun { query: Vec }, EmitCd { args: Vec }, Index { roots: Vec }, Init, Reset, + RestartDaemon, Search { query: Vec }, ShellInit { shell: Shell }, } @@ -60,10 +63,26 @@ pub fn parse_invocation( return Ok(Invocation::EmitCd { args }); } + if matches.get_flag("daemon") { + return Ok(Invocation::Daemon { run_once: false }); + } + + if matches.get_flag("daemon-once") { + return Ok(Invocation::Daemon { run_once: true }); + } + if matches.get_flag("dir-type-count") { return Ok(Invocation::DirectoryTypeCount); } + if matches.contains_id("dry-run") { + let query = matches + .get_many::("dry-run") + .map(|query| query.cloned().collect()) + .unwrap_or_default(); + return Ok(Invocation::DryRun { query }); + } + if matches.get_flag("init") { return Ok(Invocation::Init); } @@ -72,6 +91,10 @@ pub fn parse_invocation( return Ok(Invocation::Reset); } + if matches.get_flag("restart-daemon") { + return Ok(Invocation::RestartDaemon); + } + if matches.contains_id("index") { let roots = matches .get_many::("index") @@ -105,15 +128,42 @@ pub fn command() -> Command { .about("cd with semantic search") .version(clap::crate_version!()) .disable_help_subcommand(true) + .group(ArgGroup::new("mode").args([ + "daemon", + "daemon-once", + "dir-type-count", + "dry-run", + "emit-cd", + "index", + "init", + "reset", + "restart-daemon", + "search", + "shell-init", + ])) + .arg( + Arg::new("daemon") + .long("daemon") + .help("Run the cds indexing daemon in the foreground") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("daemon-once") + .long("daemon-once") + .hide(true) + .action(ArgAction::SetTrue), + ) .arg( Arg::new("init") .long("init") .help("Create the default config and database, then index configured roots") .conflicts_with_all([ "dir-type-count", + "dry-run", "emit-cd", "index", "reset", + "restart-daemon", "search", "shell-init", ]) @@ -123,7 +173,15 @@ pub fn command() -> Command { Arg::new("dir-type-count") .long("dir-type-count") .help("Print detected directory type counts") - .conflicts_with_all(["emit-cd", "index", "init", "reset", "search", "shell-init"]) + .conflicts_with_all([ + "dry-run", + "emit-cd", + "index", + "init", + "reset", + "search", + "shell-init", + ]) .action(ArgAction::SetTrue), ) .arg( @@ -132,9 +190,29 @@ pub fn command() -> Command { .help("Delete all indexed data from the cds database") .conflicts_with_all([ "dir-type-count", + "dry-run", "emit-cd", "index", "init", + "restart-daemon", + "search", + "shell-init", + ]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("restart-daemon") + .long("restart-daemon") + .help("Kill existing cds daemons and start a fresh one") + .conflicts_with_all([ + "daemon", + "daemon-once", + "dir-type-count", + "dry-run", + "emit-cd", + "index", + "init", + "reset", "search", "shell-init", ]) @@ -154,10 +232,12 @@ pub fn command() -> Command { .default_missing_value("") .conflicts_with_all([ "dir-type-count", + "dry-run", "emit-cd", "index", "init", "reset", + "restart-daemon", "search", ]) .value_parser(OsStringValueParser::new()), @@ -172,11 +252,31 @@ pub fn command() -> Command { paths instead of the configured roots.", ) .num_args(0..) + .conflicts_with_all([ + "dir-type-count", + "dry-run", + "emit-cd", + "init", + "reset", + "restart-daemon", + "search", + "shell-init", + ]) + .value_parser(OsStringValueParser::new()), + ) + .arg( + Arg::new("dry-run") + .long("dry-run") + .value_name("QUERY") + .help("Print SQL candidates, embedding scores, and the winning directory") + .num_args(1..) .conflicts_with_all([ "dir-type-count", "emit-cd", + "index", "init", "reset", + "restart-daemon", "search", "shell-init", ]) @@ -190,10 +290,12 @@ pub fn command() -> Command { .num_args(1..) .conflicts_with_all([ "dir-type-count", + "dry-run", "emit-cd", "index", "init", "reset", + "restart-daemon", "shell-init", ]) .value_parser(OsStringValueParser::new()), @@ -207,6 +309,7 @@ pub fn command() -> Command { .allow_hyphen_values(true) .conflicts_with_all([ "dir-type-count", + "dry-run", "index", "init", "reset", @@ -285,6 +388,30 @@ mod tests { ); } + #[test] + fn parses_restart_daemon() { + assert_eq!( + parse_invocation([os("--restart-daemon")]).unwrap(), + Invocation::RestartDaemon + ); + } + + #[test] + fn parses_daemon() { + assert_eq!( + parse_invocation([os("--daemon")]).unwrap(), + Invocation::Daemon { run_once: false } + ); + } + + #[test] + fn parses_daemon_once() { + assert_eq!( + parse_invocation([os("--daemon-once")]).unwrap(), + Invocation::Daemon { run_once: true } + ); + } + #[test] fn init_word_is_cd_input_inside_hidden_emit() { assert_eq!( @@ -333,6 +460,16 @@ mod tests { ); } + #[test] + fn parses_dry_run_query() { + assert_eq!( + parse_invocation([os("--dry-run"), os("github"), os("clone")]).unwrap(), + Invocation::DryRun { + query: vec![os("github"), os("clone")] + } + ); + } + #[test] fn search_word_is_cd_input_inside_hidden_emit() { assert_eq!( diff --git a/src/config/mod.rs b/src/config/mod.rs index d48653f..abacfd7 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -6,4 +6,5 @@ pub use error::ConfigError; pub use paths::{AppPaths, expand_tilde}; pub use settings::{ DirectoryTypeDefinition, DirectoryTypeRule, DirectoryTypeSignal, IndexSettings, Settings, + default_generic_terms, }; diff --git a/src/config/settings.rs b/src/config/settings.rs index 1cb778d..a58dfde 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -73,6 +73,8 @@ pub struct IndexSettings { pub max_depth_per_top_level_directory: usize, #[serde(default = "default_max_chunk_bytes")] pub max_chunk_bytes: usize, + #[serde(default = "default_generic_terms")] + pub generic_terms: Vec, } impl IndexSettings { @@ -123,6 +125,7 @@ impl Default for IndexSettings { max_entries_per_directory: 80, max_depth_per_top_level_directory: default_max_depth_per_top_level_directory(), max_chunk_bytes: default_max_chunk_bytes(), + generic_terms: default_generic_terms(), } } } @@ -135,6 +138,46 @@ const fn default_max_chunk_bytes() -> usize { 4_096 } +pub fn default_generic_terms() -> Vec { + [ + "a", + "an", + "and", + "app", + "application", + "code", + "dir", + "directory", + "for", + "from", + "game", + "in", + "last", + "me", + "my", + "of", + "on", + "program", + "project", + "repo", + "repository", + "search", + "site", + "that", + "the", + "thing", + "to", + "tool", + "web", + "website", + "with", + "work", + ] + .into_iter() + .map(str::to_string) + .collect() +} + fn matches_exclude_pattern(pattern: &str, name: &str) -> bool { if pattern == name { return true; @@ -289,6 +332,7 @@ mod tests { assert_eq!(parsed.index.max_depth_per_top_level_directory, 3); assert_eq!(parsed.index.max_chunk_bytes, 4096); + assert_eq!(parsed.index.generic_terms, default_generic_terms()); assert!(parsed.detectors.is_empty()); } diff --git a/src/db/document.rs b/src/db/document.rs index d67ab6b..770c856 100644 --- a/src/db/document.rs +++ b/src/db/document.rs @@ -49,10 +49,32 @@ pub struct FileChunkMatch { pub directory_path: String, pub content: String, pub embedding: Vec, + pub is_current: bool, pub file_modified_unix_seconds: i64, pub directory_modified_unix_seconds: i64, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ModifiedTimeRange { + pub start_unix_seconds: Option, + pub end_unix_seconds: Option, +} + +impl ModifiedTimeRange { + pub fn contains(self, unix_seconds: i64) -> bool { + if self + .start_unix_seconds + .is_some_and(|start| unix_seconds < start) + { + return false; + } + if self.end_unix_seconds.is_some_and(|end| unix_seconds >= end) { + return false; + } + true + } +} + #[derive(Debug, Clone, PartialEq)] pub struct DirectoryClassification { pub directory_path: String, diff --git a/src/db/error.rs b/src/db/error.rs index d80cc50..a1686b9 100644 --- a/src/db/error.rs +++ b/src/db/error.rs @@ -74,6 +74,13 @@ pub enum DbError { source: sqlx::Error, }, + #[error("failed to delete indexed file {path}")] + DeleteFile { + path: String, + #[source] + source: sqlx::Error, + }, + #[error("failed to insert indexed chunk {path}#{chunk_index}")] InsertFileChunk { path: String, @@ -82,6 +89,14 @@ pub enum DbError { source: sqlx::Error, }, + #[error("failed to insert historical indexed chunk {path}#{chunk_index}")] + InsertFileChunkHistory { + path: String, + chunk_index: u32, + #[source] + source: sqlx::Error, + }, + #[error("failed to commit indexed file batch")] CommitFileBatch { #[source] diff --git a/src/db/mod.rs b/src/db/mod.rs index 2e3952f..aa42641 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -8,11 +8,11 @@ use std::path::Path; use sqlx::migrate::Migrator; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteRow}; -use sqlx::{Row, SqlitePool}; +use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool, Transaction}; pub use document::{ DirectoryClassification, DirectoryTypeCount, DocumentKind, FileChunkMatch, IndexedDocument, - IndexedFile, IndexedFileChunk, + IndexedFile, IndexedFileChunk, ModifiedTimeRange, }; pub use error::DbError; pub use vector::{decode_embedding, encode_embedding}; @@ -363,6 +363,45 @@ impl Database { Ok(()) } + pub async fn delete_current_file(&self, file_path: &str) -> Result<()> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| DbError::DeleteFile { + path: file_path.to_string(), + source, + })?; + + sqlx::query("DELETE FROM indexed_file_chunks WHERE file_path = ?") + .bind(file_path) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::DeleteFileChunks { + path: file_path.to_string(), + source, + })?; + + sqlx::query("DELETE FROM indexed_files WHERE path = ?") + .bind(file_path) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::DeleteFile { + path: file_path.to_string(), + source, + })?; + + transaction + .commit() + .await + .map_err(|source| DbError::DeleteFile { + path: file_path.to_string(), + source, + })?; + + Ok(()) + } + pub async fn upsert_files_with_chunks( &self, files: &[(&IndexedFile, &[IndexedFileChunk])], @@ -476,6 +515,8 @@ impl Database { chunk_index: chunk.chunk_index, source, })?; + + insert_file_chunk_history(&mut transaction, file, chunk).await?; } } @@ -561,6 +602,40 @@ impl Database { source, })?; + sqlx::query( + " + DELETE FROM indexed_file_chunk_history + WHERE file_path = ? + OR directory_path = ? + OR ( + length(file_path) > ? + AND substr(file_path, 1, ?) = ? + AND substr(file_path, ? + 1, 1) = '/' + ) + OR ( + length(directory_path) > ? + AND substr(directory_path, 1, ?) = ? + AND substr(directory_path, ? + 1, 1) = '/' + ) + ", + ) + .bind(path) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .execute(&self.pool) + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + sqlx::query( " DELETE FROM indexed_file_chunks @@ -674,6 +749,10 @@ impl Database { .execute(&mut *transaction) .await .map_err(|source| DbError::ResetDatabase { source })?; + sqlx::query("DELETE FROM indexed_file_chunk_history") + .execute(&mut *transaction) + .await + .map_err(|source| DbError::ResetDatabase { source })?; sqlx::query("DELETE FROM indexed_files") .execute(&mut *transaction) .await @@ -776,15 +855,153 @@ impl Database { .map_err(|source| DbError::ReadDirectoryDocuments { source }) } - pub async fn file_chunk_matches(&self) -> Result> { + pub async fn directory_search_documents(&self) -> Result> { let rows = sqlx::query( " + SELECT + path, + name, + kind, + parent_path, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE kind = 'directory' + ", + ) + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryDocuments { source })?; + + rows.into_iter() + .map(decode_directory_search_row) + .collect::, _>>() + .map_err(|source| DbError::ReadDirectoryDocuments { source }) + } + + pub async fn directory_candidates_by_terms( + &self, + terms: &[String], + ) -> Result> { + if terms.is_empty() { + return Ok(Vec::new()); + } + + let mut query = QueryBuilder::::new( + " + SELECT + path, + name, + kind, + parent_path, + searchable_text, + embedding, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE kind = 'directory' AND ( + ", + ); + push_directory_term_filter(&mut query, terms); + query.push(") ORDER BY path"); + + let rows = query + .build() + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryDocuments { source })?; + + rows.into_iter() + .map(decode_document_row) + .collect::, _>>() + .map_err(|source| DbError::ReadDirectoryDocuments { source }) + } + + pub async fn directory_search_candidates_by_terms( + &self, + terms: &[String], + ) -> Result> { + if terms.is_empty() { + return Ok(Vec::new()); + } + + let mut query = QueryBuilder::::new( + " + SELECT + path, + name, + kind, + parent_path, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE kind = 'directory' AND ( + ", + ); + push_directory_term_filter(&mut query, terms); + query.push(") ORDER BY path"); + + let rows = query + .build() + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryDocuments { source })?; + + rows.into_iter() + .map(decode_directory_search_row) + .collect::, _>>() + .map_err(|source| DbError::ReadDirectoryDocuments { source }) + } + + pub async fn file_chunk_matches(&self) -> Result> { + self.file_chunk_matches_with_modified_range(None).await + } + + pub async fn file_chunk_matches_with_modified_range( + &self, + modified_range: Option, + ) -> Result> { + let mut query = QueryBuilder::::new( + " + SELECT + file_path, + file_name, + directory_path, + content, + embedding, + 0 AS is_current, + file_modified_unix_seconds, + directory_modified_unix_seconds + FROM indexed_file_chunk_history + ", + ); + if let Some(range) = modified_range { + query.push(" WHERE "); + push_modified_time_filter(&mut query, "file_modified_unix_seconds", range); + } + query.push( + " + UNION ALL SELECT chunk.file_path, file.name AS file_name, chunk.directory_path, chunk.content, chunk.embedding, + 1 AS is_current, file.modified_unix_seconds AS file_modified_unix_seconds, directory.modified_unix_seconds AS directory_modified_unix_seconds FROM indexed_file_chunks AS chunk @@ -794,10 +1011,92 @@ impl Database { ON directory.path = chunk.directory_path WHERE directory.kind = 'directory' ", - ) - .fetch_all(&self.pool) - .await - .map_err(|source| DbError::ReadFileChunks { source })?; + ); + if let Some(range) = modified_range { + query.push(" AND "); + push_modified_time_filter(&mut query, "file.modified_unix_seconds", range); + } + + let rows = query + .build() + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadFileChunks { source })?; + + rows.into_iter() + .map(decode_file_chunk_match_row) + .collect::, _>>() + .map_err(|source| DbError::ReadFileChunks { source }) + } + + pub async fn file_chunk_matches_in_directory_trees( + &self, + directory_paths: &[String], + ) -> Result> { + self.file_chunk_matches_in_directory_trees_with_modified_range(directory_paths, None) + .await + } + + pub async fn file_chunk_matches_in_directory_trees_with_modified_range( + &self, + directory_paths: &[String], + modified_range: Option, + ) -> Result> { + if directory_paths.is_empty() { + return Ok(Vec::new()); + } + + let mut query = QueryBuilder::::new( + " + SELECT + file_path, + file_name, + directory_path, + content, + embedding, + 0 AS is_current, + file_modified_unix_seconds, + directory_modified_unix_seconds + FROM indexed_file_chunk_history + WHERE + ", + ); + push_directory_tree_filter(&mut query, "directory_path", directory_paths); + if let Some(range) = modified_range { + query.push(" AND "); + push_modified_time_filter(&mut query, "file_modified_unix_seconds", range); + } + query.push( + " + UNION ALL + SELECT + chunk.file_path, + file.name AS file_name, + chunk.directory_path, + chunk.content, + chunk.embedding, + 1 AS is_current, + file.modified_unix_seconds AS file_modified_unix_seconds, + directory.modified_unix_seconds AS directory_modified_unix_seconds + FROM indexed_file_chunks AS chunk + INNER JOIN indexed_files AS file + ON file.path = chunk.file_path + INNER JOIN indexed_documents AS directory + ON directory.path = chunk.directory_path + WHERE directory.kind = 'directory' AND + ", + ); + push_directory_tree_filter(&mut query, "chunk.directory_path", directory_paths); + if let Some(range) = modified_range { + query.push(" AND "); + push_modified_time_filter(&mut query, "file.modified_unix_seconds", range); + } + + let rows = query + .build() + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadFileChunks { source })?; rows.into_iter() .map(decode_file_chunk_match_row) @@ -933,13 +1232,90 @@ async fn migrate_pool(pool: &SqlitePool) -> Result<()> { .run(pool) .await .map_err(|source| DbError::Migrate { source })?; - sqlx::query("PRAGMA user_version = 4") + sqlx::query("PRAGMA user_version = 5") .execute(pool) .await .map_err(|source| DbError::PrepareLegacyMigration { source })?; Ok(()) } +fn push_directory_term_filter(query: &mut QueryBuilder<'_, Sqlite>, terms: &[String]) { + for (index, term) in terms.iter().enumerate() { + if index > 0 { + query.push(" OR "); + } + let pattern = format!("%{}%", escape_like_pattern(term)); + query.push("("); + query.push("lower(name) LIKE "); + query.push_bind(pattern.clone()); + query.push(" ESCAPE '\\' OR lower(path) LIKE "); + query.push_bind(pattern); + query.push(" ESCAPE '\\')"); + } +} + +fn push_directory_tree_filter( + query: &mut QueryBuilder<'_, Sqlite>, + column: &'static str, + directory_paths: &[String], +) { + for (index, path) in directory_paths.iter().enumerate() { + if index > 0 { + query.push(" OR "); + } + let child_pattern = format!("{}/%", escape_like_pattern(path)); + query.push("("); + query.push(column); + query.push(" = "); + query.push_bind(path.clone()); + query.push(" OR "); + query.push(column); + query.push(" LIKE "); + query.push_bind(child_pattern); + query.push(" ESCAPE '\\')"); + } +} + +fn push_modified_time_filter( + query: &mut QueryBuilder<'_, Sqlite>, + column: &'static str, + range: ModifiedTimeRange, +) { + let mut has_previous_filter = false; + if let Some(start) = range.start_unix_seconds { + query.push(column); + query.push(" >= "); + query.push_bind(start); + has_previous_filter = true; + } + if let Some(end) = range.end_unix_seconds { + if has_previous_filter { + query.push(" AND "); + } + query.push(column); + query.push(" < "); + query.push_bind(end); + has_previous_filter = true; + } + if !has_previous_filter { + query.push("1 = 1"); + } +} + +fn escape_like_pattern(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' | '%' | '_' => { + escaped.push('\\'); + escaped.push(ch); + } + _ => escaped.push(ch), + } + } + escaped +} + async fn prepare_legacy_schema(pool: &SqlitePool) -> Result<()> { let has_legacy_documents: Option = sqlx::query_scalar( " @@ -1022,6 +1398,60 @@ async fn add_legacy_column_if_missing( Ok(()) } +async fn insert_file_chunk_history( + transaction: &mut Transaction<'_, Sqlite>, + file: &IndexedFile, + chunk: &IndexedFileChunk, +) -> Result<()> { + sqlx::query( + " + INSERT OR IGNORE INTO indexed_file_chunk_history ( + file_path, + file_name, + directory_path, + chunk_index, + content, + embedding, + embedding_dim, + start_byte, + end_byte, + content_fingerprint, + file_modified_unix_seconds, + directory_modified_unix_seconds, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&chunk.file_path) + .bind(&file.name) + .bind(&chunk.directory_path) + .bind(i64::from(chunk.chunk_index)) + .bind(&chunk.content) + .bind(encode_embedding(&chunk.embedding)) + .bind( + i64::try_from(chunk.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind( + i64::try_from(chunk.start_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(i64::try_from(chunk.end_byte).map_err(|source| DbError::MetadataSizeOverflow { source })?) + .bind(&file.content_fingerprint) + .bind(file.modified_unix_seconds) + .bind(file.modified_unix_seconds) + .bind(chunk.indexed_unix_seconds) + .execute(&mut **transaction) + .await + .map_err(|source| DbError::InsertFileChunkHistory { + path: chunk.file_path.clone(), + chunk_index: chunk.chunk_index, + source, + })?; + + Ok(()) +} + fn decode_classification_row( row: SqliteRow, ) -> std::result::Result { @@ -1070,6 +1500,28 @@ fn decode_document_row(row: SqliteRow) -> std::result::Result std::result::Result { + let kind: String = row.try_get("kind")?; + let size_bytes: i64 = row.try_get("size_bytes")?; + Ok(IndexedDocument { + path: row.try_get("path")?, + name: row.try_get("name")?, + kind: DocumentKind::from_db_value(&kind), + parent_path: row.try_get("parent_path")?, + searchable_text: String::new(), + embedding: Vec::new(), + metadata_fingerprint: String::new(), + size_bytes: u64::try_from(size_bytes).map_err(|err| sqlx::Error::Decode(Box::new(err)))?, + created_unix_seconds: row.try_get("created_unix_seconds")?, + modified_unix_seconds: row.try_get("modified_unix_seconds")?, + accessed_unix_seconds: row.try_get("accessed_unix_seconds")?, + readonly: row.try_get("readonly")?, + indexed_unix_seconds: row.try_get("indexed_unix_seconds")?, + }) +} + fn decode_file_chunk_match_row(row: SqliteRow) -> std::result::Result { let embedding: Vec = row.try_get("embedding")?; Ok(FileChunkMatch { @@ -1079,6 +1531,7 @@ fn decode_file_chunk_match_row(row: SqliteRow) -> std::result::Result("is_current")? != 0, file_modified_unix_seconds: row.try_get("file_modified_unix_seconds")?, directory_modified_unix_seconds: row.try_get("directory_modified_unix_seconds")?, }) @@ -1151,7 +1604,7 @@ mod tests { .await .unwrap(); - assert_eq!(version, 4); + assert_eq!(version, 5); db.upsert_document(&IndexedDocument { path: "/tmp/project".to_string(), name: "project".to_string(), @@ -1332,6 +1785,7 @@ mod tests { assert_eq!(db.document_count().await.unwrap(), 0); assert_eq!(table_count(&db, "indexed_files").await, 0); assert_eq!(table_count(&db, "indexed_file_chunks").await, 0); + assert_eq!(table_count(&db, "indexed_file_chunk_history").await, 0); assert_eq!(table_count(&db, "directory_classifications").await, 0); assert!(db.directory_type_counts().await.unwrap().is_empty()); } diff --git a/src/index/mod.rs b/src/index/mod.rs index fdd18ac..a7e35e1 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -46,7 +46,7 @@ where progress: &mut P, ) -> Result where - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, { let roots = self .settings @@ -60,13 +60,31 @@ where self.index_roots_with_progress(roots, &mut progress).await } + pub async fn index_file_changes( + &self, + changed_files: Vec, + deleted_files: Vec, + ) -> Result { + let mut report = IndexReport::default(); + scanner::index_file_changes( + changed_files, + deleted_files, + self.settings, + self.database, + self.embedder, + &mut report, + ) + .await?; + Ok(report) + } + pub async fn index_roots_with_progress

( &self, roots: Vec, progress: &mut P, ) -> Result where - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, { let mut report = IndexReport::default(); @@ -258,6 +276,45 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn indexes_only_changed_files() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let app = root.join("app"); + fs::create_dir_all(&app).unwrap(); + let readme = app.join("README.md"); + let cargo = app.join("Cargo.toml"); + fs::write(&readme, "initial readme content").unwrap(); + fs::write(&cargo, "[package]\nname = \"app\"").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::new(16); + let indexer = Indexer::new(&settings, &database, &embedder); + indexer.index_roots(vec![root]).await.unwrap(); + + fs::write(&readme, "changed readme content").unwrap(); + let report = indexer + .index_file_changes( + vec![readme.clone()], + vec![cargo.to_string_lossy().into_owned()], + ) + .await + .unwrap(); + + assert_eq!(report.roots_scanned, 0); + assert_eq!(report.text_files_indexed, 1); + assert_eq!(report.file_chunks_indexed, 1); + + let chunks = database.file_chunk_matches().await.unwrap(); + assert!(chunks.iter().any(|chunk| chunk.content.contains("changed"))); + assert!( + !chunks + .iter() + .any(|chunk| chunk.is_current && chunk.file_path == cargo.to_string_lossy()) + ); + } + #[tokio::test] async fn skips_hidden_roots_and_prunes_stale_rows() { let temp = tempfile::tempdir().unwrap(); diff --git a/src/index/progress.rs b/src/index/progress.rs index 4636b9a..774bda0 100644 --- a/src/index/progress.rs +++ b/src/index/progress.rs @@ -2,6 +2,8 @@ use std::path::Path; pub trait IndexProgress { fn directory_started(&mut self, directory: &Path); + + fn status(&mut self, _message: &str) {} } #[derive(Debug, Default)] diff --git a/src/index/scanner.rs b/src/index/scanner.rs index ec26213..f2abe10 100644 --- a/src/index/scanner.rs +++ b/src/index/scanner.rs @@ -1,6 +1,7 @@ use std::collections::VecDeque; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::mpsc; use std::sync::{Arc, Condvar, Mutex}; use std::thread; @@ -61,9 +62,26 @@ pub async fn scan_root_with_progress( ) -> Result<()> where E: Embedder, - P: IndexProgress + ?Sized, + P: IndexProgress + Send + ?Sized, { - let mut scan = scan_root_concurrently(root, settings)?; + let progress = Mutex::new(progress); + progress + .lock() + .expect("index progress lock is poisoned") + .directory_started(root); + let (progress_tx, progress_rx) = mpsc::channel::(); + let mut scan = thread::scope(|scope| { + let progress = &progress; + scope.spawn(move || { + while let Ok(directory) = progress_rx.recv() { + progress + .lock() + .expect("index progress lock is poisoned") + .directory_started(&directory); + } + }); + scan_root_concurrently(root, settings, Some(progress_tx)) + })?; scan.directories .sort_by(|left, right| left.document.path.cmp(&right.document.path)); scan.files @@ -71,6 +89,12 @@ where scan.pruned_paths.sort(); scan.pruned_paths.dedup(); + if !scan.pruned_paths.is_empty() { + progress + .lock() + .expect("index progress lock is poisoned") + .status("Pruning excluded paths"); + } for path in &scan.pruned_paths { database .delete_path_tree(path) @@ -81,10 +105,10 @@ where })?; } - for directory in &scan.directories { - progress.directory_started(Path::new(&directory.document.path)); - } - + progress + .lock() + .expect("index progress lock is poisoned") + .status("Writing directory metadata"); let directory_refs = scan .directories .iter() @@ -99,15 +123,156 @@ where })?; let batch = FileIndexBatch::new(database, embedder); + let has_files = !scan.files.is_empty(); + if has_files { + progress + .lock() + .expect("index progress lock is poisoned") + .status("Embedding file chunks"); + } for file in scan.files { batch.push(file).await?; } + if has_files { + progress + .lock() + .expect("index progress lock is poisoned") + .status("Writing file chunks"); + } batch.flush().await?; report.merge(scan.report); Ok(()) } +pub async fn index_file_changes( + changed_files: Vec, + deleted_files: Vec, + settings: &Settings, + database: &Database, + embedder: &E, + report: &mut IndexReport, +) -> Result<()> +where + E: Embedder, +{ + let mut parent_directories = Vec::::new(); + let batch = FileIndexBatch::new(database, embedder); + + for file_path in deleted_files { + database + .delete_current_file(&file_path) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: file_path, + source: Box::new(source), + })?; + } + + for path in changed_files { + let name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + if settings.index.is_excluded_name(&name) { + database + .delete_current_file(&path.to_string_lossy()) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: path.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + report.entries_skipped += 1; + continue; + } + + report.files_seen += 1; + let Some(directory) = path.parent() else { + report.entries_skipped += 1; + continue; + }; + + match file::prepare_text_file(&path, directory, settings)? { + Some(indexed_file) => { + parent_directories.push(directory.to_path_buf()); + report.text_files_indexed += 1; + report.file_chunks_indexed += + u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); + batch.push(indexed_file).await?; + } + None => { + database + .delete_current_file(&path.to_string_lossy()) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: path.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + report.entries_skipped += 1; + } + } + } + + parent_directories.sort(); + parent_directories.dedup(); + ensure_parent_directories(parent_directories, settings, database, report).await?; + batch.flush().await?; + Ok(()) +} + +async fn ensure_parent_directories( + directories: Vec, + settings: &Settings, + database: &Database, + report: &mut IndexReport, +) -> Result<()> { + let mut missing = Vec::new(); + for directory in directories { + let path = directory.to_string_lossy(); + if database + .get_document(&path) + .await + .map_err(|source| IndexError::StoreDocument { + path: path.to_string(), + source: Box::new(source), + })? + .is_none() + { + let document = + summary::summarize_directory(&directory, settings).map_err(|source| { + IndexError::SummarizeDirectory { + path: directory.clone(), + source: Box::new(source), + } + })?; + let classifications = classify::classify_directory(&directory, settings)?; + missing.push(ScannedDirectory { + document, + classifications, + }); + } + } + + if missing.is_empty() { + return Ok(()); + } + + let directory_refs = missing + .iter() + .map(|directory| (&directory.document, directory.classifications.as_slice())) + .collect::>(); + database + .upsert_directories_with_classifications(&directory_refs) + .await + .map_err(|source| IndexError::StoreDocument { + path: "".to_string(), + source: Box::new(source), + })?; + report.directories_indexed += u64::try_from(missing.len()).unwrap_or(u64::MAX); + + Ok(()) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum DepthPosition { IndexRoot, @@ -175,7 +340,11 @@ struct ScanQueueState { stopped: bool, } -fn scan_root_concurrently(root: &Path, settings: &Settings) -> Result { +fn scan_root_concurrently( + root: &Path, + settings: &Settings, + progress_tx: Option>, +) -> Result { let worker_count = scan_worker_count(); let queue = Arc::new(ScanQueue { state: Mutex::new(ScanQueueState { @@ -198,8 +367,9 @@ fn scan_root_concurrently(root: &Path, settings: &Settings) -> Result, scan: Arc>, error: Arc>>, + progress_tx: Option>, ) { loop { let Some(job) = next_scan_job(&queue) else { return; }; + if let Some(progress_tx) = &progress_tx { + let _ = progress_tx.send(job.directory.clone()); + } + let result = scan_directory_job(&job, &settings); match result { Ok(output) => finish_scan_job(&queue, &scan, output), diff --git a/src/lib.rs b/src/lib.rs index 1feaa89..5ff9ee3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,16 @@ pub fn emit_cd_script(args: &[OsString]) -> Vec { request.to_shell_script() } +pub fn emit_cds_command_script(args: &[OsString]) -> Vec { + let mut script = b"command cds".to_vec(); + for arg in args { + script.push(b' '); + script.extend(shell_quote(arg)); + } + script.push(b'\n'); + script +} + pub fn shell_init(shell: cli::Shell) -> &'static str { match shell { cli::Shell::Bash => BASH_ZSH_INIT, @@ -85,7 +95,7 @@ const BASH_ZSH_INIT: &str = r#"cds() { local __cds_status case "${1-}" in - --dir-type-count|--init|--index|--reset|--search|--help|-h|--version|-V|--shell-init) + --daemon|--dir-type-count|--init|--index|--reset|--restart-daemon|--search|--help|-h|--version|-V|--shell-init) command cds "$@" return $? ;; @@ -120,6 +130,12 @@ mod tests { assert_eq!(script, b"builtin cd '-P' '../some path'\n"); } + #[test] + fn emits_cds_command_script() { + let script = emit_cds_command_script(&[os("--restart-daemon")]); + assert_eq!(script, b"command cds '--restart-daemon'\n"); + } + #[test] fn preserves_double_dash() { let script = emit_cd_script(&[os("--"), os("-directory")]); diff --git a/src/main.rs b/src/main.rs index b5693c3..4f14959 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use cds::app; use cds::cli::{Invocation, parse_invocation}; use cds::index::IndexProgress; use cds::{error, shell_init}; +use chrono::{Local, TimeZone}; use color_eyre::eyre::{Result, WrapErr}; #[tokio::main] @@ -47,11 +48,22 @@ fn install_error_reporter() -> Result<()> { async fn run(invocation: Invocation) -> error::Result<()> { match invocation { + Invocation::Daemon { run_once } => { + if run_once { + app::daemon_once().await?; + } else { + app::daemon().await?; + } + } Invocation::DirectoryTypeCount => { for count in app::directory_type_counts().await? { println!("{}\t{}", count.count, count.label); } } + Invocation::DryRun { query } => { + let report = app::dry_run(query, 10).await?; + print_dry_run(report); + } Invocation::EmitCd { args } => { let mut animation = app::implied_search_query(&args) .is_some() @@ -86,6 +98,12 @@ async fn run(invocation: Invocation) -> error::Result<()> { println!("reset cancelled"); } } + Invocation::RestartDaemon => { + let report = app::restart_daemon()?; + println!("killed {} cds daemon(s)", report.killed_daemons); + println!("started cds daemon with pid {}", report.pid); + println!("daemon log: {}", report.log_file.display()); + } Invocation::Search { query } => { let mut animation = SearchAnimation::start(); let results = app::search(query, 10).await; @@ -101,6 +119,103 @@ async fn run(invocation: Invocation) -> error::Result<()> { Ok(()) } +fn print_dry_run(report: cds::search::SearchDryRun) { + println!("query: {}", report.query); + println!(); + + println!("directory cache:"); + println!(" status: {}", report.cache.status.as_str()); + println!(" directories: {}", report.cache.directory_count); + println!(); + + println!("temporal parse:"); + match &report.temporal.matched_phrase { + Some(phrase) => println!(" matched phrase: {phrase}"), + None => println!(" matched phrase: (none)"), + } + println!(" cleaned query: {}", report.temporal.cleaned_query); + println!(" semantic query: {}", report.temporal.semantic_query); + print_temporal_bound(" modified start", report.temporal.start_unix_seconds); + print_temporal_bound(" modified end", report.temporal.end_unix_seconds); + println!(); + + print_string_section("candidate terms", &report.candidate_terms); + print_string_section( + "sql directory candidates", + &report.sql_candidate_directories, + ); + print_string_section( + "fuzzy/partial directory candidates added", + &report.fuzzy_candidate_directories, + ); + + println!("embedding scores ({}):", report.embedding_scores.len()); + if report.embedding_scores.is_empty() { + println!(" (none)"); + } else { + for score in &report.embedding_scores { + let state = if score.is_current { + "current" + } else { + "history" + }; + println!( + " {:.6}\t{}\tdir={}\tfile={}\t{}", + score.cosine_score, + state, + score.directory_path, + score.file_path, + score.content_preview + ); + } + } + println!(); + + println!("final scores ({}):", report.results.len()); + if report.results.is_empty() { + println!(" (none)"); + } else { + for result in &report.results { + println!(" {:.3}\t{}", result.score, result.path); + } + } + println!(); + + println!("winner:"); + if let Some(winner) = report.results.first() { + println!(" {:.3}\t{}", winner.score, winner.path); + } else { + println!(" (none)"); + } +} + +fn print_temporal_bound(label: &str, value: Option) { + match value { + Some(value) => println!("{label}: {value} ({})", format_unix_seconds(value)), + None => println!("{label}: (none)"), + } +} + +fn format_unix_seconds(value: i64) -> String { + Local + .timestamp_opt(value, 0) + .single() + .map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) + .unwrap_or_else(|| "invalid local time".to_string()) +} + +fn print_string_section(label: &str, values: &[String]) { + println!("{label} ({}):", values.len()); + if values.is_empty() { + println!(" (none)"); + } else { + for value in values { + println!(" {value}"); + } + } + println!(); +} + fn write_stdout(bytes: &[u8]) -> error::Result<()> { io::stdout().write_all(bytes).map_err(error::Error::Stdout) } @@ -324,6 +439,10 @@ impl Drop for TerminalIndexProgress { impl IndexProgress for TerminalIndexProgress { fn directory_started(&mut self, directory: &Path) { + self.status(&directory.display().to_string()); + } + + fn status(&mut self, message: &str) { if !self.enabled { return; } @@ -332,7 +451,7 @@ impl IndexProgress for TerminalIndexProgress { return; }; - state.current_directory = Some(directory.display().to_string()); + state.current_directory = Some(message.to_string()); state.tick = 0; render_progress_line(&mut state); } diff --git a/src/search/mod.rs b/src/search/mod.rs index ff5428b..9e3430d 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -1,25 +1,111 @@ mod error; use std::cmp::Ordering; -use std::collections::HashMap; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::collections::{HashMap, HashSet}; -use crate::db::{Database, DirectoryClassification}; +use crate::config::{Settings, default_generic_terms}; +use crate::db::{ + Database, DirectoryClassification, FileChunkMatch, IndexedDocument, ModifiedTimeRange, +}; use crate::embed::Embedder; +use chrono::{DateTime, Datelike, Duration, Local, LocalResult, NaiveDate, TimeZone}; +use serde::{Deserialize, Serialize}; pub use error::SearchError; pub type Result = std::result::Result; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SearchResult { pub path: String, pub score: f32, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SearchDryRun { + pub query: String, + pub temporal: TemporalDryRun, + pub cache: CacheDryRun, + pub candidate_terms: Vec, + pub sql_candidate_directories: Vec, + pub fuzzy_candidate_directories: Vec, + pub embedding_scores: Vec, + pub results: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CacheDryRun { + pub status: CacheDryRunStatus, + pub directory_count: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CacheDryRunStatus { + Hit, + Miss, + NotUsed, +} + +impl CacheDryRunStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::Miss => "miss", + Self::NotUsed => "not used", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TemporalDryRun { + pub cleaned_query: String, + pub semantic_query: String, + pub matched_phrase: Option, + pub start_unix_seconds: Option, + pub end_unix_seconds: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EmbeddingScore { + pub directory_path: String, + pub file_path: String, + pub file_name: String, + pub is_current: bool, + pub cosine_score: f32, + pub content_preview: String, +} + pub struct Searcher<'a, E> { database: &'a Database, embedder: &'a E, + generic_terms: HashSet, +} + +#[derive(Debug, Clone)] +pub struct SearchCache { + generic_terms: HashSet, + directories: Vec, + path_term_stats: PathTermStats, +} + +impl SearchCache { + pub async fn load(database: &Database, generic_terms: &HashSet) -> Result { + let directories = database + .directory_search_documents() + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + let path_term_stats = PathTermStats::from_directories(&directories, generic_terms); + + Ok(Self { + generic_terms: generic_terms.clone(), + directories, + path_term_stats, + }) + } + + pub fn matches_generic_terms(&self, generic_terms: &HashSet) -> bool { + &self.generic_terms == generic_terms + } } impl<'a, E> Searcher<'a, E> @@ -27,35 +113,166 @@ where E: Embedder, { pub fn new(database: &'a Database, embedder: &'a E) -> Self { - Self { database, embedder } + Self::new_with_generic_terms(database, embedder, default_generic_terms()) + } + + pub fn new_with_settings(database: &'a Database, embedder: &'a E, settings: &Settings) -> Self { + Self::new_with_generic_terms(database, embedder, settings.index.generic_terms.clone()) + } + + pub fn new_with_generic_terms( + database: &'a Database, + embedder: &'a E, + generic_terms: impl IntoIterator, + ) -> Self { + Self { + database, + embedder, + generic_terms: generic_terms + .into_iter() + .map(|term| term.to_ascii_lowercase()) + .collect(), + } } pub async fn search(&self, query: &str, limit: usize) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + + self.search_with_temporal_query(TemporalQuery::from_query(query), limit, None) + .await + } + + pub async fn search_with_cache( + &self, + query: &str, + limit: usize, + cache: &SearchCache, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + + self.search_with_temporal_query(TemporalQuery::from_query(query), limit, Some(cache)) + .await + } + + async fn search_with_temporal_query( + &self, + temporal_query: TemporalQuery, + limit: usize, + cache: Option<&SearchCache>, + ) -> Result> { + let temporal_filter = temporal_query.filter; + let search_query = temporal_query.search_query(); + let query_terms = query_terms(search_query); + let path_terms = path_query_terms(search_query); + let loaded_cache; + let cache = if let Some(cache) = cache { + cache + } else { + loaded_cache = SearchCache::load(self.database, &self.generic_terms).await?; + &loaded_cache + }; + let all_directories = &cache.directories; + let path_term_stats = &cache.path_term_stats; + let candidate_terms = candidate_filter_terms(&path_terms, &self.generic_terms); + let mut candidate_directories = if candidate_terms.is_empty() { + Vec::new() + } else { + self.database + .directory_search_candidates_by_terms(&candidate_terms) + .await + .map_err(|source| SearchError::LoadDirectories { source })? + }; + add_fuzzy_directory_candidates( + &mut candidate_directories, + all_directories, + &candidate_terms, + ); + if !temporal_filter.has_range() + && let Some(directory) = + high_confidence_directory_candidate(&candidate_directories, &candidate_terms) + { + let mut fast_target_cache = HashMap::new(); + let target = navigation_target( + self.database, + &mut fast_target_cache, + &directory.path, + &path_terms, + &self.generic_terms, + ) + .await?; + let path_match = path_match_boost( + &path_terms, + &directory.path, + Some(&directory.name), + path_term_stats, + ); + + return Ok(vec![SearchResult { + path: target, + score: 10.0 + path_match.score, + }]); + } + + let (chunks, directories) = if candidate_directories.is_empty() { + let chunks = self + .database + .file_chunk_matches_with_modified_range(temporal_filter.modified_range()) + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + (chunks, all_directories.clone()) + } else { + let directory_paths = candidate_directories + .iter() + .map(|directory| directory.path.clone()) + .collect::>(); + let chunks = self + .database + .file_chunk_matches_in_directory_trees_with_modified_range( + &directory_paths, + temporal_filter.modified_range(), + ) + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + (chunks, candidate_directories) + }; let query_embedding = self .embedder - .embed_query(query) + .embed_query(search_query) .map_err(|source| SearchError::EmbedQuery { source })?; - let chunks = self - .database - .file_chunk_matches() - .await - .map_err(|source| SearchError::LoadDirectories { source })?; - let temporal_filter = TemporalFilter::from_query(query); - let use_deep_target = wants_deep_target(query); - let query_terms = query_terms(query); + let temporal_directory_paths = temporal_filter + .has_range() + .then(|| temporal_matching_directory_paths(&chunks)); - let mut scores = HashMap::::new(); + let mut scores = HashMap::::new(); + let mut ancestor_classification_cache = + HashMap::>::new(); + let mut directory_classification_cache = + HashMap::>::new(); + let mut target_cache = HashMap::::new(); for chunk in chunks { let mut score = cosine_similarity(&query_embedding, &chunk.embedding); if score <= 0.0 { continue; } - let classifications = self - .database - .ancestor_classifications(&chunk.directory_path) - .await - .map_err(|source| SearchError::LoadDirectories { source })?; + let classifications = if let Some(classifications) = + ancestor_classification_cache.get(&chunk.directory_path) + { + classifications.clone() + } else { + let classifications = self + .database + .ancestor_classifications(&chunk.directory_path) + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + ancestor_classification_cache + .insert(chunk.directory_path.clone(), classifications.clone()); + classifications + }; score += lexical_boost( &query_terms, [ @@ -64,160 +281,1063 @@ where chunk.directory_path.as_str(), ], ); + let path_match = + path_match_boost(&path_terms, &chunk.directory_path, None, path_term_stats); + score += path_match.score; + let path_matched_terms = path_match.matched_terms.clone(); + let mut matched_terms = path_match.matched_terms; + matched_terms.extend(matched_terms_in( + &path_terms, + [ + chunk.content.as_str(), + chunk.file_name.as_str(), + chunk.directory_path.as_str(), + ], + )); score += classification_boost(&query_terms, &classifications); score *= temporal_filter.score_multiplier( chunk .file_modified_unix_seconds .max(chunk.directory_modified_unix_seconds), ); + if !chunk.is_current { + score *= 0.55; + } - let target = if use_deep_target { - chunk.directory_path - } else { - self.database - .general_indexed_directory(&chunk.directory_path) - .await - .map_err(|source| SearchError::LoadDirectories { source })? - }; + let target = navigation_target( + self.database, + &mut target_cache, + &chunk.directory_path, + &path_terms, + &self.generic_terms, + ) + .await?; + score += direct_target_match_boost(&target, &chunk.directory_path, &path_matched_terms); - scores - .entry(target) - .and_modify(|existing| *existing = existing.max(score)) - .or_insert(score); + update_candidate_score( + &mut scores, + target, + score, + matched_terms, + path_matched_terms, + &chunk.directory_path, + ); } - for directory in self - .database - .directory_documents() - .await - .map_err(|source| SearchError::LoadDirectories { source })? - { - let classifications = self - .database - .directory_classifications(&directory.path) - .await - .map_err(|source| SearchError::LoadDirectories { source })?; + for directory in directories { + if let Some(matching_paths) = &temporal_directory_paths + && !temporal_filter.matches(directory.modified_unix_seconds) + && !directory_tree_has_temporal_match(&directory.path, matching_paths) + { + continue; + } + + let classifications = if let Some(classifications) = + directory_classification_cache.get(&directory.path) + { + classifications.clone() + } else { + let classifications = self + .database + .directory_classifications(&directory.path) + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + directory_classification_cache + .insert(directory.path.clone(), classifications.clone()); + classifications + }; + let path_match = path_match_boost( + &path_terms, + &directory.path, + Some(&directory.name), + path_term_stats, + ); let score = classification_boost(&query_terms, &classifications) + lexical_boost( &query_terms, [directory.path.as_str(), directory.name.as_str()], - ); + ) + + path_match.score; if score <= 0.0 { continue; } + let path_matched_terms = path_match.matched_terms.clone(); + let mut matched_terms = path_match.matched_terms; + matched_terms.extend(matched_terms_in( + &path_terms, + [directory.path.as_str(), directory.name.as_str()], + )); - let target = if use_deep_target { - directory.path - } else { - self.database - .general_indexed_directory(&directory.path) - .await - .map_err(|source| SearchError::LoadDirectories { source })? - }; + let target = navigation_target( + self.database, + &mut target_cache, + &directory.path, + &path_terms, + &self.generic_terms, + ) + .await?; + let score = + score + direct_target_match_boost(&target, &directory.path, &path_matched_terms); - scores - .entry(target) - .and_modify(|existing| *existing = existing.max(score)) - .or_insert(score); + update_candidate_score( + &mut scores, + target, + score, + matched_terms, + path_matched_terms, + &directory.path, + ); } let mut results = scores .into_iter() - .filter_map(|(path, score)| (score > 0.0).then_some(SearchResult { path, score })) + .filter_map(|(path, score)| { + let score = score.final_score(&path_terms, path_term_stats); + (score > 0.0).then_some(SearchResult { path, score }) + }) .collect::>(); results.sort_by(compare_results); results.truncate(limit); Ok(results) } + + pub async fn dry_run(&self, query: &str, limit: usize) -> Result { + self.dry_run_with_optional_cache(query, limit, None).await + } + + pub async fn dry_run_with_cache( + &self, + query: &str, + limit: usize, + cache: &SearchCache, + ) -> Result { + self.dry_run_with_cache_status(query, limit, cache, CacheDryRunStatus::Hit) + .await + } + + pub async fn dry_run_with_cache_status( + &self, + query: &str, + limit: usize, + cache: &SearchCache, + cache_status: CacheDryRunStatus, + ) -> Result { + self.dry_run_with_optional_cache(query, limit, Some((cache, cache_status))) + .await + } + + async fn dry_run_with_optional_cache( + &self, + query: &str, + limit: usize, + cache: Option<(&SearchCache, CacheDryRunStatus)>, + ) -> Result { + let temporal_query = TemporalQuery::from_query(query); + let search_query = temporal_query.search_query(); + let path_terms = path_query_terms(search_query); + let loaded_cache; + let (cache, cache_status) = if let Some((cache, cache_status)) = cache { + (cache, cache_status) + } else { + loaded_cache = SearchCache::load(self.database, &self.generic_terms).await?; + (&loaded_cache, CacheDryRunStatus::Miss) + }; + let all_directories = &cache.directories; + let candidate_terms = candidate_filter_terms(&path_terms, &self.generic_terms); + let sql_candidate_documents = if candidate_terms.is_empty() { + Vec::new() + } else { + self.database + .directory_search_candidates_by_terms(&candidate_terms) + .await + .map_err(|source| SearchError::LoadDirectories { source })? + }; + let sql_candidate_paths = sql_candidate_documents + .iter() + .map(|directory| directory.path.clone()) + .collect::>(); + let sql_candidate_directories = sql_candidate_documents + .iter() + .map(|directory| directory.path.clone()) + .collect::>(); + + let mut candidate_directories = sql_candidate_documents; + add_fuzzy_directory_candidates( + &mut candidate_directories, + all_directories, + &candidate_terms, + ); + let fuzzy_candidate_directories = candidate_directories + .iter() + .filter(|directory| !sql_candidate_paths.contains(&directory.path)) + .map(|directory| directory.path.clone()) + .collect::>(); + + let chunks = if candidate_directories.is_empty() { + self.database + .file_chunk_matches_with_modified_range(temporal_query.filter.modified_range()) + .await + .map_err(|source| SearchError::LoadDirectories { source })? + } else { + let directory_paths = candidate_directories + .iter() + .map(|directory| directory.path.clone()) + .collect::>(); + self.database + .file_chunk_matches_in_directory_trees_with_modified_range( + &directory_paths, + temporal_query.filter.modified_range(), + ) + .await + .map_err(|source| SearchError::LoadDirectories { source })? + }; + let query_embedding = self + .embedder + .embed_query(search_query) + .map_err(|source| SearchError::EmbedQuery { source })?; + let mut embedding_scores = chunks + .into_iter() + .map(|chunk| EmbeddingScore { + directory_path: chunk.directory_path, + file_path: chunk.file_path, + file_name: chunk.file_name, + is_current: chunk.is_current, + cosine_score: cosine_similarity(&query_embedding, &chunk.embedding), + content_preview: content_preview(&chunk.content), + }) + .collect::>(); + embedding_scores.sort_by(|left, right| { + right + .cosine_score + .partial_cmp(&left.cosine_score) + .unwrap_or(Ordering::Equal) + .then_with(|| left.directory_path.cmp(&right.directory_path)) + .then_with(|| left.file_path.cmp(&right.file_path)) + }); + + let results = self + .search_with_temporal_query(temporal_query.clone(), limit, Some(cache)) + .await?; + + Ok(SearchDryRun { + query: query.to_string(), + temporal: TemporalDryRun::from_query(&temporal_query), + cache: CacheDryRun { + status: cache_status, + directory_count: all_directories.len(), + }, + candidate_terms, + sql_candidate_directories, + fuzzy_candidate_directories, + embedding_scores, + results, + }) + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct TemporalFilter { - min_modified_unix_seconds: Option, +fn high_confidence_directory_candidate<'a>( + directories: &'a [IndexedDocument], + candidate_terms: &[String], +) -> Option<&'a IndexedDocument> { + if directories.len() != 1 || candidate_terms.is_empty() { + return None; + } + + let directory = &directories[0]; + path_candidate_matches_all_terms(candidate_terms, &directory.path, &directory.name) + .then_some(directory) } -impl TemporalFilter { +fn add_fuzzy_directory_candidates( + candidates: &mut Vec, + all_directories: &[IndexedDocument], + candidate_terms: &[String], +) { + if candidate_terms.is_empty() { + return; + } + + let mut seen_paths = candidates + .iter() + .map(|directory| directory.path.clone()) + .collect::>(); + for directory in all_directories { + if seen_paths.contains(&directory.path) { + continue; + } + if path_candidate_matches_any_term(candidate_terms, &directory.path, &directory.name) { + seen_paths.insert(directory.path.clone()); + candidates.push(directory.clone()); + } + } +} + +fn temporal_matching_directory_paths(chunks: &[FileChunkMatch]) -> HashSet { + chunks + .iter() + .map(|chunk| chunk.directory_path.clone()) + .collect() +} + +fn directory_tree_has_temporal_match(path: &str, matching_paths: &HashSet) -> bool { + let child_prefix = format!("{path}/"); + matching_paths + .iter() + .any(|matching_path| matching_path == path || matching_path.starts_with(&child_prefix)) +} + +#[derive(Debug, Default)] +struct CandidateScore { + score: f32, + matched_terms: HashSet, + path_matched_terms: HashSet, + direct_path_matched_terms: HashSet, + nested_path_matched_terms: HashSet, +} + +impl CandidateScore { + fn update( + &mut self, + score: f32, + matched_terms: HashSet, + path_matched_terms: HashSet, + direct_path_match: bool, + ) { + self.score = self.score.max(score); + self.matched_terms.extend(matched_terms); + if direct_path_match { + self.direct_path_matched_terms + .extend(path_matched_terms.iter().cloned()); + } else { + self.nested_path_matched_terms + .extend(path_matched_terms.iter().cloned()); + } + self.path_matched_terms.extend(path_matched_terms); + } + + fn final_score(&self, query_terms: &[String], stats: &PathTermStats) -> f32 { + if query_terms.is_empty() { + return self.score; + } + + let total_weight = query_terms + .iter() + .map(|term| stats.term_weight(term)) + .sum::(); + if total_weight <= 0.0 { + return self.score; + } + + let matched_weight = query_terms + .iter() + .filter(|term| self.matched_terms.contains(term.as_str())) + .map(|term| stats.term_weight(term)) + .sum::(); + let coverage = matched_weight / total_weight; + let path_matched_weight = query_terms + .iter() + .filter(|term| self.path_matched_terms.contains(term.as_str())) + .map(|term| stats.term_weight(term)) + .sum::(); + let path_coverage = path_matched_weight / total_weight; + let direct_path_matched_weight = query_terms + .iter() + .filter(|term| self.direct_path_matched_terms.contains(term.as_str())) + .map(|term| stats.term_weight(term)) + .sum::(); + let nested_only_path_matched_weight = query_terms + .iter() + .filter(|term| { + self.nested_path_matched_terms.contains(term.as_str()) + && !self.direct_path_matched_terms.contains(term.as_str()) + }) + .map(|term| stats.term_weight(term)) + .sum::(); + let direct_path_coverage = direct_path_matched_weight / total_weight; + let nested_only_path_coverage = nested_only_path_matched_weight / total_weight; + let full_coverage_bonus = if self.matched_terms.len() >= query_terms.len() { + 1.25 + } else { + 0.0 + }; + let content_only_penalty = if path_matched_weight <= 0.0 && matched_weight > 0.0 { + 0.75 + } else { + 0.0 + }; + + self.score + + coverage * 1.4 + + path_coverage * 4.0 + + full_coverage_bonus + + direct_path_coverage * 4.0 + - nested_only_path_coverage * 3.0 + - content_only_penalty + } +} + +fn update_candidate_score( + scores: &mut HashMap, + target: String, + score: f32, + matched_terms: HashSet, + path_matched_terms: HashSet, + evidence_path: &str, +) { + let direct_path_match = target == evidence_path; + scores.entry(target).or_default().update( + score, + matched_terms, + path_matched_terms, + direct_path_match, + ); +} + +fn direct_target_match_boost( + target_path: &str, + evidence_path: &str, + path_matched_terms: &HashSet, +) -> f32 { + if path_matched_terms.is_empty() { + 0.0 + } else if target_path == evidence_path { + 1.75 + } else { + -2.0 + } +} + +async fn navigation_target( + database: &Database, + target_cache: &mut HashMap, + path: &str, + path_terms: &[String], + generic_terms: &HashSet, +) -> Result { + if should_navigate_to_path(path_terms, path, generic_terms) { + return Ok(path.to_string()); + } + + if let Some(target) = target_cache.get(path) { + return Ok(target.clone()); + } + + let target = database + .general_indexed_directory(path) + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + target_cache.insert(path.to_string(), target.clone()); + Ok(target) +} + +fn should_navigate_to_path( + path_terms: &[String], + path: &str, + generic_terms: &HashSet, +) -> bool { + let Some(name) = PathName::from_path(path) else { + return false; + }; + let name_tokens = path_tokens(name); + let query_terms = path_terms + .iter() + .filter(|term| !is_generic_nav_term(term, generic_terms)) + .cloned() + .collect::>(); + let distinct_query_terms = query_terms.iter().collect::>(); + if distinct_query_terms.len() < 2 { + return false; + } + + let leaf_matches = query_terms.iter().any(|term| { + name_tokens.iter().any(|token| { + token == term + || is_fuzzy_path_token_match(term, token) + || is_partial_path_token_match(term, token) + }) + }); + + leaf_matches && path_candidate_matches_all_terms(&query_terms, path, name) +} + +fn path_candidate_matches_any_term(query_terms: &[String], path: &str, name: &str) -> bool { + let mut candidate_tokens = path_tokens(path); + candidate_tokens.extend(path_tokens(name)); + query_terms.iter().any(|term| { + candidate_tokens.iter().any(|token| { + token == term + || is_fuzzy_path_token_match(term, token) + || is_partial_path_token_match(term, token) + }) + }) +} + +fn path_candidate_matches_all_terms(query_terms: &[String], path: &str, name: &str) -> bool { + let mut candidate_tokens = path_tokens(path); + candidate_tokens.extend(path_tokens(name)); + query_terms.iter().all(|term| { + candidate_tokens.iter().any(|token| { + token == term + || is_fuzzy_path_token_match(term, token) + || is_partial_path_token_match(term, token) + }) + }) +} + +#[derive(Debug, Clone)] +struct PathTermStats { + directory_count: usize, + directory_name_frequency: HashMap, + generic_terms: HashSet, +} + +impl PathTermStats { + fn from_directories(directories: &[IndexedDocument], generic_terms: &HashSet) -> Self { + let mut directory_name_frequency = HashMap::new(); + for directory in directories { + let unique_terms = path_tokens(&directory.name) + .into_iter() + .collect::>(); + for term in unique_terms { + *directory_name_frequency.entry(term).or_insert(0) += 1; + } + } + + Self { + directory_count: directories.len(), + directory_name_frequency, + generic_terms: generic_terms.clone(), + } + } + + fn term_weight(&self, term: &str) -> f32 { + let document_count = self.directory_count.max(1) as f32; + let frequency = self + .directory_name_frequency + .get(term) + .copied() + .unwrap_or(0) as f32; + let rarity = ((document_count + 1.0) / (frequency + 1.0)) + .ln() + .clamp(0.0, 2.0); + let generic_multiplier = if is_generic_nav_term(term, &self.generic_terms) { + 0.45 + } else { + 1.0 + }; + let length_bonus = if term.len() >= 5 { 0.2 } else { 0.0 }; + + (1.0 + rarity + length_bonus) * generic_multiplier + } +} + +fn is_generic_nav_term(term: &str, generic_terms: &HashSet) -> bool { + generic_terms.contains(term) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TemporalQuery { + original_query: String, + cleaned_query: String, + matched_phrase: Option, + filter: TemporalFilter, +} + +impl TemporalQuery { fn from_query(query: &str) -> Self { + Self::from_query_at(query, Local::now()) + } + + fn from_query_at(query: &str, now: DateTime) -> Self + where + Tz: TimeZone, + { let lower = query.to_ascii_lowercase(); - let seconds = unix_now(); - - let min_modified_unix_seconds = if lower.contains("today") { - Some(seconds - 24 * 60 * 60) - } else if lower.contains("yesterday") { - Some(seconds - 2 * 24 * 60 * 60) - } else if lower.contains("last week") { - Some(seconds - 14 * 24 * 60 * 60) - } else if lower.contains("recent") || lower.contains("recently") { - Some(seconds - 30 * 24 * 60 * 60) + let current_date = now.date_naive(); + let timezone = now.timezone(); + + if lower.contains("last month") { + let (year, month) = previous_month(current_date.year(), current_date.month()); + let start_date = first_day_of_month(year, month); + let end_date = first_day_of_month(current_date.year(), current_date.month()); + return Self::with_range(query, "last month", start_date, end_date, &timezone); + } + + if lower.contains("this month") { + let start_date = first_day_of_month(current_date.year(), current_date.month()); + let (next_year, next_month) = next_month(current_date.year(), current_date.month()); + let end_date = first_day_of_month(next_year, next_month); + return Self::with_range(query, "this month", start_date, end_date, &timezone); + } + + if lower.contains("last week") { + let current_week_start = current_date + - Duration::days(i64::from(current_date.weekday().num_days_from_monday())); + let start_date = current_week_start - Duration::days(7); + return Self::with_range( + query, + "last week", + start_date, + current_week_start, + &timezone, + ); + } + + if lower.contains("yesterday") { + let start_date = current_date - Duration::days(1); + return Self::with_range(query, "yesterday", start_date, current_date, &timezone); + } + + if lower.contains("today") { + let end_date = current_date + Duration::days(1); + return Self::with_range(query, "today", current_date, end_date, &timezone); + } + + if lower.contains("recently") { + return Self::with_open_range(query, "recently", now.timestamp() - 30 * 24 * 60 * 60); + } + + if lower.contains("recent") { + return Self::with_open_range(query, "recent", now.timestamp() - 30 * 24 * 60 * 60); + } + + Self { + original_query: query.to_string(), + cleaned_query: normalize_query_whitespace(query), + matched_phrase: None, + filter: TemporalFilter::default(), + } + } + + fn with_range( + query: &str, + phrase: &str, + start_date: NaiveDate, + end_date: NaiveDate, + timezone: &Tz, + ) -> Self + where + Tz: TimeZone, + { + Self { + original_query: query.to_string(), + cleaned_query: remove_temporal_phrase(query, phrase), + matched_phrase: Some(phrase.to_string()), + filter: TemporalFilter { + start_unix_seconds: Some(local_midnight_unix_seconds(start_date, timezone)), + end_unix_seconds: Some(local_midnight_unix_seconds(end_date, timezone)), + }, + } + } + + fn with_open_range(query: &str, phrase: &str, start_unix_seconds: i64) -> Self { + Self { + original_query: query.to_string(), + cleaned_query: remove_temporal_phrase(query, phrase), + matched_phrase: Some(phrase.to_string()), + filter: TemporalFilter { + start_unix_seconds: Some(start_unix_seconds), + end_unix_seconds: None, + }, + } + } + + fn search_query(&self) -> &str { + if self.cleaned_query.trim().is_empty() { + &self.original_query } else { - None - }; + &self.cleaned_query + } + } +} +impl TemporalDryRun { + fn from_query(query: &TemporalQuery) -> Self { Self { - min_modified_unix_seconds, + cleaned_query: query.cleaned_query.clone(), + semantic_query: query.search_query().to_string(), + matched_phrase: query.matched_phrase.clone(), + start_unix_seconds: query.filter.start_unix_seconds, + end_unix_seconds: query.filter.end_unix_seconds, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct TemporalFilter { + start_unix_seconds: Option, + end_unix_seconds: Option, +} + +impl TemporalFilter { + fn has_range(self) -> bool { + self.start_unix_seconds.is_some() || self.end_unix_seconds.is_some() + } + + fn modified_range(self) -> Option { + self.has_range().then_some(ModifiedTimeRange { + start_unix_seconds: self.start_unix_seconds, + end_unix_seconds: self.end_unix_seconds, + }) + } + + fn matches(self, modified_unix_seconds: i64) -> bool { + self.modified_range() + .is_none_or(|range| range.contains(modified_unix_seconds)) + } + + fn score_multiplier(self, modified_unix_seconds: i64) -> f32 { + if !self.has_range() { + return 1.0; + } + + if self.matches(modified_unix_seconds) { + 1.35 + } else { + 0.25 } } +} + +fn remove_temporal_phrase(query: &str, phrase: &str) -> String { + let lower = query.to_ascii_lowercase(); + let Some(start) = lower.find(phrase) else { + return normalize_query_whitespace(query); + }; + let end = start + phrase.len(); + normalize_query_whitespace(&format!("{} {}", &query[..start], &query[end..])) +} + +fn normalize_query_whitespace(query: &str) -> String { + query.split_whitespace().collect::>().join(" ") +} + +fn previous_month(year: i32, month: u32) -> (i32, u32) { + if month == 1 { + (year - 1, 12) + } else { + (year, month - 1) + } +} + +fn next_month(year: i32, month: u32) -> (i32, u32) { + if month == 12 { + (year + 1, 1) + } else { + (year, month + 1) + } +} + +fn first_day_of_month(year: i32, month: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(year, month, 1).expect("valid first day of month") +} + +fn local_midnight_unix_seconds(date: NaiveDate, timezone: &Tz) -> i64 +where + Tz: TimeZone, +{ + let midnight = date + .and_hms_opt(0, 0, 0) + .expect("midnight should be a valid naive time"); + match timezone.from_local_datetime(&midnight) { + LocalResult::Single(time) => time.timestamp(), + LocalResult::Ambiguous(first, second) => first.timestamp().min(second.timestamp()), + LocalResult::None => timezone.from_utc_datetime(&midnight).timestamp(), + } +} + +fn query_terms(query: &str) -> Vec { + query + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|term| term.len() > 2) + .map(|term| term.to_ascii_lowercase()) + .collect() +} + +fn path_query_terms(query: &str) -> Vec { + query + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|term| term.len() > 1) + .map(|term| term.to_ascii_lowercase()) + .collect() +} + +fn candidate_filter_terms(path_terms: &[String], generic_terms: &HashSet) -> Vec { + path_terms + .iter() + .filter(|term| { + !is_generic_nav_term(term, generic_terms) && (term.len() > 2 || term.as_str() == "cd") + }) + .cloned() + .collect() +} + +fn lexical_boost<'a>(query_terms: &[String], haystacks: impl IntoIterator) -> f32 { + let haystack = haystacks + .into_iter() + .map(str::to_ascii_lowercase) + .collect::>() + .join("\n"); + + query_terms + .iter() + .filter(|term| haystack.contains(term.as_str())) + .count() as f32 + * 0.04 +} + +fn matched_terms_in<'a>( + query_terms: &[String], + haystacks: impl IntoIterator, +) -> HashSet { + let haystack = haystacks + .into_iter() + .map(str::to_ascii_lowercase) + .collect::>() + .join("\n"); + + query_terms + .iter() + .filter(|term| haystack.contains(term.as_str())) + .cloned() + .collect() +} + +#[derive(Debug, Default)] +struct PathMatch { + score: f32, + matched_terms: HashSet, +} + +fn path_match_boost( + query_terms: &[String], + path: &str, + name: Option<&str>, + stats: &PathTermStats, +) -> PathMatch { + if query_terms.is_empty() { + return PathMatch::default(); + } + + let candidate_path_tokens = path_tokens(path); + let name_tokens = name.map(path_tokens).unwrap_or_else(|| { + PathName::from_path(path) + .map(path_tokens) + .unwrap_or_default() + }); + + let path_exact_matches = query_terms + .iter() + .filter(|term| candidate_path_tokens.contains(term)) + .cloned() + .collect::>(); + let name_exact_matches = query_terms + .iter() + .filter(|term| name_tokens.contains(term)) + .cloned() + .collect::>(); + let path_fuzzy_matches = fuzzy_matched_terms(query_terms, &candidate_path_tokens); + let name_fuzzy_matches = fuzzy_matched_terms(query_terms, &name_tokens); + let path_partial_matches = partial_matched_terms(query_terms, &candidate_path_tokens); + let name_partial_matches = partial_matched_terms(query_terms, &name_tokens); + let path_lower = path.to_ascii_lowercase(); + let substring_matches = query_terms + .iter() + .filter(|term| path_lower.contains(term.as_str())) + .cloned() + .collect::>(); + let mut matched_terms = HashSet::new(); + matched_terms.extend(path_exact_matches.iter().cloned()); + matched_terms.extend(name_exact_matches.iter().cloned()); + matched_terms.extend(substring_matches.iter().cloned()); + matched_terms.extend(path_fuzzy_matches.iter().cloned()); + matched_terms.extend(name_fuzzy_matches.iter().cloned()); + matched_terms.extend(path_partial_matches.iter().cloned()); + matched_terms.extend(name_partial_matches.iter().cloned()); + + let path_exact_score = path_exact_matches + .iter() + .map(|term| stats.term_weight(term) * 0.45) + .sum::(); + let name_exact_score = name_exact_matches + .iter() + .map(|term| stats.term_weight(term) * 0.65) + .sum::(); + let substring_score = substring_matches + .iter() + .map(|term| stats.term_weight(term) * 0.08) + .sum::(); + let path_fuzzy_score = path_fuzzy_matches + .iter() + .map(|term| stats.term_weight(term) * 0.32) + .sum::(); + let name_fuzzy_score = name_fuzzy_matches + .iter() + .map(|term| stats.term_weight(term) * 0.48) + .sum::(); + let path_partial_score = path_partial_matches + .iter() + .map(|term| stats.term_weight(term) * 0.24) + .sum::(); + let name_partial_score = name_partial_matches + .iter() + .map(|term| stats.term_weight(term) * 0.36) + .sum::(); + let mut score = path_exact_score + + name_exact_score + + substring_score + + path_fuzzy_score + + name_fuzzy_score + + path_partial_score + + name_partial_score; + + if path_exact_matches.len() == query_terms.len() { + score += 1.5; + } + if name_exact_matches.len() == query_terms.len() { + score += 2.5; + } + if matched_terms.len() == query_terms.len() && !path_fuzzy_matches.is_empty() { + score += 1.0; + } + if matched_terms.len() == query_terms.len() && !name_fuzzy_matches.is_empty() { + score += 1.5; + } + if matched_terms.len() == query_terms.len() && !path_partial_matches.is_empty() { + score += 0.75; + } + if matched_terms.len() == query_terms.len() && !name_partial_matches.is_empty() { + score += 1.0; + } + + PathMatch { + score, + matched_terms, + } +} + +fn fuzzy_matched_terms(query_terms: &[String], candidate_tokens: &[String]) -> HashSet { + query_terms + .iter() + .filter(|term| { + candidate_tokens + .iter() + .any(|candidate| is_fuzzy_path_token_match(term, candidate)) + }) + .cloned() + .collect() +} + +fn partial_matched_terms(query_terms: &[String], candidate_tokens: &[String]) -> HashSet { + query_terms + .iter() + .filter(|term| { + candidate_tokens + .iter() + .any(|candidate| is_partial_path_token_match(term, candidate)) + }) + .cloned() + .collect() +} + +fn is_fuzzy_path_token_match(query_term: &str, candidate_token: &str) -> bool { + if query_term == candidate_token || query_term.len() < 4 || candidate_token.len() < 4 { + return false; + } + + let length_delta = query_term.len().abs_diff(candidate_token.len()); + if length_delta > 2 { + return false; + } + + let distance = damerau_levenshtein_distance(query_term, candidate_token); + if distance <= 1 { + return true; + } + + query_term.len().min(candidate_token.len()) >= 5 + && query_term.len().max(candidate_token.len()) >= 6 + && distance == 2 + && has_common_bigram(query_term, candidate_token) +} + +fn is_partial_path_token_match(query_term: &str, candidate_token: &str) -> bool { + if query_term == candidate_token || query_term.len() < 4 || candidate_token.len() < 3 { + return false; + } + if is_fuzzy_path_token_match(query_term, candidate_token) { + return false; + } + + common_prefix_len(query_term, candidate_token) >= 3 +} + +fn common_prefix_len(left: &str, right: &str) -> usize { + left.chars() + .zip(right.chars()) + .take_while(|(left, right)| left == right) + .count() +} + +fn has_common_bigram(left: &str, right: &str) -> bool { + let right_bigrams = right.as_bytes().windows(2).collect::>(); + + left.as_bytes() + .windows(2) + .any(|bigram| right_bigrams.contains(bigram)) +} + +fn damerau_levenshtein_distance(left: &str, right: &str) -> usize { + let left = left.chars().collect::>(); + let right = right.chars().collect::>(); + let mut previous_previous = Vec::::new(); + let mut previous = (0..=right.len()).collect::>(); + + for (left_index, left_char) in left.iter().enumerate() { + let mut current = Vec::with_capacity(right.len() + 1); + current.push(left_index + 1); + + for (right_index, right_char) in right.iter().enumerate() { + let deletion = previous[right_index + 1] + 1; + let insertion = current[right_index] + 1; + let substitution = previous[right_index] + usize::from(left_char != right_char); + let mut distance = deletion.min(insertion).min(substitution); - fn score_multiplier(self, modified_unix_seconds: i64) -> f32 { - let Some(min_modified_unix_seconds) = self.min_modified_unix_seconds else { - return 1.0; - }; + if left_index > 0 + && right_index > 0 + && left[left_index] == right[right_index - 1] + && left[left_index - 1] == right[right_index] + { + distance = distance.min(previous_previous[right_index - 1] + 1); + } - if modified_unix_seconds >= min_modified_unix_seconds { - 1.35 - } else { - 0.25 + current.push(distance); } + + previous_previous = previous; + previous = current; } + + previous[right.len()] } -fn wants_deep_target(query: &str) -> bool { - let lower = query.to_ascii_lowercase(); - [ - "migration", - "migrations", - "test", - "tests", - "spec", - "schema", - "schemas", - "component", - "components", - "route", - "routes", - "controller", - "controllers", - "model", - "models", - "config", - ] - .iter() - .any(|signal| lower.contains(signal)) +struct PathName; + +impl PathName { + fn from_path(path: &str) -> Option<&str> { + path.rsplit(['/', '\\']).find(|part| !part.is_empty()) + } } -fn query_terms(query: &str) -> Vec { - query +fn path_tokens(value: &str) -> Vec { + value .split(|ch: char| !ch.is_alphanumeric()) - .filter(|term| term.len() > 2) + .filter(|term| term.len() > 1) .map(|term| term.to_ascii_lowercase()) .collect() } -fn lexical_boost<'a>(query_terms: &[String], haystacks: impl IntoIterator) -> f32 { - let haystack = haystacks - .into_iter() - .map(str::to_ascii_lowercase) - .collect::>() - .join("\n"); - - query_terms - .iter() - .filter(|term| haystack.contains(term.as_str())) - .count() as f32 - * 0.04 -} - fn classification_boost( query_terms: &[String], classifications: &[DirectoryClassification], @@ -250,13 +1370,6 @@ fn query_terms_from_label(label: &str) -> Vec { query_terms(label) } -fn unix_now() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) - .unwrap_or(0) -} - fn compare_results(left: &SearchResult, right: &SearchResult) -> Ordering { right .score @@ -287,16 +1400,114 @@ fn cosine_similarity(left: &[f32], right: &[f32]) -> f32 { dot / (left_norm.sqrt() * right_norm.sqrt()) } +fn content_preview(content: &str) -> String { + let preview = content.split_whitespace().collect::>().join(" "); + preview.chars().take(120).collect() +} + #[cfg(test)] mod tests { use std::fs; use super::*; use crate::config::Settings; - use crate::db::Database; - use crate::embed::FakeEmbedder; + use crate::db::{Database, DocumentKind, IndexedFile, IndexedFileChunk}; + use crate::embed::{EmbedError, FakeEmbedder}; use crate::index::Indexer; + struct PanicEmbedder; + + impl Embedder for PanicEmbedder { + fn dimensions(&self) -> usize { + 32 + } + + fn embed(&self, _text: &str) -> crate::embed::Result> { + panic!("high-confidence directory match should not embed the query") + } + + fn embed_query(&self, _text: &str) -> crate::embed::Result> { + Err(EmbedError::Model { + message: "high-confidence directory match should not embed the query".to_string(), + }) + } + } + + fn fixed_time( + year: i32, + month: u32, + day: u32, + hour: u32, + ) -> chrono::DateTime { + use chrono::TimeZone as _; + + chrono::FixedOffset::east_opt(0) + .unwrap() + .with_ymd_and_hms(year, month, day, hour, 0, 0) + .single() + .unwrap() + } + + async fn insert_directory_with_chunk( + database: &Database, + embedder: &FakeEmbedder, + directory_path: &str, + file_modified_unix_seconds: i64, + content: &str, + ) { + let directory_name = directory_path + .rsplit('/') + .next() + .unwrap_or(directory_path) + .to_string(); + database + .upsert_document(&IndexedDocument { + path: directory_path.to_string(), + name: directory_name, + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: directory_path.to_string(), + embedding: embedder.embed(directory_path).unwrap(), + metadata_fingerprint: format!("directory:{directory_path}"), + size_bytes: 0, + created_unix_seconds: None, + modified_unix_seconds: 1, + accessed_unix_seconds: None, + readonly: false, + indexed_unix_seconds: file_modified_unix_seconds, + }) + .await + .unwrap(); + + let file = IndexedFile { + path: format!("{directory_path}/README.md"), + directory_path: directory_path.to_string(), + name: "README.md".to_string(), + extension: Some("md".to_string()), + size_bytes: u64::try_from(content.len()).unwrap(), + created_unix_seconds: None, + modified_unix_seconds: file_modified_unix_seconds, + accessed_unix_seconds: None, + readonly: false, + content_fingerprint: format!("mtime:{file_modified_unix_seconds}:{content}"), + indexed_unix_seconds: file_modified_unix_seconds, + }; + let chunks = vec![IndexedFileChunk { + file_path: file.path.clone(), + directory_path: file.directory_path.clone(), + chunk_index: 0, + content: content.to_string(), + embedding: embedder.embed(content).unwrap(), + start_byte: 0, + end_byte: u64::try_from(content.len()).unwrap(), + indexed_unix_seconds: file_modified_unix_seconds, + }]; + database + .upsert_files_with_chunks(&[(&file, chunks.as_slice())]) + .await + .unwrap(); + } + #[tokio::test] async fn returns_best_directory_match() { let temp = tempfile::tempdir().unwrap(); @@ -328,6 +1539,75 @@ mod tests { assert_eq!(results.first().unwrap().path, chrome.to_string_lossy()); } + #[test] + fn temporal_parser_converts_last_month_to_calendar_range() { + let parsed = + TemporalQuery::from_query_at("that thing I did last month", fixed_time(2026, 6, 4, 12)); + let dry_run = TemporalDryRun::from_query(&parsed); + + assert_eq!(parsed.cleaned_query, "that thing I did"); + assert_eq!(dry_run.matched_phrase.as_deref(), Some("last month")); + assert_eq!(dry_run.semantic_query, "that thing I did"); + assert_eq!( + parsed.filter.modified_range(), + Some(ModifiedTimeRange { + start_unix_seconds: Some(fixed_time(2026, 5, 1, 0).timestamp()), + end_unix_seconds: Some(fixed_time(2026, 6, 1, 0).timestamp()), + }) + ); + } + + #[test] + fn temporal_parser_converts_yesterday_to_day_range() { + let parsed = + TemporalQuery::from_query_at("project from yesterday", fixed_time(2026, 6, 4, 12)); + + assert_eq!(parsed.cleaned_query, "project from"); + assert_eq!( + parsed.filter.modified_range(), + Some(ModifiedTimeRange { + start_unix_seconds: Some(fixed_time(2026, 6, 3, 0).timestamp()), + end_unix_seconds: Some(fixed_time(2026, 6, 4, 0).timestamp()), + }) + ); + } + + #[tokio::test] + async fn temporal_query_filters_chunks_by_modified_time_before_ranking() { + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + insert_directory_with_chunk( + &database, + &embedder, + "/tmp/may-project", + fixed_time(2026, 5, 15, 12).timestamp(), + "project notes", + ) + .await; + insert_directory_with_chunk( + &database, + &embedder, + "/tmp/april-project", + fixed_time(2026, 4, 15, 12).timestamp(), + "project notes", + ) + .await; + + let temporal_query = + TemporalQuery::from_query_at("project last month", fixed_time(2026, 6, 4, 12)); + let results = Searcher::new(&database, &embedder) + .search_with_temporal_query(temporal_query, 5, None) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, "/tmp/may-project"); + assert!( + !results + .iter() + .any(|result| result.path == "/tmp/april-project") + ); + } + #[tokio::test] async fn returns_directory_by_deterministic_type_classification() { let temp = tempfile::tempdir().unwrap(); @@ -383,4 +1663,327 @@ mod tests { assert_eq!(results.first().unwrap().path, migrations.to_string_lossy()); } + + #[tokio::test] + async fn prefers_deeper_directory_when_leaf_name_matches_query() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let app = root.join("chrome-extension"); + let widgets = app.join("src/widgets"); + fs::create_dir_all(&widgets).unwrap(); + fs::write( + widgets.join("README.md"), + "toolbar widgets for chrome extension popup", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("widgets chrome extension", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, widgets.to_string_lossy()); + } + + #[tokio::test] + async fn collapses_single_leaf_term_to_general_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("opencode"); + let nested = expected.join("github"); + fs::create_dir_all(&nested).unwrap(); + fs::write( + nested.join("README.md"), + "github workflow configuration and repository automation", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("github thing", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, expected.to_string_lossy()); + } + + #[tokio::test] + async fn exact_path_tokens_beat_weaker_semantic_matches() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("cd-with-llm-search"); + let misleading = root.join("gollm"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&misleading).unwrap(); + fs::write(expected.join("README.md"), "semantic cd directory search").unwrap(); + fs::write( + misleading.join("README.md"), + "language model workspace with unrelated content", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("cd with llm", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, expected.to_string_lossy()); + } + + #[tokio::test] + async fn fuzzy_path_tokens_tolerate_simple_misspellings() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("cd-with-llm-search"); + let misleading = root.join("opencode-llm"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&misleading).unwrap(); + fs::write(expected.join("README.md"), "semantic cd directory search").unwrap(); + fs::write( + misleading.join("README.md"), + "llm srach assistant workspace with exact typo terms", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("llm srach", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, expected.to_string_lossy()); + } + + #[tokio::test] + async fn partial_path_tokens_match_query_dependent_prefixes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("gitplub"); + let misleading = root.join("opencode").join("github"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&misleading).unwrap(); + fs::write(expected.join("README.md"), "small project launcher").unwrap(); + fs::write( + misleading.join("README.md"), + "github clone repository github clone command documentation", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("github clone", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, expected.to_string_lossy()); + } + + #[tokio::test] + async fn dry_run_reports_candidates_embedding_scores_and_winner() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("gitplub"); + let misleading = root.join("opencode").join("github"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&misleading).unwrap(); + fs::write(expected.join("README.md"), "small project launcher").unwrap(); + fs::write( + misleading.join("README.md"), + "github clone repository github clone command documentation", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let report = Searcher::new(&database, &embedder) + .dry_run("github clone", 3) + .await + .unwrap(); + + assert_eq!(report.cache.status, CacheDryRunStatus::Miss); + assert!(report.cache.directory_count > 0); + assert_eq!(report.temporal.matched_phrase, None); + assert_eq!(report.temporal.semantic_query, "github clone"); + assert_eq!(report.candidate_terms, vec!["github", "clone"]); + assert!( + report + .sql_candidate_directories + .iter() + .any(|path| path == &misleading.to_string_lossy()) + ); + assert!( + report + .fuzzy_candidate_directories + .iter() + .any(|path| path == &expected.to_string_lossy()) + ); + assert!(!report.embedding_scores.is_empty()); + assert_eq!( + report.results.first().unwrap().path, + expected.to_string_lossy() + ); + } + + #[tokio::test] + async fn distinctive_path_term_beats_generic_category_term() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("racoon-dash"); + let misleading = root.join("fuckin-game"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&misleading).unwrap(); + fs::write(expected.join("README.md"), "dash runner score levels").unwrap(); + fs::write( + misleading.join("README.md"), + "arcade game engine sprites levels controller", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("racoon game", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, expected.to_string_lossy()); + } + + #[tokio::test] + async fn single_high_confidence_directory_match_skips_semantic_search() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let expected = root.join("racoon-dash"); + let other = root.join("arcade-game"); + fs::create_dir_all(&expected).unwrap(); + fs::create_dir_all(&other).unwrap(); + fs::write(expected.join("README.md"), "dash runner score levels").unwrap(); + fs::write(other.join("README.md"), "arcade game engine levels").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let index_embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &index_embedder) + .index_roots(vec![root]) + .await + .unwrap(); + + let results = Searcher::new(&database, &PanicEmbedder) + .search("racoon dash", 3) + .await + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].path, expected.to_string_lossy()); + assert!(results[0].score > 0.0); + } + + #[test] + fn fuzzy_path_token_match_handles_typo_and_transposition() { + assert!(is_fuzzy_path_token_match("srach", "search")); + assert!(is_fuzzy_path_token_match("serach", "search")); + assert!(is_fuzzy_path_token_match("gihub", "github")); + assert!(!is_fuzzy_path_token_match("app", "api")); + assert!(!is_fuzzy_path_token_match("clone", "code")); + assert!(!is_fuzzy_path_token_match("code", "search")); + } + + #[test] + fn partial_path_token_match_handles_shared_prefixes() { + assert!( + is_fuzzy_path_token_match("github", "gitplub") + || is_partial_path_token_match("github", "gitplub") + ); + assert!(is_partial_path_token_match("photography", "photo")); + assert!(is_partial_path_token_match("typescript", "typefully")); + assert!(!is_partial_path_token_match("clone", "gitplub")); + assert!(!is_partial_path_token_match("code", "gitplub")); + assert!(!is_partial_path_token_match("app", "apple")); + } + + #[test] + fn candidate_filter_terms_uses_configured_generic_terms() { + let terms = vec!["racoon".to_string(), "game".to_string()]; + let generic_terms = HashSet::from(["game".to_string()]); + + assert_eq!( + candidate_filter_terms(&terms, &generic_terms), + vec!["racoon".to_string()] + ); + assert_eq!(candidate_filter_terms(&terms, &HashSet::new()), terms); + } + + #[tokio::test] + async fn searches_historical_file_chunk_versions() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let app = root.join("app"); + fs::create_dir_all(&app).unwrap(); + fs::write(app.join("README.md"), "legacy photon dashboard").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().await.unwrap(); + let embedder = FakeEmbedder::default(); + let indexer = Indexer::new(&settings, &database, &embedder); + indexer.index_roots(vec![root.clone()]).await.unwrap(); + + fs::write(app.join("README.md"), "modern analytics workspace").unwrap(); + indexer.index_roots(vec![root]).await.unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("legacy photon", 3) + .await + .unwrap(); + + assert_eq!(results.first().unwrap().path, app.to_string_lossy()); + } } diff --git a/start-daemon.sh b/start-daemon.sh new file mode 100755 index 0000000..04a8624 --- /dev/null +++ b/start-daemon.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ -n "${CDS_BIN:-}" ]; then + cds_bin="$CDS_BIN" +elif command -v cds >/dev/null 2>&1; then + cds_bin="$(command -v cds)" +elif [ -x "$repo_dir/target/debug/cds" ]; then + cds_bin="$repo_dir/target/debug/cds" +elif [ -x "$repo_dir/target/release/cds" ]; then + cds_bin="$repo_dir/target/release/cds" +else + echo "error: could not find cds binary" >&2 + echo "Run 'cargo build' or './install.sh' first, or set CDS_BIN=/path/to/cds." >&2 + exit 1 +fi + +exec "$cds_bin" --restart-daemon diff --git a/tests/install_script.rs b/tests/install_script.rs index 0cc109f..2f325cb 100644 --- a/tests/install_script.rs +++ b/tests/install_script.rs @@ -14,10 +14,11 @@ fn install_script_writes_shell_integration_idempotently() { fs::write( &fake_cargo, format!( - "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> '{}'\nmkdir -p '{}'\ntouch '{}/cds'\nchmod +x '{}/cds'\n", + "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> '{}'\nmkdir -p '{}'\ncat > '{}/cds' <<'CDS'\n#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"${{1-}}\" = \"--daemon\" ]; then sleep 60; fi\nCDS\nchmod +x '{}/cds'\n", temp.path().join("cargo-args.log").display(), cargo_home.join("bin").display(), cargo_home.join("bin").display(), + temp.path().join("cds-args.log").display(), cargo_home.join("bin").display(), ), ) @@ -51,6 +52,9 @@ fn install_script_writes_shell_integration_idempotently() { let cargo_args = fs::read_to_string(temp.path().join("cargo-args.log")).unwrap(); assert_eq!(cargo_args.matches("--force").count(), 0); + + let cds_args = fs::read_to_string(temp.path().join("cds-args.log")).unwrap(); + assert_eq!(cds_args.matches("--restart-daemon").count(), 2); } #[test] @@ -65,10 +69,11 @@ fn install_script_force_passes_force_to_cargo_install() { fs::write( &fake_cargo, format!( - "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" > '{}'\nmkdir -p '{}'\ntouch '{}/cds'\nchmod +x '{}/cds'\n", + "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" > '{}'\nmkdir -p '{}'\ncat > '{}/cds' <<'CDS'\n#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"${{1-}}\" = \"--daemon\" ]; then sleep 60; fi\nCDS\nchmod +x '{}/cds'\n", temp.path().join("cargo-args.log").display(), cargo_home.join("bin").display(), cargo_home.join("bin").display(), + temp.path().join("cds-args.log").display(), cargo_home.join("bin").display(), ), ) @@ -97,4 +102,6 @@ fn install_script_force_passes_force_to_cargo_install() { let cargo_args = fs::read_to_string(temp.path().join("cargo-args.log")).unwrap(); assert!(cargo_args.contains("install --path")); assert!(cargo_args.contains("--force")); + let cds_args = fs::read_to_string(temp.path().join("cds-args.log")).unwrap(); + assert!(cds_args.contains("--restart-daemon")); } diff --git a/tests/search_integration.rs b/tests/search_integration.rs index 6b90f6c..847cb3a 100644 --- a/tests/search_integration.rs +++ b/tests/search_integration.rs @@ -111,6 +111,42 @@ fn dir_type_count_lists_detected_directory_types() { assert!(stdout.contains("1\trust project"), "{stdout}"); } +#[test] +fn daemon_once_indexes_configured_roots() { + let fixture = SearchFixture::new(); + + let daemon = fixture.cds().arg("--daemon-once").output().unwrap(); + + assert!( + daemon.status.success(), + "stderr: {}", + String::from_utf8_lossy(&daemon.stderr) + ); + + let search = fixture + .cds() + .arg("--search") + .arg("manifest") + .output() + .unwrap(); + assert!( + search.status.success(), + "stderr: {}", + String::from_utf8_lossy(&search.stderr) + ); + assert!( + String::from_utf8_lossy(&search.stderr) + .contains("cds: warning: daemon unavailable or busy; searching locally"), + "stderr: {}", + String::from_utf8_lossy(&search.stderr) + ); + assert!( + String::from_utf8(search.stdout) + .unwrap() + .contains("chrome-extension") + ); +} + #[test] fn reset_prompts_and_deletes_indexed_data_when_confirmed() { let fixture = SearchFixture::new(); From 306e9a280f1301127af048208c50486b2856c941 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 19:13:02 -0400 Subject: [PATCH 8/9] Invalidate search cache with index revision --- migrations/0004_create_index_metadata.sql | 8 + src/app/mod.rs | 54 ++++++- src/db/error.rs | 12 ++ src/db/mod.rs | 173 ++++++++++++++++++++-- src/search/mod.rs | 44 ++++-- 5 files changed, 265 insertions(+), 26 deletions(-) create mode 100644 migrations/0004_create_index_metadata.sql diff --git a/migrations/0004_create_index_metadata.sql b/migrations/0004_create_index_metadata.sql new file mode 100644 index 0000000..a29835d --- /dev/null +++ b/migrations/0004_create_index_metadata.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS index_metadata ( + key TEXT PRIMARY KEY NOT NULL, + value INTEGER NOT NULL +); + +INSERT INTO index_metadata (key, value) +VALUES ('revision', 0) +ON CONFLICT(key) DO NOTHING; diff --git a/src/app/mod.rs b/src/app/mod.rs index 7bb3af0..53d255a 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -791,9 +791,10 @@ async fn search_cache_for<'a>( search_cache: &'a mut Option, generic_terms: &HashSet, ) -> Result<(&'a SearchCache, CacheDryRunStatus)> { + let revision = database.current_revision().await?; let status = if search_cache .as_ref() - .is_none_or(|cache| !cache.matches_generic_terms(generic_terms)) + .is_none_or(|cache| !cache.matches_revision_and_generic_terms(revision, generic_terms)) { *search_cache = Some(SearchCache::load(database, generic_terms).await?); CacheDryRunStatus::Miss @@ -1254,6 +1255,57 @@ mod tests { } } + #[tokio::test] + async fn search_cache_misses_after_database_revision_changes() { + use crate::db::{DocumentKind, IndexedDocument}; + + let database = Database::open_in_memory().await.unwrap(); + let generic_terms = HashSet::new(); + let mut search_cache = None; + + let first_status = { + let (_, status) = search_cache_for(&database, &mut search_cache, &generic_terms) + .await + .unwrap(); + status + }; + let second_status = { + let (_, status) = search_cache_for(&database, &mut search_cache, &generic_terms) + .await + .unwrap(); + status + }; + assert_eq!(first_status, CacheDryRunStatus::Miss); + assert_eq!(second_status, CacheDryRunStatus::Hit); + + database + .upsert_document(&IndexedDocument { + path: "/tmp/project".to_string(), + name: "project".to_string(), + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: "project readme cargo".to_string(), + embedding: vec![0.1, 0.2, 0.3], + metadata_fingerprint: "fingerprint".to_string(), + size_bytes: 4096, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: false, + indexed_unix_seconds: 34, + }) + .await + .unwrap(); + + let stale_status = { + let (_, status) = search_cache_for(&database, &mut search_cache, &generic_terms) + .await + .unwrap(); + status + }; + assert_eq!(stale_status, CacheDryRunStatus::Miss); + } + #[test] fn semantic_query_joins_plain_words() { let temp = tempfile::tempdir().unwrap(); diff --git a/src/db/error.rs b/src/db/error.rs index a1686b9..d15a914 100644 --- a/src/db/error.rs +++ b/src/db/error.rs @@ -131,6 +131,18 @@ pub enum DbError { source: sqlx::Error, }, + #[error("failed to read index revision")] + ReadIndexRevision { + #[source] + source: sqlx::Error, + }, + + #[error("failed to bump index revision")] + BumpIndexRevision { + #[source] + source: sqlx::Error, + }, + #[error("failed to count indexed documents")] CountDocuments { #[source] diff --git a/src/db/mod.rs b/src/db/mod.rs index aa42641..bf3e76d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -62,7 +62,29 @@ impl Database { Ok(Self { pool }) } + pub async fn current_revision(&self) -> Result { + sqlx::query_scalar( + " + SELECT value + FROM index_metadata + WHERE key = 'revision' + ", + ) + .fetch_one(&self.pool) + .await + .map_err(|source| DbError::ReadIndexRevision { source }) + } + pub async fn upsert_document(&self, document: &IndexedDocument) -> Result<()> { + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; + sqlx::query( " INSERT INTO indexed_documents ( @@ -117,17 +139,35 @@ impl Database { .bind(document.accessed_unix_seconds) .bind(document.readonly) .bind(document.indexed_unix_seconds) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::UpsertDocument { path: document.path.clone(), source, })?; + bump_index_revision(&mut transaction).await?; + transaction + .commit() + .await + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; + Ok(()) } pub async fn upsert_file(&self, file: &IndexedFile) -> Result<()> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| DbError::UpsertFile { + path: file.path.clone(), + source, + })?; + sqlx::query( " INSERT INTO indexed_files ( @@ -170,13 +210,22 @@ impl Database { .bind(file.readonly) .bind(&file.content_fingerprint) .bind(file.indexed_unix_seconds) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::UpsertFile { path: file.path.clone(), source, })?; + bump_index_revision(&mut transaction).await?; + transaction + .commit() + .await + .map_err(|source| DbError::UpsertFile { + path: file.path.clone(), + source, + })?; + Ok(()) } @@ -295,6 +344,7 @@ impl Database { } } + bump_index_revision(&mut transaction).await?; transaction .commit() .await @@ -308,9 +358,18 @@ impl Database { file_path: &str, chunks: &[IndexedFileChunk], ) -> Result<()> { + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::DeleteFileChunks { + path: file_path.to_string(), + source, + })?; + sqlx::query("DELETE FROM indexed_file_chunks WHERE file_path = ?") .bind(file_path) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeleteFileChunks { path: file_path.to_string(), @@ -351,7 +410,7 @@ impl Database { .map_err(|source| DbError::MetadataSizeOverflow { source })?, ) .bind(chunk.indexed_unix_seconds) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::InsertFileChunk { path: chunk.file_path.clone(), @@ -360,6 +419,12 @@ impl Database { })?; } + bump_index_revision(&mut transaction).await?; + transaction + .commit() + .await + .map_err(|source| DbError::CommitFileBatch { source })?; + Ok(()) } @@ -391,6 +456,7 @@ impl Database { source, })?; + bump_index_revision(&mut transaction).await?; transaction .commit() .await @@ -520,6 +586,7 @@ impl Database { } } + bump_index_revision(&mut transaction).await?; transaction .commit() .await @@ -533,9 +600,18 @@ impl Database { directory_path: &str, classifications: &[DirectoryClassification], ) -> Result<()> { + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::ReplaceDirectoryClassifications { + path: directory_path.to_string(), + source, + })?; + sqlx::query("DELETE FROM directory_classifications WHERE directory_path = ?") .bind(directory_path) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::ReplaceDirectoryClassifications { path: directory_path.to_string(), @@ -563,7 +639,7 @@ impl Database { .bind(&classification.evidence_path) .bind(&classification.evidence_summary) .bind(classification.detected_unix_seconds) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::InsertDirectoryClassification { path: classification.directory_path.clone(), @@ -572,12 +648,29 @@ impl Database { })?; } + bump_index_revision(&mut transaction).await?; + transaction + .commit() + .await + .map_err(|source| DbError::ReplaceDirectoryClassifications { + path: directory_path.to_string(), + source, + })?; + Ok(()) } pub async fn delete_path_tree(&self, path: &str) -> Result<()> { let path_len = i64::try_from(path.len()) .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?; + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; sqlx::query( " @@ -595,7 +688,7 @@ impl Database { .bind(path_len) .bind(path) .bind(path_len) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeletePathTree { path: path.to_string(), @@ -629,7 +722,7 @@ impl Database { .bind(path_len) .bind(path) .bind(path_len) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeletePathTree { path: path.to_string(), @@ -663,7 +756,7 @@ impl Database { .bind(path_len) .bind(path) .bind(path_len) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeletePathTree { path: path.to_string(), @@ -697,7 +790,7 @@ impl Database { .bind(path_len) .bind(path) .bind(path_len) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeletePathTree { path: path.to_string(), @@ -720,13 +813,22 @@ impl Database { .bind(path_len) .bind(path) .bind(path_len) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(|source| DbError::DeletePathTree { path: path.to_string(), source, })?; + bump_index_revision(&mut transaction).await?; + transaction + .commit() + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + Ok(()) } @@ -762,6 +864,7 @@ impl Database { .await .map_err(|source| DbError::ResetDatabase { source })?; + bump_index_revision(&mut transaction).await?; transaction .commit() .await @@ -1232,7 +1335,7 @@ async fn migrate_pool(pool: &SqlitePool) -> Result<()> { .run(pool) .await .map_err(|source| DbError::Migrate { source })?; - sqlx::query("PRAGMA user_version = 5") + sqlx::query("PRAGMA user_version = 6") .execute(pool) .await .map_err(|source| DbError::PrepareLegacyMigration { source })?; @@ -1398,6 +1501,21 @@ async fn add_legacy_column_if_missing( Ok(()) } +async fn bump_index_revision(transaction: &mut Transaction<'_, Sqlite>) -> Result<()> { + sqlx::query( + " + INSERT INTO index_metadata (key, value) + VALUES ('revision', 1) + ON CONFLICT(key) DO UPDATE SET value = value + 1 + ", + ) + .execute(&mut **transaction) + .await + .map_err(|source| DbError::BumpIndexRevision { source })?; + + Ok(()) +} + async fn insert_file_chunk_history( transaction: &mut Transaction<'_, Sqlite>, file: &IndexedFile, @@ -1569,6 +1687,34 @@ mod tests { ); } + #[tokio::test] + async fn indexed_mutations_bump_revision() { + let db = Database::open_in_memory().await.unwrap(); + let start = db.current_revision().await.unwrap(); + + db.upsert_document(&IndexedDocument { + path: "/tmp/project".to_string(), + name: "project".to_string(), + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: "project readme cargo".to_string(), + embedding: vec![0.1, 0.2, 0.3], + metadata_fingerprint: "fingerprint".to_string(), + size_bytes: 4096, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: false, + indexed_unix_seconds: 34, + }) + .await + .unwrap(); + assert_eq!(db.current_revision().await.unwrap(), start + 1); + + db.reset().await.unwrap(); + assert_eq!(db.current_revision().await.unwrap(), start + 2); + } + #[tokio::test] async fn migrates_v1_database_to_metadata_schema() { let temp = tempfile::tempdir().unwrap(); @@ -1604,7 +1750,8 @@ mod tests { .await .unwrap(); - assert_eq!(version, 5); + assert_eq!(version, 6); + assert_eq!(db.current_revision().await.unwrap(), 0); db.upsert_document(&IndexedDocument { path: "/tmp/project".to_string(), name: "project".to_string(), diff --git a/src/search/mod.rs b/src/search/mod.rs index 9e3430d..51eb7c7 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -84,27 +84,47 @@ pub struct Searcher<'a, E> { #[derive(Debug, Clone)] pub struct SearchCache { generic_terms: HashSet, + revision: i64, directories: Vec, path_term_stats: PathTermStats, } impl SearchCache { pub async fn load(database: &Database, generic_terms: &HashSet) -> Result { - let directories = database - .directory_search_documents() - .await - .map_err(|source| SearchError::LoadDirectories { source })?; - let path_term_stats = PathTermStats::from_directories(&directories, generic_terms); + loop { + let revision = database + .current_revision() + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + let directories = database + .directory_search_documents() + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + let current_revision = database + .current_revision() + .await + .map_err(|source| SearchError::LoadDirectories { source })?; + if revision != current_revision { + continue; + } - Ok(Self { - generic_terms: generic_terms.clone(), - directories, - path_term_stats, - }) + let path_term_stats = PathTermStats::from_directories(&directories, generic_terms); + + return Ok(Self { + generic_terms: generic_terms.clone(), + revision, + directories, + path_term_stats, + }); + } } - pub fn matches_generic_terms(&self, generic_terms: &HashSet) -> bool { - &self.generic_terms == generic_terms + pub fn matches_revision_and_generic_terms( + &self, + revision: i64, + generic_terms: &HashSet, + ) -> bool { + self.revision == revision && &self.generic_terms == generic_terms } } From 83165c57eb62aa6ade73420283ff8a9926ea92c8 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 19:17:57 -0400 Subject: [PATCH 9/9] Remove Docker cd equivalence test --- AGENTS.md | 17 --- README.md | 20 ---- tests/docker_cd_equivalence.rs | 75 ------------ tests/docker_cd_equivalence.sh | 205 --------------------------------- 4 files changed, 317 deletions(-) delete mode 100644 tests/docker_cd_equivalence.rs delete mode 100644 tests/docker_cd_equivalence.sh diff --git a/AGENTS.md b/AGENTS.md index fca71fd..c462612 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,27 +176,10 @@ Add tests for: - indexing change detection - ranking behavior with deterministic fake embeddings - shell integration output -- Docker-backed `cd` equivalence for Bash behavior Do not make tests depend on downloading or running the real embedding model unless they are explicitly marked as slow/integration tests. Use a fake embedder for normal tests. -The Docker equivalence test runs the project inside a Rust container and compares `cds` -against Bash's built-in `cd` for status, stdout, stderr, `PWD`, `OLDPWD`, and physical path. -Keep these details in mind when editing it: - -- CI currently uses `CDS_DOCKER_IMAGE=rust:1`. -- The Docker image must include Cargo. The test builds `cds` with `--no-default-features` - and `CDS_EMBEDDER=fake`, so it does not need the FastEmbed/ONNX dependency stack. -- The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some container - invocations otherwise fail with `cargo: command not found`. -- Bash includes source line numbers in `cd` diagnostics. Normalize only those line numbers - before comparing stderr, while keeping the actual diagnostic text exact. -- The Docker test can skip locally when Docker is unavailable. Use - `CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence -- --nocapture` - with Docker access when changing equivalence behavior. Set `CDS_DOCKER_RANDOM_CASES` - to increase or decrease the randomized sample count. - ## Repository Hygiene - Keep generated databases, model files, caches, and local indexes out of git. diff --git a/README.md b/README.md index 8ade3a3..9e83d7b 100644 --- a/README.md +++ b/README.md @@ -62,26 +62,6 @@ To replace an existing installed binary: cargo test --all ``` -The test suite includes a Docker-backed equivalence test. When Docker is available, it -mounts this project into a Rust container, builds `cds` without the real embedding -runtime, generates a fresh random filesystem tree, and compares `cds` against the shell's -built-in `cd` for exit status, stdout, stderr, `PWD`, `OLDPWD`, and physical path -behavior. - -Use a specific container image with: - -```sh -CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence -``` - -Adjust the number of randomized cases with: - -```sh -CDS_DOCKER_RANDOM_CASES=200 cargo test --test docker_cd_equivalence -``` - -The Docker image must include Cargo. The default image is `rust:1`. - Default builds include the real local embedding runtime. Use `--no-default-features` with `CDS_EMBEDDER=fake` for shell-only checks that do not need `bge-small-en-v1.5`. diff --git a/tests/docker_cd_equivalence.rs b/tests/docker_cd_equivalence.rs deleted file mode 100644 index 97e1708..0000000 --- a/tests/docker_cd_equivalence.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -const DOCKER_TEST_SCRIPT: &str = r#" -set -eu -export PATH="/usr/local/cargo/bin:${CARGO_HOME:-/usr/local/cargo}/bin:${PATH}" - -if ! command -v cargo >/dev/null 2>&1; then - echo "cargo was not found in the Docker image. Use a Rust image that includes Cargo, or set CDS_DOCKER_IMAGE." >&2 - exit 127 -fi - -CDS_EMBEDDER=fake cargo build --quiet --no-default-features --bin cds -bash tests/docker_cd_equivalence.sh -"#; - -#[test] -fn docker_cd_equivalence_random_tree() { - if !docker_is_available() { - eprintln!("skipping docker cd equivalence test because Docker is not available"); - return; - } - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let image = std::env::var("CDS_DOCKER_IMAGE").unwrap_or_else(|_| "rust:1".to_string()); - let cache_root = std::env::var_os("CDS_DOCKER_CACHE_DIR") - .map(PathBuf::from) - .unwrap_or_else(|| manifest_dir.join("target/docker-cache")); - let cargo_cache = cache_root.join("cargo"); - let target_cache = cache_root.join("target"); - - fs::create_dir_all(&cargo_cache).expect("docker cargo cache directory is created"); - fs::create_dir_all(&target_cache).expect("docker target cache directory is created"); - - let output = Command::new("docker") - .arg("run") - .arg("--rm") - .arg("--volume") - .arg(format!("{}:/workspace", manifest_dir.display())) - .arg("--volume") - .arg(format!("{}:/tmp/cargo", cargo_cache.display())) - .arg("--volume") - .arg(format!("{}:/tmp/cds-target", target_cache.display())) - .arg("--workdir") - .arg("/workspace") - .arg("--env") - .arg("CARGO_HOME=/tmp/cargo") - .arg("--env") - .arg("CARGO_TARGET_DIR=/tmp/cds-target") - .arg("--env") - .arg("CDS_EMBEDDER=fake") - .arg(image) - .arg("bash") - .arg("-lc") - .arg(DOCKER_TEST_SCRIPT) - .output() - .expect("docker cd equivalence test starts"); - - assert!( - output.status.success(), - "docker cd equivalence test failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); -} - -fn docker_is_available() -> bool { - Command::new("docker") - .arg("info") - .arg("--format") - .arg("{{.ServerVersion}}") - .output() - .is_ok_and(|output| output.status.success()) -} diff --git a/tests/docker_cd_equivalence.sh b/tests/docker_cd_equivalence.sh deleted file mode 100644 index 7459785..0000000 --- a/tests/docker_cd_equivalence.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -export PATH="/tmp/cds-target/debug:$PATH" - -workspace="$(mktemp -d)" -home_dir="$workspace/home" -tree_root="$workspace/tree" -mkdir -p "$home_dir" "$tree_root" - -seed="${CDS_RANDOM_SEED:-$RANDOM-$RANDOM-$(date +%s%N)}" -echo "cds docker cd equivalence seed: $seed" - -declare -a dirs -dirs=("$tree_root") - -random_label() { - printf 'dir_%s_%04d_%04d' "$1" "$RANDOM" "$RANDOM" -} - -for i in $(seq 1 36); do - parent_index=$((RANDOM % ${#dirs[@]})) - parent="${dirs[$parent_index]}" - child="$parent/$(random_label "$i")" - mkdir -p "$child" - dirs+=("$child") - - if (( RANDOM % 3 == 0 )); then - nested="$child/$(random_label "${i}_nested")" - mkdir -p "$nested" - dirs+=("$nested") - fi -done - -space_dir="$tree_root/dir with spaces" -quote_dir="$tree_root/team's app" -leading_dash_parent="$tree_root/leading" -leading_dash_dir="$leading_dash_parent/-starts-with-dash" -cdpath_root="$workspace/cdpath" -cdpath_target="$cdpath_root/cdpath_target" -logical_real="$tree_root/real/path" -logical_link="$tree_root/logical-link" - -mkdir -p \ - "$space_dir" \ - "$quote_dir" \ - "$leading_dash_dir" \ - "$cdpath_target" \ - "$logical_real/child" -ln -s "$logical_real" "$logical_link" - -dirs+=( - "$space_dir" - "$quote_dir" - "$leading_dash_parent" - "$leading_dash_dir" - "$cdpath_target" - "$logical_real" - "$logical_real/child" - "$logical_link" - "$logical_link/child" -) - -for dir in "${dirs[@]}"; do - printf 'fixture file for %s\n' "$dir" > "$dir/file_$RANDOM.txt" -done - -cds_init="$(cds --shell-init bash)" - -setup_cds() { - eval "$cds_init" -} - -normalize_stderr() { - local file="$1" - local normalized - normalized="$(mktemp)" - - sed -E \ - 's#^(tests/docker_cd_equivalence\.sh: line )[0-9]+(: cd:)#\1\2#' \ - "$file" > "$normalized" - mv "$normalized" "$file" -} - -run_case() { - local mode="$1" - local start="$2" - local oldpwd="$3" - local cdpath="$4" - local label="$5" - shift 5 - - local out_file err_file state_file - out_file="$(mktemp)" - err_file="$(mktemp)" - state_file="$(mktemp)" - - ( - set +e - export HOME="$home_dir" - export CDPATH="$cdpath" - cd "$start" || exit 98 - export OLDPWD="$oldpwd" - - if [ "$mode" = "cds" ]; then - setup_cds - cds "$@" - else - builtin cd "$@" - fi - - status=$? - physical_pwd="$(pwd -P 2>/dev/null)" - physical_status=$? - if [ "$physical_status" -ne 0 ]; then - physical_pwd="" - fi - - { - printf 'label=%s\n' "$label" - printf 'status=%s\n' "$status" - printf 'PWD=%s\n' "$PWD" - printf 'OLDPWD=%s\n' "${OLDPWD-}" - printf 'PHYSICAL_PWD=%s\n' "$physical_pwd" - } > "$state_file" - ) > "$out_file" 2> "$err_file" - - normalize_stderr "$err_file" - - printf '%s\n%s\n%s\n' "$state_file" "$out_file" "$err_file" -} - -compare_case() { - local label="$1" - local start="$2" - local oldpwd="$3" - local cdpath="$4" - shift 4 - - local cd_capture cds_capture - cd_capture="$(run_case cd "$start" "$oldpwd" "$cdpath" "$label" "$@")" - cds_capture="$(run_case cds "$start" "$oldpwd" "$cdpath" "$label" "$@")" - - local cd_state cd_out cd_err cds_state cds_out cds_err - cd_state="$(printf '%s\n' "$cd_capture" | sed -n '1p')" - cd_out="$(printf '%s\n' "$cd_capture" | sed -n '2p')" - cd_err="$(printf '%s\n' "$cd_capture" | sed -n '3p')" - cds_state="$(printf '%s\n' "$cds_capture" | sed -n '1p')" - cds_out="$(printf '%s\n' "$cds_capture" | sed -n '2p')" - cds_err="$(printf '%s\n' "$cds_capture" | sed -n '3p')" - - if ! cmp -s "$cd_state" "$cds_state" || \ - ! cmp -s "$cd_out" "$cds_out" || \ - ! cmp -s "$cd_err" "$cds_err"; then - echo "cd equivalence failed: $label" >&2 - echo "args: $(printf '<%s> ' "$@")" >&2 - echo "--- cd state" >&2 - cat "$cd_state" >&2 - echo "--- cds state" >&2 - cat "$cds_state" >&2 - echo "--- cd stdout" >&2 - cat "$cd_out" >&2 - echo "--- cds stdout" >&2 - cat "$cds_out" >&2 - echo "--- cd stderr" >&2 - cat "$cd_err" >&2 - echo "--- cds stderr" >&2 - cat "$cds_err" >&2 - exit 1 - fi -} - -compare_case "no args goes home" "$tree_root" "$space_dir" "" -compare_case "absolute random directory" "$tree_root" "$space_dir" "" "${dirs[$((RANDOM % ${#dirs[@]}))]}" -compare_case "relative child" "$tree_root" "$space_dir" "" "$(basename "$space_dir")" -compare_case "relative parent" "$space_dir" "$tree_root" "" .. -compare_case "directory with quote" "$tree_root" "$space_dir" "" "$quote_dir" -compare_case "directory with spaces" "$tree_root" "$quote_dir" "" "$space_dir" -compare_case "leading dash via double dash" "$leading_dash_parent" "$tree_root" "" -- "-starts-with-dash" -compare_case "logical symlink default" "$tree_root" "$space_dir" "" "$logical_link/child" -compare_case "logical parent with -L" "$logical_link/child" "$tree_root" "" -L .. -compare_case "physical parent with -P" "$logical_link/child" "$tree_root" "" -P .. -compare_case "cdpath lookup" "$tree_root" "$space_dir" "$cdpath_root" cdpath_target -compare_case "cd dash" "$tree_root" "$space_dir" "" - -compare_case "invalid directory" "$tree_root" "$space_dir" "" "$tree_root/does-not-exist" -compare_case "too many args" "$tree_root" "$space_dir" "" "$space_dir" "$quote_dir" - -random_cases="${CDS_DOCKER_RANDOM_CASES:-60}" -for i in $(seq 1 "$random_cases"); do - start="${dirs[$((RANDOM % ${#dirs[@]}))]}" - oldpwd="${dirs[$((RANDOM % ${#dirs[@]}))]}" - target="${dirs[$((RANDOM % ${#dirs[@]}))]}" - - case $((RANDOM % 7)) in - 0) compare_case "random absolute $i" "$start" "$oldpwd" "" "$target" ;; - 1) compare_case "random dash $i" "$start" "$oldpwd" "" - ;; - 2) compare_case "random no args $i" "$start" "$oldpwd" "" ;; - 3) compare_case "random logical flag $i" "$logical_link/child" "$oldpwd" "" -L .. ;; - 4) compare_case "random physical flag $i" "$logical_link/child" "$oldpwd" "" -P .. ;; - 5) compare_case "random invalid $i" "$start" "$oldpwd" "" "$tree_root/missing_$RANDOM" ;; - 6) compare_case "random relative parent $i" "$start" "$oldpwd" "" .. ;; - esac -done - -echo "cds matched builtin cd for randomized Docker fixture"