diff --git a/README.md b/README.md index 5da4d2a..250f209 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,16 @@ The initial config looks like this: "*.xcassets", "*.imageset", "*.appiconset", - "*.colorset" + "*.colorset", + "*.svg", + "*.png", + "*.jpg", + "*.jpeg", + "*.gif", + "*.webp", + "*.ico", + "*.pdf", + "*.zip" ], "max_file_bytes": 65536, "max_excerpt_bytes": 4096, @@ -145,11 +154,14 @@ Or index explicit roots: cds --index ~/Projects ~/work ``` -The current indexer stores directory metadata and text-file content separately in SQLite. -Directory rows store structured filesystem metadata: name, type, parent path, size in bytes, -created time, modified time, accessed time, readonly status, and index time. Text files are -stored as file metadata plus embedded content chunks linked back to their containing -directories. Directory metadata itself is not embedded. +The current indexer stores directory metadata and high-signal text-file content separately +in SQLite. Directory rows store structured filesystem metadata: name, type, parent path, +size in bytes, created time, modified time, accessed time, readonly status, and index time. +High-signal files such as READMEs, package manifests, project config files, and SQL schemas +are stored as file metadata plus embedded content chunks linked back to their containing +directories. Low-signal arbitrary text files are skipped by default to keep initial indexing +fast, and media/asset/archive files such as SVGs, images, PDFs, and ZIPs are ignored. +Directory metadata itself is not embedded. Hidden directories are always skipped and pruned from the local index. Each configured index root is treated as a container, and each top-level directory inside it is indexed only through `max_depth_per_top_level_directory` levels. diff --git a/src/config/settings.rs b/src/config/settings.rs index bf4c35a..1cb778d 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -108,6 +108,15 @@ impl Default for IndexSettings { "*.imageset".to_string(), "*.appiconset".to_string(), "*.colorset".to_string(), + "*.svg".to_string(), + "*.png".to_string(), + "*.jpg".to_string(), + "*.jpeg".to_string(), + "*.gif".to_string(), + "*.webp".to_string(), + "*.ico".to_string(), + "*.pdf".to_string(), + "*.zip".to_string(), ], max_file_bytes: 65_536, max_excerpt_bytes: 4_096, @@ -147,6 +156,46 @@ fn is_low_signal_name(name: &str) -> bool { || lower.ends_with(".imageset") || lower.ends_with(".appiconset") || lower.ends_with(".colorset") + || is_low_signal_asset_extension(&lower) +} + +fn is_low_signal_asset_extension(name: &str) -> bool { + matches!( + name.rsplit_once('.').map(|(_, extension)| extension), + Some( + "svg" + | "png" + | "jpg" + | "jpeg" + | "gif" + | "webp" + | "ico" + | "bmp" + | "tif" + | "tiff" + | "pdf" + | "zip" + | "tar" + | "gz" + | "tgz" + | "bz2" + | "xz" + | "7z" + | "rar" + | "dmg" + | "mp3" + | "mp4" + | "mov" + | "avi" + | "webm" + | "wav" + | "flac" + | "woff" + | "woff2" + | "ttf" + | "otf" + ) + ) } fn is_hidden_name(name: &str) -> bool { @@ -286,6 +335,9 @@ mod tests { assert!(index.is_excluded_name("private.key")); assert!(index.is_excluded_name("Assets.xcassets")); assert!(index.is_excluded_name("Custom Icon.appiconset")); + assert!(index.is_excluded_name("logo.svg")); + assert!(index.is_excluded_name("screenshot.png")); + assert!(index.is_excluded_name("archive.zip")); assert!(index.is_excluded_directory_name(".vscode")); assert!(!index.is_excluded_name(".vscode")); assert!(!index.is_excluded_name("README.md")); diff --git a/src/db/mod.rs b/src/db/mod.rs index 28bf15a..2e3952f 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -180,6 +180,129 @@ impl Database { Ok(()) } + pub async fn upsert_directories_with_classifications( + &self, + directories: &[(&IndexedDocument, &[DirectoryClassification])], + ) -> Result<()> { + let mut transaction = + self.pool + .begin() + .await + .map_err(|source| DbError::UpsertDocument { + path: "".to_string(), + source, + })?; + + for (document, classifications) in directories { + sqlx::query( + " + INSERT INTO indexed_documents ( + path, + name, + kind, + parent_path, + searchable_text, + embedding, + embedding_dim, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + name = excluded.name, + kind = excluded.kind, + parent_path = excluded.parent_path, + searchable_text = excluded.searchable_text, + embedding = excluded.embedding, + embedding_dim = excluded.embedding_dim, + metadata_fingerprint = excluded.metadata_fingerprint, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + ) + .bind(&document.path) + .bind(&document.name) + .bind(document.kind.as_str()) + .bind(&document.parent_path) + .bind(&document.searchable_text) + .bind(encode_embedding(&document.embedding)) + .bind( + i64::try_from(document.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind(&document.metadata_fingerprint) + .bind( + i64::try_from(document.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(document.created_unix_seconds) + .bind(document.modified_unix_seconds) + .bind(document.accessed_unix_seconds) + .bind(document.readonly) + .bind(document.indexed_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; + + sqlx::query("DELETE FROM directory_classifications WHERE directory_path = ?") + .bind(&document.path) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::ReplaceDirectoryClassifications { + path: document.path.clone(), + source, + })?; + + for classification in *classifications { + sqlx::query( + " + INSERT INTO directory_classifications ( + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&classification.directory_path) + .bind(&classification.label) + .bind(f64::from(classification.confidence)) + .bind(&classification.detector) + .bind(&classification.evidence_path) + .bind(&classification.evidence_summary) + .bind(classification.detected_unix_seconds) + .execute(&mut *transaction) + .await + .map_err(|source| DbError::InsertDirectoryClassification { + path: classification.directory_path.clone(), + label: classification.label.clone(), + source, + })?; + } + } + + transaction + .commit() + .await + .map_err(|source| DbError::CommitFileBatch { source })?; + + Ok(()) + } + pub async fn replace_file_chunks( &self, file_path: &str, diff --git a/src/index/file.rs b/src/index/file.rs index 51848af..eb97482 100644 --- a/src/index/file.rs +++ b/src/index/file.rs @@ -76,6 +76,15 @@ pub fn prepare_text_file( return Ok(None); } + let name = path + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_string_lossy() + .into_owned(); + if !is_high_signal_content_file(&name) { + return Ok(None); + } + let bytes = fs::read(path).map_err(|source| IndexError::ReadFile { path: path.to_path_buf(), source, @@ -94,11 +103,6 @@ pub fn prepare_text_file( let modified_unix_seconds = unix_seconds(metadata.modified().unwrap_or(UNIX_EPOCH)); let file_path = path_to_string(path); let directory_path = path_to_string(directory); - let name = path - .file_name() - .unwrap_or_else(|| OsStr::new("")) - .to_string_lossy() - .into_owned(); let file = IndexedFile { path: file_path.clone(), @@ -180,6 +184,38 @@ fn normalize_whitespace(value: &str) -> String { value.split_whitespace().collect::>().join(" ") } +fn is_high_signal_content_file(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + if lower.starts_with("readme") || lower == "gemfile" || lower == "dockerfile" { + return true; + } + + matches!( + lower.as_str(), + "cargo.toml" + | "package.json" + | "manifest.json" + | "pyproject.toml" + | "requirements.txt" + | "setup.py" + | "go.mod" + | "makefile" + | "docker-compose.yml" + | "compose.yml" + | "compose.yaml" + | "tsconfig.json" + | "vite.config.js" + | "vite.config.ts" + | "next.config.js" + | "next.config.ts" + | "tailwind.config.js" + | "tailwind.config.ts" + ) || matches!( + lower.rsplit_once('.').map(|(_, extension)| extension), + Some("md" | "markdown" | "toml" | "yaml" | "yml" | "sql") + ) +} + fn unix_seconds(time: SystemTime) -> i64 { time.duration_since(UNIX_EPOCH) .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) @@ -210,4 +246,40 @@ mod tests { assert_eq!(chunks[1].text, "ef gh"); assert_eq!(chunks[2].text, "i"); } + + #[test] + fn skips_low_signal_text_files() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("notes.txt"); + fs::write(&path, "large low-signal notes").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap(); + + assert_eq!(prepared, None); + } + + #[test] + fn skips_svg_assets_even_though_they_are_text() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("logo.svg"); + fs::write(&path, "Logo").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap(); + + assert_eq!(prepared, None); + } + + #[test] + fn prepares_high_signal_text_files() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("README.md"); + fs::write(&path, "high signal project summary").unwrap(); + + let prepared = prepare_text_file(&path, temp.path(), &Settings::default()) + .unwrap() + .expect("README.md is indexed"); + + assert_eq!(prepared.file.name, "README.md"); + assert_eq!(prepared.chunks.len(), 1); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index c527c1b..fdd18ac 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -241,9 +241,9 @@ mod tests { let root = temp.path().join("Projects"); let app = root.join("app"); fs::create_dir_all(&app).unwrap(); - fs::write(app.join("one.txt"), "alpha").unwrap(); - fs::write(app.join("two.txt"), "bravo").unwrap(); - fs::write(app.join("three.txt"), "charlie").unwrap(); + fs::write(app.join("README.md"), "alpha").unwrap(); + fs::write(app.join("Cargo.toml"), "bravo").unwrap(); + fs::write(app.join("schema.sql"), "charlie").unwrap(); let settings = Settings::default(); let database = Database::open_in_memory().await.unwrap(); diff --git a/src/index/scanner.rs b/src/index/scanner.rs index 5c97da6..ec26213 100644 --- a/src/index/scanner.rs +++ b/src/index/scanner.rs @@ -1,8 +1,8 @@ +use std::collections::VecDeque; use std::fs; -use std::future::Future; -use std::path::Path; -use std::pin::Pin; -use std::sync::Mutex; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; use super::{classify, file, summary}; use crate::config::Settings; @@ -38,6 +38,17 @@ impl IndexReport { self.roots_not_directories, ) } + + fn merge(&mut self, other: Self) { + self.roots_scanned += other.roots_scanned; + self.roots_missing += other.roots_missing; + self.roots_not_directories += other.roots_not_directories; + self.directories_indexed += other.directories_indexed; + self.files_seen += other.files_seen; + self.text_files_indexed += other.text_files_indexed; + self.file_chunks_indexed += other.file_chunks_indexed; + self.entries_skipped += other.entries_skipped; + } } pub async fn scan_root_with_progress( @@ -52,17 +63,49 @@ where E: Embedder, P: IndexProgress + ?Sized, { + let mut scan = scan_root_concurrently(root, settings)?; + scan.directories + .sort_by(|left, right| left.document.path.cmp(&right.document.path)); + scan.files + .sort_by(|left, right| left.file.path.cmp(&right.file.path)); + scan.pruned_paths.sort(); + scan.pruned_paths.dedup(); + + for path in &scan.pruned_paths { + database + .delete_path_tree(path) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: path.clone(), + source: Box::new(source), + })?; + } + + for directory in &scan.directories { + progress.directory_started(Path::new(&directory.document.path)); + } + + let directory_refs = scan + .directories + .iter() + .map(|directory| (&directory.document, directory.classifications.as_slice())) + .collect::>(); + database + .upsert_directories_with_classifications(&directory_refs) + .await + .map_err(|source| IndexError::StoreDocument { + path: root.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + let batch = FileIndexBatch::new(database, embedder); - scan_directory( - root, - settings, - &batch, - report, - progress, - DepthPosition::IndexRoot, - ) - .await?; - batch.flush().await + for file in scan.files { + batch.push(file).await?; + } + batch.flush().await?; + + report.merge(scan.report); + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -82,138 +125,267 @@ impl DepthPosition { } } -fn scan_directory<'a, E, P>( - directory: &'a Path, - settings: &'a Settings, - batch: &'a FileIndexBatch<'a, E>, - report: &'a mut IndexReport, - progress: &'a mut P, +fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { + match depth_position { + DepthPosition::IndexRoot => false, + DepthPosition::TopLevelDirectory { depth } => { + depth > settings.index.max_depth_per_top_level_directory + } + } +} + +#[derive(Debug, Clone)] +struct ScanJob { + directory: PathBuf, depth_position: DepthPosition, -) -> Pin> + 'a>> -where - E: Embedder + 'a, - P: IndexProgress + ?Sized + 'a, -{ - Box::pin(async move { - progress.directory_started(directory); +} - let document = summary::summarize_directory(directory, settings).map_err(|source| { - IndexError::SummarizeDirectory { - path: directory.to_path_buf(), - source: Box::new(source), +#[derive(Debug, Default)] +struct ConcurrentScan { + directories: Vec, + files: Vec, + pruned_paths: Vec, + report: IndexReport, +} + +#[derive(Debug)] +struct ScannedDirectory { + document: IndexedDocument, + classifications: Vec, +} + +#[derive(Debug)] +struct ScanOutput { + directory: ScannedDirectory, + files: Vec, + child_jobs: Vec, + pruned_paths: Vec, + report: IndexReport, +} + +struct ScanQueue { + state: Mutex, + changed: Condvar, +} + +#[derive(Debug)] +struct ScanQueueState { + pending: VecDeque, + active: usize, + stopped: bool, +} + +fn scan_root_concurrently(root: &Path, settings: &Settings) -> Result { + let worker_count = scan_worker_count(); + let queue = Arc::new(ScanQueue { + state: Mutex::new(ScanQueueState { + pending: VecDeque::from([ScanJob { + directory: root.to_path_buf(), + depth_position: DepthPosition::IndexRoot, + }]), + active: 0, + stopped: false, + }), + changed: Condvar::new(), + }); + let settings = Arc::new(settings.clone()); + let scan = Arc::new(Mutex::new(ConcurrentScan::default())); + let error = Arc::new(Mutex::new(None)); + + thread::scope(|scope| { + for _ in 0..worker_count { + let queue = Arc::clone(&queue); + let settings = Arc::clone(&settings); + let scan = Arc::clone(&scan); + let error = Arc::clone(&error); + + scope.spawn(move || worker_loop(queue, settings, scan, error)); + } + }); + + if let Some(error) = error.lock().expect("scan error lock is poisoned").take() { + return Err(error); + } + + let scan = Arc::try_unwrap(scan) + .expect("scan workers have stopped") + .into_inner() + .expect("scan result lock is poisoned"); + Ok(scan) +} + +fn worker_loop( + queue: Arc, + settings: Arc, + scan: Arc>, + error: Arc>>, +) { + loop { + let Some(job) = next_scan_job(&queue) else { + return; + }; + + let result = scan_directory_job(&job, &settings); + match result { + Ok(output) => finish_scan_job(&queue, &scan, output), + Err(source) => { + *error.lock().expect("scan error lock is poisoned") = Some(source); + stop_scan_workers(&queue); + return; } - })?; + } + } +} - batch - .database - .upsert_document(&document) - .await - .map_err(|source| IndexError::StoreDocument { - path: document.path.clone(), - source: Box::new(source), - })?; - let classifications = classify::classify_directory(directory, settings)?; - batch - .database - .replace_directory_classifications(&document.path, &classifications) - .await - .map_err(|source| IndexError::StoreDirectoryClassifications { - path: document.path.clone(), - source: Box::new(source), - })?; - report.directories_indexed += 1; +fn next_scan_job(queue: &ScanQueue) -> Option { + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); - let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { - path: directory.to_path_buf(), - source, - })?; + loop { + if state.stopped { + return None; + } + + if let Some(job) = state.pending.pop_front() { + state.active += 1; + return Some(job); + } + + if state.active == 0 { + state.stopped = true; + queue.changed.notify_all(); + return None; + } - for entry in entries { + state = queue + .changed + .wait(state) + .expect("scan queue lock is poisoned"); + } +} + +fn finish_scan_job(queue: &ScanQueue, scan: &Mutex, output: ScanOutput) { + { + let mut scan = scan.lock().expect("scan result lock is poisoned"); + scan.directories.push(output.directory); + scan.files.extend(output.files); + scan.pruned_paths.extend(output.pruned_paths); + scan.report.merge(output.report); + } + + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); + state.pending.extend(output.child_jobs); + state.active = state.active.saturating_sub(1); + queue.changed.notify_all(); +} + +fn stop_scan_workers(queue: &ScanQueue) { + let mut state = queue.state.lock().expect("scan queue lock is poisoned"); + state.stopped = true; + state.pending.clear(); + queue.changed.notify_all(); +} + +fn scan_directory_job(job: &ScanJob, settings: &Settings) -> Result { + let document = summary::summarize_directory(&job.directory, settings).map_err(|source| { + IndexError::SummarizeDirectory { + path: job.directory.clone(), + source: Box::new(source), + } + })?; + let classifications = classify::classify_directory(&job.directory, settings)?; + let mut output = ScanOutput { + directory: ScannedDirectory { + document, + classifications, + }, + files: Vec::new(), + child_jobs: Vec::new(), + pruned_paths: Vec::new(), + report: IndexReport { + directories_indexed: 1, + ..IndexReport::default() + }, + }; + + let mut entries = fs::read_dir(&job.directory) + .map_err(|source| IndexError::ReadDirectory { + path: job.directory.clone(), + source, + })? + .map(|entry| { let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { - path: directory.to_path_buf(), + path: job.directory.clone(), source, })?; let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - + let name = entry.file_name().to_string_lossy().into_owned(); let file_type = entry .file_type() .map_err(|source| IndexError::ReadFileType { path: path.clone(), source, })?; + Ok((path, name, file_type)) + }) + .collect::>>()?; + entries.sort_by(|left, right| left.0.cmp(&right.0)); + + for (path, name, file_type) in entries { + if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; + continue; + } - if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { - batch - .database - .delete_path_tree(&path.to_string_lossy()) - .await - .map_err(|source| IndexError::PruneExcludedPath { - path: path.to_string_lossy().into_owned(), - source: Box::new(source), - })?; - report.entries_skipped += 1; + if file_type.is_dir() { + let Some(child_position) = job.depth_position.child_position() else { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; continue; - } + }; - if file_type.is_dir() { - let Some(child_position) = depth_position.child_position() else { - prune_unindexed_directory(batch.database, report, &path).await?; - continue; - }; - - if exceeds_max_depth(settings, child_position) { - prune_unindexed_directory(batch.database, report, &path).await?; - continue; - } - - scan_directory(&path, settings, batch, report, progress, child_position).await?; - } else if file_type.is_file() { - if settings.index.is_excluded_name(&name) { - report.entries_skipped += 1; - continue; - } - - report.files_seen += 1; - if let Some(indexed_file) = file::prepare_text_file(&path, directory, settings)? { - report.text_files_indexed += 1; - report.file_chunks_indexed += - u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); - batch.push(indexed_file).await?; - } - } else { - report.entries_skipped += 1; + if exceeds_max_depth(settings, child_position) { + output + .pruned_paths + .push(path.to_string_lossy().into_owned()); + output.report.entries_skipped += 1; + continue; } - } - Ok(()) - }) -} + output.child_jobs.push(ScanJob { + directory: path, + depth_position: child_position, + }); + } else if file_type.is_file() { + if settings.index.is_excluded_name(&name) { + output.report.entries_skipped += 1; + continue; + } -fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { - match depth_position { - DepthPosition::IndexRoot => false, - DepthPosition::TopLevelDirectory { depth } => { - depth > settings.index.max_depth_per_top_level_directory + output.report.files_seen += 1; + if let Some(indexed_file) = file::prepare_text_file(&path, &job.directory, settings)? { + output.report.text_files_indexed += 1; + output.report.file_chunks_indexed += + u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); + output.files.push(indexed_file); + } + } else { + output.report.entries_skipped += 1; } } + + Ok(output) } -async fn prune_unindexed_directory( - database: &Database, - report: &mut IndexReport, - path: &Path, -) -> Result<()> { - database - .delete_path_tree(&path.to_string_lossy()) - .await - .map_err(|source| IndexError::PruneExcludedPath { - path: path.to_string_lossy().into_owned(), - source: Box::new(source), - })?; - report.entries_skipped += 1; - Ok(()) +fn scan_worker_count() -> usize { + thread::available_parallelism() + .map(usize::from) + .unwrap_or(4) + .clamp(2, 8) } struct FileIndexBatch<'a, E> { diff --git a/src/index/summary.rs b/src/index/summary.rs index 58421c4..08a4422 100644 --- a/src/index/summary.rs +++ b/src/index/summary.rs @@ -218,6 +218,11 @@ mod tests { fs::create_dir(temp.path().join(".vscode")).unwrap(); fs::write(temp.path().join("README.md"), "Chrome extension workspace").unwrap(); fs::write(temp.path().join(".env"), "SECRET=true").unwrap(); + fs::write( + temp.path().join("logo.svg"), + "Logo", + ) + .unwrap(); let settings = Settings::default(); let document = summarize_directory(temp.path(), &settings).unwrap(); @@ -235,6 +240,7 @@ mod tests { assert!(document.modified_unix_seconds > 0); assert!(document.searchable_text.contains("child directories: src")); assert!(!document.searchable_text.contains(".vscode")); + assert!(!document.searchable_text.contains("logo.svg")); assert!(document.searchable_text.contains("README.md")); assert!(document.searchable_text.contains("type: directory")); assert!(document.searchable_text.contains("size bytes:"));