Optimize initial indexing

README.md

@@ -121,7 +121,16 @@ The initial config looks like this:
       "*.xcassets",
       "*.imageset",
       "*.appiconset",
-      "*.colorset"
+      "*.colorset",
+      "*.svg",
+      "*.png",
+      "*.jpg",
+      "*.jpeg",
+      "*.gif",
+      "*.webp",
+      "*.ico",
+      "*.pdf",
+      "*.zip"
     ],
     "max_file_bytes": 65536,
     "max_excerpt_bytes": 4096,
@@ -145,11 +154,14 @@ Or index explicit roots:
 cds --index ~/Projects ~/work
 ```
 
-The current indexer stores directory metadata and text-file content separately in SQLite.
-Directory rows store structured filesystem metadata: name, type, parent path, size in bytes,
-created time, modified time, accessed time, readonly status, and index time. Text files are
-stored as file metadata plus embedded content chunks linked back to their containing
-directories. Directory metadata itself is not embedded.
+The current indexer stores directory metadata and high-signal text-file content separately
+in SQLite. Directory rows store structured filesystem metadata: name, type, parent path,
+size in bytes, created time, modified time, accessed time, readonly status, and index time.
+High-signal files such as READMEs, package manifests, project config files, and SQL schemas
+are stored as file metadata plus embedded content chunks linked back to their containing
+directories. Low-signal arbitrary text files are skipped by default to keep initial indexing
+fast, and media/asset/archive files such as SVGs, images, PDFs, and ZIPs are ignored.
+Directory metadata itself is not embedded.
 Hidden directories are always skipped and pruned from the local index. Each configured index
 root is treated as a container, and each top-level directory inside it is indexed only through
 `max_depth_per_top_level_directory` levels.

src/config/settings.rs

@@ -108,6 +108,15 @@ impl Default for IndexSettings {
                 "*.imageset".to_string(),
                 "*.appiconset".to_string(),
                 "*.colorset".to_string(),
+                "*.svg".to_string(),
+                "*.png".to_string(),
+                "*.jpg".to_string(),
+                "*.jpeg".to_string(),
+                "*.gif".to_string(),
+                "*.webp".to_string(),
+                "*.ico".to_string(),
+                "*.pdf".to_string(),
+                "*.zip".to_string(),
             ],
             max_file_bytes: 65_536,
             max_excerpt_bytes: 4_096,
@@ -147,6 +156,46 @@ fn is_low_signal_name(name: &str) -> bool {
         || lower.ends_with(".imageset")
         || lower.ends_with(".appiconset")
         || lower.ends_with(".colorset")
+        || is_low_signal_asset_extension(&lower)
+}
+
+fn is_low_signal_asset_extension(name: &str) -> bool {
+    matches!(
+        name.rsplit_once('.').map(|(_, extension)| extension),
+        Some(
+            "svg"
+                | "png"
+                | "jpg"
+                | "jpeg"
+                | "gif"
+                | "webp"
+                | "ico"
+                | "bmp"
+                | "tif"
+                | "tiff"
+                | "pdf"
+                | "zip"
+                | "tar"
+                | "gz"
+                | "tgz"
+                | "bz2"
+                | "xz"
+                | "7z"
+                | "rar"
+                | "dmg"
+                | "mp3"
+                | "mp4"
+                | "mov"
+                | "avi"
+                | "webm"
+                | "wav"
+                | "flac"
+                | "woff"
+                | "woff2"
+                | "ttf"
+                | "otf"
+        )
+    )
 }
 
 fn is_hidden_name(name: &str) -> bool {
@@ -286,6 +335,9 @@ mod tests {
         assert!(index.is_excluded_name("private.key"));
         assert!(index.is_excluded_name("Assets.xcassets"));
         assert!(index.is_excluded_name("Custom Icon.appiconset"));
+        assert!(index.is_excluded_name("logo.svg"));
+        assert!(index.is_excluded_name("screenshot.png"));
+        assert!(index.is_excluded_name("archive.zip"));
         assert!(index.is_excluded_directory_name(".vscode"));
         assert!(!index.is_excluded_name(".vscode"));
         assert!(!index.is_excluded_name("README.md"));

src/db/mod.rs

@@ -180,6 +180,129 @@ impl Database {
         Ok(())
     }
 
+    pub async fn upsert_directories_with_classifications(
+        &self,
+        directories: &[(&IndexedDocument, &[DirectoryClassification])],
+    ) -> Result<()> {
+        let mut transaction =
+            self.pool
+                .begin()
+                .await
+                .map_err(|source| DbError::UpsertDocument {
+                    path: "<directory batch>".to_string(),
+                    source,
+                })?;
+
+        for (document, classifications) in directories {
+            sqlx::query(
+                "
+                INSERT INTO indexed_documents (
+                    path,
+                    name,
+                    kind,
+                    parent_path,
+                    searchable_text,
+                    embedding,
+                    embedding_dim,
+                    metadata_fingerprint,
+                    size_bytes,
+                    created_unix_seconds,
+                    modified_unix_seconds,
+                    accessed_unix_seconds,
+                    readonly,
+                    indexed_unix_seconds
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(path) DO UPDATE SET
+                    name = excluded.name,
+                    kind = excluded.kind,
+                    parent_path = excluded.parent_path,
+                    searchable_text = excluded.searchable_text,
+                    embedding = excluded.embedding,
+                    embedding_dim = excluded.embedding_dim,
+                    metadata_fingerprint = excluded.metadata_fingerprint,
+                    size_bytes = excluded.size_bytes,
+                    created_unix_seconds = excluded.created_unix_seconds,
+                    modified_unix_seconds = excluded.modified_unix_seconds,
+                    accessed_unix_seconds = excluded.accessed_unix_seconds,
+                    readonly = excluded.readonly,
+                    indexed_unix_seconds = excluded.indexed_unix_seconds
+                ",
+            )
+            .bind(&document.path)
+            .bind(&document.name)
+            .bind(document.kind.as_str())
+            .bind(&document.parent_path)
+            .bind(&document.searchable_text)
+            .bind(encode_embedding(&document.embedding))
+            .bind(
+                i64::try_from(document.embedding.len())
+                    .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?,
+            )
+            .bind(&document.metadata_fingerprint)
+            .bind(
+                i64::try_from(document.size_bytes)
+                    .map_err(|source| DbError::MetadataSizeOverflow { source })?,
+            )
+            .bind(document.created_unix_seconds)
+            .bind(document.modified_unix_seconds)
+            .bind(document.accessed_unix_seconds)
+            .bind(document.readonly)
+            .bind(document.indexed_unix_seconds)
+            .execute(&mut *transaction)
+            .await
+            .map_err(|source| DbError::UpsertDocument {
+                path: document.path.clone(),
+                source,
+            })?;
+
+            sqlx::query("DELETE FROM directory_classifications WHERE directory_path = ?")
+                .bind(&document.path)
+                .execute(&mut *transaction)
+                .await
+                .map_err(|source| DbError::ReplaceDirectoryClassifications {
+                    path: document.path.clone(),
+                    source,
+                })?;
+
+            for classification in *classifications {
+                sqlx::query(
+                    "
+                    INSERT INTO directory_classifications (
+                        directory_path,
+                        label,
+                        confidence,
+                        detector,
+                        evidence_path,
+                        evidence_summary,
+                        detected_unix_seconds
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                    ",
+                )
+                .bind(&classification.directory_path)
+                .bind(&classification.label)
+                .bind(f64::from(classification.confidence))
+                .bind(&classification.detector)
+                .bind(&classification.evidence_path)
+                .bind(&classification.evidence_summary)
+                .bind(classification.detected_unix_seconds)
+                .execute(&mut *transaction)
+                .await
+                .map_err(|source| DbError::InsertDirectoryClassification {
+                    path: classification.directory_path.clone(),
+                    label: classification.label.clone(),
+                    source,
+                })?;
+            }
+        }
+
+        transaction
+            .commit()
+            .await
+            .map_err(|source| DbError::CommitFileBatch { source })?;
+
+        Ok(())
+    }
+
     pub async fn replace_file_chunks(
         &self,
         file_path: &str,

src/index/file.rs

@@ -76,6 +76,15 @@ pub fn prepare_text_file(
         return Ok(None);
     }
 
+    let name = path
+        .file_name()
+        .unwrap_or_else(|| OsStr::new(""))
+        .to_string_lossy()
+        .into_owned();
+    if !is_high_signal_content_file(&name) {
+        return Ok(None);
+    }
+
     let bytes = fs::read(path).map_err(|source| IndexError::ReadFile {
         path: path.to_path_buf(),
         source,
@@ -94,11 +103,6 @@ pub fn prepare_text_file(
     let modified_unix_seconds = unix_seconds(metadata.modified().unwrap_or(UNIX_EPOCH));
     let file_path = path_to_string(path);
     let directory_path = path_to_string(directory);
-    let name = path
-        .file_name()
-        .unwrap_or_else(|| OsStr::new(""))
-        .to_string_lossy()
-        .into_owned();
 
     let file = IndexedFile {
         path: file_path.clone(),
@@ -180,6 +184,38 @@ fn normalize_whitespace(value: &str) -> String {
     value.split_whitespace().collect::<Vec<_>>().join(" ")
 }
 
+fn is_high_signal_content_file(name: &str) -> bool {
+    let lower = name.to_ascii_lowercase();
+    if lower.starts_with("readme") || lower == "gemfile" || lower == "dockerfile" {
+        return true;
+    }
+
+    matches!(
+        lower.as_str(),
+        "cargo.toml"
+            | "package.json"
+            | "manifest.json"
+            | "pyproject.toml"
+            | "requirements.txt"
+            | "setup.py"
+            | "go.mod"
+            | "makefile"
+            | "docker-compose.yml"
+            | "compose.yml"
+            | "compose.yaml"
+            | "tsconfig.json"
+            | "vite.config.js"
+            | "vite.config.ts"
+            | "next.config.js"
+            | "next.config.ts"
+            | "tailwind.config.js"
+            | "tailwind.config.ts"
+    ) || matches!(
+        lower.rsplit_once('.').map(|(_, extension)| extension),
+        Some("md" | "markdown" | "toml" | "yaml" | "yml" | "sql")
+    )
+}
+
 fn unix_seconds(time: SystemTime) -> i64 {
     time.duration_since(UNIX_EPOCH)
         .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX))
@@ -210,4 +246,40 @@ mod tests {
         assert_eq!(chunks[1].text, "ef gh");
         assert_eq!(chunks[2].text, "i");
     }
+
+    #[test]
+    fn skips_low_signal_text_files() {
+        let temp = tempfile::tempdir().unwrap();
+        let path = temp.path().join("notes.txt");
+        fs::write(&path, "large low-signal notes").unwrap();
+
+        let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap();
+
+        assert_eq!(prepared, None);
+    }
+
+    #[test]
+    fn skips_svg_assets_even_though_they_are_text() {
+        let temp = tempfile::tempdir().unwrap();
+        let path = temp.path().join("logo.svg");
+        fs::write(&path, "<svg><title>Logo</title></svg>").unwrap();
+
+        let prepared = prepare_text_file(&path, temp.path(), &Settings::default()).unwrap();
+
+        assert_eq!(prepared, None);
+    }
+
+    #[test]
+    fn prepares_high_signal_text_files() {
+        let temp = tempfile::tempdir().unwrap();
+        let path = temp.path().join("README.md");
+        fs::write(&path, "high signal project summary").unwrap();
+
+        let prepared = prepare_text_file(&path, temp.path(), &Settings::default())
+            .unwrap()
+            .expect("README.md is indexed");
+
+        assert_eq!(prepared.file.name, "README.md");
+        assert_eq!(prepared.chunks.len(), 1);
+    }
 }

src/index/mod.rs

@@ -241,9 +241,9 @@ mod tests {
         let root = temp.path().join("Projects");
         let app = root.join("app");
         fs::create_dir_all(&app).unwrap();
-        fs::write(app.join("one.txt"), "alpha").unwrap();
-        fs::write(app.join("two.txt"), "bravo").unwrap();
-        fs::write(app.join("three.txt"), "charlie").unwrap();
+        fs::write(app.join("README.md"), "alpha").unwrap();
+        fs::write(app.join("Cargo.toml"), "bravo").unwrap();
+        fs::write(app.join("schema.sql"), "charlie").unwrap();
 
         let settings = Settings::default();
         let database = Database::open_in_memory().await.unwrap();