diff --git a/AGENTS.md b/AGENTS.md index d268447..a143a78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,7 @@ user's current shell. Prefer stable, well-maintained crates: - CLI parsing: `clap` -- SQLite: `sqlx` with SQLite +- SQLite: `rusqlite` - Serialization: `serde`, `serde_json` - Error handling: `color_eyre` for binaries, `thiserror` for reusable library errors - Filesystem walking: `ignore` or `walkdir`; prefer `ignore` because it respects common @@ -92,21 +92,26 @@ Embedding storage options may include: If using a SQLite extension, preserve a fallback or clear setup error so the CLI does not fail opaquely. -## Command Shape +## Current Command Shape -Likely commands: +All explicit `cds` commands should be long flags so they cannot be confused with normal +`cd` operands. Do not add bare subcommands such as `cds init`; those should remain valid +directory names when routed through the shell integration. + +Current user-facing commands: ```sh -cds init -cds index [PATH] -cds search QUERY... -cds go QUERY... -cds explain QUERY... -cds doctor +cds --init +cds --index [PATH...] +cds --search QUERY... +cds --dir-type-count +cds --reset +cds --shell-init [bash|zsh] ``` -Shell integration should make the common path concise. For example, the installed shell -function may call `cds go "$@"` and `cd` to the emitted path. +The shell integration makes the common path concise. Plain invocations such as +`cds Projects` should preserve `cd` behavior unless they are clearly semantic searches. +The hidden shell-machine mode currently uses `--cds-emit`. The Rust binary should not print human-oriented decoration in machine-readable shell integration mode. Keep stdout parseable and put diagnostics on stderr. @@ -166,10 +171,24 @@ Add tests for: - indexing change detection - ranking behavior with deterministic fake embeddings - shell integration output +- Docker-backed `cd` equivalence for Bash behavior Do not make tests depend on downloading or running the real embedding model unless they are explicitly marked as slow/integration tests. Use a fake embedder for normal tests. +The Docker equivalence test runs the project inside a Rust container and compares `cds` +against Bash's built-in `cd` for status, stdout, stderr, `PWD`, `OLDPWD`, and physical path. +Keep these details in mind when editing it: + +- CI currently uses `CDS_DOCKER_IMAGE=rust:1-slim`. +- The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some slim image + invocations otherwise fail with `cargo: command not found`. +- Bash includes source line numbers in `cd` diagnostics. Normalize only those line numbers + before comparing stderr, while keeping the actual diagnostic text exact. +- The Docker test can skip locally when Docker is unavailable, so use + `CDS_DOCKER_IMAGE=rust:1-slim cargo test --test docker_cd_equivalence -- --nocapture` + with Docker access when changing equivalence behavior. + ## Repository Hygiene - Keep generated databases, model files, caches, and local indexes out of git. diff --git a/Cargo.lock b/Cargo.lock index 7ca8c1d..ec2fdf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "anstream" version = "1.0.0" @@ -52,13 +67,62 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "bitflags" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cds" version = "0.1.0" dependencies = [ "clap", + "color-eyre", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror", ] +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + [[package]] name = "clap" version = "4.6.1" @@ -86,36 +150,594 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "color-eyre" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91632f3b4fb6bd1d72aa3d78f41ffecfcf2b1a6648d8c241dbe7dbfaf4875e15" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rusqlite" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de23c3319433716cf134eed225fe9986bc24f63bed9be9f20c329029e672dc7" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -130,3 +752,103 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index caf4334..83f2960 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,11 @@ edition = "2024" [dependencies] clap = { version = "4.5.40", features = ["cargo"] } +color-eyre = "0.6.5" +rusqlite = { version = "0.36.0", features = ["bundled"] } +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.140" +thiserror = "2.0.17" + +[dev-dependencies] +tempfile = "3.20.0" diff --git a/README.md b/README.md index 18c3c24..7d430e7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,20 @@ cds solves this problem. `cds the chrome extension I worked on last month` will take you right there. -This all runs locally via an sqlite backed vector database and a small, local embeddings model. +## How does it work? + +When you first download cds, you define directories that you'd like to index. `~/Projects` for example. + +When you then run `cds --init`, cds will scan all of the folders/files in those directories (excluding hidden directories +and sensitive files) and generate embeddings for them that are stored in a local SQLite database. + +When you later run `cds the chrome extension I worked on last month` cds will search that embeddings data to find +the directory you're looking for and invoke cd to navigate you there. + +#### Why not my entire system? +Embeddings get expensive fast and I have to use a very tiny model (bge-small-en-v1.5) to keep this lean. +If we have a ton of parameteres in the database A: The db would be *huge* and B: The search would take a lot of time. + ## Build @@ -19,6 +32,30 @@ This all runs locally via an sqlite backed vector database and a small, local em cargo build ``` +## Install + +From the repository root: + +```sh +./install.sh +``` + +The installer runs `cargo install --path .` and adds an idempotent shell integration +block to your `~/.zshrc` or `~/.bashrc`. After it finishes: + +```sh +source ~/.zshrc +cds --init +``` + +Use `source ~/.bashrc` instead if you use bash. + +To replace an existing installed binary: + +```sh +./install.sh --force +``` + ## Test ```sh @@ -36,18 +73,154 @@ Use a specific container image with: CDS_DOCKER_IMAGE=rust:1-slim cargo test --test docker_cd_equivalence ``` +## Indexing + +`cds` only indexes directories you configure. It does not crawl the whole filesystem. + +Create the default config and SQLite database, then index the configured roots: + +```sh +cds --init +``` + +By default this creates: + +```text +~/.config/cds/config.json +~/.local/share/cds/cds.sqlite +``` + +The initial config looks like this: + +```json +{ + "index": { + "roots": ["~/Projects"], + "exclude": [ + ".git", + "node_modules", + "target", + "dist", + "build", + ".next", + ".cache", + ".venv", + "venv", + "vendor", + "*.xcassets", + "*.imageset", + "*.appiconset", + "*.colorset" + ], + "max_file_bytes": 65536, + "max_excerpt_bytes": 4096, + "max_entries_per_directory": 80, + "max_depth_per_top_level_directory": 3, + "max_chunk_bytes": 4096 + }, + "detectors": [] +} +``` + +Run an indexing pass over configured roots: + +```sh +cds --index +``` + +Or index explicit roots: + +```sh +cds --index ~/Projects ~/work +``` + +The current indexer stores directory metadata and text-file content separately in SQLite. +Directory rows store structured filesystem metadata: name, type, parent path, size in bytes, +created time, modified time, accessed time, readonly status, and index time. Text files are +stored as file metadata plus embedded content chunks linked back to their containing +directories. Directory metadata itself is not embedded. +Hidden directories are always skipped and pruned from the local index. Each configured index +root is treated as a container, and each top-level directory inside it is indexed only through +`max_depth_per_top_level_directory` levels. + +The embedder is still deterministic and fake for now. That is deliberate: it lets the config, +schema, scanning, and ranking plumbing settle before the real `bge-small-en-v1.5` runtime is +wired in. + +## Directory Types + +Directory type inference is rule-based and data-driven. Built-in types such as `rust project` +and `chrome extension` are defined as JSON. Custom/community detectors can be added to the +`detectors` array in `~/.config/cds/config.json`; `cds --index` loads them every time it +indexes. + +Example detector: + +```json +{ + "label": "rust project", + "rules": [ + { + "id": "cargo_toml_package", + "confidence": 0.98, + "evidence_summary": "Cargo.toml contains [package] or [workspace]", + "signals": [ + { + "kind": "file_contains", + "path": "Cargo.toml", + "contains_any": ["[package]", "[workspace]"] + } + ] + } + ] +} +``` + +Supported signal kinds are `file_exists`, `file_contains`, `directory_name`, and `child_name`. +Each matched rule stores the label, confidence, detector id, evidence path, and evidence +summary in SQLite. + +Search indexed directories: + +```sh +cds --search chrome extension +``` + +List detected directory types: + +```sh +cds --dir-type-count +``` + +Delete all indexed data from the SQLite database: + +```sh +cds --reset +``` + +This prompts before deleting directory metadata, file metadata, content chunks, and directory +type classifications. The database file and schema are kept in place. + +When the shell integration is installed, `cds` also tries semantic search automatically for +plain directory changes that do not look like local `cd` usage. For example, `cds Projects` +first checks the current directory for any folder starting with `p`. If such a folder exists, +`cds` delegates to the shell's built-in `cd` exactly as usual. If no local folder starts with +that character, `cds` searches the index and emits a `cd` to the best existing indexed +directory. Flags, `-`, `--`, `~`, `.`, `..`, and paths containing `/` always use normal `cd` +behavior. + ## Shell Setup Install the shell integration in your current shell: ```sh -eval "$(target/debug/cds --shell-init zsh)" +eval "$(command target/debug/cds --shell-init zsh)" ``` For bash: ```sh -eval "$(target/debug/cds --shell-init bash)" +eval "$(command target/debug/cds --shell-init bash)" ``` After that, use `cds` like `cd`: @@ -59,5 +232,9 @@ cds -P ../somewhere cds -- -directory-starting-with-dash ``` -To make this permanent, add the appropriate `eval` line to your shell profile after -`cds` is installed somewhere on `PATH`. +To make this permanent after `cds` is installed somewhere on `PATH`, add the appropriate +line to your shell profile: + +```sh +eval "$(command cds --shell-init zsh)" +``` diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..a407a9b --- /dev/null +++ b/install.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +shell_name="$(basename "${SHELL:-}")" +force_install=0 + +usage() { + cat <&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +if ! command -v cargo >/dev/null 2>&1; then + echo "error: cargo is required to install cds" >&2 + echo "Install Rust from https://rustup.rs, then rerun ./install.sh" >&2 + exit 1 +fi + +case "$shell_name" in + zsh) + profile="${ZDOTDIR:-$HOME}/.zshrc" + init_shell="zsh" + ;; + bash) + profile="$HOME/.bashrc" + init_shell="bash" + ;; + *) + profile="${ZDOTDIR:-$HOME}/.zshrc" + init_shell="zsh" + echo "warning: unsupported shell '${shell_name:-unknown}', defaulting setup to zsh" >&2 + ;; +esac + +echo "Installing cds with cargo..." +cargo_args=(install --path "$repo_dir") +if [ "$force_install" -eq 1 ]; then + cargo_args+=(--force) +fi +cargo "${cargo_args[@]}" + +cargo_bin="${CARGO_HOME:-$HOME/.cargo}/bin" +mkdir -p "$(dirname "$profile")" +touch "$profile" + +if ! grep -Fq '# >>> cds init >>>' "$profile"; then + cat >> "$profile" <>> cds init >>> +if [ -x "$cargo_bin/cds" ]; then + eval "\$(command "$cargo_bin/cds" --shell-init $init_shell)" +fi +# <<< cds init <<< +EOF + echo "Added cds shell integration to $profile" +else + echo "cds shell integration already exists in $profile" +fi + +if ! command -v cds >/dev/null 2>&1; then + echo "warning: cds is installed at $cargo_bin/cds, but $cargo_bin is not on PATH" >&2 + echo "Add this to your shell profile before the cds integration block:" >&2 + echo "export PATH=\"$cargo_bin:\$PATH\"" >&2 +fi + +cat < Result { + let mut progress = NoopProgress; + init_with_progress(&mut progress) +} + +pub fn init_with_progress

(progress: &mut P) -> Result +where + P: IndexProgress + ?Sized, +{ + let paths = AppPaths::discover()?; + let settings = Settings::load_or_create(&paths.config_file)?; + let database = Database::open(&paths.database_file)?; + let embedder = FakeEmbedder::default(); + let indexer = Indexer::new(&settings, &database, &embedder); + let index = indexer.index_configured_roots_with_progress(progress)?; + + Ok(InitReport { + config_file: paths.config_file, + database_file: paths.database_file, + index, + }) +} + +pub fn index(roots: Vec) -> Result { + let mut progress = NoopProgress; + index_with_progress(roots, &mut progress) +} + +pub fn index_with_progress

(roots: Vec, progress: &mut P) -> Result +where + P: IndexProgress + ?Sized, +{ + let paths = AppPaths::discover()?; + let settings = Settings::load_or_create(&paths.config_file)?; + let database = Database::open(&paths.database_file)?; + let embedder = FakeEmbedder::default(); + let indexer = Indexer::new(&settings, &database, &embedder); + + if roots.is_empty() { + return Ok(indexer.index_configured_roots_with_progress(progress)?); + } + + let roots = roots + .iter() + .map(|root| { + let root = root + .to_str() + .ok_or_else(|| app_err(AppError::InvalidIndexRootUtf8 { root: root.clone() }))?; + expand_tilde(root).map_err(config_err) + }) + .collect::>>()?; + + Ok(indexer.index_roots_with_progress(roots, progress)?) +} + +pub fn search(query: Vec, limit: usize) -> Result> { + let query = join_query(query)?; + search_text(&query, limit) +} + +pub fn search_text(query: &str, limit: usize) -> Result> { + let paths = AppPaths::discover()?; + let database = Database::open_existing(&paths.database_file)?; + let embedder = FakeEmbedder::default(); + let searcher = Searcher::new(&database, &embedder); + Ok(searcher.search(query, limit)?) +} + +pub fn directory_type_counts() -> Result> { + let paths = AppPaths::discover()?; + let database = Database::open_existing(&paths.database_file)?; + Ok(database.directory_type_counts()?) +} + +pub fn reset_database() -> Result<()> { + let paths = AppPaths::discover()?; + let database = Database::open_existing(&paths.database_file)?; + Ok(database.reset()?) +} + +pub fn resolve_cd_script(args: Vec) -> Vec { + if let Some(query) = semantic_query(&args) + && let Ok(results) = search_text(&query, 5) + && let Some(result) = results + .into_iter() + .find(|result| Path::new(&result.path).is_dir()) + { + return crate::emit_cd_script(&[OsString::from(result.path)]); + } + + crate::emit_cd_script(&args) +} + +fn join_query(query: Vec) -> Result { + let mut parts = Vec::with_capacity(query.len()); + + for part in query { + let part = part.to_str().ok_or_else(|| { + app_err(AppError::InvalidSearchQueryUtf8 { + query: part.clone(), + }) + })?; + parts.push(part.to_string()); + } + + Ok(parts.join(" ")) +} + +fn semantic_query(args: &[OsString]) -> Option { + semantic_query_in(args, current_shell_directory().as_deref()) +} + +fn semantic_query_in(args: &[OsString], directory: Option<&Path>) -> Option { + if args.is_empty() { + return None; + } + + let mut parts = Vec::with_capacity(args.len()); + for arg in args { + let arg = arg.to_str()?; + if is_cd_syntax(arg) { + return None; + } + parts.push(arg); + } + + let query = parts.join(" "); + let first = query.chars().find(|ch| !ch.is_whitespace())?; + + if local_directory_starts_with(first, directory) { + return None; + } + + Some(query) +} + +fn is_cd_syntax(arg: &str) -> bool { + arg.is_empty() + || arg == "-" + || arg == "--" + || arg == "." + || arg == ".." + || arg == "~" + || arg.starts_with('-') + || arg.starts_with("~/") + || arg.contains('/') +} + +fn local_directory_starts_with(first: char, directory: Option<&Path>) -> bool { + let Some(directory) = directory else { + return true; + }; + + let Ok(entries) = fs::read_dir(directory) else { + return true; + }; + + let first = first.to_lowercase().to_string(); + + entries.filter_map(std::result::Result::ok).any(|entry| { + if !entry.path().is_dir() { + return false; + } + + entry + .file_name() + .to_string_lossy() + .to_lowercase() + .starts_with(&first) + }) +} + +fn current_shell_directory() -> Option { + env::var_os("PWD") + .map(PathBuf::from) + .filter(|path| path.is_dir()) + .or_else(|| env::current_dir().ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn os(value: &str) -> OsString { + OsString::from(value) + } + + #[test] + fn semantic_query_is_allowed_when_no_local_directory_prefix_matches() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("src")).unwrap(); + + assert_eq!( + semantic_query_in(&[os("Projects")], Some(temp.path())), + Some("Projects".to_string()) + ); + } + + #[test] + fn semantic_query_is_blocked_when_local_directory_prefix_matches() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("playground")).unwrap(); + + assert_eq!( + semantic_query_in(&[os("Projects")], Some(temp.path())), + None + ); + } + + #[test] + fn semantic_query_is_blocked_for_cd_syntax() { + let temp = tempfile::tempdir().unwrap(); + + assert_eq!( + semantic_query_in(&[os("-P"), os("Projects")], Some(temp.path())), + None + ); + assert_eq!( + semantic_query_in(&[os("--"), os("-dir")], Some(temp.path())), + None + ); + assert_eq!( + semantic_query_in(&[os("../Projects")], Some(temp.path())), + None + ); + assert_eq!(semantic_query_in(&[os("~")], Some(temp.path())), None); + } + + #[test] + fn semantic_query_joins_plain_words() { + let temp = tempfile::tempdir().unwrap(); + + assert_eq!( + semantic_query_in(&[os("chrome"), os("extension")], Some(temp.path())), + Some("chrome extension".to_string()) + ); + } +} diff --git a/src/cli.rs b/src/cli.rs index 560cf37..b922327 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,7 +6,12 @@ use clap::{Arg, ArgAction, Command, error::ErrorKind}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Invocation { + DirectoryTypeCount, EmitCd { args: Vec }, + Index { roots: Vec }, + Init, + Reset, + Search { query: Vec }, ShellInit { shell: Shell }, } @@ -46,14 +51,43 @@ pub fn parse_invocation( let matches = command().try_get_matches_from(argv)?; - if let Some(matches) = matches.subcommand_matches("__cds_emit") { - let args = matches - .get_many::("args") + if matches.contains_id("emit-cd") { + let mut args = matches + .get_many::("emit-cd") .map(|args| args.cloned().collect()) .unwrap_or_default(); + strip_internal_separator(&mut args); return Ok(Invocation::EmitCd { args }); } + if matches.get_flag("dir-type-count") { + return Ok(Invocation::DirectoryTypeCount); + } + + if matches.get_flag("init") { + return Ok(Invocation::Init); + } + + if matches.get_flag("reset") { + return Ok(Invocation::Reset); + } + + if matches.contains_id("index") { + let roots = matches + .get_many::("index") + .map(|roots| roots.cloned().collect()) + .unwrap_or_default(); + return Ok(Invocation::Index { roots }); + } + + if matches.contains_id("search") { + let query = matches + .get_many::("search") + .map(|query| query.cloned().collect()) + .unwrap_or_default(); + return Ok(Invocation::Search { query }); + } + if matches.contains_id("shell-init") { let shell = matches .get_one::("shell-init") @@ -71,6 +105,41 @@ pub fn command() -> Command { .about("cd with semantic search") .version(clap::crate_version!()) .disable_help_subcommand(true) + .arg( + Arg::new("init") + .long("init") + .help("Create the default config and database, then index configured roots") + .conflicts_with_all([ + "dir-type-count", + "emit-cd", + "index", + "reset", + "search", + "shell-init", + ]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("dir-type-count") + .long("dir-type-count") + .help("Print detected directory type counts") + .conflicts_with_all(["emit-cd", "index", "init", "reset", "search", "shell-init"]) + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("reset") + .long("reset") + .help("Delete all indexed data from the cds database") + .conflicts_with_all([ + "dir-type-count", + "emit-cd", + "index", + "init", + "search", + "shell-init", + ]) + .action(ArgAction::SetTrue), + ) .arg( Arg::new("shell-init") .long("shell-init") @@ -83,21 +152,79 @@ pub fn command() -> Command { .action(ArgAction::Set) .num_args(0..=1) .default_missing_value("") + .conflicts_with_all([ + "dir-type-count", + "emit-cd", + "index", + "init", + "reset", + "search", + ]) .value_parser(OsStringValueParser::new()), ) - .subcommand( - Command::new("__cds_emit").hide(true).arg( - Arg::new("args") - .num_args(0..) - .trailing_var_arg(true) - .allow_hyphen_values(true) - .value_parser(OsStringValueParser::new()), - ), + .arg( + Arg::new("index") + .long("index") + .value_name("ROOT") + .help("Index configured roots or explicit paths") + .long_help( + "Index configured roots, or pass one or more ROOT values to index explicit \ + paths instead of the configured roots.", + ) + .num_args(0..) + .conflicts_with_all([ + "dir-type-count", + "emit-cd", + "init", + "reset", + "search", + "shell-init", + ]) + .value_parser(OsStringValueParser::new()), + ) + .arg( + Arg::new("search") + .long("search") + .value_name("QUERY") + .help("Search indexed directories") + .num_args(1..) + .conflicts_with_all([ + "dir-type-count", + "emit-cd", + "index", + "init", + "reset", + "shell-init", + ]) + .value_parser(OsStringValueParser::new()), + ) + .arg( + Arg::new("emit-cd") + .long("cds-emit") + .hide(true) + .num_args(0..) + .trailing_var_arg(true) + .allow_hyphen_values(true) + .conflicts_with_all([ + "dir-type-count", + "index", + "init", + "reset", + "search", + "shell-init", + ]) + .value_parser(OsStringValueParser::new()), ) } fn direct_invocation_error() -> &'static str { - "install the shell integration first: eval \"$(cds --shell-init zsh)\"" + "install the shell integration first: eval \"$(command cds --shell-init zsh)\"" +} + +fn strip_internal_separator(args: &mut Vec) { + if args.first().is_some_and(|arg| arg == "--") { + args.remove(0); + } } #[cfg(test)] @@ -110,7 +237,7 @@ mod tests { #[test] fn parses_internal_emit_separator() { - let parsed = parse_invocation([os("__cds_emit"), os("--"), os("-")]).unwrap(); + let parsed = parse_invocation([os("--cds-emit"), os("--"), os("-")]).unwrap(); assert_eq!( parsed, Invocation::EmitCd { @@ -122,7 +249,7 @@ mod tests { #[test] fn preserves_emit_args_that_look_like_options() { let parsed = - parse_invocation([os("__cds_emit"), os("--"), os("-P"), os("--"), os("-dir")]).unwrap(); + parse_invocation([os("--cds-emit"), os("--"), os("-P"), os("--"), os("-dir")]).unwrap(); assert_eq!( parsed, Invocation::EmitCd { @@ -137,6 +264,85 @@ mod tests { assert_eq!(parsed, Invocation::ShellInit { shell: Shell::Bash }); } + #[test] + fn parses_init() { + assert_eq!(parse_invocation([os("--init")]).unwrap(), Invocation::Init); + } + + #[test] + fn parses_directory_type_count() { + assert_eq!( + parse_invocation([os("--dir-type-count")]).unwrap(), + Invocation::DirectoryTypeCount + ); + } + + #[test] + fn parses_reset() { + assert_eq!( + parse_invocation([os("--reset")]).unwrap(), + Invocation::Reset + ); + } + + #[test] + fn init_word_is_cd_input_inside_hidden_emit() { + assert_eq!( + parse_invocation([os("--cds-emit"), os("--"), os("init")]).unwrap(), + Invocation::EmitCd { + args: vec![os("init")] + } + ); + } + + #[test] + fn index_word_is_cd_input_inside_hidden_emit() { + assert_eq!( + parse_invocation([os("--cds-emit"), os("--"), os("index")]).unwrap(), + Invocation::EmitCd { + args: vec![os("index")] + } + ); + } + + #[test] + fn parses_index_roots() { + assert_eq!( + parse_invocation([os("--index"), os("~/Projects"), os("/tmp/work")]).unwrap(), + Invocation::Index { + roots: vec![os("~/Projects"), os("/tmp/work")] + } + ); + } + + #[test] + fn parses_index_without_roots() { + assert_eq!( + parse_invocation([os("--index")]).unwrap(), + Invocation::Index { roots: Vec::new() } + ); + } + + #[test] + fn parses_search_query() { + assert_eq!( + parse_invocation([os("--search"), os("chrome"), os("extension")]).unwrap(), + Invocation::Search { + query: vec![os("chrome"), os("extension")] + } + ); + } + + #[test] + fn search_word_is_cd_input_inside_hidden_emit() { + assert_eq!( + parse_invocation([os("--cds-emit"), os("--"), os("search")]).unwrap(), + Invocation::EmitCd { + args: vec![os("search")] + } + ); + } + #[test] fn infers_shell_when_init_shell_is_omitted() { let parsed = parse_invocation([os("--shell-init")]).unwrap(); diff --git a/src/config/error.rs b/src/config/error.rs new file mode 100644 index 0000000..b1bf3dc --- /dev/null +++ b/src/config/error.rs @@ -0,0 +1,50 @@ +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConfigError { + #[error("could not determine {kind} directory; set {env_var}")] + MissingDirectory { + kind: &'static str, + env_var: &'static str, + }, + + #[error("could not expand {value} because HOME is not set")] + HomeNotSet { value: &'static str }, + + #[error("failed to read config file {path}")] + ReadConfig { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to parse JSON config {path}")] + ParseConfig { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + + #[error("failed to create config directory {path}")] + CreateConfigDir { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to serialize default config")] + SerializeDefault { + #[source] + source: serde_json::Error, + }, + + #[error("failed to write {path}")] + WriteConfig { + path: PathBuf, + #[source] + source: io::Error, + }, +} diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..d48653f --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,9 @@ +mod error; +mod paths; +mod settings; + +pub use error::ConfigError; +pub use paths::{AppPaths, expand_tilde}; +pub use settings::{ + DirectoryTypeDefinition, DirectoryTypeRule, DirectoryTypeSignal, IndexSettings, Settings, +}; diff --git a/src/config/paths.rs b/src/config/paths.rs new file mode 100644 index 0000000..011992a --- /dev/null +++ b/src/config/paths.rs @@ -0,0 +1,87 @@ +use std::env; +use std::path::PathBuf; + +use super::ConfigError; + +pub type Result = std::result::Result; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppPaths { + pub config_dir: PathBuf, + pub config_file: PathBuf, + pub data_dir: PathBuf, + pub database_file: PathBuf, + pub cache_dir: PathBuf, +} + +impl AppPaths { + pub fn discover() -> Result { + let config_dir = env_path("CDS_CONFIG_DIR") + .or_else(|| home_dir().map(|home| home.join(".config/cds"))) + .ok_or(ConfigError::MissingDirectory { + kind: "config", + env_var: "CDS_CONFIG_DIR", + })?; + + let data_dir = env_path("CDS_DATA_DIR") + .or_else(|| home_dir().map(|home| home.join(".local/share/cds"))) + .ok_or(ConfigError::MissingDirectory { + kind: "data", + env_var: "CDS_DATA_DIR", + })?; + + let cache_dir = env_path("CDS_CACHE_DIR") + .or_else(|| home_dir().map(|home| home.join(".cache/cds"))) + .ok_or(ConfigError::MissingDirectory { + kind: "cache", + env_var: "CDS_CACHE_DIR", + })?; + + Ok(Self { + config_file: config_dir.join("config.json"), + database_file: data_dir.join("cds.sqlite"), + config_dir, + data_dir, + cache_dir, + }) + } +} + +pub fn expand_tilde(value: &str) -> Result { + if value == "~" { + return home_dir().ok_or(ConfigError::HomeNotSet { value: "~" }); + } + + if let Some(rest) = value.strip_prefix("~/") { + return home_dir() + .map(|home| home.join(rest)) + .ok_or(ConfigError::HomeNotSet { value: "~/" }); + } + + Ok(PathBuf::from(value)) +} + +fn env_path(name: &str) -> Option { + env::var_os(name) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn home_dir() -> Option { + env::var_os("HOME") + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leaves_normal_paths_alone() { + assert_eq!( + expand_tilde("/tmp/projects").unwrap(), + PathBuf::from("/tmp/projects") + ); + } +} diff --git a/src/config/settings.rs b/src/config/settings.rs new file mode 100644 index 0000000..bf4c35a --- /dev/null +++ b/src/config/settings.rs @@ -0,0 +1,293 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use super::{ConfigError, paths::expand_tilde}; + +pub type Result = std::result::Result; + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Settings { + #[serde(default)] + pub index: IndexSettings, + #[serde(default)] + pub detectors: Vec, +} + +impl Settings { + pub fn load(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|source| ConfigError::ReadConfig { + path: path.to_path_buf(), + source, + })?; + serde_json::from_str(&text).map_err(|source| ConfigError::ParseConfig { + path: path.to_path_buf(), + source, + }) + } + + pub fn load_or_create(path: &Path) -> Result { + if path.exists() { + return Self::load(path); + } + + let settings = Self::default(); + settings.write(path)?; + Ok(settings) + } + + pub fn write(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|source| ConfigError::CreateConfigDir { + path: parent.to_path_buf(), + source, + })?; + } + + let text = serde_json::to_string_pretty(self) + .map_err(|source| ConfigError::SerializeDefault { source })?; + fs::write(path, text).map_err(|source| ConfigError::WriteConfig { + path: path.to_path_buf(), + source, + }) + } + + pub fn expanded_roots(&self) -> Result> { + self.index + .roots + .iter() + .map(|root| expand_tilde(root)) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IndexSettings { + pub roots: Vec, + pub exclude: Vec, + pub max_file_bytes: u64, + pub max_excerpt_bytes: usize, + pub max_entries_per_directory: usize, + #[serde(default = "default_max_depth_per_top_level_directory")] + pub max_depth_per_top_level_directory: usize, + #[serde(default = "default_max_chunk_bytes")] + pub max_chunk_bytes: usize, +} + +impl IndexSettings { + pub fn is_excluded_name(&self, name: &str) -> bool { + self.exclude + .iter() + .any(|excluded| matches_exclude_pattern(excluded, name)) + || is_low_signal_name(name) + || is_secret_name(name) + } + + pub fn is_excluded_directory_name(&self, name: &str) -> bool { + is_hidden_name(name) || self.is_excluded_name(name) + } +} + +impl Default for IndexSettings { + fn default() -> Self { + Self { + roots: vec!["~/Projects".to_string()], + exclude: vec![ + ".git".to_string(), + "node_modules".to_string(), + "target".to_string(), + "dist".to_string(), + "build".to_string(), + ".next".to_string(), + ".cache".to_string(), + ".venv".to_string(), + "venv".to_string(), + "vendor".to_string(), + "*.xcassets".to_string(), + "*.imageset".to_string(), + "*.appiconset".to_string(), + "*.colorset".to_string(), + ], + max_file_bytes: 65_536, + max_excerpt_bytes: 4_096, + max_entries_per_directory: 80, + max_depth_per_top_level_directory: default_max_depth_per_top_level_directory(), + max_chunk_bytes: default_max_chunk_bytes(), + } + } +} + +const fn default_max_depth_per_top_level_directory() -> usize { + 3 +} + +const fn default_max_chunk_bytes() -> usize { + 4_096 +} + +fn matches_exclude_pattern(pattern: &str, name: &str) -> bool { + if pattern == name { + return true; + } + + let pattern = pattern.to_ascii_lowercase(); + let name = name.to_ascii_lowercase(); + + if let Some(suffix) = pattern.strip_prefix('*') { + return name.ends_with(suffix); + } + + pattern == name +} + +fn is_low_signal_name(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower.ends_with(".xcassets") + || lower.ends_with(".imageset") + || lower.ends_with(".appiconset") + || lower.ends_with(".colorset") +} + +fn is_hidden_name(name: &str) -> bool { + name.starts_with('.') && name != "." && name != ".." +} + +fn is_secret_name(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower == ".env" + || lower.starts_with(".env.") + || lower.ends_with(".pem") + || lower.ends_with(".key") + || lower.contains("secret") + || lower.contains("credential") +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DirectoryTypeDefinition { + pub label: String, + #[serde(default)] + pub rules: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DirectoryTypeRule { + pub id: String, + pub confidence: f32, + #[serde(default)] + pub evidence_summary: Option, + #[serde(default)] + pub signals: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DirectoryTypeSignal { + FileExists { + path: String, + }, + FileContains { + path: String, + #[serde(default)] + contains_any: Vec, + #[serde(default)] + contains_all: Vec, + }, + DirectoryName { + #[serde(default)] + equals_any: Vec, + #[serde(default)] + contains_any: Vec, + }, + ChildName { + #[serde(default)] + contains_any: Vec, + #[serde(default)] + starts_with_any: Vec, + #[serde(default)] + ends_with_any: Vec, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_config_round_trips_through_json() { + let settings = Settings::default(); + let text = serde_json::to_string(&settings).unwrap(); + let parsed: Settings = serde_json::from_str(&text).unwrap(); + assert_eq!(parsed, settings); + } + + #[test] + fn older_json_config_defaults_missing_fields() { + let parsed: Settings = serde_json::from_str( + r#" +{ + "index": { + "roots": ["~/Projects"], + "exclude": [], + "max_file_bytes": 65536, + "max_excerpt_bytes": 4096, + "max_entries_per_directory": 80 + } +} +"#, + ) + .unwrap(); + + assert_eq!(parsed.index.max_depth_per_top_level_directory, 3); + assert_eq!(parsed.index.max_chunk_bytes, 4096); + assert!(parsed.detectors.is_empty()); + } + + #[test] + fn json_config_accepts_inline_detectors() { + let parsed: Settings = serde_json::from_str( + r#" +{ + "index": { + "roots": ["~/Projects"], + "exclude": [], + "max_file_bytes": 65536, + "max_excerpt_bytes": 4096, + "max_entries_per_directory": 80 + }, + "detectors": [ + { + "label": "custom project", + "rules": [ + { + "id": "marker", + "confidence": 0.99, + "signals": [ + { "kind": "file_exists", "path": "custom.marker" } + ] + } + ] + } + ] +} +"#, + ) + .unwrap(); + + assert_eq!(parsed.detectors.len(), 1); + assert_eq!(parsed.detectors[0].label, "custom project"); + } + + #[test] + fn excludes_default_noise_and_secrets() { + let index = IndexSettings::default(); + assert!(index.is_excluded_name(".git")); + assert!(index.is_excluded_name(".env.local")); + assert!(index.is_excluded_name("private.key")); + assert!(index.is_excluded_name("Assets.xcassets")); + assert!(index.is_excluded_name("Custom Icon.appiconset")); + assert!(index.is_excluded_directory_name(".vscode")); + assert!(!index.is_excluded_name(".vscode")); + assert!(!index.is_excluded_name("README.md")); + } +} diff --git a/src/db/document.rs b/src/db/document.rs new file mode 100644 index 0000000..d67ab6b --- /dev/null +++ b/src/db/document.rs @@ -0,0 +1,93 @@ +#[derive(Debug, Clone, PartialEq)] +pub struct IndexedDocument { + pub path: String, + pub name: String, + pub kind: DocumentKind, + pub parent_path: Option, + pub searchable_text: String, + pub embedding: Vec, + pub metadata_fingerprint: String, + pub size_bytes: u64, + pub created_unix_seconds: Option, + pub modified_unix_seconds: i64, + pub accessed_unix_seconds: Option, + pub readonly: bool, + pub indexed_unix_seconds: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedFile { + pub path: String, + pub directory_path: String, + pub name: String, + pub extension: Option, + pub size_bytes: u64, + pub created_unix_seconds: Option, + pub modified_unix_seconds: i64, + pub accessed_unix_seconds: Option, + pub readonly: bool, + pub content_fingerprint: String, + pub indexed_unix_seconds: i64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct IndexedFileChunk { + pub file_path: String, + pub directory_path: String, + pub chunk_index: u32, + pub content: String, + pub embedding: Vec, + pub start_byte: u64, + pub end_byte: u64, + pub indexed_unix_seconds: i64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FileChunkMatch { + pub file_path: String, + pub file_name: String, + pub directory_path: String, + pub content: String, + pub embedding: Vec, + pub file_modified_unix_seconds: i64, + pub directory_modified_unix_seconds: i64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DirectoryClassification { + pub directory_path: String, + pub label: String, + pub confidence: f32, + pub detector: String, + pub evidence_path: Option, + pub evidence_summary: String, + pub detected_unix_seconds: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryTypeCount { + pub label: String, + pub count: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocumentKind { + Directory, + File, +} + +impl DocumentKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Directory => "directory", + Self::File => "file", + } + } + + pub fn from_db_value(value: &str) -> Self { + match value { + "file" => Self::File, + _ => Self::Directory, + } + } +} diff --git a/src/db/error.rs b/src/db/error.rs new file mode 100644 index 0000000..e6e7fdb --- /dev/null +++ b/src/db/error.rs @@ -0,0 +1,207 @@ +use std::io; +use std::num::TryFromIntError; +use std::path::PathBuf; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum DbError { + #[error("failed to create database directory {path}")] + CreateDatabaseDir { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to open database {path}")] + OpenDatabase { + path: PathBuf, + #[source] + source: rusqlite::Error, + }, + + #[error("database does not exist at {path}; run `cds --init` first")] + MissingDatabase { path: PathBuf }, + + #[error("failed to open in-memory database")] + OpenInMemory { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to migrate database")] + Migrate { + #[source] + source: Box, + }, + + #[error("failed to read schema version")] + ReadSchemaVersion { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to update schema version")] + UpdateSchemaVersion { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to create schema v1")] + CreateSchemaV1 { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to create schema v2")] + CreateSchemaV2 { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to create schema v3")] + CreateSchemaV3 { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to create schema v4")] + CreateSchemaV4 { + #[source] + source: rusqlite::Error, + }, + + #[error("embedding dimension overflows i64")] + EmbeddingDimensionOverflow { + #[source] + source: TryFromIntError, + }, + + #[error("metadata size overflows i64")] + MetadataSizeOverflow { + #[source] + source: TryFromIntError, + }, + + #[error("failed to upsert indexed document {path}")] + UpsertDocument { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to upsert indexed file {path}")] + UpsertFile { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to delete indexed chunks for {path}")] + DeleteFileChunks { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to insert indexed chunk {path}#{chunk_index}")] + InsertFileChunk { + path: String, + chunk_index: u32, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to replace classifications for {path}")] + ReplaceDirectoryClassifications { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to insert classification {label} for {path}")] + InsertDirectoryClassification { + path: String, + label: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to delete indexed path tree {path}")] + DeletePathTree { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to reset database content")] + ResetDatabase { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to count indexed documents")] + CountDocuments { + #[source] + source: rusqlite::Error, + }, + + #[error("document count was negative")] + NegativeDocumentCount { + #[source] + source: TryFromIntError, + }, + + #[error("failed to prepare document lookup")] + PrepareDocumentLookup { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to prepare directory document scan")] + PrepareDirectoryDocumentScan { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to look up indexed document {path}")] + LookupDocument { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to decode indexed document {path}")] + DecodeDocument { + path: String, + #[source] + source: rusqlite::Error, + }, + + #[error("failed to read directory documents")] + ReadDirectoryDocuments { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to read file chunk embeddings")] + ReadFileChunks { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to read directory classifications")] + ReadDirectoryClassifications { + #[source] + source: rusqlite::Error, + }, + + #[error("failed to read directory type counts")] + ReadDirectoryTypeCounts { + #[source] + source: rusqlite::Error, + }, + + #[error("embedding blob length {len} is not divisible by 4")] + InvalidEmbeddingBlobLength { len: usize }, +} diff --git a/src/db/mod.rs b/src/db/mod.rs new file mode 100644 index 0000000..8ef0729 --- /dev/null +++ b/src/db/mod.rs @@ -0,0 +1,911 @@ +mod document; +mod error; +mod schema; +mod vector; + +use std::fs; +use std::path::Path; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub use document::{ + DirectoryClassification, DirectoryTypeCount, DocumentKind, FileChunkMatch, IndexedDocument, + IndexedFile, IndexedFileChunk, +}; +pub use error::DbError; +pub use vector::{decode_embedding, encode_embedding}; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub struct Database { + connection: Connection, +} + +impl Database { + pub fn open(path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|source| DbError::CreateDatabaseDir { + path: parent.to_path_buf(), + source, + })?; + } + + let connection = Connection::open(path).map_err(|source| DbError::OpenDatabase { + path: path.to_path_buf(), + source, + })?; + schema::migrate(&connection).map_err(|source| DbError::Migrate { + source: Box::new(source), + })?; + Ok(Self { connection }) + } + + pub fn open_existing(path: &Path) -> Result { + if !path.exists() { + return Err(DbError::MissingDatabase { + path: path.to_path_buf(), + }); + } + + let connection = Connection::open(path).map_err(|source| DbError::OpenDatabase { + path: path.to_path_buf(), + source, + })?; + schema::migrate(&connection).map_err(|source| DbError::Migrate { + source: Box::new(source), + })?; + Ok(Self { connection }) + } + + pub fn open_in_memory() -> Result { + let connection = + Connection::open_in_memory().map_err(|source| DbError::OpenInMemory { source })?; + schema::migrate(&connection).map_err(|source| DbError::Migrate { + source: Box::new(source), + })?; + Ok(Self { connection }) + } + + pub fn upsert_document(&self, document: &IndexedDocument) -> Result<()> { + self.connection + .execute( + " + INSERT INTO indexed_documents ( + path, + name, + kind, + parent_path, + searchable_text, + embedding, + embedding_dim, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) + ON CONFLICT(path) DO UPDATE SET + name = excluded.name, + kind = excluded.kind, + parent_path = excluded.parent_path, + searchable_text = excluded.searchable_text, + embedding = excluded.embedding, + embedding_dim = excluded.embedding_dim, + metadata_fingerprint = excluded.metadata_fingerprint, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + params![ + document.path, + document.name, + document.kind.as_str(), + document.parent_path, + document.searchable_text, + encode_embedding(&document.embedding), + i64::try_from(document.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + document.metadata_fingerprint, + i64::try_from(document.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + document.created_unix_seconds, + document.modified_unix_seconds, + document.accessed_unix_seconds, + document.readonly, + document.indexed_unix_seconds, + ], + ) + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; + + Ok(()) + } + + pub fn upsert_file(&self, file: &IndexedFile) -> Result<()> { + self.connection + .execute( + " + INSERT INTO indexed_files ( + path, + directory_path, + name, + extension, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + content_fingerprint, + indexed_unix_seconds + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(path) DO UPDATE SET + directory_path = excluded.directory_path, + name = excluded.name, + extension = excluded.extension, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + content_fingerprint = excluded.content_fingerprint, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + params![ + file.path, + file.directory_path, + file.name, + file.extension, + i64::try_from(file.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + file.created_unix_seconds, + file.modified_unix_seconds, + file.accessed_unix_seconds, + file.readonly, + file.content_fingerprint, + file.indexed_unix_seconds, + ], + ) + .map_err(|source| DbError::UpsertFile { + path: file.path.clone(), + source, + })?; + + Ok(()) + } + + pub fn replace_file_chunks(&self, file_path: &str, chunks: &[IndexedFileChunk]) -> Result<()> { + self.connection + .execute( + "DELETE FROM indexed_file_chunks WHERE file_path = ?1", + [file_path], + ) + .map_err(|source| DbError::DeleteFileChunks { + path: file_path.to_string(), + source, + })?; + + for chunk in chunks { + self.connection + .execute( + " + INSERT INTO indexed_file_chunks ( + file_path, + directory_path, + chunk_index, + content, + embedding, + embedding_dim, + start_byte, + end_byte, + indexed_unix_seconds + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) + ", + params![ + chunk.file_path, + chunk.directory_path, + i64::from(chunk.chunk_index), + chunk.content, + encode_embedding(&chunk.embedding), + i64::try_from(chunk.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + i64::try_from(chunk.start_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + i64::try_from(chunk.end_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + chunk.indexed_unix_seconds, + ], + ) + .map_err(|source| DbError::InsertFileChunk { + path: chunk.file_path.clone(), + chunk_index: chunk.chunk_index, + source, + })?; + } + + Ok(()) + } + + pub fn replace_directory_classifications( + &self, + directory_path: &str, + classifications: &[DirectoryClassification], + ) -> Result<()> { + self.connection + .execute( + "DELETE FROM directory_classifications WHERE directory_path = ?1", + [directory_path], + ) + .map_err(|source| DbError::ReplaceDirectoryClassifications { + path: directory_path.to_string(), + source, + })?; + + for classification in classifications { + self.connection + .execute( + " + INSERT INTO directory_classifications ( + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ", + params![ + classification.directory_path, + classification.label, + classification.confidence, + classification.detector, + classification.evidence_path, + classification.evidence_summary, + classification.detected_unix_seconds, + ], + ) + .map_err(|source| DbError::InsertDirectoryClassification { + path: classification.directory_path.clone(), + label: classification.label.clone(), + source, + })?; + } + + Ok(()) + } + + pub fn delete_path_tree(&self, path: &str) -> Result<()> { + let path_len = i64::try_from(path.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?; + + self.connection + .execute( + " + DELETE FROM directory_classifications + WHERE directory_path = ?1 + OR ( + length(directory_path) > ?2 + AND substr(directory_path, 1, ?2) = ?1 + AND substr(directory_path, ?2 + 1, 1) = '/' + ) + ", + params![path, path_len], + ) + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + + self.connection + .execute( + " + DELETE FROM indexed_file_chunks + WHERE file_path = ?1 + OR directory_path = ?1 + OR ( + length(file_path) > ?2 + AND substr(file_path, 1, ?2) = ?1 + AND substr(file_path, ?2 + 1, 1) = '/' + ) + OR ( + length(directory_path) > ?2 + AND substr(directory_path, 1, ?2) = ?1 + AND substr(directory_path, ?2 + 1, 1) = '/' + ) + ", + params![path, path_len], + ) + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + + self.connection + .execute( + " + DELETE FROM indexed_files + WHERE path = ?1 + OR directory_path = ?1 + OR ( + length(path) > ?2 + AND substr(path, 1, ?2) = ?1 + AND substr(path, ?2 + 1, 1) = '/' + ) + OR ( + length(directory_path) > ?2 + AND substr(directory_path, 1, ?2) = ?1 + AND substr(directory_path, ?2 + 1, 1) = '/' + ) + ", + params![path, path_len], + ) + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + + self.connection + .execute( + " + DELETE FROM indexed_documents + WHERE path = ?1 + OR ( + length(path) > ?2 + AND substr(path, 1, ?2) = ?1 + AND substr(path, ?2 + 1, 1) = '/' + ) + ", + params![path, path_len], + ) + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; + + Ok(()) + } + + pub fn reset(&self) -> Result<()> { + self.connection + .execute_batch( + " + BEGIN; + DELETE FROM directory_classifications; + DELETE FROM indexed_file_chunks; + DELETE FROM indexed_files; + DELETE FROM indexed_documents; + COMMIT; + ", + ) + .map_err(|source| DbError::ResetDatabase { source })?; + + Ok(()) + } + + pub fn document_count(&self) -> Result { + let count: i64 = self + .connection + .query_row("SELECT COUNT(*) FROM indexed_documents", [], |row| { + row.get(0) + }) + .map_err(|source| DbError::CountDocuments { source })?; + u64::try_from(count).map_err(|source| DbError::NegativeDocumentCount { source }) + } + + pub fn get_document(&self, path: &str) -> Result> { + let mut statement = self + .connection + .prepare( + " + SELECT + path, + name, + kind, + parent_path, + searchable_text, + embedding, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE path = ?1 + ", + ) + .map_err(|source| DbError::PrepareDocumentLookup { source })?; + + let mut rows = statement + .query_map([path], decode_document_row) + .map_err(|source| DbError::LookupDocument { + path: path.to_string(), + source, + })?; + + rows.next() + .transpose() + .map_err(|source| DbError::DecodeDocument { + path: path.to_string(), + source, + }) + } + + pub fn directory_documents(&self) -> Result> { + let mut statement = self + .connection + .prepare( + " + SELECT + path, + name, + kind, + parent_path, + searchable_text, + embedding, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE kind = 'directory' + ", + ) + .map_err(|source| DbError::PrepareDirectoryDocumentScan { source })?; + + let rows = statement + .query_map([], decode_document_row) + .map_err(|source| DbError::ReadDirectoryDocuments { source })?; + + rows.collect::, _>>() + .map_err(|source| DbError::ReadDirectoryDocuments { source }) + } + + pub fn file_chunk_matches(&self) -> Result> { + let mut statement = self + .connection + .prepare( + " + SELECT + chunk.file_path, + file.name, + chunk.directory_path, + chunk.content, + chunk.embedding, + file.modified_unix_seconds, + directory.modified_unix_seconds + FROM indexed_file_chunks AS chunk + INNER JOIN indexed_files AS file + ON file.path = chunk.file_path + INNER JOIN indexed_documents AS directory + ON directory.path = chunk.directory_path + WHERE directory.kind = 'directory' + ", + ) + .map_err(|source| DbError::ReadFileChunks { source })?; + + let rows = statement + .query_map([], |row| { + let embedding: Vec = row.get(4)?; + Ok(FileChunkMatch { + file_path: row.get(0)?, + file_name: row.get(1)?, + directory_path: row.get(2)?, + content: row.get(3)?, + embedding: decode_embedding(&embedding).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure( + 4, + rusqlite::types::Type::Blob, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + err.to_string(), + )), + ) + })?, + file_modified_unix_seconds: row.get(5)?, + directory_modified_unix_seconds: row.get(6)?, + }) + }) + .map_err(|source| DbError::ReadFileChunks { source })?; + + rows.collect::, _>>() + .map_err(|source| DbError::ReadFileChunks { source }) + } + + pub fn directory_classifications( + &self, + directory_path: &str, + ) -> Result> { + let mut statement = self + .connection + .prepare( + " + SELECT + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + FROM directory_classifications + WHERE directory_path = ?1 + ", + ) + .map_err(|source| DbError::ReadDirectoryClassifications { source })?; + + let rows = statement + .query_map([directory_path], decode_classification_row) + .map_err(|source| DbError::ReadDirectoryClassifications { source })?; + + rows.collect::, _>>() + .map_err(|source| DbError::ReadDirectoryClassifications { source }) + } + + pub fn ancestor_classifications( + &self, + directory_path: &str, + ) -> Result> { + let mut classifications = Vec::new(); + for ancestor in self.indexed_ancestors(directory_path)? { + classifications.extend(self.directory_classifications(&ancestor)?); + } + Ok(classifications) + } + + pub fn directory_type_counts(&self) -> Result> { + let mut statement = self + .connection + .prepare( + " + SELECT label, COUNT(DISTINCT directory_path) AS directory_count + FROM directory_classifications + GROUP BY label + ORDER BY directory_count DESC, label ASC + ", + ) + .map_err(|source| DbError::ReadDirectoryTypeCounts { source })?; + + let rows = statement + .query_map([], |row| { + let count: i64 = row.get(1)?; + Ok(DirectoryTypeCount { + label: row.get(0)?, + count: u64::try_from(count).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure( + 1, + rusqlite::types::Type::Integer, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + err.to_string(), + )), + ) + })?, + }) + }) + .map_err(|source| DbError::ReadDirectoryTypeCounts { source })?; + + rows.collect::, _>>() + .map_err(|source| DbError::ReadDirectoryTypeCounts { source }) + } + + pub fn general_indexed_directory(&self, path: &str) -> Result { + let ancestors = self.indexed_ancestors(path)?; + + if ancestors.len() >= 2 { + return Ok(ancestors[ancestors.len() - 2].clone()); + } + + Ok(ancestors + .first() + .cloned() + .unwrap_or_else(|| path.to_string())) + } + + pub fn indexed_ancestors(&self, path: &str) -> Result> { + let mut current = Path::new(path); + let mut ancestors = Vec::new(); + + loop { + let current_path = current.to_string_lossy(); + let exists = self + .connection + .query_row( + " + SELECT path + FROM indexed_documents + WHERE path = ?1 AND kind = 'directory' + ", + [current_path.as_ref()], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|source| DbError::LookupDocument { + path: current_path.into_owned(), + source, + })?; + + if let Some(path) = exists { + ancestors.push(path); + } + + let Some(parent) = current.parent() else { + break; + }; + current = parent; + } + + Ok(ancestors) + } +} + +fn decode_classification_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(DirectoryClassification { + directory_path: row.get(0)?, + label: row.get(1)?, + confidence: row.get(2)?, + detector: row.get(3)?, + evidence_path: row.get(4)?, + evidence_summary: row.get(5)?, + detected_unix_seconds: row.get(6)?, + }) +} + +fn decode_document_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let kind: String = row.get(2)?; + let embedding: Vec = row.get(5)?; + let size_bytes: i64 = row.get(7)?; + Ok(IndexedDocument { + path: row.get(0)?, + name: row.get(1)?, + kind: DocumentKind::from_db_value(&kind), + parent_path: row.get(3)?, + searchable_text: row.get(4)?, + embedding: decode_embedding(&embedding).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure( + 5, + rusqlite::types::Type::Blob, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + err.to_string(), + )), + ) + })?, + metadata_fingerprint: row.get(6)?, + size_bytes: u64::try_from(size_bytes).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure( + 7, + rusqlite::types::Type::Integer, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + err.to_string(), + )), + ) + })?, + created_unix_seconds: row.get(8)?, + modified_unix_seconds: row.get(9)?, + accessed_unix_seconds: row.get(10)?, + readonly: row.get(11)?, + indexed_unix_seconds: row.get(12)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upserts_and_reads_document() { + let db = Database::open_in_memory().unwrap(); + let document = IndexedDocument { + path: "/tmp/project".to_string(), + name: "project".to_string(), + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: "project readme cargo".to_string(), + embedding: vec![0.1, 0.2, 0.3], + metadata_fingerprint: "fingerprint".to_string(), + size_bytes: 4096, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: false, + indexed_unix_seconds: 34, + }; + + db.upsert_document(&document).unwrap(); + + assert_eq!(db.document_count().unwrap(), 1); + assert_eq!(db.get_document("/tmp/project").unwrap(), Some(document)); + } + + #[test] + fn migrates_v1_database_to_metadata_schema() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("cds.sqlite"); + let connection = Connection::open(&path).unwrap(); + connection + .execute_batch( + " + CREATE TABLE indexed_documents ( + path TEXT PRIMARY KEY NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('directory', 'file')), + parent_path TEXT, + searchable_text TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL, + metadata_fingerprint TEXT NOT NULL, + modified_unix_seconds INTEGER NOT NULL, + indexed_unix_seconds INTEGER NOT NULL + ); + PRAGMA user_version = 1; + ", + ) + .unwrap(); + drop(connection); + + let db = Database::open(&path).unwrap(); + let version: i64 = db + .connection + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(version, 4); + db.upsert_document(&IndexedDocument { + path: "/tmp/project".to_string(), + name: "project".to_string(), + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: "project readme cargo".to_string(), + embedding: vec![0.1, 0.2, 0.3], + metadata_fingerprint: "fingerprint".to_string(), + size_bytes: 4096, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: true, + indexed_unix_seconds: 34, + }) + .unwrap(); + + let document = db.get_document("/tmp/project").unwrap().unwrap(); + assert_eq!(document.name, "project"); + assert_eq!(document.size_bytes, 4096); + assert_eq!(document.created_unix_seconds, Some(10)); + assert_eq!(document.accessed_unix_seconds, Some(14)); + assert!(document.readonly); + } + + #[test] + fn replaces_and_reads_directory_classifications() { + let db = Database::open_in_memory().unwrap(); + let classification = DirectoryClassification { + directory_path: "/tmp/project".to_string(), + label: "rust project".to_string(), + confidence: 0.98, + detector: "cargo_toml_package".to_string(), + evidence_path: Some("/tmp/project/Cargo.toml".to_string()), + evidence_summary: "Cargo.toml contains [package]".to_string(), + detected_unix_seconds: 100, + }; + + db.replace_directory_classifications("/tmp/project", std::slice::from_ref(&classification)) + .unwrap(); + assert_eq!( + db.directory_classifications("/tmp/project").unwrap(), + vec![classification] + ); + + db.replace_directory_classifications("/tmp/project", &[]) + .unwrap(); + assert!( + db.directory_classifications("/tmp/project") + .unwrap() + .is_empty() + ); + } + + #[test] + fn counts_distinct_directories_by_type() { + let db = Database::open_in_memory().unwrap(); + let rust_one = DirectoryClassification { + directory_path: "/tmp/one".to_string(), + label: "rust project".to_string(), + confidence: 0.98, + detector: "cargo".to_string(), + evidence_path: None, + evidence_summary: "Cargo.toml exists".to_string(), + detected_unix_seconds: 100, + }; + let rust_two = DirectoryClassification { + directory_path: "/tmp/two".to_string(), + label: "rust project".to_string(), + confidence: 0.98, + detector: "cargo".to_string(), + evidence_path: None, + evidence_summary: "Cargo.toml exists".to_string(), + detected_unix_seconds: 100, + }; + let chrome = DirectoryClassification { + directory_path: "/tmp/three".to_string(), + label: "chrome extension".to_string(), + confidence: 1.0, + detector: "manifest".to_string(), + evidence_path: None, + evidence_summary: "manifest.json exists".to_string(), + detected_unix_seconds: 100, + }; + + db.replace_directory_classifications("/tmp/one", &[rust_one]) + .unwrap(); + db.replace_directory_classifications("/tmp/two", &[rust_two]) + .unwrap(); + db.replace_directory_classifications("/tmp/three", &[chrome]) + .unwrap(); + + assert_eq!( + db.directory_type_counts().unwrap(), + vec![ + DirectoryTypeCount { + label: "rust project".to_string(), + count: 2, + }, + DirectoryTypeCount { + label: "chrome extension".to_string(), + count: 1, + }, + ] + ); + } + + #[test] + fn reset_clears_indexed_content() { + let db = Database::open_in_memory().unwrap(); + db.upsert_document(&IndexedDocument { + path: "/tmp/project".to_string(), + name: "project".to_string(), + kind: DocumentKind::Directory, + parent_path: Some("/tmp".to_string()), + searchable_text: "project readme cargo".to_string(), + embedding: Vec::new(), + metadata_fingerprint: "fingerprint".to_string(), + size_bytes: 4096, + created_unix_seconds: Some(10), + modified_unix_seconds: 12, + accessed_unix_seconds: Some(14), + readonly: false, + indexed_unix_seconds: 34, + }) + .unwrap(); + db.replace_directory_classifications( + "/tmp/project", + &[DirectoryClassification { + directory_path: "/tmp/project".to_string(), + label: "rust project".to_string(), + confidence: 0.98, + detector: "cargo".to_string(), + evidence_path: None, + evidence_summary: "Cargo.toml exists".to_string(), + detected_unix_seconds: 100, + }], + ) + .unwrap(); + + db.reset().unwrap(); + + assert_eq!(db.document_count().unwrap(), 0); + assert!(db.directory_type_counts().unwrap().is_empty()); + } +} diff --git a/src/db/schema.rs b/src/db/schema.rs new file mode 100644 index 0000000..8e2dfaa --- /dev/null +++ b/src/db/schema.rs @@ -0,0 +1,157 @@ +use rusqlite::Connection; + +use super::DbError; + +const CURRENT_SCHEMA_VERSION: i64 = 4; + +pub fn migrate(connection: &Connection) -> Result<(), DbError> { + let version: i64 = connection + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(|source| DbError::ReadSchemaVersion { source })?; + + if version < 1 { + migrate_to_v1(connection)?; + } + + if version < 2 { + migrate_to_v2(connection)?; + } + + if version < 3 { + migrate_to_v3(connection)?; + } + + if version < 4 { + migrate_to_v4(connection)?; + } + + connection + .pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION) + .map_err(|source| DbError::UpdateSchemaVersion { source })?; + + Ok(()) +} + +fn migrate_to_v1(connection: &Connection) -> Result<(), DbError> { + connection + .execute_batch( + " + CREATE TABLE IF NOT EXISTS indexed_documents ( + path TEXT PRIMARY KEY NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('directory', 'file')), + parent_path TEXT, + searchable_text TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL, + metadata_fingerprint TEXT NOT NULL, + modified_unix_seconds INTEGER NOT NULL, + indexed_unix_seconds INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_indexed_documents_kind + ON indexed_documents(kind); + + CREATE INDEX IF NOT EXISTS idx_indexed_documents_parent_path + ON indexed_documents(parent_path); + + CREATE INDEX IF NOT EXISTS idx_indexed_documents_indexed_time + ON indexed_documents(indexed_unix_seconds); + ", + ) + .map_err(|source| DbError::CreateSchemaV1 { source }) +} + +fn migrate_to_v2(connection: &Connection) -> Result<(), DbError> { + connection + .execute_batch( + " + ALTER TABLE indexed_documents + ADD COLUMN name TEXT NOT NULL DEFAULT ''; + + ALTER TABLE indexed_documents + ADD COLUMN size_bytes INTEGER NOT NULL DEFAULT 0; + + ALTER TABLE indexed_documents + ADD COLUMN created_unix_seconds INTEGER; + + ALTER TABLE indexed_documents + ADD COLUMN accessed_unix_seconds INTEGER; + + ALTER TABLE indexed_documents + ADD COLUMN readonly INTEGER NOT NULL DEFAULT 0; + + CREATE INDEX IF NOT EXISTS idx_indexed_documents_name + ON indexed_documents(name); + ", + ) + .map_err(|source| DbError::CreateSchemaV2 { source }) +} + +fn migrate_to_v3(connection: &Connection) -> Result<(), DbError> { + connection + .execute_batch( + " + CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY NOT NULL, + directory_path TEXT NOT NULL, + name TEXT NOT NULL, + extension TEXT, + size_bytes INTEGER NOT NULL, + created_unix_seconds INTEGER, + modified_unix_seconds INTEGER NOT NULL, + accessed_unix_seconds INTEGER, + readonly INTEGER NOT NULL, + content_fingerprint TEXT NOT NULL, + indexed_unix_seconds INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_indexed_files_directory_path + ON indexed_files(directory_path); + + CREATE INDEX IF NOT EXISTS idx_indexed_files_modified_time + ON indexed_files(modified_unix_seconds); + + CREATE TABLE IF NOT EXISTS indexed_file_chunks ( + file_path TEXT NOT NULL, + directory_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL, + start_byte INTEGER NOT NULL, + end_byte INTEGER NOT NULL, + indexed_unix_seconds INTEGER NOT NULL, + PRIMARY KEY (file_path, chunk_index) + ); + + CREATE INDEX IF NOT EXISTS idx_indexed_file_chunks_directory_path + ON indexed_file_chunks(directory_path); + ", + ) + .map_err(|source| DbError::CreateSchemaV3 { source }) +} + +fn migrate_to_v4(connection: &Connection) -> Result<(), DbError> { + connection + .execute_batch( + " + CREATE TABLE IF NOT EXISTS directory_classifications ( + directory_path TEXT NOT NULL, + label TEXT NOT NULL, + confidence REAL NOT NULL, + detector TEXT NOT NULL, + evidence_path TEXT, + evidence_summary TEXT NOT NULL, + detected_unix_seconds INTEGER NOT NULL, + PRIMARY KEY (directory_path, label, detector) + ); + + CREATE INDEX IF NOT EXISTS idx_directory_classifications_label + ON directory_classifications(label); + + CREATE INDEX IF NOT EXISTS idx_directory_classifications_directory_path + ON directory_classifications(directory_path); + ", + ) + .map_err(|source| DbError::CreateSchemaV4 { source }) +} diff --git a/src/db/vector.rs b/src/db/vector.rs new file mode 100644 index 0000000..61e6313 --- /dev/null +++ b/src/db/vector.rs @@ -0,0 +1,34 @@ +use super::DbError; + +pub fn encode_embedding(values: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(values.len() * 4); + for value in values { + bytes.extend(value.to_le_bytes()); + } + bytes +} + +pub fn decode_embedding(bytes: &[u8]) -> Result, DbError> { + if !bytes.len().is_multiple_of(4) { + return Err(DbError::InvalidEmbeddingBlobLength { len: bytes.len() }); + } + + Ok(bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn embedding_round_trips() { + let values = vec![0.25, -0.5, 2.0]; + assert_eq!( + decode_embedding(&encode_embedding(&values)).unwrap(), + values + ); + } +} diff --git a/src/embed/error.rs b/src/embed/error.rs new file mode 100644 index 0000000..0928c6e --- /dev/null +++ b/src/embed/error.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum EmbedError { + #[error("embedding model failed: {message}")] + Model { message: String }, +} diff --git a/src/embed/fake.rs b/src/embed/fake.rs new file mode 100644 index 0000000..aa49fc6 --- /dev/null +++ b/src/embed/fake.rs @@ -0,0 +1,81 @@ +use super::{Embedder, Result}; + +#[derive(Debug, Clone)] +pub struct FakeEmbedder { + dimensions: usize, +} + +impl FakeEmbedder { + pub fn new(dimensions: usize) -> Self { + Self { dimensions } + } +} + +impl Default for FakeEmbedder { + fn default() -> Self { + Self::new(32) + } +} + +impl Embedder for FakeEmbedder { + fn dimensions(&self) -> usize { + self.dimensions + } + + fn embed(&self, text: &str) -> Result> { + let mut values = vec![0.0; self.dimensions]; + + for token in text.split(|ch: char| !ch.is_alphanumeric()) { + if token.is_empty() { + continue; + } + + let hash = stable_hash(token.as_bytes()); + let index = hash as usize % self.dimensions; + values[index] += 1.0; + } + + normalize(&mut values); + Ok(values) + } +} + +fn stable_hash(bytes: &[u8]) -> u64 { + let mut hash = 14_695_981_039_346_656_037_u64; + for byte in bytes { + hash ^= u64::from(byte.to_ascii_lowercase()); + hash = hash.wrapping_mul(1_099_511_628_211); + } + hash +} + +fn normalize(values: &mut [f32]) { + let magnitude = values.iter().map(|value| value * value).sum::().sqrt(); + if magnitude == 0.0 { + return; + } + + for value in values { + *value /= magnitude; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fake_embeddings_are_deterministic() { + let embedder = FakeEmbedder::new(8); + assert_eq!( + embedder.embed("Chrome Extension").unwrap(), + embedder.embed("Chrome Extension").unwrap() + ); + } + + #[test] + fn fake_embeddings_use_requested_dimension() { + let embedder = FakeEmbedder::new(12); + assert_eq!(embedder.embed("hello").unwrap().len(), 12); + } +} diff --git a/src/embed/mod.rs b/src/embed/mod.rs new file mode 100644 index 0000000..34b6f5c --- /dev/null +++ b/src/embed/mod.rs @@ -0,0 +1,12 @@ +mod error; +mod fake; + +pub use error::EmbedError; +pub use fake::FakeEmbedder; + +pub type Result = std::result::Result; + +pub trait Embedder { + fn dimensions(&self) -> usize; + fn embed(&self, text: &str) -> Result>; +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..32cb93e --- /dev/null +++ b/src/error.rs @@ -0,0 +1,99 @@ +use std::io; + +use thiserror::Error; + +use crate::app::AppError; +use crate::config::ConfigError; +use crate::db::DbError; +use crate::embed::EmbedError; +use crate::index::IndexError; +use crate::search::SearchError; + +pub type Result = std::result::Result; + +#[derive(Debug, Error)] +pub enum Error { + #[error(transparent)] + App(Box), + + #[error(transparent)] + Config(Box), + + #[error(transparent)] + Db(Box), + + #[error(transparent)] + Embed(Box), + + #[error(transparent)] + Index(Box), + + #[error(transparent)] + Search(Box), + + #[error("failed to write stdout")] + Stdout(#[source] io::Error), + + #[error("failed to read stdin")] + Stdin(#[source] io::Error), +} + +impl From for Error { + fn from(error: AppError) -> Self { + Self::App(Box::new(error)) + } +} + +impl From for Error { + fn from(error: ConfigError) -> Self { + Self::Config(Box::new(error)) + } +} + +impl From for Error { + fn from(error: DbError) -> Self { + Self::Db(Box::new(error)) + } +} + +impl From for Error { + fn from(error: EmbedError) -> Self { + Self::Embed(Box::new(error)) + } +} + +impl From for Error { + fn from(error: IndexError) -> Self { + Self::Index(Box::new(error)) + } +} + +impl From for Error { + fn from(error: SearchError) -> Self { + Self::Search(Box::new(error)) + } +} + +pub fn app_err(error: AppError) -> Error { + Error::from(error) +} + +pub fn config_err(error: ConfigError) -> Error { + Error::from(error) +} + +pub fn db_err(error: DbError) -> Error { + Error::from(error) +} + +pub fn embed_err(error: EmbedError) -> Error { + Error::from(error) +} + +pub fn index_err(error: IndexError) -> Error { + Error::from(error) +} + +pub fn search_err(error: SearchError) -> Error { + Error::from(error) +} diff --git a/src/index/classify.rs b/src/index/classify.rs new file mode 100644 index 0000000..54de824 --- /dev/null +++ b/src/index/classify.rs @@ -0,0 +1,279 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::config::{DirectoryTypeDefinition, DirectoryTypeRule, DirectoryTypeSignal, Settings}; +use crate::db::DirectoryClassification; +use crate::index::{IndexError, Result}; + +const BUILTIN_DEFINITIONS: &[(&str, &str)] = &[ + ( + "chrome_extension.json", + include_str!("directory_types/chrome_extension.json"), + ), + ("rust.json", include_str!("directory_types/rust.json")), + ("node.json", include_str!("directory_types/node.json")), + ("next.json", include_str!("directory_types/next.json")), + ("python.json", include_str!("directory_types/python.json")), + ("rails.json", include_str!("directory_types/rails.json")), + ( + "migrations.json", + include_str!("directory_types/migrations.json"), + ), +]; + +pub fn classify_directory( + directory: &Path, + settings: &Settings, +) -> Result> { + let definitions = load_definitions(settings)?; + let detected_unix_seconds = unix_seconds(SystemTime::now()); + let directory_path = path_to_string(directory); + let mut classifications = Vec::new(); + + for definition in definitions { + for rule in definition.rules { + let Some(evidence) = matches_rule(directory, &rule) else { + continue; + }; + + classifications.push(DirectoryClassification { + directory_path: directory_path.clone(), + label: definition.label.clone(), + confidence: rule.confidence, + detector: rule.id, + evidence_path: evidence.evidence_path.map(|path| path_to_string(&path)), + evidence_summary: rule + .evidence_summary + .unwrap_or_else(|| evidence.summaries.join("; ")), + detected_unix_seconds, + }); + } + } + + classifications.sort_by(|left, right| { + left.label + .cmp(&right.label) + .then_with(|| right.confidence.total_cmp(&left.confidence)) + .then_with(|| left.detector.cmp(&right.detector)) + }); + classifications.dedup_by(|left, right| left.label == right.label); + + Ok(classifications) +} + +fn load_definitions(settings: &Settings) -> Result> { + let mut definitions = Vec::new(); + + for (name, text) in BUILTIN_DEFINITIONS { + definitions.push(parse_definition(Path::new(name), text)?); + } + + definitions.extend(settings.detectors.clone()); + + Ok(definitions) +} + +fn parse_definition(path: &Path, text: &str) -> Result { + serde_json::from_str(text).map_err(|source| IndexError::ParseDirectoryTypeDefinition { + path: path.to_path_buf(), + source, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RuleEvidence { + evidence_path: Option, + summaries: Vec, +} + +fn matches_rule(directory: &Path, rule: &DirectoryTypeRule) -> Option { + let mut evidence_path = None; + let mut summaries = Vec::new(); + + for signal in &rule.signals { + let signal_match = matches_signal(directory, signal)?; + if evidence_path.is_none() { + evidence_path = signal_match.evidence_path; + } + summaries.push(signal_match.summary); + } + + Some(RuleEvidence { + evidence_path, + summaries, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct SignalMatch { + evidence_path: Option, + summary: String, +} + +fn matches_signal(directory: &Path, signal: &DirectoryTypeSignal) -> Option { + match signal { + DirectoryTypeSignal::FileExists { path } => { + let path = directory.join(path); + path.exists().then(|| SignalMatch { + evidence_path: Some(path), + summary: "required file exists".to_string(), + }) + } + DirectoryTypeSignal::FileContains { + path, + contains_any, + contains_all, + } => { + let path = directory.join(path); + let text = read_small_text(&path)?.to_ascii_lowercase(); + let any_matches = contains_any.is_empty() + || contains_any + .iter() + .any(|needle| text.contains(&needle.to_ascii_lowercase())); + let all_match = contains_all + .iter() + .all(|needle| text.contains(&needle.to_ascii_lowercase())); + + (any_matches && all_match).then(|| SignalMatch { + evidence_path: Some(path), + summary: "file contains required text".to_string(), + }) + } + DirectoryTypeSignal::DirectoryName { + equals_any, + contains_any, + } => { + let name = directory + .file_name() + .map(|name| name.to_string_lossy().to_ascii_lowercase()) + .unwrap_or_default(); + let equals = !equals_any.is_empty() + && equals_any + .iter() + .any(|candidate| name == candidate.to_ascii_lowercase()); + let contains = contains_any + .iter() + .any(|candidate| name.contains(&candidate.to_ascii_lowercase())); + + (equals || contains).then(|| SignalMatch { + evidence_path: None, + summary: format!("directory name matched {name}"), + }) + } + DirectoryTypeSignal::ChildName { + contains_any, + starts_with_any, + ends_with_any, + } => { + let entries = fs::read_dir(directory).ok()?; + for entry in entries.filter_map(std::result::Result::ok) { + let name = entry.file_name().to_string_lossy().to_ascii_lowercase(); + let contains = contains_any + .iter() + .any(|candidate| name.contains(&candidate.to_ascii_lowercase())); + let starts = starts_with_any + .iter() + .any(|candidate| name.starts_with(&candidate.to_ascii_lowercase())); + let ends = ends_with_any + .iter() + .any(|candidate| name.ends_with(&candidate.to_ascii_lowercase())); + + if contains || starts || ends { + return Some(SignalMatch { + evidence_path: Some(entry.path()), + summary: format!("child name matched {name}"), + }); + } + } + + None + } + } +} + +fn read_small_text(path: &Path) -> Option { + let metadata = fs::metadata(path).ok()?; + if !metadata.is_file() || metadata.len() > 131_072 { + return None; + } + + let bytes = fs::read(path).ok()?; + if bytes.contains(&0) { + return None; + } + + Some(String::from_utf8_lossy(&bytes).into_owned()) +} + +fn unix_seconds(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_chrome_extension_from_builtin_json() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("manifest.json"), + r#"{"manifest_version":3,"action":{},"permissions":["storage"]}"#, + ) + .unwrap(); + + let labels = classify_directory(temp.path(), &Settings::default()) + .unwrap() + .into_iter() + .map(|classification| classification.label) + .collect::>(); + + assert!(labels.contains(&"chrome extension".to_string())); + } + + #[test] + fn detects_custom_directory_type_from_config_array() { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + fs::create_dir_all(&project).unwrap(); + fs::write(project.join("custom.marker"), "yes").unwrap(); + + let settings = Settings { + detectors: serde_json::from_str( + r#" +[ +{ + "label": "custom project", + "rules": [ + { + "id": "marker", + "confidence": 0.99, + "signals": [ + { "kind": "file_exists", "path": "custom.marker" } + ] + } + ] +} +] +"#, + ) + .unwrap(), + ..Settings::default() + }; + + let labels = classify_directory(&project, &settings) + .unwrap() + .into_iter() + .map(|classification| classification.label) + .collect::>(); + + assert!(labels.contains(&"custom project".to_string())); + } +} diff --git a/src/index/directory_types/chrome_extension.json b/src/index/directory_types/chrome_extension.json new file mode 100644 index 0000000..1d6724a --- /dev/null +++ b/src/index/directory_types/chrome_extension.json @@ -0,0 +1,24 @@ +{ + "label": "chrome extension", + "rules": [ + { + "id": "manifest_json", + "confidence": 1.0, + "evidence_summary": "manifest.json contains manifest_version and browser extension fields", + "signals": [ + { + "kind": "file_contains", + "path": "manifest.json", + "contains_all": ["\"manifest_version\""], + "contains_any": [ + "\"permissions\"", + "\"action\"", + "\"browser_action\"", + "\"background\"", + "\"content_scripts\"" + ] + } + ] + } + ] +} diff --git a/src/index/directory_types/migrations.json b/src/index/directory_types/migrations.json new file mode 100644 index 0000000..98053cc --- /dev/null +++ b/src/index/directory_types/migrations.json @@ -0,0 +1,28 @@ +{ + "label": "database migrations", + "rules": [ + { + "id": "directory_name", + "confidence": 0.95, + "evidence_summary": "directory name is migrations", + "signals": [ + { + "kind": "directory_name", + "equals_any": ["migrations", "migration"] + } + ] + }, + { + "id": "migration_file_names", + "confidence": 0.75, + "evidence_summary": "directory contains SQL or migration-like file names", + "signals": [ + { + "kind": "child_name", + "contains_any": ["migration"], + "ends_with_any": [".sql"] + } + ] + } + ] +} diff --git a/src/index/directory_types/next.json b/src/index/directory_types/next.json new file mode 100644 index 0000000..2d5d1a2 --- /dev/null +++ b/src/index/directory_types/next.json @@ -0,0 +1,28 @@ +{ + "label": "next.js app", + "rules": [ + { + "id": "next_dependency", + "confidence": 0.95, + "evidence_summary": "package.json references Next.js", + "signals": [ + { + "kind": "file_contains", + "path": "package.json", + "contains_any": ["\"next\""] + } + ] + }, + { + "id": "next_config", + "confidence": 0.95, + "evidence_summary": "next.config file exists", + "signals": [ + { + "kind": "child_name", + "starts_with_any": ["next.config"] + } + ] + } + ] +} diff --git a/src/index/directory_types/node.json b/src/index/directory_types/node.json new file mode 100644 index 0000000..b98f79d --- /dev/null +++ b/src/index/directory_types/node.json @@ -0,0 +1,21 @@ +{ + "label": "node project", + "rules": [ + { + "id": "package_json", + "confidence": 0.9, + "evidence_summary": "package.json contains scripts or dependencies", + "signals": [ + { + "kind": "file_contains", + "path": "package.json", + "contains_any": [ + "\"scripts\"", + "\"dependencies\"", + "\"devDependencies\"" + ] + } + ] + } + ] +} diff --git a/src/index/directory_types/python.json b/src/index/directory_types/python.json new file mode 100644 index 0000000..dcda56b --- /dev/null +++ b/src/index/directory_types/python.json @@ -0,0 +1,21 @@ +{ + "label": "python project", + "rules": [ + { + "id": "pyproject_toml", + "confidence": 0.92, + "evidence_summary": "pyproject.toml exists", + "signals": [ + { "kind": "file_exists", "path": "pyproject.toml" } + ] + }, + { + "id": "requirements_txt", + "confidence": 0.85, + "evidence_summary": "requirements.txt exists", + "signals": [ + { "kind": "file_exists", "path": "requirements.txt" } + ] + } + ] +} diff --git a/src/index/directory_types/rails.json b/src/index/directory_types/rails.json new file mode 100644 index 0000000..847cde4 --- /dev/null +++ b/src/index/directory_types/rails.json @@ -0,0 +1,18 @@ +{ + "label": "rails app", + "rules": [ + { + "id": "gemfile_routes", + "confidence": 0.98, + "evidence_summary": "Gemfile mentions Rails and config/routes.rb exists", + "signals": [ + { + "kind": "file_contains", + "path": "Gemfile", + "contains_any": ["rails"] + }, + { "kind": "file_exists", "path": "config/routes.rb" } + ] + } + ] +} diff --git a/src/index/directory_types/rust.json b/src/index/directory_types/rust.json new file mode 100644 index 0000000..29f94cf --- /dev/null +++ b/src/index/directory_types/rust.json @@ -0,0 +1,17 @@ +{ + "label": "rust project", + "rules": [ + { + "id": "cargo_toml_package", + "confidence": 0.98, + "evidence_summary": "Cargo.toml contains [package] or [workspace]", + "signals": [ + { + "kind": "file_contains", + "path": "Cargo.toml", + "contains_any": ["[package]", "[workspace]"] + } + ] + } + ] +} diff --git a/src/index/error.rs b/src/index/error.rs new file mode 100644 index 0000000..8e923ea --- /dev/null +++ b/src/index/error.rs @@ -0,0 +1,122 @@ +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +use crate::config::ConfigError; +use crate::db::DbError; +use crate::embed::EmbedError; + +#[derive(Debug, Error)] +pub enum IndexError { + #[error("failed to expand configured index roots")] + ExpandConfiguredRoots { + #[source] + source: ConfigError, + }, + + #[error("failed to index root {root}")] + ScanRoot { + root: PathBuf, + #[source] + source: Box, + }, + + #[error("failed to summarize {path}")] + SummarizeDirectory { + path: PathBuf, + #[source] + source: Box, + }, + + #[error("failed to store indexed document {path}")] + StoreDocument { + path: String, + #[source] + source: Box, + }, + + #[error("failed to store indexed file {path}")] + StoreFile { + path: String, + #[source] + source: Box, + }, + + #[error("failed to store indexed chunks for {path}")] + StoreFileChunks { + path: String, + #[source] + source: Box, + }, + + #[error("failed to store directory classifications for {path}")] + StoreDirectoryClassifications { + path: String, + #[source] + source: Box, + }, + + #[error("failed to prune excluded indexed path {path}")] + PruneExcludedPath { + path: String, + #[source] + source: Box, + }, + + #[error("failed to embed summary for {path}")] + EmbedSummary { + path: PathBuf, + #[source] + source: EmbedError, + }, + + #[error("failed to read metadata for {path}")] + ReadMetadata { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to read directory {path}")] + ReadDirectory { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to read entry in {path}")] + ReadDirectoryEntry { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to read file type for {path}")] + ReadFileType { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to stat {path}")] + StatFile { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to read {path}")] + ReadFile { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("failed to parse directory type definition {path}")] + ParseDirectoryTypeDefinition { + path: PathBuf, + #[source] + source: serde_json::Error, + }, +} diff --git a/src/index/file.rs b/src/index/file.rs new file mode 100644 index 0000000..c4b1572 --- /dev/null +++ b/src/index/file.rs @@ -0,0 +1,177 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::config::Settings; +use crate::db::{IndexedFile, IndexedFileChunk}; +use crate::embed::Embedder; +use crate::index::{IndexError, Result}; + +#[derive(Debug, Clone, PartialEq)] +pub struct IndexedFileData { + pub file: IndexedFile, + pub chunks: Vec, +} + +pub fn index_text_file( + path: &Path, + directory: &Path, + settings: &Settings, + embedder: &E, +) -> Result> +where + E: Embedder, +{ + let metadata = fs::metadata(path).map_err(|source| IndexError::StatFile { + path: path.to_path_buf(), + source, + })?; + + if metadata.len() > settings.index.max_file_bytes { + return Ok(None); + } + + let bytes = fs::read(path).map_err(|source| IndexError::ReadFile { + path: path.to_path_buf(), + source, + })?; + if bytes.contains(&0) { + return Ok(None); + } + + let content = String::from_utf8_lossy(&bytes).into_owned(); + let normalized = normalize_whitespace(&content); + if normalized.is_empty() { + return Ok(None); + } + + let indexed_unix_seconds = unix_seconds(SystemTime::now()); + let modified_unix_seconds = unix_seconds(metadata.modified().unwrap_or(UNIX_EPOCH)); + let file_path = path_to_string(path); + let directory_path = path_to_string(directory); + let name = path + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_string_lossy() + .into_owned(); + + let file = IndexedFile { + path: file_path.clone(), + directory_path: directory_path.clone(), + name, + extension: path + .extension() + .map(|extension| extension.to_string_lossy().into_owned()), + size_bytes: metadata.len(), + created_unix_seconds: metadata.created().ok().map(unix_seconds), + modified_unix_seconds, + accessed_unix_seconds: metadata.accessed().ok().map(unix_seconds), + readonly: metadata.permissions().readonly(), + content_fingerprint: format!( + "mtime:{modified_unix_seconds}:len:{}:hash:{:016x}", + metadata.len(), + fnv1a64(&bytes), + ), + indexed_unix_seconds, + }; + + let mut chunks = Vec::new(); + for (chunk_index, chunk) in chunk_text(&normalized, settings.index.max_chunk_bytes) + .into_iter() + .enumerate() + { + let embedding = embedder + .embed(chunk.text) + .map_err(|source| IndexError::EmbedSummary { + path: path.to_path_buf(), + source, + })?; + + chunks.push(IndexedFileChunk { + file_path: file_path.clone(), + directory_path: directory_path.clone(), + chunk_index: u32::try_from(chunk_index).unwrap_or(u32::MAX), + content: chunk.text.to_string(), + embedding, + start_byte: chunk.start_byte, + end_byte: chunk.end_byte, + indexed_unix_seconds, + }); + } + + Ok(Some(IndexedFileData { file, chunks })) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TextChunk<'a> { + text: &'a str, + start_byte: u64, + end_byte: u64, +} + +fn chunk_text(text: &str, max_chunk_bytes: usize) -> Vec> { + let max_chunk_bytes = max_chunk_bytes.max(1); + let mut chunks = Vec::new(); + let mut start = 0; + + while start < text.len() { + let mut end = (start + max_chunk_bytes).min(text.len()); + while end > start && !text.is_char_boundary(end) { + end -= 1; + } + + if end == start { + end = text[start..] + .char_indices() + .nth(1) + .map(|(offset, _)| start + offset) + .unwrap_or(text.len()); + } + + chunks.push(TextChunk { + text: &text[start..end], + start_byte: u64::try_from(start).unwrap_or(u64::MAX), + end_byte: u64::try_from(end).unwrap_or(u64::MAX), + }); + start = end; + } + + chunks +} + +fn normalize_whitespace(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn unix_seconds(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chunks_text_on_char_boundaries() { + let chunks = chunk_text("abc def ghi", 5); + assert_eq!(chunks[0].text, "abc d"); + assert_eq!(chunks[1].text, "ef gh"); + assert_eq!(chunks[2].text, "i"); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs new file mode 100644 index 0000000..2e53e09 --- /dev/null +++ b/src/index/mod.rs @@ -0,0 +1,320 @@ +mod classify; +mod error; +mod file; +mod progress; +mod scanner; +mod summary; + +use std::path::PathBuf; + +use crate::config::Settings; +use crate::db::Database; +use crate::embed::Embedder; + +pub use error::IndexError; +pub use progress::{IndexProgress, NoopProgress}; +pub use scanner::IndexReport; + +pub type Result = std::result::Result; + +pub struct Indexer<'a, E> { + settings: &'a Settings, + database: &'a Database, + embedder: &'a E, +} + +impl<'a, E> Indexer<'a, E> +where + E: Embedder, +{ + pub fn new(settings: &'a Settings, database: &'a Database, embedder: &'a E) -> Self { + Self { + settings, + database, + embedder, + } + } + + pub fn index_configured_roots(&self) -> Result { + let mut progress = NoopProgress; + self.index_configured_roots_with_progress(&mut progress) + } + + pub fn index_configured_roots_with_progress

(&self, progress: &mut P) -> Result + where + P: IndexProgress + ?Sized, + { + let roots = self + .settings + .expanded_roots() + .map_err(|source| IndexError::ExpandConfiguredRoots { source })?; + self.index_roots_with_progress(roots, progress) + } + + pub fn index_roots(&self, roots: Vec) -> Result { + let mut progress = NoopProgress; + self.index_roots_with_progress(roots, &mut progress) + } + + pub fn index_roots_with_progress

( + &self, + roots: Vec, + progress: &mut P, + ) -> Result + where + P: IndexProgress + ?Sized, + { + let mut report = IndexReport::default(); + + for root in roots { + if !root.exists() { + report.roots_missing += 1; + continue; + } + + if !root.is_dir() { + report.roots_not_directories += 1; + continue; + } + + if is_excluded_index_root(self.settings, &root) { + self.database + .delete_path_tree(&root.to_string_lossy()) + .map_err(|source| IndexError::PruneExcludedPath { + path: root.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + report.entries_skipped += 1; + continue; + } + + report.roots_scanned += 1; + scanner::scan_root_with_progress( + &root, + self.settings, + self.database, + self.embedder, + &mut report, + progress, + ) + .map_err(|source| IndexError::ScanRoot { + root: root.clone(), + source: Box::new(source), + })?; + } + + Ok(report) + } +} + +fn is_excluded_index_root(settings: &Settings, root: &std::path::Path) -> bool { + root.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| settings.index.is_excluded_directory_name(name)) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use crate::db::{Database, DocumentKind, IndexedDocument}; + use crate::embed::FakeEmbedder; + + #[test] + fn indexes_directory_summaries_and_skips_excluded_paths() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let app = root.join("chrome-extension"); + let ignored = app.join("node_modules"); + fs::create_dir_all(&ignored).unwrap(); + fs::write(app.join("README.md"), "Chrome extension manifest tools").unwrap(); + fs::write(ignored.join("package.json"), "should not appear").unwrap(); + let asset_catalog = app.join("macos/Assets.xcassets/Custom Icon.appiconset"); + fs::create_dir_all(&asset_catalog).unwrap(); + fs::write(asset_catalog.join("Contents.json"), "should not be indexed").unwrap(); + let hidden = app.join(".vscode"); + fs::create_dir_all(&hidden).unwrap(); + fs::write(hidden.join("settings.json"), "should not be indexed").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + database + .upsert_document(&IndexedDocument { + path: asset_catalog.to_string_lossy().into_owned(), + name: "Custom Icon.appiconset".to_string(), + kind: DocumentKind::Directory, + parent_path: asset_catalog + .parent() + .map(|path| path.to_string_lossy().into_owned()), + searchable_text: "stale custom icon asset catalog".to_string(), + embedding: vec![1.0; 16], + metadata_fingerprint: "stale".to_string(), + size_bytes: 0, + created_unix_seconds: None, + modified_unix_seconds: 0, + accessed_unix_seconds: None, + readonly: false, + indexed_unix_seconds: 0, + }) + .unwrap(); + database + .upsert_document(&IndexedDocument { + path: hidden.to_string_lossy().into_owned(), + name: ".vscode".to_string(), + kind: DocumentKind::Directory, + parent_path: hidden + .parent() + .map(|path| path.to_string_lossy().into_owned()), + searchable_text: "stale hidden editor config".to_string(), + embedding: vec![1.0; 16], + metadata_fingerprint: "stale".to_string(), + size_bytes: 0, + created_unix_seconds: None, + modified_unix_seconds: 0, + accessed_unix_seconds: None, + readonly: false, + indexed_unix_seconds: 0, + }) + .unwrap(); + let embedder = FakeEmbedder::new(16); + let indexer = Indexer::new(&settings, &database, &embedder); + + let report = indexer.index_roots(vec![root.clone()]).unwrap(); + + assert_eq!(report.roots_scanned, 1); + assert_eq!(report.directories_indexed, 3); + assert_eq!(report.text_files_indexed, 1); + assert_eq!(report.file_chunks_indexed, 1); + assert_eq!(report.entries_skipped, 3); + assert_eq!(database.document_count().unwrap(), 3); + + let app_document = database + .get_document(&app.to_string_lossy()) + .unwrap() + .expect("app directory is indexed"); + assert!( + app_document + .searchable_text + .contains("Chrome extension manifest tools") + ); + assert!(!app_document.searchable_text.contains("should not appear")); + assert!(!app_document.searchable_text.contains(".vscode")); + assert_eq!( + database + .get_document(&asset_catalog.to_string_lossy()) + .unwrap(), + None + ); + assert_eq!( + database.get_document(&hidden.to_string_lossy()).unwrap(), + None + ); + } + + #[test] + fn skips_hidden_roots_and_prunes_stale_rows() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join(".hidden-project"); + fs::create_dir_all(&root).unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + database + .upsert_document(&IndexedDocument { + path: root.to_string_lossy().into_owned(), + name: ".hidden-project".to_string(), + kind: DocumentKind::Directory, + parent_path: root + .parent() + .map(|path| path.to_string_lossy().into_owned()), + searchable_text: "stale hidden root".to_string(), + embedding: vec![1.0; 16], + metadata_fingerprint: "stale".to_string(), + size_bytes: 0, + created_unix_seconds: None, + modified_unix_seconds: 0, + accessed_unix_seconds: None, + readonly: false, + indexed_unix_seconds: 0, + }) + .unwrap(); + let embedder = FakeEmbedder::new(16); + let indexer = Indexer::new(&settings, &database, &embedder); + + let report = indexer.index_roots(vec![root.clone()]).unwrap(); + + assert_eq!(report.roots_scanned, 0); + assert_eq!(report.entries_skipped, 1); + assert_eq!( + database.get_document(&root.to_string_lossy()).unwrap(), + None + ); + } + + #[test] + fn limits_recursion_depth_per_top_level_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let top_level = root.join("app"); + let depth_one = top_level.join("packages"); + let depth_two = depth_one.join("web"); + let depth_three = depth_two.join("src"); + let too_deep = depth_three.join("components"); + fs::create_dir_all(&too_deep).unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + database + .upsert_document(&IndexedDocument { + path: too_deep.to_string_lossy().into_owned(), + name: "components".to_string(), + kind: DocumentKind::Directory, + parent_path: too_deep + .parent() + .map(|path| path.to_string_lossy().into_owned()), + searchable_text: "stale deeply nested component directory".to_string(), + embedding: vec![1.0; 16], + metadata_fingerprint: "stale".to_string(), + size_bytes: 0, + created_unix_seconds: None, + modified_unix_seconds: 0, + accessed_unix_seconds: None, + readonly: false, + indexed_unix_seconds: 0, + }) + .unwrap(); + let embedder = FakeEmbedder::new(16); + let indexer = Indexer::new(&settings, &database, &embedder); + + let report = indexer.index_roots(vec![root.clone()]).unwrap(); + + assert_eq!(report.roots_scanned, 1); + assert_eq!(report.directories_indexed, 5); + assert_eq!(report.text_files_indexed, 0); + assert_eq!(report.file_chunks_indexed, 0); + assert_eq!(report.entries_skipped, 1); + assert!( + database + .get_document(&root.to_string_lossy()) + .unwrap() + .is_some() + ); + assert!( + database + .get_document(&top_level.to_string_lossy()) + .unwrap() + .is_some() + ); + assert!( + database + .get_document(&depth_three.to_string_lossy()) + .unwrap() + .is_some() + ); + assert_eq!( + database.get_document(&too_deep.to_string_lossy()).unwrap(), + None + ); + } +} diff --git a/src/index/progress.rs b/src/index/progress.rs new file mode 100644 index 0000000..4636b9a --- /dev/null +++ b/src/index/progress.rs @@ -0,0 +1,12 @@ +use std::path::Path; + +pub trait IndexProgress { + fn directory_started(&mut self, directory: &Path); +} + +#[derive(Debug, Default)] +pub struct NoopProgress; + +impl IndexProgress for NoopProgress { + fn directory_started(&mut self, _directory: &Path) {} +} diff --git a/src/index/scanner.rs b/src/index/scanner.rs new file mode 100644 index 0000000..636240a --- /dev/null +++ b/src/index/scanner.rs @@ -0,0 +1,227 @@ +use std::fs; +use std::path::Path; + +use super::{classify, file, summary}; +use crate::config::Settings; +use crate::db::{Database, IndexedDocument}; +use crate::embed::Embedder; +use crate::index::{IndexError, IndexProgress, Result}; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct IndexReport { + pub roots_scanned: u64, + pub roots_missing: u64, + pub roots_not_directories: u64, + pub directories_indexed: u64, + pub files_seen: u64, + pub text_files_indexed: u64, + pub file_chunks_indexed: u64, + pub entries_skipped: u64, +} + +impl IndexReport { + pub fn human_summary(&self) -> String { + format!( + "indexed {} directories and {} text file chunks from {} text files across {} roots ({} files seen, {} skipped, {} missing roots, {} non-directory roots)", + self.directories_indexed, + self.file_chunks_indexed, + self.text_files_indexed, + self.roots_scanned, + self.files_seen, + self.entries_skipped, + self.roots_missing, + self.roots_not_directories, + ) + } +} + +pub fn scan_root_with_progress( + root: &Path, + settings: &Settings, + database: &Database, + embedder: &E, + report: &mut IndexReport, + progress: &mut P, +) -> Result<()> +where + E: Embedder, + P: IndexProgress + ?Sized, +{ + scan_directory( + root, + settings, + database, + embedder, + report, + progress, + DepthPosition::IndexRoot, + ) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DepthPosition { + IndexRoot, + TopLevelDirectory { depth: usize }, +} + +impl DepthPosition { + fn child_position(self) -> Option { + match self { + Self::IndexRoot => Some(Self::TopLevelDirectory { depth: 0 }), + Self::TopLevelDirectory { depth } => depth + .checked_add(1) + .map(|depth| Self::TopLevelDirectory { depth }), + } + } +} + +fn scan_directory( + directory: &Path, + settings: &Settings, + database: &Database, + embedder: &E, + report: &mut IndexReport, + progress: &mut P, + depth_position: DepthPosition, +) -> Result<()> +where + E: Embedder, + P: IndexProgress + ?Sized, +{ + progress.directory_started(directory); + + let document = summary::summarize_directory(directory, settings).map_err(|source| { + IndexError::SummarizeDirectory { + path: directory.to_path_buf(), + source: Box::new(source), + } + })?; + + database + .upsert_document(&document) + .map_err(|source| IndexError::StoreDocument { + path: document.path.clone(), + source: Box::new(source), + })?; + let classifications = classify::classify_directory(directory, settings)?; + database + .replace_directory_classifications(&document.path, &classifications) + .map_err(|source| IndexError::StoreDirectoryClassifications { + path: document.path.clone(), + source: Box::new(source), + })?; + report.directories_indexed += 1; + + let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { + path: directory.to_path_buf(), + source, + })?; + + for entry in entries { + let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { + path: directory.to_path_buf(), + source, + })?; + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_string_lossy(); + + let file_type = entry + .file_type() + .map_err(|source| IndexError::ReadFileType { + path: path.clone(), + source, + })?; + + if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { + database + .delete_path_tree(&path.to_string_lossy()) + .map_err(|source| IndexError::PruneExcludedPath { + path: path.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + report.entries_skipped += 1; + continue; + } + + if file_type.is_dir() { + let Some(child_position) = depth_position.child_position() else { + prune_unindexed_directory(database, report, &path)?; + continue; + }; + + if exceeds_max_depth(settings, child_position) { + prune_unindexed_directory(database, report, &path)?; + continue; + } + + scan_directory( + &path, + settings, + database, + embedder, + report, + progress, + child_position, + )?; + } else if file_type.is_file() { + if settings.index.is_excluded_name(&name) { + report.entries_skipped += 1; + continue; + } + + report.files_seen += 1; + if let Some(indexed_file) = file::index_text_file(&path, directory, settings, embedder)? + { + report.text_files_indexed += 1; + report.file_chunks_indexed += + u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); + database.upsert_file(&indexed_file.file).map_err(|source| { + IndexError::StoreFile { + path: indexed_file.file.path.clone(), + source: Box::new(source), + } + })?; + database + .replace_file_chunks(&indexed_file.file.path, &indexed_file.chunks) + .map_err(|source| IndexError::StoreFileChunks { + path: indexed_file.file.path, + source: Box::new(source), + })?; + } + } else { + report.entries_skipped += 1; + } + } + + Ok(()) +} + +fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { + match depth_position { + DepthPosition::IndexRoot => false, + DepthPosition::TopLevelDirectory { depth } => { + depth > settings.index.max_depth_per_top_level_directory + } + } +} + +fn prune_unindexed_directory( + database: &Database, + report: &mut IndexReport, + path: &Path, +) -> Result<()> { + database + .delete_path_tree(&path.to_string_lossy()) + .map_err(|source| IndexError::PruneExcludedPath { + path: path.to_string_lossy().into_owned(), + source: Box::new(source), + })?; + report.entries_skipped += 1; + Ok(()) +} + +#[allow(dead_code)] +fn _assert_document_is_send(document: IndexedDocument) -> IndexedDocument { + document +} diff --git a/src/index/summary.rs b/src/index/summary.rs new file mode 100644 index 0000000..58421c4 --- /dev/null +++ b/src/index/summary.rs @@ -0,0 +1,249 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::config::Settings; +use crate::db::{DocumentKind, IndexedDocument}; +use crate::index::{IndexError, Result}; + +pub fn summarize_directory(directory: &Path, settings: &Settings) -> Result { + let metadata = fs::metadata(directory).map_err(|source| IndexError::ReadMetadata { + path: directory.to_path_buf(), + source, + })?; + let name = directory + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_string_lossy() + .into_owned(); + let size_bytes = metadata.len(); + let created_unix_seconds = metadata.created().ok().map(unix_seconds); + let modified_unix_seconds = unix_seconds(metadata.modified().unwrap_or(UNIX_EPOCH)); + let accessed_unix_seconds = metadata.accessed().ok().map(unix_seconds); + let readonly = metadata.permissions().readonly(); + let indexed_unix_seconds = unix_seconds(SystemTime::now()); + let searchable_text = searchable_text( + directory, + &name, + &DirectoryMetadata { + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + }, + settings, + )?; + Ok(IndexedDocument { + path: path_to_string(directory), + name, + kind: DocumentKind::Directory, + parent_path: directory.parent().map(path_to_string), + searchable_text, + embedding: Vec::new(), + metadata_fingerprint: format!( + "mtime:{modified_unix_seconds}:ctime:{created}:atime:{accessed}:len:{size_bytes}:readonly:{readonly}", + created = created_unix_seconds.unwrap_or(0), + accessed = accessed_unix_seconds.unwrap_or(0), + ), + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds, + }) +} + +#[derive(Debug, Clone, Copy)] +struct DirectoryMetadata { + size_bytes: u64, + created_unix_seconds: Option, + modified_unix_seconds: i64, + accessed_unix_seconds: Option, + readonly: bool, +} + +fn searchable_text( + directory: &Path, + name: &str, + metadata: &DirectoryMetadata, + settings: &Settings, +) -> Result { + let mut lines = vec![ + format!("directory: {}", path_to_string(directory)), + format!("name: {name}"), + "type: directory".to_string(), + format!("size bytes: {}", metadata.size_bytes), + format!( + "created unix seconds: {}", + metadata + .created_unix_seconds + .map(|seconds| seconds.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ), + format!("modified unix seconds: {}", metadata.modified_unix_seconds), + format!( + "accessed unix seconds: {}", + metadata + .accessed_unix_seconds + .map(|seconds| seconds.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ), + format!("readonly: {}", metadata.readonly), + ]; + + let mut child_dirs = Vec::new(); + let mut child_files = Vec::new(); + let mut excerpts = Vec::new(); + + let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { + path: directory.to_path_buf(), + source, + })?; + + for entry in entries.take(settings.index.max_entries_per_directory) { + let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { + path: directory.to_path_buf(), + source, + })?; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().to_string(); + + let file_type = entry + .file_type() + .map_err(|source| IndexError::ReadFileType { + path: path.clone(), + source, + })?; + + if file_type.is_dir() { + if settings.index.is_excluded_directory_name(&name) { + continue; + } + + child_dirs.push(name); + } else if file_type.is_file() { + if settings.index.is_excluded_name(&name) { + continue; + } + + child_files.push(name.clone()); + if is_high_signal_file(&name) + && let Some(excerpt) = read_text_excerpt( + &path, + settings.index.max_file_bytes, + settings.index.max_excerpt_bytes, + )? + { + excerpts.push(format!("file {name}: {excerpt}")); + } + } + } + + child_dirs.sort(); + child_files.sort(); + excerpts.sort(); + + if !child_dirs.is_empty() { + lines.push(format!("child directories: {}", child_dirs.join(", "))); + } + + if !child_files.is_empty() { + lines.push(format!("child files: {}", child_files.join(", "))); + } + + lines.extend(excerpts); + Ok(lines.join("\n")) +} + +fn is_high_signal_file(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower == "cargo.toml" + || lower == "package.json" + || lower == "pyproject.toml" + || lower == "go.mod" + || lower == "gemfile" + || lower.starts_with("readme") +} + +fn read_text_excerpt( + path: &Path, + max_file_bytes: u64, + max_excerpt_bytes: usize, +) -> Result> { + let metadata = fs::metadata(path).map_err(|source| IndexError::StatFile { + path: path.to_path_buf(), + source, + })?; + if metadata.len() > max_file_bytes { + return Ok(None); + } + + let bytes = fs::read(path).map_err(|source| IndexError::ReadFile { + path: path.to_path_buf(), + source, + })?; + if bytes.contains(&0) { + return Ok(None); + } + + let mut excerpt = String::from_utf8_lossy(&bytes).into_owned(); + excerpt = excerpt.chars().take(max_excerpt_bytes).collect(); + Ok(Some(normalize_whitespace(&excerpt))) +} + +fn normalize_whitespace(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn unix_seconds(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn directory_summary_includes_names_and_high_signal_excerpt() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("src")).unwrap(); + fs::create_dir(temp.path().join(".vscode")).unwrap(); + fs::write(temp.path().join("README.md"), "Chrome extension workspace").unwrap(); + fs::write(temp.path().join(".env"), "SECRET=true").unwrap(); + + let settings = Settings::default(); + let document = summarize_directory(temp.path(), &settings).unwrap(); + + assert_eq!( + document.name, + temp.path() + .file_name() + .unwrap_or_else(|| OsStr::new("")) + .to_string_lossy() + ); + assert_eq!(document.kind, DocumentKind::Directory); + assert!(document.embedding.is_empty()); + assert!(document.size_bytes > 0); + assert!(document.modified_unix_seconds > 0); + assert!(document.searchable_text.contains("child directories: src")); + assert!(!document.searchable_text.contains(".vscode")); + assert!(document.searchable_text.contains("README.md")); + assert!(document.searchable_text.contains("type: directory")); + assert!(document.searchable_text.contains("size bytes:")); + assert!(document.searchable_text.contains("modified unix seconds:")); + assert!( + document + .searchable_text + .contains("Chrome extension workspace") + ); + assert!(!document.searchable_text.contains("SECRET=true")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6134ad9..1feaa89 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,13 @@ use std::ffi::{OsStr, OsString}; +pub mod app; pub mod cli; +pub mod config; +pub mod db; +pub mod embed; +pub mod error; +pub mod index; +pub mod search; pub fn emit_cd_script(args: &[OsString]) -> Vec { let request = CdRequest::new(args.to_vec()); @@ -77,7 +84,14 @@ const BASH_ZSH_INIT: &str = r#"cds() { local __cds_script local __cds_status - __cds_script="$(command cds __cds_emit -- "$@")" + case "${1-}" in + --dir-type-count|--init|--index|--reset|--search|--help|-h|--version|-V|--shell-init) + command cds "$@" + return $? + ;; + esac + + __cds_script="$(command cds --cds-emit -- "$@")" __cds_status=$? if [ "$__cds_status" -ne 0 ]; then return "$__cds_status" diff --git a/src/main.rs b/src/main.rs index 7a418cd..8a07ce2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,30 +1,199 @@ use std::env; use std::io::{self, Write}; +use std::path::Path; use std::process::ExitCode; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, +}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; +use cds::app; use cds::cli::{Invocation, parse_invocation}; -use cds::{emit_cd_script, shell_init}; -use clap::error::ErrorKind; +use cds::index::IndexProgress; +use cds::{error, shell_init}; +use color_eyre::eyre::{Result, WrapErr}; fn main() -> ExitCode { - match run() { - Ok(()) => ExitCode::SUCCESS, + if let Err(err) = install_error_reporter() { + eprintln!("cds: failed to install error reporter: {err}"); + return ExitCode::FAILURE; + } + + let invocation = match parse_invocation(env::args_os().skip(1)) { + Ok(invocation) => invocation, Err(err) => err.exit(), + }; + + match run(invocation) { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + let report = color_eyre::eyre::Report::from(err); + eprintln!("{report:?}"); + ExitCode::FAILURE + } + } +} + +fn install_error_reporter() -> Result<()> { + color_eyre::config::HookBuilder::default() + .display_location_section(false) + .display_env_section(false) + .install() + .wrap_err("failed to install color-eyre error reporter") +} + +fn run(invocation: Invocation) -> error::Result<()> { + match invocation { + Invocation::DirectoryTypeCount => { + for count in app::directory_type_counts()? { + println!("{}\t{}", count.count, count.label); + } + } + Invocation::EmitCd { args } => write_stdout(&app::resolve_cd_script(args))?, + Invocation::ShellInit { shell } => write_stdout(shell_init(shell).as_bytes())?, + Invocation::Init => { + let mut progress = TerminalIndexProgress::start(); + let report = app::init_with_progress(&mut progress)?; + progress.finish(); + println!("config ready: {}", report.config_file.display()); + println!("database ready: {}", report.database_file.display()); + println!("{}", report.index.human_summary()); + } + Invocation::Index { roots } => { + let mut progress = TerminalIndexProgress::start(); + let report = app::index_with_progress(roots, &mut progress)?; + progress.finish(); + println!("{}", report.human_summary()); + } + Invocation::Reset => { + if confirm_reset()? { + app::reset_database()?; + println!("database reset"); + } else { + println!("reset cancelled"); + } + } + Invocation::Search { query } => { + let results = app::search(query, 10)?; + for result in results { + println!("{:.3}\t{}", result.score, result.path); + } + } + } + + Ok(()) +} + +fn write_stdout(bytes: &[u8]) -> error::Result<()> { + io::stdout().write_all(bytes).map_err(error::Error::Stdout) +} + +fn confirm_reset() -> error::Result { + print!( + "This will delete all data in the cds database and it is irreversable. Continue [y/n]? " + ); + io::stdout().flush().map_err(error::Error::Stdout)?; + + let mut answer = String::new(); + io::stdin() + .read_line(&mut answer) + .map_err(error::Error::Stdin)?; + + Ok(matches!( + answer.trim().to_ascii_lowercase().as_str(), + "y" | "yes" + )) +} + +#[derive(Debug, Default)] +struct ProgressState { + current_directory: Option, + tick: usize, + last_len: usize, +} + +struct TerminalIndexProgress { + state: Arc>, + stop: Arc, + worker: Option>, +} + +impl TerminalIndexProgress { + fn start() -> Self { + let state = Arc::new(Mutex::new(ProgressState::default())); + let stop = Arc::new(AtomicBool::new(false)); + let worker = Some(spawn_progress_worker(Arc::clone(&state), Arc::clone(&stop))); + + Self { + state, + stop, + worker, + } + } + + fn finish(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } } } -fn run() -> Result<(), clap::Error> { - match parse_invocation(env::args_os().skip(1))? { - Invocation::EmitCd { args } => write_stdout(&emit_cd_script(&args)), - Invocation::ShellInit { shell } => write_stdout(shell_init(shell).as_bytes()), +impl Drop for TerminalIndexProgress { + fn drop(&mut self) { + self.finish(); } - .map_err(write_error) } -fn write_stdout(bytes: &[u8]) -> io::Result<()> { - io::stdout().write_all(bytes) +impl IndexProgress for TerminalIndexProgress { + fn directory_started(&mut self, directory: &Path) { + let Ok(mut state) = self.state.lock() else { + return; + }; + + state.current_directory = Some(directory.display().to_string()); + state.tick = 0; + render_progress_line(&mut state); + } } -fn write_error(err: io::Error) -> clap::Error { - clap::Error::raw(ErrorKind::Io, format!("cds: failed to write stdout: {err}")) +fn spawn_progress_worker( + state: Arc>, + stop: Arc, +) -> JoinHandle<()> { + thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if let Ok(mut state) = state.lock() { + render_progress_line(&mut state); + } + + thread::sleep(Duration::from_millis(180)); + } + + if let Ok(mut state) = state.lock() + && state.last_len > 0 + { + eprint!("\r{}\r", " ".repeat(state.last_len)); + let _ = io::stderr().flush(); + state.last_len = 0; + } + }) +} + +fn render_progress_line(state: &mut ProgressState) { + let Some(directory) = &state.current_directory else { + return; + }; + + let dots = ".".repeat((state.tick % 3) + 1); + let line = format!("Indexing: {directory}{dots}"); + let padding = " ".repeat(state.last_len.saturating_sub(line.len())); + + eprint!("\r{line}{padding}"); + let _ = io::stderr().flush(); + + state.last_len = line.len(); + state.tick = state.tick.wrapping_add(1); } diff --git a/src/search/error.rs b/src/search/error.rs new file mode 100644 index 0000000..6447487 --- /dev/null +++ b/src/search/error.rs @@ -0,0 +1,19 @@ +use thiserror::Error; + +use crate::db::DbError; +use crate::embed::EmbedError; + +#[derive(Debug, Error)] +pub enum SearchError { + #[error("failed to embed query")] + EmbedQuery { + #[source] + source: EmbedError, + }, + + #[error("failed to load indexed directories")] + LoadDirectories { + #[source] + source: DbError, + }, +} diff --git a/src/search/mod.rs b/src/search/mod.rs new file mode 100644 index 0000000..18e664f --- /dev/null +++ b/src/search/mod.rs @@ -0,0 +1,374 @@ +mod error; + +use std::cmp::Ordering; +use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::db::{Database, DirectoryClassification}; +use crate::embed::Embedder; + +pub use error::SearchError; + +pub type Result = std::result::Result; + +#[derive(Debug, Clone, PartialEq)] +pub struct SearchResult { + pub path: String, + pub score: f32, +} + +pub struct Searcher<'a, E> { + database: &'a Database, + embedder: &'a E, +} + +impl<'a, E> Searcher<'a, E> +where + E: Embedder, +{ + pub fn new(database: &'a Database, embedder: &'a E) -> Self { + Self { database, embedder } + } + + pub fn search(&self, query: &str, limit: usize) -> Result> { + let query_embedding = self + .embedder + .embed(query) + .map_err(|source| SearchError::EmbedQuery { source })?; + let chunks = self + .database + .file_chunk_matches() + .map_err(|source| SearchError::LoadDirectories { source })?; + let temporal_filter = TemporalFilter::from_query(query); + let use_deep_target = wants_deep_target(query); + let query_terms = query_terms(query); + + let mut scores = HashMap::::new(); + for chunk in chunks { + let mut score = cosine_similarity(&query_embedding, &chunk.embedding); + if score <= 0.0 { + continue; + } + + let classifications = self + .database + .ancestor_classifications(&chunk.directory_path) + .map_err(|source| SearchError::LoadDirectories { source })?; + score += lexical_boost( + &query_terms, + [ + chunk.content.as_str(), + chunk.file_name.as_str(), + chunk.directory_path.as_str(), + ], + ); + score += classification_boost(&query_terms, &classifications); + score *= temporal_filter.score_multiplier( + chunk + .file_modified_unix_seconds + .max(chunk.directory_modified_unix_seconds), + ); + + let target = if use_deep_target { + chunk.directory_path + } else { + self.database + .general_indexed_directory(&chunk.directory_path) + .map_err(|source| SearchError::LoadDirectories { source })? + }; + + scores + .entry(target) + .and_modify(|existing| *existing = existing.max(score)) + .or_insert(score); + } + + for directory in self + .database + .directory_documents() + .map_err(|source| SearchError::LoadDirectories { source })? + { + let classifications = self + .database + .directory_classifications(&directory.path) + .map_err(|source| SearchError::LoadDirectories { source })?; + let score = classification_boost(&query_terms, &classifications) + + lexical_boost( + &query_terms, + [directory.path.as_str(), directory.name.as_str()], + ); + if score <= 0.0 { + continue; + } + + let target = if use_deep_target { + directory.path + } else { + self.database + .general_indexed_directory(&directory.path) + .map_err(|source| SearchError::LoadDirectories { source })? + }; + + scores + .entry(target) + .and_modify(|existing| *existing = existing.max(score)) + .or_insert(score); + } + + let mut results = scores + .into_iter() + .filter_map(|(path, score)| (score > 0.0).then_some(SearchResult { path, score })) + .collect::>(); + + results.sort_by(compare_results); + results.truncate(limit); + Ok(results) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TemporalFilter { + min_modified_unix_seconds: Option, +} + +impl TemporalFilter { + fn from_query(query: &str) -> Self { + let lower = query.to_ascii_lowercase(); + let seconds = unix_now(); + + let min_modified_unix_seconds = if lower.contains("today") { + Some(seconds - 24 * 60 * 60) + } else if lower.contains("yesterday") { + Some(seconds - 2 * 24 * 60 * 60) + } else if lower.contains("last week") { + Some(seconds - 14 * 24 * 60 * 60) + } else if lower.contains("recent") || lower.contains("recently") { + Some(seconds - 30 * 24 * 60 * 60) + } else { + None + }; + + Self { + min_modified_unix_seconds, + } + } + + fn score_multiplier(self, modified_unix_seconds: i64) -> f32 { + let Some(min_modified_unix_seconds) = self.min_modified_unix_seconds else { + return 1.0; + }; + + if modified_unix_seconds >= min_modified_unix_seconds { + 1.35 + } else { + 0.25 + } + } +} + +fn wants_deep_target(query: &str) -> bool { + let lower = query.to_ascii_lowercase(); + [ + "migration", + "migrations", + "test", + "tests", + "spec", + "schema", + "schemas", + "component", + "components", + "route", + "routes", + "controller", + "controllers", + "model", + "models", + "config", + ] + .iter() + .any(|signal| lower.contains(signal)) +} + +fn query_terms(query: &str) -> Vec { + query + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|term| term.len() > 2) + .map(|term| term.to_ascii_lowercase()) + .collect() +} + +fn lexical_boost<'a>(query_terms: &[String], haystacks: impl IntoIterator) -> f32 { + let haystack = haystacks + .into_iter() + .map(str::to_ascii_lowercase) + .collect::>() + .join("\n"); + + query_terms + .iter() + .filter(|term| haystack.contains(term.as_str())) + .count() as f32 + * 0.04 +} + +fn classification_boost( + query_terms: &[String], + classifications: &[DirectoryClassification], +) -> f32 { + classifications + .iter() + .map(|classification| { + let label_terms = query_terms_from_label(&classification.label); + if label_terms.is_empty() { + return 0.0; + } + + let matched_terms = label_terms + .iter() + .filter(|term| query_terms.contains(term)) + .count(); + + if matched_terms == label_terms.len() { + classification.confidence * 1.2 + } else if matched_terms > 0 { + classification.confidence * 0.25 * matched_terms as f32 + } else { + 0.0 + } + }) + .sum() +} + +fn query_terms_from_label(label: &str) -> Vec { + query_terms(label) +} + +fn unix_now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +fn compare_results(left: &SearchResult, right: &SearchResult) -> Ordering { + right + .score + .partial_cmp(&left.score) + .unwrap_or(Ordering::Equal) + .then_with(|| left.path.cmp(&right.path)) +} + +fn cosine_similarity(left: &[f32], right: &[f32]) -> f32 { + if left.is_empty() || right.is_empty() || left.len() != right.len() { + return 0.0; + } + + let mut dot = 0.0; + let mut left_norm = 0.0; + let mut right_norm = 0.0; + + for (left, right) in left.iter().zip(right) { + dot += left * right; + left_norm += left * left; + right_norm += right * right; + } + + if left_norm == 0.0 || right_norm == 0.0 { + return 0.0; + } + + dot / (left_norm.sqrt() * right_norm.sqrt()) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use crate::config::Settings; + use crate::db::Database; + use crate::embed::FakeEmbedder; + use crate::index::Indexer; + + #[test] + fn returns_best_directory_match() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let chrome = root.join("chrome-extension"); + let notes = root.join("meeting-notes"); + fs::create_dir_all(&chrome).unwrap(); + fs::create_dir_all(¬es).unwrap(); + fs::write( + chrome.join("README.md"), + "Chrome extension manifest browser popup", + ) + .unwrap(); + fs::write(notes.join("README.md"), "Meeting notes calendar agenda").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("chrome extension", 3) + .unwrap(); + + assert_eq!(results.first().unwrap().path, chrome.to_string_lossy()); + } + + #[test] + fn returns_directory_by_deterministic_type_classification() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let rust = root.join("cds-rust"); + let notes = root.join("notes"); + fs::create_dir_all(&rust).unwrap(); + fs::create_dir_all(¬es).unwrap(); + fs::write(rust.join("Cargo.toml"), "[package]\nname = \"cds\"").unwrap(); + fs::write(notes.join("README.md"), "rust colored notes without cargo").unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("rust project", 3) + .unwrap(); + + assert_eq!(results.first().unwrap().path, rust.to_string_lossy()); + } + + #[test] + fn prefers_deeper_directory_when_query_has_strong_signal() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("Projects"); + let chrome = root.join("chrome-extension"); + let migrations = chrome.join("server/db/migrations"); + fs::create_dir_all(&migrations).unwrap(); + fs::write( + migrations.join("001_create_popup_events.sql"), + "create table popup_events for chrome extension telemetry migrations", + ) + .unwrap(); + + let settings = Settings::default(); + let database = Database::open_in_memory().unwrap(); + let embedder = FakeEmbedder::default(); + Indexer::new(&settings, &database, &embedder) + .index_roots(vec![root]) + .unwrap(); + + let results = Searcher::new(&database, &embedder) + .search("migrations chrome extension", 3) + .unwrap(); + + assert_eq!(results.first().unwrap().path, migrations.to_string_lossy()); + } +} diff --git a/tests/docker_cd_equivalence.rs b/tests/docker_cd_equivalence.rs index 709cfca..6221ba4 100644 --- a/tests/docker_cd_equivalence.rs +++ b/tests/docker_cd_equivalence.rs @@ -2,6 +2,19 @@ use std::fs; use std::path::PathBuf; use std::process::Command; +const DOCKER_TEST_SCRIPT: &str = r#" +set -eu +export PATH="/usr/local/cargo/bin:${CARGO_HOME:-/usr/local/cargo}/bin:${PATH}" + +if ! command -v cargo >/dev/null 2>&1; then + echo "cargo was not found in the Docker image. Use a Rust image that includes Cargo, or set CDS_DOCKER_IMAGE." >&2 + exit 127 +fi + +cargo build --quiet +bash tests/docker_cd_equivalence.sh +"#; + #[test] fn docker_cd_equivalence_random_tree() { if !docker_is_available() { @@ -38,7 +51,7 @@ fn docker_cd_equivalence_random_tree() { .arg(image) .arg("bash") .arg("-lc") - .arg("cargo build --quiet && bash tests/docker_cd_equivalence.sh") + .arg(DOCKER_TEST_SCRIPT) .output() .expect("docker cd equivalence test starts"); diff --git a/tests/docker_cd_equivalence.sh b/tests/docker_cd_equivalence.sh index 26b3e4b..871bd5d 100644 --- a/tests/docker_cd_equivalence.sh +++ b/tests/docker_cd_equivalence.sh @@ -69,6 +69,17 @@ setup_cds() { eval "$(cds --shell-init bash)" } +normalize_stderr() { + local file="$1" + local normalized + normalized="$(mktemp)" + + sed -E \ + 's#^(tests/docker_cd_equivalence\.sh: line )[0-9]+(: cd:)#\1\2#' \ + "$file" > "$normalized" + mv "$normalized" "$file" +} + run_case() { local mode="$1" local start="$2" @@ -112,6 +123,8 @@ run_case() { } > "$state_file" ) > "$out_file" 2> "$err_file" + normalize_stderr "$err_file" + printf '%s\n%s\n%s\n' "$state_file" "$out_file" "$err_file" } diff --git a/tests/install_script.rs b/tests/install_script.rs new file mode 100644 index 0000000..0cc109f --- /dev/null +++ b/tests/install_script.rs @@ -0,0 +1,100 @@ +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::process::Command; + +#[test] +fn install_script_writes_shell_integration_idempotently() { + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + let cargo_home = temp.path().join("cargo-home"); + fs::create_dir_all(&bin).unwrap(); + fs::create_dir_all(cargo_home.join("bin")).unwrap(); + + let fake_cargo = bin.join("cargo"); + fs::write( + &fake_cargo, + format!( + "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >> '{}'\nmkdir -p '{}'\ntouch '{}/cds'\nchmod +x '{}/cds'\n", + temp.path().join("cargo-args.log").display(), + cargo_home.join("bin").display(), + cargo_home.join("bin").display(), + cargo_home.join("bin").display(), + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&fake_cargo).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&fake_cargo, permissions).unwrap(); + + for _ in 0..2 { + let output = Command::new("bash") + .arg("install.sh") + .env("HOME", temp.path()) + .env("CARGO_HOME", &cargo_home) + .env("SHELL", "/bin/zsh") + .env("PATH", format!("{}:/usr/bin:/bin", bin.display())) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + let zshrc = fs::read_to_string(temp.path().join(".zshrc")).unwrap(); + assert_eq!(zshrc.matches("# >>> cds init >>>").count(), 1); + assert!(zshrc.contains("command \"")); + assert!(zshrc.contains("--shell-init zsh")); + + let cargo_args = fs::read_to_string(temp.path().join("cargo-args.log")).unwrap(); + assert_eq!(cargo_args.matches("--force").count(), 0); +} + +#[test] +fn install_script_force_passes_force_to_cargo_install() { + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + let cargo_home = temp.path().join("cargo-home"); + fs::create_dir_all(&bin).unwrap(); + fs::create_dir_all(cargo_home.join("bin")).unwrap(); + + let fake_cargo = bin.join("cargo"); + fs::write( + &fake_cargo, + format!( + "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" > '{}'\nmkdir -p '{}'\ntouch '{}/cds'\nchmod +x '{}/cds'\n", + temp.path().join("cargo-args.log").display(), + cargo_home.join("bin").display(), + cargo_home.join("bin").display(), + cargo_home.join("bin").display(), + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&fake_cargo).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&fake_cargo, permissions).unwrap(); + + let output = Command::new("bash") + .arg("install.sh") + .arg("--force") + .env("HOME", temp.path()) + .env("CARGO_HOME", &cargo_home) + .env("SHELL", "/bin/zsh") + .env("PATH", format!("{}:/usr/bin:/bin", bin.display())) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let cargo_args = fs::read_to_string(temp.path().join("cargo-args.log")).unwrap(); + assert!(cargo_args.contains("install --path")); + assert!(cargo_args.contains("--force")); +} diff --git a/tests/search_integration.rs b/tests/search_integration.rs new file mode 100644 index 0000000..dc1d61f --- /dev/null +++ b/tests/search_integration.rs @@ -0,0 +1,164 @@ +use std::io::Write; +use std::process::{Command, Stdio}; + +#[test] +fn hidden_emit_searches_when_local_prefix_is_absent() { + let fixture = SearchFixture::new(); + fixture.init(); + + let cwd = fixture.temp.path().join("cwd"); + std::fs::create_dir_all(&cwd).unwrap(); + std::fs::create_dir(cwd.join("alpha")).unwrap(); + + let output = fixture + .cds() + .arg("--cds-emit") + .arg("--") + .arg("manifest") + .current_dir(&cwd) + .env("PWD", &cwd) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let expected = format!( + "builtin cd '{}'\n", + fixture.project_dir.to_string_lossy().replace('\'', "'\\''") + ); + assert_eq!(String::from_utf8(output.stdout).unwrap(), expected); +} + +#[test] +fn hidden_emit_preserves_cd_when_local_prefix_exists() { + let fixture = SearchFixture::new(); + fixture.init(); + + let cwd = fixture.temp.path().join("cwd"); + std::fs::create_dir_all(&cwd).unwrap(); + std::fs::create_dir(cwd.join("music")).unwrap(); + + let output = fixture + .cds() + .arg("--cds-emit") + .arg("--") + .arg("manifest") + .current_dir(&cwd) + .env("PWD", &cwd) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "builtin cd 'manifest'\n" + ); +} + +#[test] +fn dir_type_count_lists_detected_directory_types() { + let fixture = SearchFixture::new(); + fixture.init(); + + let output = fixture.cds().arg("--dir-type-count").output().unwrap(); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert!(stdout.contains("1\tchrome extension"), "{stdout}"); + assert!(stdout.contains("1\trust project"), "{stdout}"); +} + +#[test] +fn reset_prompts_and_deletes_indexed_data_when_confirmed() { + let fixture = SearchFixture::new(); + fixture.init(); + + let mut child = fixture + .cds() + .arg("--reset") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.as_mut().unwrap().write_all(b"y\n").unwrap(); + let output = child.wait_with_output().unwrap(); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert!(stdout.contains( + "This will delete all data in the cds database and it is irreversable. Continue [y/n]?" + )); + assert!(stdout.contains("database reset")); + + let counts = fixture.cds().arg("--dir-type-count").output().unwrap(); + assert!(counts.status.success()); + assert_eq!(String::from_utf8(counts.stdout).unwrap(), ""); +} + +struct SearchFixture { + temp: tempfile::TempDir, + project_dir: std::path::PathBuf, +} + +impl SearchFixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let projects = temp.path().join("Projects"); + let project_dir = projects.join("chrome-extension"); + std::fs::create_dir_all(project_dir.join("src")).unwrap(); + std::fs::write( + project_dir.join("README.md"), + "Chrome extension manifest browser popup", + ) + .unwrap(); + std::fs::write( + project_dir.join("manifest.json"), + r#"{"manifest_version":3,"action":{},"permissions":["storage"]}"#, + ) + .unwrap(); + let rust_dir = projects.join("rust-tool"); + std::fs::create_dir_all(&rust_dir).unwrap(); + std::fs::write( + rust_dir.join("Cargo.toml"), + "[package]\nname = \"rust-tool\"", + ) + .unwrap(); + + Self { temp, project_dir } + } + + fn cds(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_cds")); + command + .env("HOME", self.temp.path()) + .env("CDS_CONFIG_DIR", self.temp.path().join("config")) + .env("CDS_DATA_DIR", self.temp.path().join("data")) + .env("CDS_CACHE_DIR", self.temp.path().join("cache")); + command + } + + fn init(&self) { + let init = self.cds().arg("--init").output().unwrap(); + assert!( + init.status.success(), + "init stderr: {}", + String::from_utf8_lossy(&init.stderr) + ); + } +} diff --git a/tests/shell_integration.rs b/tests/shell_integration.rs index 5e157f6..d9cd919 100644 --- a/tests/shell_integration.rs +++ b/tests/shell_integration.rs @@ -33,11 +33,20 @@ fn run_shell_cd_smoke_test(shell: &str, init_shell: &str) { r#" set -e eval "$("{bin}" --shell-init {init_shell})" +eval "$(cds --shell-init {init_shell})" tmp="$(mktemp -d)" -mkdir -p "$tmp/a/b" "$tmp/cpath/d" +mkdir -p "$tmp/a/b" "$tmp/cpath/d" "$tmp/search" "$tmp/index" ln -s "$tmp/a" "$tmp/linka" +cd "$tmp" +cds search +[ "$(pwd -P)" = "$(cd "$tmp/search" && pwd -P)" ] + +cd "$tmp" +cds index +[ "$(pwd -P)" = "$(cd "$tmp/index" && pwd -P)" ] + cd "$tmp/a" cds "$tmp/linka/b" [ "$PWD" = "$tmp/linka/b" ] @@ -59,6 +68,13 @@ cd "$tmp/cpath" >/dev/null cds - >/dev/null expected="$(cd "$tmp/a" && pwd -P)" [ "$(pwd -P)" = "$expected" ] + +home="$tmp/home" +mkdir -p "$home/Projects/cli-init-test" +printf 'shell integration init project\n' > "$home/Projects/cli-init-test/README.md" +HOME="$home" CDS_CONFIG_DIR="$tmp/config" CDS_DATA_DIR="$tmp/data" CDS_CACHE_DIR="$tmp/cache" cds --init >/tmp/cds-shell-init.out +[ -f "$tmp/config/config.json" ] +[ -f "$tmp/data/cds.sqlite" ] "# );