diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf9e358..fa69d4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,5 +91,5 @@ jobs: - name: Run Docker cd equivalence test env: - CDS_DOCKER_IMAGE: rust:1-slim + CDS_DOCKER_IMAGE: rust:1 run: cargo test --test docker_cd_equivalence -- --nocapture diff --git a/AGENTS.md b/AGENTS.md index df028d0..840a049 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,11 +44,11 @@ Prefer stable, well-maintained crates: For embeddings, prefer an implementation path that keeps the tool locally usable: -- Use `bge-small-en-v1.5` through a local model runtime such as ONNX Runtime, Candle, or - another Rust-friendly inference backend. +- Use `BAAI/bge-small-en-v1.5` through FastEmbed/ONNX for production text embeddings. - Do not introduce a hosted embedding API as the default path. - If model download/setup is needed, make it explicit and cache model files outside the repo in a user cache directory. +- Keep deterministic fake embeddings available for normal tests with `CDS_EMBEDDER=fake`. ## Suggested Architecture @@ -183,13 +183,16 @@ The Docker equivalence test runs the project inside a Rust container and compare against Bash's built-in `cd` for status, stdout, stderr, `PWD`, `OLDPWD`, and physical path. Keep these details in mind when editing it: -- CI currently uses `CDS_DOCKER_IMAGE=rust:1-slim`. -- The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some slim image +- CI currently uses `CDS_DOCKER_IMAGE=rust:1`. +- The Docker image must include Cargo and C++ standard library/linker support. The + FastEmbed/ONNX dependency links against `libstdc++`, so very small images such as + `rust:1-slim` can fail with `unable to find library -lstdc++`. +- The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some container invocations otherwise fail with `cargo: command not found`. - Bash includes source line numbers in `cd` diagnostics. Normalize only those line numbers before comparing stderr, while keeping the actual diagnostic text exact. - The Docker test can skip locally when Docker is unavailable, so use - `CDS_DOCKER_IMAGE=rust:1-slim cargo test --test docker_cd_equivalence -- --nocapture` + `CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence -- --nocapture` with Docker access when changing equivalence behavior. ## Repository Hygiene diff --git a/Cargo.lock b/Cargo.lock index c1713cf..677f017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,29 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -88,6 +111,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" @@ -109,6 +138,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -139,6 +174,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "byteorder" version = "1.5.0" @@ -151,6 +192,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.63" @@ -167,6 +217,7 @@ version = "0.1.0" dependencies = [ "clap", "color-eyre", + "fastembed", "serde", "serde_json", "sqlx", @@ -181,6 +232,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" version = "4.6.1" @@ -241,6 +298,21 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -250,12 +322,53 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -280,6 +393,34 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -305,6 +446,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "der" version = "0.7.10" @@ -316,6 +501,46 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -328,6 +553,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -339,6 +585,15 @@ dependencies = [ "syn", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -354,6 +609,12 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.2" @@ -370,6 +631,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etcetera" version = "0.8.0" @@ -402,6 +669,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fastembed" +version = "5.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65faeb787d66e812248f992693268eee8a43a95e3e43e0c33547f1c8391ecce" +dependencies = [ + "anyhow", + "hf-hub", + "ndarray", + "ort", + "safetensors", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -414,6 +697,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -425,12 +718,24 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -484,6 +789,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -504,6 +820,7 @@ checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -528,8 +845,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -540,7 +873,7 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] @@ -559,7 +892,20 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -589,6 +935,26 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs", + "http", + "indicatif", + "libc", + "log", + "rand 0.9.4", + "reqwest", + "serde", + "serde_json", + "thiserror", + "ureq", + "windows-sys 0.61.2", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -607,6 +973,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "home" version = "0.5.12" @@ -617,30 +989,128 @@ dependencies = [ ] [[package]] -name = "icu_collections" -version = "2.2.0" +name = "http" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", + "bytes", + "itoa", ] [[package]] -name = "icu_locale_core" -version = "2.2.0" +name = "http-body" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] @@ -704,6 +1174,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -743,18 +1219,58 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -817,6 +1333,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -832,6 +1354,44 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -848,6 +1408,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -855,6 +1421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -868,6 +1435,53 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -879,11 +1493,26 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.6", "smallvec", "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -935,6 +1564,58 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -970,6 +1651,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1024,6 +1711,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1033,6 +1735,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1061,6 +1769,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -1070,6 +1833,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -1083,8 +1852,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1094,7 +1873,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1106,6 +1895,52 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1124,6 +1959,101 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rsa" version = "0.9.10" @@ -1137,7 +2067,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -1150,6 +2080,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "1.1.4" @@ -1163,12 +2099,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5" +dependencies = [ + "hashbrown 0.16.1", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1280,9 +2269,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "slab" version = "0.4.12" @@ -1308,6 +2303,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.9.8" @@ -1327,6 +2333,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlx" version = "0.8.6" @@ -1346,7 +2364,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", @@ -1417,7 +2435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -1439,7 +2457,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.6", "rsa", "sha1", "sha2", @@ -1458,7 +2476,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "crc", @@ -1476,7 +2494,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.6", "serde", "serde_json", "sha2", @@ -1518,6 +2536,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" version = "0.1.5" @@ -1552,6 +2576,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1605,6 +2638,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1630,6 +2694,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -1656,6 +2753,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -1667,6 +2774,64 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -1721,6 +2886,12 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.1" @@ -1748,18 +2919,90 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -1772,6 +3015,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -1802,6 +3051,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1832,6 +3090,61 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + [[package]] name = "wasm-encoder" version = "0.244.0" @@ -1854,6 +3167,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -1866,6 +3192,35 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "1.6.1" @@ -1876,6 +3231,28 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -1888,7 +3265,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -1906,13 +3292,29 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -1921,42 +3323,90 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index d31edcf..540b35a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] clap = { version = "4.5.40", features = ["cargo"] } color-eyre = "0.6.5" +fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "sqlite", "migrate", "macros"] } diff --git a/README.md b/README.md index 7d430e7..98a09ad 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ the directory you're looking for and invoke cd to navigate you there. Embeddings get expensive fast and I have to use a very tiny model (bge-small-en-v1.5) to keep this lean. If we have a ton of parameteres in the database A: The db would be *huge* and B: The search would take a lot of time. - ## Build ```sh @@ -70,9 +69,12 @@ stdout, stderr, `PWD`, `OLDPWD`, and physical path behavior. Use a specific container image with: ```sh -CDS_DOCKER_IMAGE=rust:1-slim cargo test --test docker_cd_equivalence +CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence ``` +The Docker image must include Cargo and the C++ standard library/linker support required by +the embedding runtime. The default image is `rust:1`. + ## Indexing `cds` only indexes directories you configure. It does not crawl the whole filesystem. @@ -143,16 +145,37 @@ Hidden directories are always skipped and pruned from the local index. Each conf root is treated as a container, and each top-level directory inside it is indexed only through `max_depth_per_top_level_directory` levels. -The embedder is still deterministic and fake for now. That is deliberate: it lets the config, -schema, scanning, and ranking plumbing settle before the real `bge-small-en-v1.5` runtime is -wired in. +Text-file chunks are embedded locally with `BAAI/bge-small-en-v1.5` through FastEmbed/ONNX. +Model files are cached under `~/.cache/cds/models` by default, or under `CDS_CACHE_DIR/models` +when that environment variable is set. Tests can force deterministic fake embeddings with +`CDS_EMBEDDER=fake`. ## Directory Types -Directory type inference is rule-based and data-driven. Built-in types such as `rust project` -and `chrome extension` are defined as JSON. Custom/community detectors can be added to the -`detectors` array in `~/.config/cds/config.json`; `cds --index` loads them every time it -indexes. +Directory type inference is rule-based and data-driven. `cds` does not ask the embedding +model to guess whether a directory is a `rust project`, `chrome extension`, `rails app`, etc. +Instead, it runs JSON detectors against each indexed directory during `cds --init` and +`cds --index`. + +Built-in detectors live in `src/index/directory_types/` and currently cover: + +- `chrome extension` +- `database migrations` +- `next.js app` +- `node project` +- `python project` +- `rails app` +- `rust project` + +Each detector has a `label` and one or more `rules`. A rule matches only when all of its +`signals` match. If a detector has multiple matching rules for the same directory, `cds` +keeps one classification for that label and prefers the highest-confidence rule. Each stored +classification includes the directory path, label, confidence, rule id, evidence path, +evidence summary, and detection timestamp. + +Custom detectors are added to the `detectors` array in `~/.config/cds/config.json`. +`cds --index` loads that array every time it indexes, so editing the config and re-running +`cds --index` is enough to apply a new directory type locally. Example detector: @@ -176,9 +199,82 @@ Example detector: } ``` -Supported signal kinds are `file_exists`, `file_contains`, `directory_name`, and `child_name`. -Each matched rule stores the label, confidence, detector id, evidence path, and evidence -summary in SQLite. +Supported signal kinds: + +- `file_exists`: matches when a relative file path exists inside the directory. +- `file_contains`: reads a small text file and checks for text. Use `contains_any`, + `contains_all`, or both. +- `directory_name`: checks the directory's own name with `equals_any` and/or + `contains_any`. +- `child_name`: checks immediate child names with `contains_any`, `starts_with_any`, + and/or `ends_with_any`. + +Signals are case-insensitive. Paths are relative to the directory being classified. +`file_contains` ignores binary files and files larger than 128 KiB. + +For example, a custom detector for Terraform projects can be added directly to +`~/.config/cds/config.json`: + +```json +{ + "index": { + "roots": ["~/Projects"], + "exclude": [ + ".git", + "node_modules", + "target", + "dist", + "build", + ".next", + ".cache", + ".venv", + "venv", + "vendor", + "*.xcassets", + "*.imageset", + "*.appiconset", + "*.colorset" + ], + "max_file_bytes": 65536, + "max_excerpt_bytes": 4096, + "max_entries_per_directory": 80, + "max_depth_per_top_level_directory": 3, + "max_chunk_bytes": 4096 + }, + "detectors": [ + { + "label": "terraform project", + "rules": [ + { + "id": "terraform_files", + "confidence": 0.95, + "evidence_summary": "directory contains Terraform files", + "signals": [ + { + "kind": "child_name", + "ends_with_any": [".tf", ".tfvars"] + } + ] + } + ] + } + ] +} +``` + +After saving the config: + +```sh +cds --index +cds --dir-type-count +``` + +To add a built-in/community detector to the project itself, add a new JSON file under +`src/index/directory_types/`, then register it in `BUILTIN_DEFINITIONS` in +`src/index/classify.rs`. Prefer high-confidence signals that are difficult to trigger by +accident. For example, checking that `manifest.json` contains `"manifest_version"` and at +least one browser-extension field is better than classifying every directory with any +`manifest.json` as a Chrome extension. Search indexed directories: @@ -203,11 +299,10 @@ type classifications. The database file and schema are kept in place. When the shell integration is installed, `cds` also tries semantic search automatically for plain directory changes that do not look like local `cd` usage. For example, `cds Projects` -first checks the current directory for any folder starting with `p`. If such a folder exists, -`cds` delegates to the shell's built-in `cd` exactly as usual. If no local folder starts with -that character, `cds` searches the index and emits a `cd` to the best existing indexed -directory. Flags, `-`, `--`, `~`, `.`, `..`, and paths containing `/` always use normal `cd` -behavior. +first checks whether `./Projects` exists exactly. If it does, `cds` delegates to the shell's +built-in `cd` exactly as usual. If it does not, `cds` searches the index and emits a `cd` to +the best existing indexed directory. Flags, `-`, `--`, `~`, `.`, `..`, and paths containing +`/` always use normal `cd` behavior. ## Shell Setup diff --git a/src/app/mod.rs b/src/app/mod.rs index 2973247..0637fcf 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -2,13 +2,12 @@ mod error; use std::env; use std::ffi::OsString; -use std::fs; use std::path::{Path, PathBuf}; use crate::config::{AppPaths, Settings, expand_tilde}; use crate::db::{Database, DirectoryTypeCount}; -use crate::embed::FakeEmbedder; -use crate::error::{Result, app_err, config_err}; +use crate::embed::{BgeSmallEmbedder, Embedder, FakeEmbedder}; +use crate::error::{Result, app_err, config_err, embed_err}; use crate::index::{IndexProgress, IndexReport, Indexer, NoopProgress}; use crate::search::{SearchResult, Searcher}; @@ -33,7 +32,7 @@ where let paths = AppPaths::discover()?; let settings = Settings::load_or_create(&paths.config_file)?; let database = Database::open(&paths.database_file).await?; - let embedder = FakeEmbedder::default(); + let embedder = RuntimeEmbedder::load(&paths)?; let indexer = Indexer::new(&settings, &database, &embedder); let index = indexer .index_configured_roots_with_progress(progress) @@ -58,7 +57,7 @@ where let paths = AppPaths::discover()?; let settings = Settings::load_or_create(&paths.config_file)?; let database = Database::open(&paths.database_file).await?; - let embedder = FakeEmbedder::default(); + let embedder = RuntimeEmbedder::load(&paths)?; let indexer = Indexer::new(&settings, &database, &embedder); if roots.is_empty() { @@ -88,7 +87,7 @@ pub async fn search(query: Vec, limit: usize) -> Result Result> { let paths = AppPaths::discover()?; let database = Database::open_existing(&paths.database_file).await?; - let embedder = FakeEmbedder::default(); + let embedder = RuntimeEmbedder::load(&paths)?; let searcher = Searcher::new(&database, &embedder); Ok(searcher.search(query, limit).await?) } @@ -106,7 +105,7 @@ pub async fn reset_database() -> Result<()> { } pub async fn resolve_cd_script(args: Vec) -> Vec { - if let Some(query) = semantic_query(&args) + if let Some(query) = implied_search_query(&args) && let Ok(results) = search_text(&query, 5).await && let Some(result) = results .into_iter() @@ -118,6 +117,10 @@ pub async fn resolve_cd_script(args: Vec) -> Vec { crate::emit_cd_script(&args) } +pub fn implied_search_query(args: &[OsString]) -> Option { + semantic_query(args) +} + fn join_query(query: Vec) -> Result { let mut parts = Vec::with_capacity(query.len()); @@ -151,13 +154,12 @@ fn semantic_query_in(args: &[OsString], directory: Option<&Path>) -> Option bool { || arg.contains('/') } -fn local_directory_starts_with(first: char, directory: Option<&Path>) -> bool { +fn local_entry_exists(name: &str, directory: Option<&Path>) -> bool { let Some(directory) = directory else { return true; }; - let Ok(entries) = fs::read_dir(directory) else { - return true; - }; - - let first = first.to_lowercase().to_string(); - - entries.filter_map(std::result::Result::ok).any(|entry| { - if !entry.path().is_dir() { - return false; - } - - entry - .file_name() - .to_string_lossy() - .to_lowercase() - .starts_with(&first) - }) + directory.join(name).exists() } fn current_shell_directory() -> Option { @@ -204,6 +190,54 @@ fn current_shell_directory() -> Option { .or_else(|| env::current_dir().ok()) } +enum RuntimeEmbedder { + Bge(Box), + Fake(FakeEmbedder), +} + +impl RuntimeEmbedder { + fn load(paths: &AppPaths) -> Result { + if env::var("CDS_EMBEDDER").is_ok_and(|value| value.eq_ignore_ascii_case("fake")) { + return Ok(Self::Fake(FakeEmbedder::default())); + } + + BgeSmallEmbedder::new(&paths.cache_dir) + .map(Box::new) + .map(Self::Bge) + .map_err(embed_err) + } +} + +impl Embedder for RuntimeEmbedder { + fn dimensions(&self) -> usize { + match self { + Self::Bge(embedder) => embedder.dimensions(), + Self::Fake(embedder) => embedder.dimensions(), + } + } + + fn embed(&self, text: &str) -> crate::embed::Result> { + match self { + Self::Bge(embedder) => embedder.embed(text), + Self::Fake(embedder) => embedder.embed(text), + } + } + + fn embed_document(&self, text: &str) -> crate::embed::Result> { + match self { + Self::Bge(embedder) => embedder.embed_document(text), + Self::Fake(embedder) => embedder.embed_document(text), + } + } + + fn embed_query(&self, text: &str) -> crate::embed::Result> { + match self { + Self::Bge(embedder) => embedder.embed_query(text), + Self::Fake(embedder) => embedder.embed_query(text), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -213,9 +247,20 @@ mod tests { } #[test] - fn semantic_query_is_allowed_when_no_local_directory_prefix_matches() { + fn semantic_query_is_allowed_when_no_exact_local_entry_matches() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir(temp.path().join("src")).unwrap(); + + assert_eq!( + semantic_query_in(&[os("Projects")], Some(temp.path())), + Some("Projects".to_string()) + ); + } + + #[test] + fn semantic_query_is_allowed_when_only_local_prefix_matches() { let temp = tempfile::tempdir().unwrap(); - fs::create_dir(temp.path().join("src")).unwrap(); + std::fs::create_dir(temp.path().join("playground")).unwrap(); assert_eq!( semantic_query_in(&[os("Projects")], Some(temp.path())), @@ -224,9 +269,9 @@ mod tests { } #[test] - fn semantic_query_is_blocked_when_local_directory_prefix_matches() { + fn semantic_query_is_blocked_when_exact_local_entry_matches() { let temp = tempfile::tempdir().unwrap(); - fs::create_dir(temp.path().join("playground")).unwrap(); + std::fs::create_dir(temp.path().join("Projects")).unwrap(); assert_eq!( semantic_query_in(&[os("Projects")], Some(temp.path())), @@ -262,4 +307,11 @@ mod tests { Some("chrome extension".to_string()) ); } + + #[test] + fn implied_search_query_uses_cd_semantic_rules() { + assert_eq!(implied_search_query(&[os("-P"), os("Projects")]), None); + assert_eq!(implied_search_query(&[os("../Projects")]), None); + assert_eq!(implied_search_query(&[]), None); + } } diff --git a/src/embed/bge.rs b/src/embed/bge.rs new file mode 100644 index 0000000..124dae9 --- /dev/null +++ b/src/embed/bge.rs @@ -0,0 +1,71 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use fastembed::{EmbeddingModel, TextEmbedding, TextInitOptions}; + +use super::{EmbedError, Embedder, Result}; + +pub const BGE_SMALL_EN_V15_DIMENSIONS: usize = 384; + +pub struct BgeSmallEmbedder { + model: Mutex, +} + +impl BgeSmallEmbedder { + pub fn new(cache_dir: impl AsRef) -> Result { + let cache_dir = model_cache_dir(cache_dir.as_ref()); + fs::create_dir_all(&cache_dir).map_err(|source| EmbedError::CreateCacheDir { + path: cache_dir.clone(), + source, + })?; + + let options = TextInitOptions::new(EmbeddingModel::BGESmallENV15) + .with_cache_dir(cache_dir) + .with_show_download_progress(false) + .with_intra_threads(2); + let model = TextEmbedding::try_new(options).map_err(|source| EmbedError::Model { + message: source.to_string(), + })?; + + Ok(Self { + model: Mutex::new(model), + }) + } + + fn embed_prefixed(&self, prefix: &str, text: &str) -> Result> { + let input = format!("{prefix}: {text}"); + let mut model = self.model.lock().map_err(|_| EmbedError::Lock)?; + let mut embeddings = model + .embed([input], None) + .map_err(|source| EmbedError::Model { + message: source.to_string(), + })?; + + embeddings.pop().ok_or_else(|| EmbedError::Model { + message: "embedding model returned no vectors".to_string(), + }) + } +} + +impl Embedder for BgeSmallEmbedder { + fn dimensions(&self) -> usize { + BGE_SMALL_EN_V15_DIMENSIONS + } + + fn embed(&self, text: &str) -> Result> { + self.embed_document(text) + } + + fn embed_document(&self, text: &str) -> Result> { + self.embed_prefixed("passage", text) + } + + fn embed_query(&self, text: &str) -> Result> { + self.embed_prefixed("query", text) + } +} + +fn model_cache_dir(cache_dir: &Path) -> PathBuf { + cache_dir.join("models") +} diff --git a/src/embed/error.rs b/src/embed/error.rs index 0928c6e..dbecec2 100644 --- a/src/embed/error.rs +++ b/src/embed/error.rs @@ -1,7 +1,20 @@ +use std::io; +use std::path::PathBuf; + use thiserror::Error; #[derive(Debug, Error)] pub enum EmbedError { + #[error("failed to create embedding model cache directory {path}")] + CreateCacheDir { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("embedding model failed: {message}")] Model { message: String }, + + #[error("embedding model lock is poisoned")] + Lock, } diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 34b6f5c..8001185 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -1,6 +1,8 @@ +mod bge; mod error; mod fake; +pub use bge::{BGE_SMALL_EN_V15_DIMENSIONS, BgeSmallEmbedder}; pub use error::EmbedError; pub use fake::FakeEmbedder; @@ -9,4 +11,12 @@ pub type Result = std::result::Result; pub trait Embedder { fn dimensions(&self) -> usize; fn embed(&self, text: &str) -> Result>; + + fn embed_document(&self, text: &str) -> Result> { + self.embed(text) + } + + fn embed_query(&self, text: &str) -> Result> { + self.embed(text) + } } diff --git a/src/index/file.rs b/src/index/file.rs index c4b1572..a305f8f 100644 --- a/src/index/file.rs +++ b/src/index/file.rs @@ -81,12 +81,13 @@ where .into_iter() .enumerate() { - let embedding = embedder - .embed(chunk.text) - .map_err(|source| IndexError::EmbedSummary { - path: path.to_path_buf(), - source, - })?; + let embedding = + embedder + .embed_document(chunk.text) + .map_err(|source| IndexError::EmbedSummary { + path: path.to_path_buf(), + source, + })?; chunks.push(IndexedFileChunk { file_path: file_path.clone(), diff --git a/src/index/mod.rs b/src/index/mod.rs index 1cd4fe9..fecd305 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -135,6 +135,11 @@ mod tests { let ignored = app.join("node_modules"); fs::create_dir_all(&ignored).unwrap(); fs::write(app.join("README.md"), "Chrome extension manifest tools").unwrap(); + fs::write( + app.join(".env.production"), + "DATABASE_URL=postgres://secret", + ) + .unwrap(); fs::write(ignored.join("package.json"), "should not appear").unwrap(); let asset_catalog = app.join("macos/Assets.xcassets/Custom Icon.appiconset"); fs::create_dir_all(&asset_catalog).unwrap(); @@ -194,7 +199,7 @@ mod tests { assert_eq!(report.directories_indexed, 3); assert_eq!(report.text_files_indexed, 1); assert_eq!(report.file_chunks_indexed, 1); - assert_eq!(report.entries_skipped, 3); + assert_eq!(report.entries_skipped, 4); assert_eq!(database.document_count().await.unwrap(), 3); let app_document = database @@ -208,6 +213,7 @@ mod tests { .contains("Chrome extension manifest tools") ); assert!(!app_document.searchable_text.contains("should not appear")); + assert!(!app_document.searchable_text.contains("DATABASE_URL")); assert!(!app_document.searchable_text.contains(".vscode")); assert_eq!( database diff --git a/src/main.rs b/src/main.rs index abfff02..a3b50a8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use std::env; -use std::io::{self, Write}; +use std::io::{self, IsTerminal, Write}; use std::path::Path; use std::process::ExitCode; use std::sync::{ @@ -52,7 +52,17 @@ async fn run(invocation: Invocation) -> error::Result<()> { println!("{}\t{}", count.count, count.label); } } - Invocation::EmitCd { args } => write_stdout(&app::resolve_cd_script(args).await)?, + Invocation::EmitCd { args } => { + let mut animation = app::implied_search_query(&args) + .is_some() + .then(SearchAnimation::start); + let script = app::resolve_cd_script(args).await; + if let Some(animation) = &mut animation { + animation.finish(); + } + + write_stdout(&script)?; + } Invocation::ShellInit { shell } => write_stdout(shell_init(shell).as_bytes())?, Invocation::Init => { let mut progress = TerminalIndexProgress::start(); @@ -77,7 +87,11 @@ async fn run(invocation: Invocation) -> error::Result<()> { } } Invocation::Search { query } => { - let results = app::search(query, 10).await?; + let mut animation = SearchAnimation::start(); + let results = app::search(query, 10).await; + animation.finish(); + + let results = results?; for result in results { println!("{:.3}\t{}", result.score, result.path); } @@ -108,6 +122,68 @@ fn confirm_reset() -> error::Result { )) } +const SEARCH_LABEL: &str = "Searching"; + +struct SearchAnimation { + stop: Arc, + worker: Option>, +} + +impl SearchAnimation { + fn start() -> Self { + let stop = Arc::new(AtomicBool::new(false)); + + if !io::stderr().is_terminal() { + return Self { stop, worker: None }; + } + + let worker = Some(spawn_search_animation(Arc::clone(&stop))); + Self { stop, worker } + } + + fn finish(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl Drop for SearchAnimation { + fn drop(&mut self) { + self.finish(); + } +} + +fn spawn_search_animation(stop: Arc) -> JoinHandle<()> { + thread::spawn(move || { + let mut tick = 0; + + while !stop.load(Ordering::Relaxed) { + render_search_frame(tick); + tick = tick.wrapping_add(1); + thread::sleep(Duration::from_millis(120)); + } + + clear_search_frame(); + }) +} + +fn render_search_frame(tick: usize) { + eprint!("\r\x1b[2K{}", search_frame(tick)); + let _ = io::stderr().flush(); +} + +fn clear_search_frame() { + eprint!("\r\x1b[2K"); + let _ = io::stderr().flush(); +} + +fn search_frame(tick: usize) -> String { + let dots = ".".repeat((tick % 3) + 1); + format!("{SEARCH_LABEL}{dots}") +} + #[derive(Debug, Default)] struct ProgressState { current_directory: Option, @@ -198,3 +274,16 @@ fn render_progress_line(state: &mut ProgressState) { state.last_len = line.len(); state.tick = state.tick.wrapping_add(1); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn search_frame_cycles_dots() { + assert_eq!(search_frame(0), "Searching."); + assert_eq!(search_frame(1), "Searching.."); + assert_eq!(search_frame(2), "Searching..."); + assert_eq!(search_frame(3), "Searching."); + } +} diff --git a/src/search/mod.rs b/src/search/mod.rs index d4000f7..ff5428b 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -33,7 +33,7 @@ where pub async fn search(&self, query: &str, limit: usize) -> Result> { let query_embedding = self .embedder - .embed(query) + .embed_query(query) .map_err(|source| SearchError::EmbedQuery { source })?; let chunks = self .database diff --git a/tests/docker_cd_equivalence.rs b/tests/docker_cd_equivalence.rs index 6221ba4..7766f4b 100644 --- a/tests/docker_cd_equivalence.rs +++ b/tests/docker_cd_equivalence.rs @@ -23,7 +23,7 @@ fn docker_cd_equivalence_random_tree() { } let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let image = std::env::var("CDS_DOCKER_IMAGE").unwrap_or_else(|_| "rust:1-slim".to_string()); + let image = std::env::var("CDS_DOCKER_IMAGE").unwrap_or_else(|_| "rust:1".to_string()); let cache_root = std::env::var_os("CDS_DOCKER_CACHE_DIR") .map(PathBuf::from) .unwrap_or_else(|| manifest_dir.join("target/docker-cache")); diff --git a/tests/search_integration.rs b/tests/search_integration.rs index dc1d61f..f6ceba9 100644 --- a/tests/search_integration.rs +++ b/tests/search_integration.rs @@ -34,7 +34,7 @@ fn hidden_emit_searches_when_local_prefix_is_absent() { } #[test] -fn hidden_emit_preserves_cd_when_local_prefix_exists() { +fn hidden_emit_searches_when_only_local_prefix_exists() { let fixture = SearchFixture::new(); fixture.init(); @@ -42,6 +42,37 @@ fn hidden_emit_preserves_cd_when_local_prefix_exists() { std::fs::create_dir_all(&cwd).unwrap(); std::fs::create_dir(cwd.join("music")).unwrap(); + let output = fixture + .cds() + .arg("--cds-emit") + .arg("--") + .arg("manifest") + .current_dir(&cwd) + .env("PWD", &cwd) + .output() + .unwrap(); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let expected = format!( + "builtin cd '{}'\n", + fixture.project_dir.to_string_lossy().replace('\'', "'\\''") + ); + assert_eq!(String::from_utf8(output.stdout).unwrap(), expected); +} + +#[test] +fn hidden_emit_preserves_cd_when_exact_local_entry_exists() { + let fixture = SearchFixture::new(); + fixture.init(); + + let cwd = fixture.temp.path().join("cwd"); + std::fs::create_dir_all(&cwd).unwrap(); + std::fs::create_dir(cwd.join("manifest")).unwrap(); + let output = fixture .cds() .arg("--cds-emit") @@ -147,6 +178,7 @@ impl SearchFixture { let mut command = Command::new(env!("CARGO_BIN_EXE_cds")); command .env("HOME", self.temp.path()) + .env("CDS_EMBEDDER", "fake") .env("CDS_CONFIG_DIR", self.temp.path().join("config")) .env("CDS_DATA_DIR", self.temp.path().join("data")) .env("CDS_CACHE_DIR", self.temp.path().join("cache")); diff --git a/tests/shell_integration.rs b/tests/shell_integration.rs index d9cd919..bef8dc9 100644 --- a/tests/shell_integration.rs +++ b/tests/shell_integration.rs @@ -72,7 +72,7 @@ expected="$(cd "$tmp/a" && pwd -P)" home="$tmp/home" mkdir -p "$home/Projects/cli-init-test" printf 'shell integration init project\n' > "$home/Projects/cli-init-test/README.md" -HOME="$home" CDS_CONFIG_DIR="$tmp/config" CDS_DATA_DIR="$tmp/data" CDS_CACHE_DIR="$tmp/cache" cds --init >/tmp/cds-shell-init.out +HOME="$home" CDS_EMBEDDER="fake" CDS_CONFIG_DIR="$tmp/config" CDS_DATA_DIR="$tmp/data" CDS_CACHE_DIR="$tmp/cache" cds --init >/tmp/cds-shell-init.out [ -f "$tmp/config/config.json" ] [ -f "$tmp/data/cds.sqlite" ] "#