From a335b2202516b35b4eeed56f7ce6d7ec8154d241 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 13:23:57 -0400 Subject: [PATCH] Switch database layer to async SQLx --- .github/workflows/ci.yml | 2 + AGENTS.md | 5 +- Cargo.lock | 1379 ++++++++++++++++++++++- Cargo.toml | 3 +- migrations/0001_create_index_schema.sql | 81 ++ src/app/mod.rs | 52 +- src/db/error.rs | 86 +- src/db/mod.rs | 1293 +++++++++++---------- src/db/schema.rs | 157 --- src/index/mod.rs | 68 +- src/index/scanner.rs | 215 ++-- src/main.rs | 19 +- src/search/mod.rs | 32 +- 13 files changed, 2387 insertions(+), 1005 deletions(-) create mode 100644 migrations/0001_create_index_schema.sql delete mode 100644 src/db/schema.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6dfa15..9986b51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: - "**/*.rs" - "Cargo.toml" - "Cargo.lock" + - "migrations/**/*.sql" - "tests/**/*.sh" - "install.sh" pull_request: @@ -15,6 +16,7 @@ on: - "**/*.rs" - "Cargo.toml" - "Cargo.lock" + - "migrations/**/*.sql" - "tests/**/*.sh" - "install.sh" workflow_dispatch: diff --git a/AGENTS.md b/AGENTS.md index a143a78..df028d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,7 @@ user's current shell. Prefer stable, well-maintained crates: - CLI parsing: `clap` -- SQLite: `rusqlite` +- SQLite: `sqlx` with SQLite - Serialization: `serde`, `serde_json` - Error handling: `color_eyre` for binaries, `thiserror` for reusable library errors - Filesystem walking: `ignore` or `walkdir`; prefer `ignore` because it respects common @@ -53,6 +53,8 @@ For embeddings, prefer an implementation path that keeps the tool locally usable ## Suggested Architecture Keep the code split between a thin binary and testable library modules. +The CLI and application pipeline are async end to end; use `tokio` for the binary runtime +and keep database access on SQLx async APIs. Suggested modules: @@ -69,6 +71,7 @@ Keep business logic out of `main.rs`. ## SQLite Guidance SQLite should be the durable source of indexed data. +Schema changes belong in top-level SQLx migration files under `migrations/`. Track at least: diff --git a/Cargo.lock b/Cargo.lock index ec2fdf2..c1713cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "1.0.0" @@ -53,7 +59,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -64,7 +70,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -73,6 +79,21 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "backtrace" version = "0.3.76" @@ -88,11 +109,47 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bitflags" version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" @@ -110,11 +167,12 @@ version = "0.1.0" dependencies = [ "clap", "color-eyre", - "rusqlite", "serde", "serde_json", + "sqlx", "tempfile", "thiserror", + "tokio", ] [[package]] @@ -183,6 +241,119 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +dependencies = [ + "serde", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -196,30 +367,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] -name = "eyre" -version = "0.6.12" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "indenter", - "once_cell", + "cfg-if", + "home", + "windows-sys 0.48.0", ] [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "event-listener" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] [[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" +name = "eyre" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] [[package]] name = "fastrand" @@ -233,12 +414,124 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -264,6 +557,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -288,12 +583,148 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indenter" version = "0.3.4" @@ -329,6 +760,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leb128fmt" @@ -342,11 +776,29 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.8.1", +] + [[package]] name = "libsqlite3-sys" -version = "0.34.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91632f3b4fb6bd1d72aa3d78f41ffecfcf2b1a6648d8c241dbe7dbfaf4875e15" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -360,14 +812,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] -name = "log" -version = "0.4.32" +name = "litemap" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] -name = "memchr" -version = "2.8.1" +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" @@ -380,6 +857,63 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "object" version = "0.37.3" @@ -407,18 +941,107 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -454,17 +1077,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "rusqlite" -version = "0.36.0" +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3de23c3319433716cf134eed225fe9986bc24f63bed9be9f20c329029e672dc7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" dependencies = [ "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", ] [[package]] @@ -483,9 +1160,21 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.28" @@ -535,6 +1224,40 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -550,11 +1273,261 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-sqlite", + "syn", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] [[package]] name = "strsim" @@ -562,6 +1535,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -573,6 +1552,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -580,10 +1570,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -615,16 +1605,91 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tracing" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -656,18 +1721,63 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -686,6 +1796,18 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -704,6 +1826,12 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-encoder" version = "0.244.0" @@ -738,12 +1866,31 @@ dependencies = [ "semver", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -753,6 +1900,63 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -847,6 +2051,115 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 83f2960..d31edcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,11 @@ edition = "2024" [dependencies] clap = { version = "4.5.40", features = ["cargo"] } color-eyre = "0.6.5" -rusqlite = { version = "0.36.0", features = ["bundled"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "sqlite", "migrate", "macros"] } thiserror = "2.0.17" +tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread"] } [dev-dependencies] tempfile = "3.20.0" diff --git a/migrations/0001_create_index_schema.sql b/migrations/0001_create_index_schema.sql new file mode 100644 index 0000000..7f5dc29 --- /dev/null +++ b/migrations/0001_create_index_schema.sql @@ -0,0 +1,81 @@ +CREATE TABLE IF NOT EXISTS indexed_documents ( + path TEXT PRIMARY KEY NOT NULL, + name TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL CHECK (kind IN ('directory', 'file')), + parent_path TEXT, + searchable_text TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL, + metadata_fingerprint TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + created_unix_seconds INTEGER, + modified_unix_seconds INTEGER NOT NULL, + accessed_unix_seconds INTEGER, + readonly INTEGER NOT NULL DEFAULT 0, + indexed_unix_seconds INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_indexed_documents_kind + ON indexed_documents(kind); + +CREATE INDEX IF NOT EXISTS idx_indexed_documents_parent_path + ON indexed_documents(parent_path); + +CREATE INDEX IF NOT EXISTS idx_indexed_documents_indexed_time + ON indexed_documents(indexed_unix_seconds); + +CREATE INDEX IF NOT EXISTS idx_indexed_documents_name + ON indexed_documents(name); + +CREATE TABLE IF NOT EXISTS indexed_files ( + path TEXT PRIMARY KEY NOT NULL, + directory_path TEXT NOT NULL, + name TEXT NOT NULL, + extension TEXT, + size_bytes INTEGER NOT NULL, + created_unix_seconds INTEGER, + modified_unix_seconds INTEGER NOT NULL, + accessed_unix_seconds INTEGER, + readonly INTEGER NOT NULL, + content_fingerprint TEXT NOT NULL, + indexed_unix_seconds INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_indexed_files_directory_path + ON indexed_files(directory_path); + +CREATE INDEX IF NOT EXISTS idx_indexed_files_modified_time + ON indexed_files(modified_unix_seconds); + +CREATE TABLE IF NOT EXISTS indexed_file_chunks ( + file_path TEXT NOT NULL, + directory_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + embedding BLOB NOT NULL, + embedding_dim INTEGER NOT NULL, + start_byte INTEGER NOT NULL, + end_byte INTEGER NOT NULL, + indexed_unix_seconds INTEGER NOT NULL, + PRIMARY KEY (file_path, chunk_index) +); + +CREATE INDEX IF NOT EXISTS idx_indexed_file_chunks_directory_path + ON indexed_file_chunks(directory_path); + +CREATE TABLE IF NOT EXISTS directory_classifications ( + directory_path TEXT NOT NULL, + label TEXT NOT NULL, + confidence REAL NOT NULL, + detector TEXT NOT NULL, + evidence_path TEXT, + evidence_summary TEXT NOT NULL, + detected_unix_seconds INTEGER NOT NULL, + PRIMARY KEY (directory_path, label, detector) +); + +CREATE INDEX IF NOT EXISTS idx_directory_classifications_label + ON directory_classifications(label); + +CREATE INDEX IF NOT EXISTS idx_directory_classifications_directory_path + ON directory_classifications(directory_path); diff --git a/src/app/mod.rs b/src/app/mod.rs index c49f2ec..2973247 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -21,21 +21,23 @@ pub struct InitReport { pub index: IndexReport, } -pub fn init() -> Result { +pub async fn init() -> Result { let mut progress = NoopProgress; - init_with_progress(&mut progress) + init_with_progress(&mut progress).await } -pub fn init_with_progress

(progress: &mut P) -> Result +pub async fn init_with_progress

(progress: &mut P) -> Result where P: IndexProgress + ?Sized, { let paths = AppPaths::discover()?; let settings = Settings::load_or_create(&paths.config_file)?; - let database = Database::open(&paths.database_file)?; + let database = Database::open(&paths.database_file).await?; let embedder = FakeEmbedder::default(); let indexer = Indexer::new(&settings, &database, &embedder); - let index = indexer.index_configured_roots_with_progress(progress)?; + let index = indexer + .index_configured_roots_with_progress(progress) + .await?; Ok(InitReport { config_file: paths.config_file, @@ -44,23 +46,25 @@ where }) } -pub fn index(roots: Vec) -> Result { +pub async fn index(roots: Vec) -> Result { let mut progress = NoopProgress; - index_with_progress(roots, &mut progress) + index_with_progress(roots, &mut progress).await } -pub fn index_with_progress

(roots: Vec, progress: &mut P) -> Result +pub async fn index_with_progress

(roots: Vec, progress: &mut P) -> Result where P: IndexProgress + ?Sized, { let paths = AppPaths::discover()?; let settings = Settings::load_or_create(&paths.config_file)?; - let database = Database::open(&paths.database_file)?; + let database = Database::open(&paths.database_file).await?; let embedder = FakeEmbedder::default(); let indexer = Indexer::new(&settings, &database, &embedder); if roots.is_empty() { - return Ok(indexer.index_configured_roots_with_progress(progress)?); + return Ok(indexer + .index_configured_roots_with_progress(progress) + .await?); } let roots = roots @@ -73,37 +77,37 @@ where }) .collect::>>()?; - Ok(indexer.index_roots_with_progress(roots, progress)?) + Ok(indexer.index_roots_with_progress(roots, progress).await?) } -pub fn search(query: Vec, limit: usize) -> Result> { +pub async fn search(query: Vec, limit: usize) -> Result> { let query = join_query(query)?; - search_text(&query, limit) + search_text(&query, limit).await } -pub fn search_text(query: &str, limit: usize) -> Result> { +pub async fn search_text(query: &str, limit: usize) -> Result> { let paths = AppPaths::discover()?; - let database = Database::open_existing(&paths.database_file)?; + let database = Database::open_existing(&paths.database_file).await?; let embedder = FakeEmbedder::default(); let searcher = Searcher::new(&database, &embedder); - Ok(searcher.search(query, limit)?) + Ok(searcher.search(query, limit).await?) } -pub fn directory_type_counts() -> Result> { +pub async fn directory_type_counts() -> Result> { let paths = AppPaths::discover()?; - let database = Database::open_existing(&paths.database_file)?; - Ok(database.directory_type_counts()?) + let database = Database::open_existing(&paths.database_file).await?; + Ok(database.directory_type_counts().await?) } -pub fn reset_database() -> Result<()> { +pub async fn reset_database() -> Result<()> { let paths = AppPaths::discover()?; - let database = Database::open_existing(&paths.database_file)?; - Ok(database.reset()?) + let database = Database::open_existing(&paths.database_file).await?; + Ok(database.reset().await?) } -pub fn resolve_cd_script(args: Vec) -> Vec { +pub async fn resolve_cd_script(args: Vec) -> Vec { if let Some(query) = semantic_query(&args) - && let Ok(results) = search_text(&query, 5) + && let Ok(results) = search_text(&query, 5).await && let Some(result) = results .into_iter() .find(|result| Path::new(&result.path).is_dir()) diff --git a/src/db/error.rs b/src/db/error.rs index e6e7fdb..756e458 100644 --- a/src/db/error.rs +++ b/src/db/error.rs @@ -17,7 +17,7 @@ pub enum DbError { OpenDatabase { path: PathBuf, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("database does not exist at {path}; run `cds --init` first")] @@ -26,49 +26,19 @@ pub enum DbError { #[error("failed to open in-memory database")] OpenInMemory { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to migrate database")] Migrate { #[source] - source: Box, + source: sqlx::migrate::MigrateError, }, - #[error("failed to read schema version")] - ReadSchemaVersion { + #[error("failed to prepare legacy database for SQLx migrations")] + PrepareLegacyMigration { #[source] - source: rusqlite::Error, - }, - - #[error("failed to update schema version")] - UpdateSchemaVersion { - #[source] - source: rusqlite::Error, - }, - - #[error("failed to create schema v1")] - CreateSchemaV1 { - #[source] - source: rusqlite::Error, - }, - - #[error("failed to create schema v2")] - CreateSchemaV2 { - #[source] - source: rusqlite::Error, - }, - - #[error("failed to create schema v3")] - CreateSchemaV3 { - #[source] - source: rusqlite::Error, - }, - - #[error("failed to create schema v4")] - CreateSchemaV4 { - #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("embedding dimension overflows i64")] @@ -87,21 +57,21 @@ pub enum DbError { UpsertDocument { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to upsert indexed file {path}")] UpsertFile { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to delete indexed chunks for {path}")] DeleteFileChunks { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to insert indexed chunk {path}#{chunk_index}")] @@ -109,14 +79,14 @@ pub enum DbError { path: String, chunk_index: u32, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to replace classifications for {path}")] ReplaceDirectoryClassifications { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to insert classification {label} for {path}")] @@ -124,26 +94,26 @@ pub enum DbError { path: String, label: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to delete indexed path tree {path}")] DeletePathTree { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to reset database content")] ResetDatabase { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to count indexed documents")] CountDocuments { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("document count was negative")] @@ -152,54 +122,42 @@ pub enum DbError { source: TryFromIntError, }, - #[error("failed to prepare document lookup")] - PrepareDocumentLookup { - #[source] - source: rusqlite::Error, - }, - - #[error("failed to prepare directory document scan")] - PrepareDirectoryDocumentScan { - #[source] - source: rusqlite::Error, - }, - #[error("failed to look up indexed document {path}")] LookupDocument { path: String, #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to decode indexed document {path}")] - DecodeDocument { + InvalidDocumentSize { path: String, #[source] - source: rusqlite::Error, + source: TryFromIntError, }, #[error("failed to read directory documents")] ReadDirectoryDocuments { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to read file chunk embeddings")] ReadFileChunks { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to read directory classifications")] ReadDirectoryClassifications { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("failed to read directory type counts")] ReadDirectoryTypeCounts { #[source] - source: rusqlite::Error, + source: sqlx::Error, }, #[error("embedding blob length {len} is not divisible by 4")] diff --git a/src/db/mod.rs b/src/db/mod.rs index 8ef0729..389b06d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,12 +1,14 @@ mod document; mod error; -mod schema; mod vector; +use std::collections::HashSet; use std::fs; use std::path::Path; -use rusqlite::{Connection, OptionalExtension, params}; +use sqlx::migrate::Migrator; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteRow}; +use sqlx::{Row, SqlitePool}; pub use document::{ DirectoryClassification, DirectoryTypeCount, DocumentKind, FileChunkMatch, IndexedDocument, @@ -17,13 +19,15 @@ pub use vector::{decode_embedding, encode_embedding}; pub type Result = std::result::Result; -#[derive(Debug)] +static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); + +#[derive(Debug, Clone)] pub struct Database { - connection: Connection, + pool: SqlitePool, } impl Database { - pub fn open(path: &Path) -> Result { + pub async fn open(path: &Path) -> Result { if let Some(parent) = path.parent() { fs::create_dir_all(parent).map_err(|source| DbError::CreateDatabaseDir { path: parent.to_path_buf(), @@ -31,578 +35,566 @@ impl Database { })?; } - let connection = Connection::open(path).map_err(|source| DbError::OpenDatabase { - path: path.to_path_buf(), - source, - })?; - schema::migrate(&connection).map_err(|source| DbError::Migrate { - source: Box::new(source), - })?; - Ok(Self { connection }) + let pool = open_file_pool(path, true).await?; + migrate_pool(&pool).await?; + Ok(Self { pool }) } - pub fn open_existing(path: &Path) -> Result { + pub async fn open_existing(path: &Path) -> Result { if !path.exists() { return Err(DbError::MissingDatabase { path: path.to_path_buf(), }); } - let connection = Connection::open(path).map_err(|source| DbError::OpenDatabase { - path: path.to_path_buf(), - source, - })?; - schema::migrate(&connection).map_err(|source| DbError::Migrate { - source: Box::new(source), - })?; - Ok(Self { connection }) + let pool = open_file_pool(path, false).await?; + migrate_pool(&pool).await?; + Ok(Self { pool }) } - pub fn open_in_memory() -> Result { - let connection = - Connection::open_in_memory().map_err(|source| DbError::OpenInMemory { source })?; - schema::migrate(&connection).map_err(|source| DbError::Migrate { - source: Box::new(source), - })?; - Ok(Self { connection }) + pub async fn open_in_memory() -> Result { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .map_err(|source| DbError::OpenInMemory { source })?; + migrate_pool(&pool).await?; + Ok(Self { pool }) } - pub fn upsert_document(&self, document: &IndexedDocument) -> Result<()> { - self.connection - .execute( - " - INSERT INTO indexed_documents ( - path, - name, - kind, - parent_path, - searchable_text, - embedding, - embedding_dim, - metadata_fingerprint, - size_bytes, - created_unix_seconds, - modified_unix_seconds, - accessed_unix_seconds, - readonly, - indexed_unix_seconds - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14) - ON CONFLICT(path) DO UPDATE SET - name = excluded.name, - kind = excluded.kind, - parent_path = excluded.parent_path, - searchable_text = excluded.searchable_text, - embedding = excluded.embedding, - embedding_dim = excluded.embedding_dim, - metadata_fingerprint = excluded.metadata_fingerprint, - size_bytes = excluded.size_bytes, - created_unix_seconds = excluded.created_unix_seconds, - modified_unix_seconds = excluded.modified_unix_seconds, - accessed_unix_seconds = excluded.accessed_unix_seconds, - readonly = excluded.readonly, - indexed_unix_seconds = excluded.indexed_unix_seconds - ", - params![ - document.path, - document.name, - document.kind.as_str(), - document.parent_path, - document.searchable_text, - encode_embedding(&document.embedding), - i64::try_from(document.embedding.len()) - .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, - document.metadata_fingerprint, - i64::try_from(document.size_bytes) - .map_err(|source| DbError::MetadataSizeOverflow { source })?, - document.created_unix_seconds, - document.modified_unix_seconds, - document.accessed_unix_seconds, - document.readonly, - document.indexed_unix_seconds, - ], - ) - .map_err(|source| DbError::UpsertDocument { - path: document.path.clone(), - source, - })?; + pub async fn upsert_document(&self, document: &IndexedDocument) -> Result<()> { + sqlx::query( + " + INSERT INTO indexed_documents ( + path, + name, + kind, + parent_path, + searchable_text, + embedding, + embedding_dim, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + name = excluded.name, + kind = excluded.kind, + parent_path = excluded.parent_path, + searchable_text = excluded.searchable_text, + embedding = excluded.embedding, + embedding_dim = excluded.embedding_dim, + metadata_fingerprint = excluded.metadata_fingerprint, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + ) + .bind(&document.path) + .bind(&document.name) + .bind(document.kind.as_str()) + .bind(&document.parent_path) + .bind(&document.searchable_text) + .bind(encode_embedding(&document.embedding)) + .bind( + i64::try_from(document.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind(&document.metadata_fingerprint) + .bind( + i64::try_from(document.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(document.created_unix_seconds) + .bind(document.modified_unix_seconds) + .bind(document.accessed_unix_seconds) + .bind(document.readonly) + .bind(document.indexed_unix_seconds) + .execute(&self.pool) + .await + .map_err(|source| DbError::UpsertDocument { + path: document.path.clone(), + source, + })?; Ok(()) } - pub fn upsert_file(&self, file: &IndexedFile) -> Result<()> { - self.connection - .execute( - " - INSERT INTO indexed_files ( - path, - directory_path, - name, - extension, - size_bytes, - created_unix_seconds, - modified_unix_seconds, - accessed_unix_seconds, - readonly, - content_fingerprint, - indexed_unix_seconds - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) - ON CONFLICT(path) DO UPDATE SET - directory_path = excluded.directory_path, - name = excluded.name, - extension = excluded.extension, - size_bytes = excluded.size_bytes, - created_unix_seconds = excluded.created_unix_seconds, - modified_unix_seconds = excluded.modified_unix_seconds, - accessed_unix_seconds = excluded.accessed_unix_seconds, - readonly = excluded.readonly, - content_fingerprint = excluded.content_fingerprint, - indexed_unix_seconds = excluded.indexed_unix_seconds - ", - params![ - file.path, - file.directory_path, - file.name, - file.extension, - i64::try_from(file.size_bytes) - .map_err(|source| DbError::MetadataSizeOverflow { source })?, - file.created_unix_seconds, - file.modified_unix_seconds, - file.accessed_unix_seconds, - file.readonly, - file.content_fingerprint, - file.indexed_unix_seconds, - ], - ) - .map_err(|source| DbError::UpsertFile { - path: file.path.clone(), - source, - })?; + pub async fn upsert_file(&self, file: &IndexedFile) -> Result<()> { + sqlx::query( + " + INSERT INTO indexed_files ( + path, + directory_path, + name, + extension, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + content_fingerprint, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + directory_path = excluded.directory_path, + name = excluded.name, + extension = excluded.extension, + size_bytes = excluded.size_bytes, + created_unix_seconds = excluded.created_unix_seconds, + modified_unix_seconds = excluded.modified_unix_seconds, + accessed_unix_seconds = excluded.accessed_unix_seconds, + readonly = excluded.readonly, + content_fingerprint = excluded.content_fingerprint, + indexed_unix_seconds = excluded.indexed_unix_seconds + ", + ) + .bind(&file.path) + .bind(&file.directory_path) + .bind(&file.name) + .bind(&file.extension) + .bind( + i64::try_from(file.size_bytes) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(file.created_unix_seconds) + .bind(file.modified_unix_seconds) + .bind(file.accessed_unix_seconds) + .bind(file.readonly) + .bind(&file.content_fingerprint) + .bind(file.indexed_unix_seconds) + .execute(&self.pool) + .await + .map_err(|source| DbError::UpsertFile { + path: file.path.clone(), + source, + })?; Ok(()) } - pub fn replace_file_chunks(&self, file_path: &str, chunks: &[IndexedFileChunk]) -> Result<()> { - self.connection - .execute( - "DELETE FROM indexed_file_chunks WHERE file_path = ?1", - [file_path], - ) + pub async fn replace_file_chunks( + &self, + file_path: &str, + chunks: &[IndexedFileChunk], + ) -> Result<()> { + sqlx::query("DELETE FROM indexed_file_chunks WHERE file_path = ?") + .bind(file_path) + .execute(&self.pool) + .await .map_err(|source| DbError::DeleteFileChunks { path: file_path.to_string(), source, })?; for chunk in chunks { - self.connection - .execute( - " - INSERT INTO indexed_file_chunks ( - file_path, - directory_path, - chunk_index, - content, - embedding, - embedding_dim, - start_byte, - end_byte, - indexed_unix_seconds - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) - ", - params![ - chunk.file_path, - chunk.directory_path, - i64::from(chunk.chunk_index), - chunk.content, - encode_embedding(&chunk.embedding), - i64::try_from(chunk.embedding.len()) - .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, - i64::try_from(chunk.start_byte) - .map_err(|source| DbError::MetadataSizeOverflow { source })?, - i64::try_from(chunk.end_byte) - .map_err(|source| DbError::MetadataSizeOverflow { source })?, - chunk.indexed_unix_seconds, - ], - ) - .map_err(|source| DbError::InsertFileChunk { - path: chunk.file_path.clone(), - chunk_index: chunk.chunk_index, - source, - })?; + sqlx::query( + " + INSERT INTO indexed_file_chunks ( + file_path, + directory_path, + chunk_index, + content, + embedding, + embedding_dim, + start_byte, + end_byte, + indexed_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&chunk.file_path) + .bind(&chunk.directory_path) + .bind(i64::from(chunk.chunk_index)) + .bind(&chunk.content) + .bind(encode_embedding(&chunk.embedding)) + .bind( + i64::try_from(chunk.embedding.len()) + .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?, + ) + .bind( + i64::try_from(chunk.start_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind( + i64::try_from(chunk.end_byte) + .map_err(|source| DbError::MetadataSizeOverflow { source })?, + ) + .bind(chunk.indexed_unix_seconds) + .execute(&self.pool) + .await + .map_err(|source| DbError::InsertFileChunk { + path: chunk.file_path.clone(), + chunk_index: chunk.chunk_index, + source, + })?; } Ok(()) } - pub fn replace_directory_classifications( + pub async fn replace_directory_classifications( &self, directory_path: &str, classifications: &[DirectoryClassification], ) -> Result<()> { - self.connection - .execute( - "DELETE FROM directory_classifications WHERE directory_path = ?1", - [directory_path], - ) + sqlx::query("DELETE FROM directory_classifications WHERE directory_path = ?") + .bind(directory_path) + .execute(&self.pool) + .await .map_err(|source| DbError::ReplaceDirectoryClassifications { path: directory_path.to_string(), source, })?; for classification in classifications { - self.connection - .execute( - " - INSERT INTO directory_classifications ( - directory_path, - label, - confidence, - detector, - evidence_path, - evidence_summary, - detected_unix_seconds - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) - ", - params![ - classification.directory_path, - classification.label, - classification.confidence, - classification.detector, - classification.evidence_path, - classification.evidence_summary, - classification.detected_unix_seconds, - ], - ) - .map_err(|source| DbError::InsertDirectoryClassification { - path: classification.directory_path.clone(), - label: classification.label.clone(), - source, - })?; + sqlx::query( + " + INSERT INTO directory_classifications ( + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ", + ) + .bind(&classification.directory_path) + .bind(&classification.label) + .bind(f64::from(classification.confidence)) + .bind(&classification.detector) + .bind(&classification.evidence_path) + .bind(&classification.evidence_summary) + .bind(classification.detected_unix_seconds) + .execute(&self.pool) + .await + .map_err(|source| DbError::InsertDirectoryClassification { + path: classification.directory_path.clone(), + label: classification.label.clone(), + source, + })?; } Ok(()) } - pub fn delete_path_tree(&self, path: &str) -> Result<()> { + pub async fn delete_path_tree(&self, path: &str) -> Result<()> { let path_len = i64::try_from(path.len()) .map_err(|source| DbError::EmbeddingDimensionOverflow { source })?; - self.connection - .execute( - " - DELETE FROM directory_classifications - WHERE directory_path = ?1 - OR ( - length(directory_path) > ?2 - AND substr(directory_path, 1, ?2) = ?1 - AND substr(directory_path, ?2 + 1, 1) = '/' - ) - ", - params![path, path_len], - ) - .map_err(|source| DbError::DeletePathTree { - path: path.to_string(), - source, - })?; + sqlx::query( + " + DELETE FROM directory_classifications + WHERE directory_path = ? + OR ( + length(directory_path) > ? + AND substr(directory_path, 1, ?) = ? + AND substr(directory_path, ? + 1, 1) = '/' + ) + ", + ) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .execute(&self.pool) + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; - self.connection - .execute( - " - DELETE FROM indexed_file_chunks - WHERE file_path = ?1 - OR directory_path = ?1 - OR ( - length(file_path) > ?2 - AND substr(file_path, 1, ?2) = ?1 - AND substr(file_path, ?2 + 1, 1) = '/' - ) - OR ( - length(directory_path) > ?2 - AND substr(directory_path, 1, ?2) = ?1 - AND substr(directory_path, ?2 + 1, 1) = '/' - ) - ", - params![path, path_len], - ) - .map_err(|source| DbError::DeletePathTree { - path: path.to_string(), - source, - })?; + sqlx::query( + " + DELETE FROM indexed_file_chunks + WHERE file_path = ? + OR directory_path = ? + OR ( + length(file_path) > ? + AND substr(file_path, 1, ?) = ? + AND substr(file_path, ? + 1, 1) = '/' + ) + OR ( + length(directory_path) > ? + AND substr(directory_path, 1, ?) = ? + AND substr(directory_path, ? + 1, 1) = '/' + ) + ", + ) + .bind(path) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .execute(&self.pool) + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; - self.connection - .execute( - " - DELETE FROM indexed_files - WHERE path = ?1 - OR directory_path = ?1 - OR ( - length(path) > ?2 - AND substr(path, 1, ?2) = ?1 - AND substr(path, ?2 + 1, 1) = '/' - ) - OR ( - length(directory_path) > ?2 - AND substr(directory_path, 1, ?2) = ?1 - AND substr(directory_path, ?2 + 1, 1) = '/' - ) - ", - params![path, path_len], - ) - .map_err(|source| DbError::DeletePathTree { - path: path.to_string(), - source, - })?; + sqlx::query( + " + DELETE FROM indexed_files + WHERE path = ? + OR directory_path = ? + OR ( + length(path) > ? + AND substr(path, 1, ?) = ? + AND substr(path, ? + 1, 1) = '/' + ) + OR ( + length(directory_path) > ? + AND substr(directory_path, 1, ?) = ? + AND substr(directory_path, ? + 1, 1) = '/' + ) + ", + ) + .bind(path) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .execute(&self.pool) + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; - self.connection - .execute( - " - DELETE FROM indexed_documents - WHERE path = ?1 - OR ( - length(path) > ?2 - AND substr(path, 1, ?2) = ?1 - AND substr(path, ?2 + 1, 1) = '/' - ) - ", - params![path, path_len], - ) - .map_err(|source| DbError::DeletePathTree { - path: path.to_string(), - source, - })?; + sqlx::query( + " + DELETE FROM indexed_documents + WHERE path = ? + OR ( + length(path) > ? + AND substr(path, 1, ?) = ? + AND substr(path, ? + 1, 1) = '/' + ) + ", + ) + .bind(path) + .bind(path_len) + .bind(path_len) + .bind(path) + .bind(path_len) + .execute(&self.pool) + .await + .map_err(|source| DbError::DeletePathTree { + path: path.to_string(), + source, + })?; Ok(()) } - pub fn reset(&self) -> Result<()> { - self.connection - .execute_batch( - " - BEGIN; - DELETE FROM directory_classifications; - DELETE FROM indexed_file_chunks; - DELETE FROM indexed_files; - DELETE FROM indexed_documents; - COMMIT; - ", - ) + pub async fn reset(&self) -> Result<()> { + sqlx::query("DELETE FROM directory_classifications") + .execute(&self.pool) + .await + .map_err(|source| DbError::ResetDatabase { source })?; + sqlx::query("DELETE FROM indexed_file_chunks") + .execute(&self.pool) + .await + .map_err(|source| DbError::ResetDatabase { source })?; + sqlx::query("DELETE FROM indexed_files") + .execute(&self.pool) + .await + .map_err(|source| DbError::ResetDatabase { source })?; + sqlx::query("DELETE FROM indexed_documents") + .execute(&self.pool) + .await .map_err(|source| DbError::ResetDatabase { source })?; Ok(()) } - pub fn document_count(&self) -> Result { - let count: i64 = self - .connection - .query_row("SELECT COUNT(*) FROM indexed_documents", [], |row| { - row.get(0) - }) + pub async fn document_count(&self) -> Result { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM indexed_documents") + .fetch_one(&self.pool) + .await .map_err(|source| DbError::CountDocuments { source })?; u64::try_from(count).map_err(|source| DbError::NegativeDocumentCount { source }) } - pub fn get_document(&self, path: &str) -> Result> { - let mut statement = self - .connection - .prepare( - " - SELECT - path, - name, - kind, - parent_path, - searchable_text, - embedding, - metadata_fingerprint, - size_bytes, - created_unix_seconds, - modified_unix_seconds, - accessed_unix_seconds, - readonly, - indexed_unix_seconds - FROM indexed_documents - WHERE path = ?1 - ", - ) - .map_err(|source| DbError::PrepareDocumentLookup { source })?; - - let mut rows = statement - .query_map([path], decode_document_row) - .map_err(|source| DbError::LookupDocument { - path: path.to_string(), - source, - })?; + pub async fn get_document(&self, path: &str) -> Result> { + let row = sqlx::query( + " + SELECT + path, + name, + kind, + parent_path, + searchable_text, + embedding, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE path = ? + ", + ) + .bind(path) + .fetch_optional(&self.pool) + .await + .map_err(|source| DbError::LookupDocument { + path: path.to_string(), + source, + })?; - rows.next() + row.map(decode_document_row) .transpose() - .map_err(|source| DbError::DecodeDocument { + .map_err(|source| DbError::LookupDocument { path: path.to_string(), source, }) } - pub fn directory_documents(&self) -> Result> { - let mut statement = self - .connection - .prepare( - " - SELECT - path, - name, - kind, - parent_path, - searchable_text, - embedding, - metadata_fingerprint, - size_bytes, - created_unix_seconds, - modified_unix_seconds, - accessed_unix_seconds, - readonly, - indexed_unix_seconds - FROM indexed_documents - WHERE kind = 'directory' - ", - ) - .map_err(|source| DbError::PrepareDirectoryDocumentScan { source })?; - - let rows = statement - .query_map([], decode_document_row) - .map_err(|source| DbError::ReadDirectoryDocuments { source })?; + pub async fn directory_documents(&self) -> Result> { + let rows = sqlx::query( + " + SELECT + path, + name, + kind, + parent_path, + searchable_text, + embedding, + metadata_fingerprint, + size_bytes, + created_unix_seconds, + modified_unix_seconds, + accessed_unix_seconds, + readonly, + indexed_unix_seconds + FROM indexed_documents + WHERE kind = 'directory' + ", + ) + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryDocuments { source })?; - rows.collect::, _>>() + rows.into_iter() + .map(decode_document_row) + .collect::, _>>() .map_err(|source| DbError::ReadDirectoryDocuments { source }) } - pub fn file_chunk_matches(&self) -> Result> { - let mut statement = self - .connection - .prepare( - " - SELECT - chunk.file_path, - file.name, - chunk.directory_path, - chunk.content, - chunk.embedding, - file.modified_unix_seconds, - directory.modified_unix_seconds - FROM indexed_file_chunks AS chunk - INNER JOIN indexed_files AS file - ON file.path = chunk.file_path - INNER JOIN indexed_documents AS directory - ON directory.path = chunk.directory_path - WHERE directory.kind = 'directory' - ", - ) - .map_err(|source| DbError::ReadFileChunks { source })?; - - let rows = statement - .query_map([], |row| { - let embedding: Vec = row.get(4)?; - Ok(FileChunkMatch { - file_path: row.get(0)?, - file_name: row.get(1)?, - directory_path: row.get(2)?, - content: row.get(3)?, - embedding: decode_embedding(&embedding).map_err(|err| { - rusqlite::Error::FromSqlConversionFailure( - 4, - rusqlite::types::Type::Blob, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - err.to_string(), - )), - ) - })?, - file_modified_unix_seconds: row.get(5)?, - directory_modified_unix_seconds: row.get(6)?, - }) - }) - .map_err(|source| DbError::ReadFileChunks { source })?; + pub async fn file_chunk_matches(&self) -> Result> { + let rows = sqlx::query( + " + SELECT + chunk.file_path, + file.name AS file_name, + chunk.directory_path, + chunk.content, + chunk.embedding, + file.modified_unix_seconds AS file_modified_unix_seconds, + directory.modified_unix_seconds AS directory_modified_unix_seconds + FROM indexed_file_chunks AS chunk + INNER JOIN indexed_files AS file + ON file.path = chunk.file_path + INNER JOIN indexed_documents AS directory + ON directory.path = chunk.directory_path + WHERE directory.kind = 'directory' + ", + ) + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadFileChunks { source })?; - rows.collect::, _>>() + rows.into_iter() + .map(decode_file_chunk_match_row) + .collect::, _>>() .map_err(|source| DbError::ReadFileChunks { source }) } - pub fn directory_classifications( + pub async fn directory_classifications( &self, directory_path: &str, ) -> Result> { - let mut statement = self - .connection - .prepare( - " - SELECT - directory_path, - label, - confidence, - detector, - evidence_path, - evidence_summary, - detected_unix_seconds - FROM directory_classifications - WHERE directory_path = ?1 - ", - ) - .map_err(|source| DbError::ReadDirectoryClassifications { source })?; - - let rows = statement - .query_map([directory_path], decode_classification_row) - .map_err(|source| DbError::ReadDirectoryClassifications { source })?; - - rows.collect::, _>>() + let rows = sqlx::query( + " + SELECT + directory_path, + label, + confidence, + detector, + evidence_path, + evidence_summary, + detected_unix_seconds + FROM directory_classifications + WHERE directory_path = ? + ", + ) + .bind(directory_path) + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryClassifications { source })?; + + rows.into_iter() + .map(decode_classification_row) + .collect::, _>>() .map_err(|source| DbError::ReadDirectoryClassifications { source }) } - pub fn ancestor_classifications( + pub async fn ancestor_classifications( &self, directory_path: &str, ) -> Result> { let mut classifications = Vec::new(); - for ancestor in self.indexed_ancestors(directory_path)? { - classifications.extend(self.directory_classifications(&ancestor)?); + for ancestor in self.indexed_ancestors(directory_path).await? { + classifications.extend(self.directory_classifications(&ancestor).await?); } Ok(classifications) } - pub fn directory_type_counts(&self) -> Result> { - let mut statement = self - .connection - .prepare( - " - SELECT label, COUNT(DISTINCT directory_path) AS directory_count - FROM directory_classifications - GROUP BY label - ORDER BY directory_count DESC, label ASC - ", - ) - .map_err(|source| DbError::ReadDirectoryTypeCounts { source })?; - - let rows = statement - .query_map([], |row| { - let count: i64 = row.get(1)?; - Ok(DirectoryTypeCount { - label: row.get(0)?, - count: u64::try_from(count).map_err(|err| { - rusqlite::Error::FromSqlConversionFailure( - 1, - rusqlite::types::Type::Integer, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - err.to_string(), - )), - ) - })?, - }) - }) - .map_err(|source| DbError::ReadDirectoryTypeCounts { source })?; + pub async fn directory_type_counts(&self) -> Result> { + let rows = sqlx::query( + " + SELECT label, COUNT(DISTINCT directory_path) AS directory_count + FROM directory_classifications + GROUP BY label + ORDER BY directory_count DESC, label ASC + ", + ) + .fetch_all(&self.pool) + .await + .map_err(|source| DbError::ReadDirectoryTypeCounts { source })?; - rows.collect::, _>>() + rows.into_iter() + .map(decode_directory_type_count_row) + .collect::, _>>() .map_err(|source| DbError::ReadDirectoryTypeCounts { source }) } - pub fn general_indexed_directory(&self, path: &str) -> Result { - let ancestors = self.indexed_ancestors(path)?; + pub async fn general_indexed_directory(&self, path: &str) -> Result { + let ancestors = self.indexed_ancestors(path).await?; if ancestors.len() >= 2 { return Ok(ancestors[ancestors.len() - 2].clone()); @@ -614,28 +606,26 @@ impl Database { .unwrap_or_else(|| path.to_string())) } - pub fn indexed_ancestors(&self, path: &str) -> Result> { + pub async fn indexed_ancestors(&self, path: &str) -> Result> { let mut current = Path::new(path); let mut ancestors = Vec::new(); loop { let current_path = current.to_string_lossy(); - let exists = self - .connection - .query_row( - " - SELECT path - FROM indexed_documents - WHERE path = ?1 AND kind = 'directory' - ", - [current_path.as_ref()], - |row| row.get::<_, String>(0), - ) - .optional() - .map_err(|source| DbError::LookupDocument { - path: current_path.into_owned(), - source, - })?; + let exists: Option = sqlx::query_scalar( + " + SELECT path + FROM indexed_documents + WHERE path = ? AND kind = 'directory' + ", + ) + .bind(current_path.as_ref()) + .fetch_optional(&self.pool) + .await + .map_err(|source| DbError::LookupDocument { + path: current_path.to_string(), + source, + })?; if let Some(path) = exists { ancestors.push(path); @@ -651,54 +641,175 @@ impl Database { } } -fn decode_classification_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { +async fn open_file_pool(path: &Path, create_if_missing: bool) -> Result { + let options = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(create_if_missing); + + SqlitePoolOptions::new() + .max_connections(5) + .connect_with(options) + .await + .map_err(|source| DbError::OpenDatabase { + path: path.to_path_buf(), + source, + }) +} + +async fn migrate_pool(pool: &SqlitePool) -> Result<()> { + prepare_legacy_schema(pool).await?; + MIGRATOR + .run(pool) + .await + .map_err(|source| DbError::Migrate { source })?; + sqlx::query("PRAGMA user_version = 4") + .execute(pool) + .await + .map_err(|source| DbError::PrepareLegacyMigration { source })?; + Ok(()) +} + +async fn prepare_legacy_schema(pool: &SqlitePool) -> Result<()> { + let has_legacy_documents: Option = sqlx::query_scalar( + " + SELECT 1 + FROM sqlite_master + WHERE type = 'table' AND name = 'indexed_documents' + ", + ) + .fetch_optional(pool) + .await + .map_err(|source| DbError::PrepareLegacyMigration { source })?; + + if has_legacy_documents.is_none() { + return Ok(()); + } + + let rows = sqlx::query("PRAGMA table_info(indexed_documents)") + .fetch_all(pool) + .await + .map_err(|source| DbError::PrepareLegacyMigration { source })?; + let columns = rows + .into_iter() + .map(|row| row.try_get::("name")) + .collect::, _>>() + .map_err(|source| DbError::PrepareLegacyMigration { source })?; + + add_legacy_column_if_missing( + pool, + &columns, + "name", + "ALTER TABLE indexed_documents ADD COLUMN name TEXT NOT NULL DEFAULT ''", + ) + .await?; + add_legacy_column_if_missing( + pool, + &columns, + "size_bytes", + "ALTER TABLE indexed_documents ADD COLUMN size_bytes INTEGER NOT NULL DEFAULT 0", + ) + .await?; + add_legacy_column_if_missing( + pool, + &columns, + "created_unix_seconds", + "ALTER TABLE indexed_documents ADD COLUMN created_unix_seconds INTEGER", + ) + .await?; + add_legacy_column_if_missing( + pool, + &columns, + "accessed_unix_seconds", + "ALTER TABLE indexed_documents ADD COLUMN accessed_unix_seconds INTEGER", + ) + .await?; + add_legacy_column_if_missing( + pool, + &columns, + "readonly", + "ALTER TABLE indexed_documents ADD COLUMN readonly INTEGER NOT NULL DEFAULT 0", + ) + .await?; + + Ok(()) +} + +async fn add_legacy_column_if_missing( + pool: &SqlitePool, + columns: &HashSet, + column: &str, + sql: &str, +) -> Result<()> { + if columns.contains(column) { + return Ok(()); + } + + sqlx::query(sql) + .execute(pool) + .await + .map_err(|source| DbError::PrepareLegacyMigration { source })?; + Ok(()) +} + +fn decode_classification_row( + row: SqliteRow, +) -> std::result::Result { + let confidence: f64 = row.try_get("confidence")?; Ok(DirectoryClassification { - directory_path: row.get(0)?, - label: row.get(1)?, - confidence: row.get(2)?, - detector: row.get(3)?, - evidence_path: row.get(4)?, - evidence_summary: row.get(5)?, - detected_unix_seconds: row.get(6)?, + directory_path: row.try_get("directory_path")?, + label: row.try_get("label")?, + confidence: confidence as f32, + detector: row.try_get("detector")?, + evidence_path: row.try_get("evidence_path")?, + evidence_summary: row.try_get("evidence_summary")?, + detected_unix_seconds: row.try_get("detected_unix_seconds")?, + }) +} + +fn decode_directory_type_count_row( + row: SqliteRow, +) -> std::result::Result { + let count: i64 = row.try_get("directory_count")?; + Ok(DirectoryTypeCount { + label: row.try_get("label")?, + count: u64::try_from(count).map_err(|err| sqlx::Error::Decode(Box::new(err)))?, }) } -fn decode_document_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { - let kind: String = row.get(2)?; - let embedding: Vec = row.get(5)?; - let size_bytes: i64 = row.get(7)?; +fn decode_document_row(row: SqliteRow) -> std::result::Result { + let path: String = row.try_get("path")?; + let kind: String = row.try_get("kind")?; + let embedding: Vec = row.try_get("embedding")?; + let size_bytes: i64 = row.try_get("size_bytes")?; Ok(IndexedDocument { - path: row.get(0)?, - name: row.get(1)?, + path, + name: row.try_get("name")?, kind: DocumentKind::from_db_value(&kind), - parent_path: row.get(3)?, - searchable_text: row.get(4)?, - embedding: decode_embedding(&embedding).map_err(|err| { - rusqlite::Error::FromSqlConversionFailure( - 5, - rusqlite::types::Type::Blob, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - err.to_string(), - )), - ) - })?, - metadata_fingerprint: row.get(6)?, - size_bytes: u64::try_from(size_bytes).map_err(|err| { - rusqlite::Error::FromSqlConversionFailure( - 7, - rusqlite::types::Type::Integer, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - err.to_string(), - )), - ) - })?, - created_unix_seconds: row.get(8)?, - modified_unix_seconds: row.get(9)?, - accessed_unix_seconds: row.get(10)?, - readonly: row.get(11)?, - indexed_unix_seconds: row.get(12)?, + parent_path: row.try_get("parent_path")?, + searchable_text: row.try_get("searchable_text")?, + embedding: decode_embedding(&embedding) + .map_err(|err| sqlx::Error::Decode(Box::new(err)))?, + metadata_fingerprint: row.try_get("metadata_fingerprint")?, + size_bytes: u64::try_from(size_bytes).map_err(|err| sqlx::Error::Decode(Box::new(err)))?, + created_unix_seconds: row.try_get("created_unix_seconds")?, + modified_unix_seconds: row.try_get("modified_unix_seconds")?, + accessed_unix_seconds: row.try_get("accessed_unix_seconds")?, + readonly: row.try_get("readonly")?, + indexed_unix_seconds: row.try_get("indexed_unix_seconds")?, + }) +} + +fn decode_file_chunk_match_row(row: SqliteRow) -> std::result::Result { + let embedding: Vec = row.try_get("embedding")?; + Ok(FileChunkMatch { + file_path: row.try_get("file_path")?, + file_name: row.try_get("file_name")?, + directory_path: row.try_get("directory_path")?, + content: row.try_get("content")?, + embedding: decode_embedding(&embedding) + .map_err(|err| sqlx::Error::Decode(Box::new(err)))?, + file_modified_unix_seconds: row.try_get("file_modified_unix_seconds")?, + directory_modified_unix_seconds: row.try_get("directory_modified_unix_seconds")?, }) } @@ -706,9 +817,9 @@ fn decode_document_row(row: &rusqlite::Row<'_>) -> rusqlite::Result Result<(), DbError> { - let version: i64 = connection - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .map_err(|source| DbError::ReadSchemaVersion { source })?; - - if version < 1 { - migrate_to_v1(connection)?; - } - - if version < 2 { - migrate_to_v2(connection)?; - } - - if version < 3 { - migrate_to_v3(connection)?; - } - - if version < 4 { - migrate_to_v4(connection)?; - } - - connection - .pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION) - .map_err(|source| DbError::UpdateSchemaVersion { source })?; - - Ok(()) -} - -fn migrate_to_v1(connection: &Connection) -> Result<(), DbError> { - connection - .execute_batch( - " - CREATE TABLE IF NOT EXISTS indexed_documents ( - path TEXT PRIMARY KEY NOT NULL, - kind TEXT NOT NULL CHECK (kind IN ('directory', 'file')), - parent_path TEXT, - searchable_text TEXT NOT NULL, - embedding BLOB NOT NULL, - embedding_dim INTEGER NOT NULL, - metadata_fingerprint TEXT NOT NULL, - modified_unix_seconds INTEGER NOT NULL, - indexed_unix_seconds INTEGER NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_indexed_documents_kind - ON indexed_documents(kind); - - CREATE INDEX IF NOT EXISTS idx_indexed_documents_parent_path - ON indexed_documents(parent_path); - - CREATE INDEX IF NOT EXISTS idx_indexed_documents_indexed_time - ON indexed_documents(indexed_unix_seconds); - ", - ) - .map_err(|source| DbError::CreateSchemaV1 { source }) -} - -fn migrate_to_v2(connection: &Connection) -> Result<(), DbError> { - connection - .execute_batch( - " - ALTER TABLE indexed_documents - ADD COLUMN name TEXT NOT NULL DEFAULT ''; - - ALTER TABLE indexed_documents - ADD COLUMN size_bytes INTEGER NOT NULL DEFAULT 0; - - ALTER TABLE indexed_documents - ADD COLUMN created_unix_seconds INTEGER; - - ALTER TABLE indexed_documents - ADD COLUMN accessed_unix_seconds INTEGER; - - ALTER TABLE indexed_documents - ADD COLUMN readonly INTEGER NOT NULL DEFAULT 0; - - CREATE INDEX IF NOT EXISTS idx_indexed_documents_name - ON indexed_documents(name); - ", - ) - .map_err(|source| DbError::CreateSchemaV2 { source }) -} - -fn migrate_to_v3(connection: &Connection) -> Result<(), DbError> { - connection - .execute_batch( - " - CREATE TABLE IF NOT EXISTS indexed_files ( - path TEXT PRIMARY KEY NOT NULL, - directory_path TEXT NOT NULL, - name TEXT NOT NULL, - extension TEXT, - size_bytes INTEGER NOT NULL, - created_unix_seconds INTEGER, - modified_unix_seconds INTEGER NOT NULL, - accessed_unix_seconds INTEGER, - readonly INTEGER NOT NULL, - content_fingerprint TEXT NOT NULL, - indexed_unix_seconds INTEGER NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_indexed_files_directory_path - ON indexed_files(directory_path); - - CREATE INDEX IF NOT EXISTS idx_indexed_files_modified_time - ON indexed_files(modified_unix_seconds); - - CREATE TABLE IF NOT EXISTS indexed_file_chunks ( - file_path TEXT NOT NULL, - directory_path TEXT NOT NULL, - chunk_index INTEGER NOT NULL, - content TEXT NOT NULL, - embedding BLOB NOT NULL, - embedding_dim INTEGER NOT NULL, - start_byte INTEGER NOT NULL, - end_byte INTEGER NOT NULL, - indexed_unix_seconds INTEGER NOT NULL, - PRIMARY KEY (file_path, chunk_index) - ); - - CREATE INDEX IF NOT EXISTS idx_indexed_file_chunks_directory_path - ON indexed_file_chunks(directory_path); - ", - ) - .map_err(|source| DbError::CreateSchemaV3 { source }) -} - -fn migrate_to_v4(connection: &Connection) -> Result<(), DbError> { - connection - .execute_batch( - " - CREATE TABLE IF NOT EXISTS directory_classifications ( - directory_path TEXT NOT NULL, - label TEXT NOT NULL, - confidence REAL NOT NULL, - detector TEXT NOT NULL, - evidence_path TEXT, - evidence_summary TEXT NOT NULL, - detected_unix_seconds INTEGER NOT NULL, - PRIMARY KEY (directory_path, label, detector) - ); - - CREATE INDEX IF NOT EXISTS idx_directory_classifications_label - ON directory_classifications(label); - - CREATE INDEX IF NOT EXISTS idx_directory_classifications_directory_path - ON directory_classifications(directory_path); - ", - ) - .map_err(|source| DbError::CreateSchemaV4 { source }) -} diff --git a/src/index/mod.rs b/src/index/mod.rs index 2e53e09..1cd4fe9 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -35,12 +35,16 @@ where } } - pub fn index_configured_roots(&self) -> Result { + pub async fn index_configured_roots(&self) -> Result { let mut progress = NoopProgress; self.index_configured_roots_with_progress(&mut progress) + .await } - pub fn index_configured_roots_with_progress

(&self, progress: &mut P) -> Result + pub async fn index_configured_roots_with_progress

( + &self, + progress: &mut P, + ) -> Result where P: IndexProgress + ?Sized, { @@ -48,15 +52,15 @@ where .settings .expanded_roots() .map_err(|source| IndexError::ExpandConfiguredRoots { source })?; - self.index_roots_with_progress(roots, progress) + self.index_roots_with_progress(roots, progress).await } - pub fn index_roots(&self, roots: Vec) -> Result { + pub async fn index_roots(&self, roots: Vec) -> Result { let mut progress = NoopProgress; - self.index_roots_with_progress(roots, &mut progress) + self.index_roots_with_progress(roots, &mut progress).await } - pub fn index_roots_with_progress

( + pub async fn index_roots_with_progress

( &self, roots: Vec, progress: &mut P, @@ -80,6 +84,7 @@ where if is_excluded_index_root(self.settings, &root) { self.database .delete_path_tree(&root.to_string_lossy()) + .await .map_err(|source| IndexError::PruneExcludedPath { path: root.to_string_lossy().into_owned(), source: Box::new(source), @@ -97,6 +102,7 @@ where &mut report, progress, ) + .await .map_err(|source| IndexError::ScanRoot { root: root.clone(), source: Box::new(source), @@ -121,8 +127,8 @@ mod tests { use crate::db::{Database, DocumentKind, IndexedDocument}; use crate::embed::FakeEmbedder; - #[test] - fn indexes_directory_summaries_and_skips_excluded_paths() { + #[tokio::test] + async fn indexes_directory_summaries_and_skips_excluded_paths() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("Projects"); let app = root.join("chrome-extension"); @@ -138,7 +144,7 @@ mod tests { fs::write(hidden.join("settings.json"), "should not be indexed").unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); database .upsert_document(&IndexedDocument { path: asset_catalog.to_string_lossy().into_owned(), @@ -157,6 +163,7 @@ mod tests { readonly: false, indexed_unix_seconds: 0, }) + .await .unwrap(); database .upsert_document(&IndexedDocument { @@ -176,21 +183,23 @@ mod tests { readonly: false, indexed_unix_seconds: 0, }) + .await .unwrap(); let embedder = FakeEmbedder::new(16); let indexer = Indexer::new(&settings, &database, &embedder); - let report = indexer.index_roots(vec![root.clone()]).unwrap(); + let report = indexer.index_roots(vec![root.clone()]).await.unwrap(); assert_eq!(report.roots_scanned, 1); assert_eq!(report.directories_indexed, 3); assert_eq!(report.text_files_indexed, 1); assert_eq!(report.file_chunks_indexed, 1); assert_eq!(report.entries_skipped, 3); - assert_eq!(database.document_count().unwrap(), 3); + assert_eq!(database.document_count().await.unwrap(), 3); let app_document = database .get_document(&app.to_string_lossy()) + .await .unwrap() .expect("app directory is indexed"); assert!( @@ -203,23 +212,27 @@ mod tests { assert_eq!( database .get_document(&asset_catalog.to_string_lossy()) + .await .unwrap(), None ); assert_eq!( - database.get_document(&hidden.to_string_lossy()).unwrap(), + database + .get_document(&hidden.to_string_lossy()) + .await + .unwrap(), None ); } - #[test] - fn skips_hidden_roots_and_prunes_stale_rows() { + #[tokio::test] + async fn skips_hidden_roots_and_prunes_stale_rows() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join(".hidden-project"); fs::create_dir_all(&root).unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); database .upsert_document(&IndexedDocument { path: root.to_string_lossy().into_owned(), @@ -238,22 +251,26 @@ mod tests { readonly: false, indexed_unix_seconds: 0, }) + .await .unwrap(); let embedder = FakeEmbedder::new(16); let indexer = Indexer::new(&settings, &database, &embedder); - let report = indexer.index_roots(vec![root.clone()]).unwrap(); + let report = indexer.index_roots(vec![root.clone()]).await.unwrap(); assert_eq!(report.roots_scanned, 0); assert_eq!(report.entries_skipped, 1); assert_eq!( - database.get_document(&root.to_string_lossy()).unwrap(), + database + .get_document(&root.to_string_lossy()) + .await + .unwrap(), None ); } - #[test] - fn limits_recursion_depth_per_top_level_directory() { + #[tokio::test] + async fn limits_recursion_depth_per_top_level_directory() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("Projects"); let top_level = root.join("app"); @@ -264,7 +281,7 @@ mod tests { fs::create_dir_all(&too_deep).unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); database .upsert_document(&IndexedDocument { path: too_deep.to_string_lossy().into_owned(), @@ -283,11 +300,12 @@ mod tests { readonly: false, indexed_unix_seconds: 0, }) + .await .unwrap(); let embedder = FakeEmbedder::new(16); let indexer = Indexer::new(&settings, &database, &embedder); - let report = indexer.index_roots(vec![root.clone()]).unwrap(); + let report = indexer.index_roots(vec![root.clone()]).await.unwrap(); assert_eq!(report.roots_scanned, 1); assert_eq!(report.directories_indexed, 5); @@ -297,23 +315,29 @@ mod tests { assert!( database .get_document(&root.to_string_lossy()) + .await .unwrap() .is_some() ); assert!( database .get_document(&top_level.to_string_lossy()) + .await .unwrap() .is_some() ); assert!( database .get_document(&depth_three.to_string_lossy()) + .await .unwrap() .is_some() ); assert_eq!( - database.get_document(&too_deep.to_string_lossy()).unwrap(), + database + .get_document(&too_deep.to_string_lossy()) + .await + .unwrap(), None ); } diff --git a/src/index/scanner.rs b/src/index/scanner.rs index 636240a..623fab0 100644 --- a/src/index/scanner.rs +++ b/src/index/scanner.rs @@ -1,5 +1,7 @@ use std::fs; +use std::future::Future; use std::path::Path; +use std::pin::Pin; use super::{classify, file, summary}; use crate::config::Settings; @@ -35,7 +37,7 @@ impl IndexReport { } } -pub fn scan_root_with_progress( +pub async fn scan_root_with_progress( root: &Path, settings: &Settings, database: &Database, @@ -56,6 +58,7 @@ where progress, DepthPosition::IndexRoot, ) + .await } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -75,126 +78,135 @@ impl DepthPosition { } } -fn scan_directory( - directory: &Path, - settings: &Settings, - database: &Database, - embedder: &E, - report: &mut IndexReport, - progress: &mut P, +fn scan_directory<'a, E, P>( + directory: &'a Path, + settings: &'a Settings, + database: &'a Database, + embedder: &'a E, + report: &'a mut IndexReport, + progress: &'a mut P, depth_position: DepthPosition, -) -> Result<()> +) -> Pin> + 'a>> where - E: Embedder, - P: IndexProgress + ?Sized, + E: Embedder + 'a, + P: IndexProgress + ?Sized + 'a, { - progress.directory_started(directory); - - let document = summary::summarize_directory(directory, settings).map_err(|source| { - IndexError::SummarizeDirectory { - path: directory.to_path_buf(), - source: Box::new(source), - } - })?; + Box::pin(async move { + progress.directory_started(directory); - database - .upsert_document(&document) - .map_err(|source| IndexError::StoreDocument { - path: document.path.clone(), - source: Box::new(source), - })?; - let classifications = classify::classify_directory(directory, settings)?; - database - .replace_directory_classifications(&document.path, &classifications) - .map_err(|source| IndexError::StoreDirectoryClassifications { - path: document.path.clone(), - source: Box::new(source), + let document = summary::summarize_directory(directory, settings).map_err(|source| { + IndexError::SummarizeDirectory { + path: directory.to_path_buf(), + source: Box::new(source), + } })?; - report.directories_indexed += 1; - let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { - path: directory.to_path_buf(), - source, - })?; + database + .upsert_document(&document) + .await + .map_err(|source| IndexError::StoreDocument { + path: document.path.clone(), + source: Box::new(source), + })?; + let classifications = classify::classify_directory(directory, settings)?; + database + .replace_directory_classifications(&document.path, &classifications) + .await + .map_err(|source| IndexError::StoreDirectoryClassifications { + path: document.path.clone(), + source: Box::new(source), + })?; + report.directories_indexed += 1; - for entry in entries { - let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { + let entries = fs::read_dir(directory).map_err(|source| IndexError::ReadDirectory { path: directory.to_path_buf(), source, })?; - let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - - let file_type = entry - .file_type() - .map_err(|source| IndexError::ReadFileType { - path: path.clone(), + + for entry in entries { + let entry = entry.map_err(|source| IndexError::ReadDirectoryEntry { + path: directory.to_path_buf(), source, })?; - - if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { - database - .delete_path_tree(&path.to_string_lossy()) - .map_err(|source| IndexError::PruneExcludedPath { - path: path.to_string_lossy().into_owned(), - source: Box::new(source), + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_string_lossy(); + + let file_type = entry + .file_type() + .map_err(|source| IndexError::ReadFileType { + path: path.clone(), + source, })?; - report.entries_skipped += 1; - continue; - } - - if file_type.is_dir() { - let Some(child_position) = depth_position.child_position() else { - prune_unindexed_directory(database, report, &path)?; - continue; - }; - if exceeds_max_depth(settings, child_position) { - prune_unindexed_directory(database, report, &path)?; - continue; - } - - scan_directory( - &path, - settings, - database, - embedder, - report, - progress, - child_position, - )?; - } else if file_type.is_file() { - if settings.index.is_excluded_name(&name) { + if file_type.is_dir() && settings.index.is_excluded_directory_name(&name) { + database + .delete_path_tree(&path.to_string_lossy()) + .await + .map_err(|source| IndexError::PruneExcludedPath { + path: path.to_string_lossy().into_owned(), + source: Box::new(source), + })?; report.entries_skipped += 1; continue; } - report.files_seen += 1; - if let Some(indexed_file) = file::index_text_file(&path, directory, settings, embedder)? - { - report.text_files_indexed += 1; - report.file_chunks_indexed += - u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); - database.upsert_file(&indexed_file.file).map_err(|source| { - IndexError::StoreFile { - path: indexed_file.file.path.clone(), - source: Box::new(source), - } - })?; - database - .replace_file_chunks(&indexed_file.file.path, &indexed_file.chunks) - .map_err(|source| IndexError::StoreFileChunks { - path: indexed_file.file.path, - source: Box::new(source), - })?; + if file_type.is_dir() { + let Some(child_position) = depth_position.child_position() else { + prune_unindexed_directory(database, report, &path).await?; + continue; + }; + + if exceeds_max_depth(settings, child_position) { + prune_unindexed_directory(database, report, &path).await?; + continue; + } + + scan_directory( + &path, + settings, + database, + embedder, + report, + progress, + child_position, + ) + .await?; + } else if file_type.is_file() { + if settings.index.is_excluded_name(&name) { + report.entries_skipped += 1; + continue; + } + + report.files_seen += 1; + if let Some(indexed_file) = + file::index_text_file(&path, directory, settings, embedder)? + { + report.text_files_indexed += 1; + report.file_chunks_indexed += + u64::try_from(indexed_file.chunks.len()).unwrap_or(u64::MAX); + database + .upsert_file(&indexed_file.file) + .await + .map_err(|source| IndexError::StoreFile { + path: indexed_file.file.path.clone(), + source: Box::new(source), + })?; + database + .replace_file_chunks(&indexed_file.file.path, &indexed_file.chunks) + .await + .map_err(|source| IndexError::StoreFileChunks { + path: indexed_file.file.path, + source: Box::new(source), + })?; + } + } else { + report.entries_skipped += 1; } - } else { - report.entries_skipped += 1; } - } - Ok(()) + Ok(()) + }) } fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool { @@ -206,13 +218,14 @@ fn exceeds_max_depth(settings: &Settings, depth_position: DepthPosition) -> bool } } -fn prune_unindexed_directory( +async fn prune_unindexed_directory( database: &Database, report: &mut IndexReport, path: &Path, ) -> Result<()> { database .delete_path_tree(&path.to_string_lossy()) + .await .map_err(|source| IndexError::PruneExcludedPath { path: path.to_string_lossy().into_owned(), source: Box::new(source), diff --git a/src/main.rs b/src/main.rs index 8a07ce2..abfff02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,8 @@ use cds::index::IndexProgress; use cds::{error, shell_init}; use color_eyre::eyre::{Result, WrapErr}; -fn main() -> ExitCode { +#[tokio::main] +async fn main() -> ExitCode { if let Err(err) = install_error_reporter() { eprintln!("cds: failed to install error reporter: {err}"); return ExitCode::FAILURE; @@ -26,7 +27,7 @@ fn main() -> ExitCode { Err(err) => err.exit(), }; - match run(invocation) { + match run(invocation).await { Ok(()) => ExitCode::SUCCESS, Err(err) => { let report = color_eyre::eyre::Report::from(err); @@ -44,18 +45,18 @@ fn install_error_reporter() -> Result<()> { .wrap_err("failed to install color-eyre error reporter") } -fn run(invocation: Invocation) -> error::Result<()> { +async fn run(invocation: Invocation) -> error::Result<()> { match invocation { Invocation::DirectoryTypeCount => { - for count in app::directory_type_counts()? { + for count in app::directory_type_counts().await? { println!("{}\t{}", count.count, count.label); } } - Invocation::EmitCd { args } => write_stdout(&app::resolve_cd_script(args))?, + Invocation::EmitCd { args } => write_stdout(&app::resolve_cd_script(args).await)?, Invocation::ShellInit { shell } => write_stdout(shell_init(shell).as_bytes())?, Invocation::Init => { let mut progress = TerminalIndexProgress::start(); - let report = app::init_with_progress(&mut progress)?; + let report = app::init_with_progress(&mut progress).await?; progress.finish(); println!("config ready: {}", report.config_file.display()); println!("database ready: {}", report.database_file.display()); @@ -63,20 +64,20 @@ fn run(invocation: Invocation) -> error::Result<()> { } Invocation::Index { roots } => { let mut progress = TerminalIndexProgress::start(); - let report = app::index_with_progress(roots, &mut progress)?; + let report = app::index_with_progress(roots, &mut progress).await?; progress.finish(); println!("{}", report.human_summary()); } Invocation::Reset => { if confirm_reset()? { - app::reset_database()?; + app::reset_database().await?; println!("database reset"); } else { println!("reset cancelled"); } } Invocation::Search { query } => { - let results = app::search(query, 10)?; + let results = app::search(query, 10).await?; for result in results { println!("{:.3}\t{}", result.score, result.path); } diff --git a/src/search/mod.rs b/src/search/mod.rs index 18e664f..d4000f7 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -30,7 +30,7 @@ where Self { database, embedder } } - pub fn search(&self, query: &str, limit: usize) -> Result> { + pub async fn search(&self, query: &str, limit: usize) -> Result> { let query_embedding = self .embedder .embed(query) @@ -38,6 +38,7 @@ where let chunks = self .database .file_chunk_matches() + .await .map_err(|source| SearchError::LoadDirectories { source })?; let temporal_filter = TemporalFilter::from_query(query); let use_deep_target = wants_deep_target(query); @@ -53,6 +54,7 @@ where let classifications = self .database .ancestor_classifications(&chunk.directory_path) + .await .map_err(|source| SearchError::LoadDirectories { source })?; score += lexical_boost( &query_terms, @@ -74,6 +76,7 @@ where } else { self.database .general_indexed_directory(&chunk.directory_path) + .await .map_err(|source| SearchError::LoadDirectories { source })? }; @@ -86,11 +89,13 @@ where for directory in self .database .directory_documents() + .await .map_err(|source| SearchError::LoadDirectories { source })? { let classifications = self .database .directory_classifications(&directory.path) + .await .map_err(|source| SearchError::LoadDirectories { source })?; let score = classification_boost(&query_terms, &classifications) + lexical_boost( @@ -106,6 +111,7 @@ where } else { self.database .general_indexed_directory(&directory.path) + .await .map_err(|source| SearchError::LoadDirectories { source })? }; @@ -291,8 +297,8 @@ mod tests { use crate::embed::FakeEmbedder; use crate::index::Indexer; - #[test] - fn returns_best_directory_match() { + #[tokio::test] + async fn returns_best_directory_match() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("Projects"); let chrome = root.join("chrome-extension"); @@ -307,21 +313,23 @@ mod tests { fs::write(notes.join("README.md"), "Meeting notes calendar agenda").unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); let embedder = FakeEmbedder::default(); Indexer::new(&settings, &database, &embedder) .index_roots(vec![root]) + .await .unwrap(); let results = Searcher::new(&database, &embedder) .search("chrome extension", 3) + .await .unwrap(); assert_eq!(results.first().unwrap().path, chrome.to_string_lossy()); } - #[test] - fn returns_directory_by_deterministic_type_classification() { + #[tokio::test] + async fn returns_directory_by_deterministic_type_classification() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("Projects"); let rust = root.join("cds-rust"); @@ -332,21 +340,23 @@ mod tests { fs::write(notes.join("README.md"), "rust colored notes without cargo").unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); let embedder = FakeEmbedder::default(); Indexer::new(&settings, &database, &embedder) .index_roots(vec![root]) + .await .unwrap(); let results = Searcher::new(&database, &embedder) .search("rust project", 3) + .await .unwrap(); assert_eq!(results.first().unwrap().path, rust.to_string_lossy()); } - #[test] - fn prefers_deeper_directory_when_query_has_strong_signal() { + #[tokio::test] + async fn prefers_deeper_directory_when_query_has_strong_signal() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("Projects"); let chrome = root.join("chrome-extension"); @@ -359,14 +369,16 @@ mod tests { .unwrap(); let settings = Settings::default(); - let database = Database::open_in_memory().unwrap(); + let database = Database::open_in_memory().await.unwrap(); let embedder = FakeEmbedder::default(); Indexer::new(&settings, &database, &embedder) .index_roots(vec![root]) + .await .unwrap(); let results = Searcher::new(&database, &embedder) .search("migrations chrome extension", 3) + .await .unwrap(); assert_eq!(results.first().unwrap().path, migrations.to_string_lossy());