From c63559cc26326ab32fc9dbbe4e12567c839f32f2 Mon Sep 17 00:00:00 2001 From: plum Date: Thu, 4 Jun 2026 15:11:04 -0400 Subject: [PATCH] Speed up Docker cd equivalence CI --- .github/workflows/ci.yml | 2 +- AGENTS.md | 10 +++++----- Cargo.toml | 6 +++++- README.md | 19 ++++++++++++++----- src/app/mod.rs | 29 ++++++++++++++++++++++++----- src/embed/mod.rs | 2 ++ tests/docker_cd_equivalence.rs | 4 +++- tests/docker_cd_equivalence.sh | 7 +++++-- 8 files changed, 59 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa69d4c..219cd45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,6 @@ jobs: cd-equivalence: name: Docker cd Equivalence runs-on: ubuntu-latest - needs: rust steps: - name: Checkout @@ -92,4 +91,5 @@ jobs: - name: Run Docker cd equivalence test env: CDS_DOCKER_IMAGE: rust:1 + CDS_DOCKER_RANDOM_CASES: 20 run: cargo test --test docker_cd_equivalence -- --nocapture diff --git a/AGENTS.md b/AGENTS.md index 840a049..9add1c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,16 +184,16 @@ against Bash's built-in `cd` for status, stdout, stderr, `PWD`, `OLDPWD`, and ph Keep these details in mind when editing it: - CI currently uses `CDS_DOCKER_IMAGE=rust:1`. -- The Docker image must include Cargo and C++ standard library/linker support. The - FastEmbed/ONNX dependency links against `libstdc++`, so very small images such as - `rust:1-slim` can fail with `unable to find library -lstdc++`. +- The Docker image must include Cargo. The test builds `cds` with `--no-default-features` + and `CDS_EMBEDDER=fake`, so it does not need the FastEmbed/ONNX dependency stack. - The test runner must explicitly add `/usr/local/cargo/bin` to `PATH`; some container invocations otherwise fail with `cargo: command not found`. - Bash includes source line numbers in `cd` diagnostics. Normalize only those line numbers before comparing stderr, while keeping the actual diagnostic text exact. -- The Docker test can skip locally when Docker is unavailable, so use +- The Docker test can skip locally when Docker is unavailable. Use `CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence -- --nocapture` - with Docker access when changing equivalence behavior. + with Docker access when changing equivalence behavior. Set `CDS_DOCKER_RANDOM_CASES` + to increase or decrease the randomized sample count. ## Repository Hygiene diff --git a/Cargo.toml b/Cargo.toml index 540b35a..0313a16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,14 @@ name = "cds" version = "0.1.0" edition = "2024" +[features] +default = ["real-embedder"] +real-embedder = ["dep:fastembed"] + [dependencies] clap = { version = "4.5.40", features = ["cargo"] } color-eyre = "0.6.5" -fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"] } +fastembed = { version = "5.15.0", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"], optional = true } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "sqlite", "migrate", "macros"] } diff --git a/README.md b/README.md index 98a09ad..5da4d2a 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ cargo test --all ``` The test suite includes a Docker-backed equivalence test. When Docker is available, it -mounts this project into a Rust container, builds `cds`, generates a fresh random -filesystem tree, and compares `cds` against the shell's built-in `cd` for exit status, -stdout, stderr, `PWD`, `OLDPWD`, and physical path behavior. +mounts this project into a Rust container, builds `cds` without the real embedding +runtime, generates a fresh random filesystem tree, and compares `cds` against the shell's +built-in `cd` for exit status, stdout, stderr, `PWD`, `OLDPWD`, and physical path +behavior. Use a specific container image with: @@ -72,8 +73,16 @@ Use a specific container image with: CDS_DOCKER_IMAGE=rust:1 cargo test --test docker_cd_equivalence ``` -The Docker image must include Cargo and the C++ standard library/linker support required by -the embedding runtime. The default image is `rust:1`. +Adjust the number of randomized cases with: + +```sh +CDS_DOCKER_RANDOM_CASES=200 cargo test --test docker_cd_equivalence +``` + +The Docker image must include Cargo. The default image is `rust:1`. + +Default builds include the real local embedding runtime. Use `--no-default-features` with +`CDS_EMBEDDER=fake` for shell-only checks that do not need `bge-small-en-v1.5`. ## Indexing diff --git a/src/app/mod.rs b/src/app/mod.rs index eac8a20..06fab66 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -6,7 +6,9 @@ use std::path::{Path, PathBuf}; use crate::config::{AppPaths, Settings, expand_tilde}; use crate::db::{Database, DirectoryTypeCount}; -use crate::embed::{BgeSmallEmbedder, Embedder, FakeEmbedder}; +#[cfg(feature = "real-embedder")] +use crate::embed::BgeSmallEmbedder; +use crate::embed::{Embedder, FakeEmbedder}; use crate::error::{Result, app_err, config_err, embed_err}; use crate::index::{IndexProgress, IndexReport, Indexer, NoopProgress}; use crate::search::{SearchResult, Searcher}; @@ -236,6 +238,7 @@ fn current_shell_directory() -> Option { } enum RuntimeEmbedder { + #[cfg(feature = "real-embedder")] Bge(Box), Fake(FakeEmbedder), } @@ -246,16 +249,14 @@ impl RuntimeEmbedder { return Ok(Self::Fake(FakeEmbedder::default())); } - BgeSmallEmbedder::new(&paths.cache_dir) - .map(Box::new) - .map(Self::Bge) - .map_err(embed_err) + load_real_embedder(paths) } } impl Embedder for RuntimeEmbedder { fn dimensions(&self) -> usize { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.dimensions(), Self::Fake(embedder) => embedder.dimensions(), } @@ -263,6 +264,7 @@ impl Embedder for RuntimeEmbedder { fn embed(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed(text), Self::Fake(embedder) => embedder.embed(text), } @@ -270,6 +272,7 @@ impl Embedder for RuntimeEmbedder { fn embed_document(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed_document(text), Self::Fake(embedder) => embedder.embed_document(text), } @@ -277,12 +280,28 @@ impl Embedder for RuntimeEmbedder { fn embed_query(&self, text: &str) -> crate::embed::Result> { match self { + #[cfg(feature = "real-embedder")] Self::Bge(embedder) => embedder.embed_query(text), Self::Fake(embedder) => embedder.embed_query(text), } } } +#[cfg(feature = "real-embedder")] +fn load_real_embedder(paths: &AppPaths) -> Result { + BgeSmallEmbedder::new(&paths.cache_dir) + .map(Box::new) + .map(RuntimeEmbedder::Bge) + .map_err(embed_err) +} + +#[cfg(not(feature = "real-embedder"))] +fn load_real_embedder(_paths: &AppPaths) -> Result { + Err(embed_err(crate::embed::EmbedError::Model { + message: "real embedder support is disabled; rebuild with the real-embedder feature or set CDS_EMBEDDER=fake".to_string(), + })) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 18eb92c..4259c8a 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -1,7 +1,9 @@ +#[cfg(feature = "real-embedder")] mod bge; mod error; mod fake; +#[cfg(feature = "real-embedder")] pub use bge::{BGE_SMALL_EN_V15_DIMENSIONS, BgeSmallEmbedder}; pub use error::EmbedError; pub use fake::FakeEmbedder; diff --git a/tests/docker_cd_equivalence.rs b/tests/docker_cd_equivalence.rs index 7766f4b..97e1708 100644 --- a/tests/docker_cd_equivalence.rs +++ b/tests/docker_cd_equivalence.rs @@ -11,7 +11,7 @@ if ! command -v cargo >/dev/null 2>&1; then exit 127 fi -cargo build --quiet +CDS_EMBEDDER=fake cargo build --quiet --no-default-features --bin cds bash tests/docker_cd_equivalence.sh "#; @@ -48,6 +48,8 @@ fn docker_cd_equivalence_random_tree() { .arg("CARGO_HOME=/tmp/cargo") .arg("--env") .arg("CARGO_TARGET_DIR=/tmp/cds-target") + .arg("--env") + .arg("CDS_EMBEDDER=fake") .arg(image) .arg("bash") .arg("-lc") diff --git a/tests/docker_cd_equivalence.sh b/tests/docker_cd_equivalence.sh index 871bd5d..7459785 100644 --- a/tests/docker_cd_equivalence.sh +++ b/tests/docker_cd_equivalence.sh @@ -65,8 +65,10 @@ for dir in "${dirs[@]}"; do printf 'fixture file for %s\n' "$dir" > "$dir/file_$RANDOM.txt" done +cds_init="$(cds --shell-init bash)" + setup_cds() { - eval "$(cds --shell-init bash)" + eval "$cds_init" } normalize_stderr() { @@ -183,7 +185,8 @@ compare_case "cd dash" "$tree_root" "$space_dir" "" - compare_case "invalid directory" "$tree_root" "$space_dir" "" "$tree_root/does-not-exist" compare_case "too many args" "$tree_root" "$space_dir" "" "$space_dir" "$quote_dir" -for i in $(seq 1 60); do +random_cases="${CDS_DOCKER_RANDOM_CASES:-60}" +for i in $(seq 1 "$random_cases"); do start="${dirs[$((RANDOM % ${#dirs[@]}))]}" oldpwd="${dirs[$((RANDOM % ${#dirs[@]}))]}" target="${dirs[$((RANDOM % ${#dirs[@]}))]}"