From bdec79d9115be612cc8619494130184f4f06905c Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 11 Sep 2026 23:41:13 +0800 Subject: [PATCH 1/4] fix(index): check quantizer params before sampling training data load_or_build_quantizer sampled and residual-transformed the full training dataset before checking whether quantizer build params were set, wasting the entire pass when they were missing. The Some(q) match arm after the early return was also dead code (the early return at the top of the function already handles that case). Move the params check before any IO and remove the dead arm. Assisted-by: GLM-5.3 --- rust/lance/src/index/vector/builder.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index a7525529db4..fa951f6e56d 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -666,6 +666,11 @@ impl IvfIndexBuilder if self.quantizer.is_some() { return Ok(self.quantizer.clone().unwrap()); } + // Fail before any sampling or residual work when there is nothing + // to build with. + if self.quantizer_params.is_none() { + return Err(Error::invalid_input("quantizer build params not set")); + } let Some(dataset) = self.dataset.as_ref() else { return Err(Error::invalid_input( @@ -720,16 +725,8 @@ impl IvfIndexBuilder info!("Start to train quantizer"); let start = std::time::Instant::now(); - let quantizer = match &self.quantizer { - Some(q) => q.clone(), - None => { - let quantizer_params = self - .quantizer_params - .as_ref() - .ok_or(Error::invalid_input("quantizer build params not set"))?; - Q::build(&training_data, DistanceType::L2, quantizer_params)? - } - }; + let quantizer_params = self.quantizer_params.as_ref().unwrap(); + let quantizer = Q::build(&training_data, DistanceType::L2, quantizer_params)?; info!( "Trained quantizer in {:02} seconds", start.elapsed().as_secs_f32() From bda950d4e02ca22b6380c57df02d4ee3dbadf154 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 18 Sep 2026 13:48:42 +0800 Subject: [PATCH 2/4] fix(index): check quantizer params before sampling training data --- rust/lance/src/index/vector/builder.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index fa951f6e56d..0c0ba065aab 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -666,21 +666,19 @@ impl IvfIndexBuilder if self.quantizer.is_some() { return Ok(self.quantizer.clone().unwrap()); } - // Fail before any sampling or residual work when there is nothing - // to build with. - if self.quantizer_params.is_none() { + // Bind the params up front: the sampling and residual work below is + // wasted if there is nothing to build with. Reaching here with no + // params means the builder was given neither params nor a quantizer. + let Some(quantizer_params) = self.quantizer_params.as_ref() else { return Err(Error::invalid_input("quantizer build params not set")); - } + }; let Some(dataset) = self.dataset.as_ref() else { return Err(Error::invalid_input( "dataset not set before loading or building quantizer", )); }; - let sample_size_hint = match &self.quantizer_params { - Some(params) => params.try_sample_size()?, - None => 256 * 256, // here it must be retrain, let's just set sample size to the default value - }; + let sample_size_hint = quantizer_params.try_sample_size()?; let start = std::time::Instant::now(); info!( @@ -725,7 +723,6 @@ impl IvfIndexBuilder info!("Start to train quantizer"); let start = std::time::Instant::now(); - let quantizer_params = self.quantizer_params.as_ref().unwrap(); let quantizer = Q::build(&training_data, DistanceType::L2, quantizer_params)?; info!( "Trained quantizer in {:02} seconds", From b3a19d88d1bd4c44a2af977e7e3a23d3bd4bea9b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 18 Sep 2026 15:23:23 +0800 Subject: [PATCH 3/4] test(index): pin the quantizer params check ahead of sampling --- rust/lance/src/index/vector/builder.rs | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index 0c0ba065aab..576437dd6f9 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -3566,6 +3566,41 @@ mod tests { .collect() } + /// The params check has to run before the sampling work, not after it. + /// Both orders end in the same error for a builder with no quantizer + /// params, so the discriminator is *which* error comes first: this builder + /// points at a column the dataset does not have, so sampling fails with + /// its own message, and only a check that runs ahead of it still reports + /// the missing params. + #[tokio::test] + async fn test_load_or_build_quantizer_checks_params_before_sampling() { + use lance_index::vector::v3::shuffler::IvfShuffler; + + let tmp = tempfile::tempdir().unwrap(); + let uri = tmp.path().to_str().unwrap(); + let dataset = write_clusters(uri, &[(8, 0.0)]).await; + let index_dir = dataset.indices_dir().child("idx"); + + let builder = IvfIndexBuilder::::new( + dataset, + "no_such_column".to_owned(), + index_dir.clone(), + DistanceType::L2, + Box::new(IvfShuffler::new(index_dir, 1)), + None, + None, + (), + None, + ) + .unwrap(); + + let err = builder.load_or_build_quantizer().await.unwrap_err(); + assert!( + err.to_string().contains("quantizer build params not set"), + "expected the params error before any sampling, got: {err}" + ); + } + fn cluster_batch( schema: &Arc, num_rows: usize, From 214d236c94e5deaec9427bf397f1663972b9da1a Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 18 Sep 2026 15:34:53 +0800 Subject: [PATCH 4/4] test(index): use the non-deprecated path join in the new test --- rust/lance/src/index/vector/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index 576437dd6f9..1e8228ebf2f 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -3579,7 +3579,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let uri = tmp.path().to_str().unwrap(); let dataset = write_clusters(uri, &[(8, 0.0)]).await; - let index_dir = dataset.indices_dir().child("idx"); + let index_dir = dataset.indices_dir().join("idx"); let builder = IvfIndexBuilder::::new( dataset,