feat(index): accept a vector index a table is too small to train

python/python/tests/test_create_empty_index.py

@@ -31,17 +31,31 @@ def test_create_empty_vector_index():
     data = pa.table({"vector": vectors})
     dataset = lance.write_dataset(data, "memory://")
 
-    # Currently, vector indices with train=False are not supported
-    try:
-        dataset.create_index(
-            "vector", "IVF_PQ", num_partitions=10, num_sub_vectors=8, train=False
-        )
-        # If we get here, the implementation has been added (unexpected for now)
-        assert False, (
-            "Expected NotImplementedError for train=False on vector index, "
-            "but succeeded"
-        )
-    except NotImplementedError as e:
-        # Expected error for unimplemented functionality
-        error_msg = str(e).lower()
-        assert "not yet implemented" in error_msg or "not implemented" in error_msg
+    dataset.create_index(
+        "vector", "IVF_PQ", num_partitions=10, num_sub_vectors=8, train=False
+    )
+
+    # Same shape the scalar case reports: listed, covering nothing yet.
+    indices = dataset.describe_indices()
+    assert len(indices) == 1
+    stats = dataset.stats.index_stats(indices[0].name)
+    assert stats["num_indexed_rows"] == 0
+    assert stats["num_unindexed_rows"] == dataset.count_rows()
+
+
+def test_create_vector_index_below_the_row_floor():
+    """A table too small to train the quantizer takes the index anyway."""
+    dim = 32
+    values = pc.random(100 * dim).cast(pa.float32())
+    vectors = pa.FixedSizeListArray.from_arrays(values, dim)
+    data = pa.table({"vector": vectors})
+    dataset = lance.write_dataset(data, "memory://")
+
+    # 100 vectors cannot train a 256-code codebook, and train defaults to True.
+    dataset.create_index("vector", "IVF_PQ", num_partitions=10, num_sub_vectors=8)
+
+    indices = dataset.describe_indices()
+    assert len(indices) == 1
+    stats = dataset.stats.index_stats(indices[0].name)
+    assert stats["num_indexed_rows"] == 0
+    assert stats["num_unindexed_rows"] == dataset.count_rows()

python/python/tests/test_ivf_centroids.py

@@ -8,7 +8,10 @@
 def test_ivf_centroids_exposed(tmp_path):
     """Verify that centroids for an IVF-based index are exposed via both"""
 
-    dim, rows, parts = 4, 256, 8
+    # A partition is trained only when the data can give it a 256-code
+    # codebook's worth of vectors, so the fixture covers every partition.
+    dim, parts = 4, 8
+    rows = parts * 256
     vecs = pa.array(
         np.random.randn(rows, dim).tolist(),
         type=pa.list_(pa.float32(), dim),

python/python/tests/test_vector_index.py

@@ -1238,18 +1238,19 @@ def test_create_ivf_rq_index():
     assert stats["indices"][0]["sub_index"]["num_bits"] == 5
     assert stats["indices"][0]["sub_index"]["packed"] is True
 
-    with pytest.raises(
-        NotImplementedError,
-        match="Creating empty vector indices with train=False is not yet implemented",
-    ):
-        ds.delete("id>=0")
-        ds = ds.create_index(
-            "vector",
-            index_type="IVF_RQ",
-            num_partitions=4,
-            num_bits=1,
-            replace=True,
-        )
+    # An emptied table still takes the index; it carries its settings and
+    # covers nothing until there is data to train on.
+    ds.delete("id>=0")
+    ds = ds.create_index(
+        "vector",
+        index_type="IVF_RQ",
+        num_partitions=4,
+        num_bits=1,
+        replace=True,
+    )
+    stats = ds.stats.index_stats("vector_idx")
+    assert stats["num_indexed_rows"] == 0
+    assert stats["num_unindexed_rows"] == 0
 
     zero_vectors = np.zeros((1000, 128)).astype(np.float32).tolist()
     tbl = pa.Table.from_pydict(
@@ -1863,7 +1864,9 @@ def query_index(ds, ntimes, q=None):
                 },
             )
 
-    tbl = create_table(nvec=1024, ndim=16)
+    # Each of the 128 partitions is trained only when the data can give it a
+    # 256-code codebook's worth of vectors.
+    tbl = create_table(nvec=128 * 256, ndim=16)
     dataset = lance.write_dataset(tbl, tmp_path / "test")
 
     dataset.create_index(
@@ -1908,7 +1911,9 @@ def query_index(ds, ntimes, q=None):
                 },
             )
 
-    tbl = create_table(nvec=1024, ndim=16)
+    # Each of the 128 partitions is trained only when the data can give it a
+    # 256-code codebook's worth of vectors.
+    tbl = create_table(nvec=128 * 256, ndim=16)
     dataset = lance.write_dataset(tbl, tmp_path / "test")
 
     dataset.create_index(

rust/lance/src/dataset/mem_wal/api.rs

@@ -823,20 +823,31 @@ async fn load_vector_index_config(
     let column = field.name.clone();
 
     // Inherit the base table's distance type so the in-memory index and the
-    // base index produce comparable distances. Surface the open error
-    // instead of silently defaulting to L2 — flushed `IVF_HNSW_SQ` files
-    // bake this metric into their on-disk metadata, so a wrong default would
-    // be durable corruption.
-    let distance_type = dataset
-        .open_vector_index(&column, &index_meta.uuid, &NoOpMetricsCollector)
-        .await
-        .map_err(|e| {
-            Error::invalid_input(format!(
-                "Failed to open base vector index '{}' to inherit distance type: {}",
-                index_name, e
-            ))
-        })?
-        .metric_type();
+    // base index produce comparable distances. The index's recorded details
+    // state it, and for an index that covers nothing they are the only source:
+    // it carries its settings with no file to open. Opening the index is the
+    // fallback for an entry whose details do not decode. Surface the failure
+    // rather than silently defaulting to L2 — flushed `IVF_HNSW_SQ` files bake this metric
+    // into their on-disk metadata, so a wrong default would be durable
+    // corruption.
+    let recorded = index_meta
+        .index_details
+        .as_deref()
+        .and_then(crate::index::vector::details::vector_params_from_details)
+        .map(|params| params.metric_type);
+    let distance_type = match recorded {
+        Some(distance_type) => distance_type,
+        None => dataset
+            .open_vector_index(&column, &index_meta.uuid, &NoOpMetricsCollector)
+            .await
+            .map_err(|e| {
+                Error::invalid_input(format!(
+                    "Failed to open base vector index '{}' to inherit distance type: {}",
+                    index_name, e
+                ))
+            })?
+            .metric_type(),
+    };
 
     Ok(match hnsw_params {
         Some(params) => MemIndexConfig::hnsw_with_params(
@@ -1050,6 +1061,53 @@ mod tests {
             .expect("a Float32 vector column is maintainable");
     }
 
+    /// An index that covers nothing is maintainable: its recorded details
+    /// state the distance type, so there is no file to open.
+    ///
+    /// A table can register WAL before it holds enough vectors to train, and
+    /// validation refusing the definition would leave the index outside the
+    /// maintained set for the life of the table — the set is a snapshot, so
+    /// training it later does not add it back.
+    #[tokio::test]
+    async fn test_validate_maintained_indexes_accepts_a_definition() {
+        use crate::index::vector::VectorIndexParams;
+        use lance_linalg::distance::DistanceType;
+
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            true,
+        )]));
+        let reader = RecordBatchIterator::new(vec![], schema.clone());
+        let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("vector_idx".to_string()),
+                &VectorIndexParams::ivf_pq(4, 8, 2, DistanceType::Cosine, 1),
+                true,
+            )
+            .await
+            .unwrap();
+        let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert!(
+            indices[0]
+                .fragment_bitmap
+                .as_ref()
+                .is_some_and(roaring::RoaringBitmap::is_empty),
+            "an empty table trains nothing, so the index covers no rows"
+        );
+
+        validate_maintained_indexes(&dataset, &["vector_idx".to_string()])
+            .await
+            .expect("a definition is maintainable");
+    }
+
     #[tokio::test]
     async fn test_validate_maintained_indexes_accepts_btree() {
         // Guards the shard-schema plumbing: validation resolves field ids against

rust/lance/src/dataset/scanner.rs

@@ -7280,7 +7280,7 @@ pub mod test_dataset {
     use uuid::Uuid;
 
     use crate::dataset::WriteParams;
-    use crate::index::vector::VectorIndexParams;
+    use crate::index::vector::{StageParams, VectorIndexParams};
 
     // Creates a dataset with 5 batches where each batch has 80 rows
     //
@@ -7379,7 +7379,13 @@ pub mod test_dataset {
         }
 
         pub async fn make_vector_index_with_metric(&mut self, metric: MetricType) -> Result<()> {
-            let params = VectorIndexParams::ivf_pq(2, 8, 2, metric, 2);
+            let mut params = VectorIndexParams::ivf_pq(2, 8, 2, metric, 2);
+            // Two partitions over 400 vectors only holds at a sample rate this
+            // fixture can cover: the trained count is capped at the vectors
+            // available per centroid.
+            if let Some(StageParams::Ivf(ivf)) = params.stages.first_mut() {
+                ivf.sample_rate = 200;
+            }
             self.dataset
                 .create_index(
                     &["vec"],
@@ -12040,10 +12046,16 @@ mod test {
         data_storage_version: LanceFileVersion,
         #[values(false, true)] stable_row_ids: bool,
     ) {
+        const PARTITIONS: usize = 4;
+        // A partition is trained only when the data can give it a codebook's
+        // worth of vectors, so the fixture has to cover every partition it asks
+        // for; the cached-entry count below counts per partition.
+        const ROWS: usize = PARTITIONS * 256;
+
         let vec_params = vec![
             // TODO: re-enable diskann test when we can tune to get reproducible results.
             // VectorIndexParams::with_diskann_params(MetricType::L2, DiskANNParams::new(10, 1.5, 10)),
-            VectorIndexParams::ivf_pq(4, 8, 2, MetricType::L2, 2),
+            VectorIndexParams::ivf_pq(PARTITIONS, 8, 2, MetricType::L2, 2),
         ];
         for params in vec_params {
             use lance_arrow::FixedSizeListArrayExt;
@@ -12066,14 +12078,14 @@ mod test {
 
             // vectors are [1, 1, 1, ...] [2, 2, 2, ...]
             let vector_values: Float32Array =
-                (0..32 * 512).map(|v| (v / 32) as f32 + 1.0).collect();
+                (0..32 * ROWS).map(|v| (v / 32) as f32 + 1.0).collect();
             let vectors = FixedSizeListArray::try_new_from_values(vector_values, 32).unwrap();
 
             let batches = vec![
                 RecordBatch::try_new(
                     schema.clone(),
                     vec![
-                        Arc::new(Int32Array::from_iter_values(0..512)),
+                        Arc::new(Int32Array::from_iter_values(0..ROWS as i32)),
                         Arc::new(vectors.clone()),
                     ],
                 )
@@ -12175,7 +12187,7 @@ mod test {
                 RecordBatch::try_new(
                     schema.clone(),
                     vec![
-                        Arc::new(Int32Array::from_iter_values(512..1024)),
+                        Arc::new(Int32Array::from_iter_values(ROWS as i32..2 * ROWS as i32)),
                         Arc::new(vectors),
                     ],
                 )
@@ -12205,7 +12217,7 @@ mod test {
                 .await
                 .unwrap();
 
-            dataset.delete("i < 512").await.unwrap();
+            dataset.delete(&format!("i < {ROWS}")).await.unwrap();
 
             let mut scan = dataset.scan();
             scan.nearest("vec", &key, 5).unwrap();
@@ -12224,7 +12236,9 @@ mod test {
             let batch = &results[0];
 
             // It should not pick up any results from the first fragment
-            let expected_i = BTreeSet::from_iter(vec![512, 513, 514, 515, 516]);
+            let first = ROWS as i32;
+            let expected_i =
+                BTreeSet::from_iter(vec![first, first + 1, first + 2, first + 3, first + 4]);
             let column_i = batch.column_by_name("i").unwrap();
             let actual_i: BTreeSet<i32> = as_primitive_array::<Int32Type>(column_i.as_ref())
                 .values()

rust/lance/src/dataset/tests/dataset_index.rs

@@ -86,7 +86,7 @@ async fn test_create_index(
         false,
     )]));
 
-    let float_arr = generate_random_array(512 * dimension as usize);
+    let float_arr = generate_random_array(2560 * dimension as usize);
     let vectors = Arc::new(
         <arrow_array::FixedSizeListArray as FixedSizeListArrayExt>::try_new_from_values(
             float_arr, dimension,
@@ -667,7 +667,7 @@ async fn test_create_int8_index(
         false,
     )]));
 
-    let int8_arr = generate_random_int8_array(512 * dimension as usize);
+    let int8_arr = generate_random_int8_array(2560 * dimension as usize);
     let vectors = Arc::new(
         <arrow_array::FixedSizeListArray as FixedSizeListArrayExt>::try_new_from_values(
             int8_arr, dimension,