feat(index)!: support covering columns in distributed vector index builds

java/lance-jni/src/utils.rs

@@ -515,6 +515,7 @@ pub fn get_vector_index_params(
                 version: IndexFileVersion::V3,
                 skip_transpose: false,
                 runtime_hints: Default::default(),
+                covering_columns: Default::default(),
             })
         },
     )?;

java/src/main/java/org/lance/index/IndexDescription.java

@@ -69,7 +69,14 @@ public String getName() {
     return name;
   }
 
-  /** Field ids that this index is built on. */
+  /**
+   * Field ids that this index is built on -- the columns it can answer queries for.
+   *
+   * <p>This is the index's <em>keyed</em> prefix only. An index may additionally carry values for
+   * columns it is not keyed on; those are deliberately absent here, because the index cannot be
+   * searched on them. They stay reachable per segment via {@link Index#coveringFields()} on the
+   * entries of {@link #getMetadata()}.
+   */
   public List<Integer> getFieldIds() {
     return fieldIds;
   }

python/src/indices.rs

@@ -674,9 +674,11 @@ pub struct PyIndexDescription {
     pub type_url: String,
     /// The short type of the index (may not be unique)
     pub index_type: String,
-    /// The ids of the fields that the index is built on
+    /// The ids of the fields the index is keyed on -- the columns it can answer queries about.
+    /// Covering ("included") columns are excluded; those are reported per-segment as
+    /// `covering_fields`.
     pub fields: Vec<u32>,
-    /// The full paths of the fields that the index is built on
+    /// The full paths of the fields the index is keyed on, matching `fields`
     /// (dotted, with backtick-quoted segments for non-identifier names)
     pub field_names: Vec<String>,
     /// The number of rows indexed by the index

rust/lance-index/src/traits.rs

@@ -344,7 +344,13 @@ pub trait IndexDescription: Send + Sync {
     /// deleted.
     fn rows_indexed(&self) -> u64;
 
-    /// Returns the ids of the fields that the index is built on.
+    /// Returns the ids of the fields that the index is built on -- the columns it can
+    /// answer queries for.
+    ///
+    /// This is the index's *keyed* prefix only. An index may additionally carry values
+    /// for columns it is not keyed on (see [`IndexMetadata::covering_fields`]); those are
+    /// deliberately absent here, because the index cannot be searched on them. They stay
+    /// reachable per segment via [`Self::metadata`].
     fn field_ids(&self) -> &[u32];
 
     /// Returns a JSON string representation of the index details

rust/lance-index/src/vector/bq/storage.rs

@@ -44,6 +44,7 @@ use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle};
 use crate::pb;
 use crate::scalar::RowIdRemapper;
 use crate::vector::ApproxMode;
+use crate::vector::PART_ID_COLUMN;
 use crate::vector::bq::dist_table_quant::{
     DistTableDequant, quantize_dist_table_into, quantize_dist_table_u16_into,
 };
@@ -78,6 +79,25 @@ pub const RABIT_EX_CODE_COLUMN: &str = "__ex_codes";
 /// older versions, which fail with a missing-column error instead of
 /// misinterpreting the bytes.
 pub const RABIT_BLOCKED_EX_CODE_COLUMN: &str = "__blocked_ex_codes";
+/// RaBitQ storage's internal (non-covering) column names: the row id, the binary code
+/// and extended-bit code columns, the per-row factor columns (some present only for
+/// higher `num_bits`), and the legacy `__ivf_part_id` column (left in pre-#3606
+/// single-partition builds; unlike PQ, RaBitQ storage does not drop it at
+/// construction). Every other column is a user-declared covering ("included") column.
+/// The single authority for this set -- the distributed shard merger classifies from
+/// it too, so a new factor column added here is excluded everywhere at once.
+pub const RABIT_INTERNAL_COLUMNS: &[&str] = &[
+    ROW_ID,
+    RABIT_CODE_COLUMN,
+    RABIT_EX_CODE_COLUMN,
+    RABIT_BLOCKED_EX_CODE_COLUMN,
+    ADD_FACTORS_COLUMN,
+    SCALE_FACTORS_COLUMN,
+    ERROR_FACTORS_COLUMN,
+    EX_ADD_FACTORS_COLUMN,
+    EX_SCALE_FACTORS_COLUMN,
+    PART_ID_COLUMN,
+];
 pub const SEGMENT_LENGTH: usize = 4;
 pub const SEGMENT_NUM_CODES: usize = 1 << SEGMENT_LENGTH;
 const RABIT_PRUNE_STATS_ENV: &str = "LANCE_RQ_PRUNE_STATS";
@@ -2040,6 +2060,9 @@ fn accumulate_filtered_distances_into_heap(
 impl VectorStore for RabitQuantizationStorage {
     type DistanceCalculator<'a> = RabitDistCalculator<'a>;
 
+    /// RaBitQ storage is `[_rowid, <code/factor cols...>, <included cols...>]`.
+    const INTERNAL_COLUMNS: &'static [&'static str] = RABIT_INTERNAL_COLUMNS;
+
     fn as_any(&self) -> &dyn std::any::Any {
         self
     }
@@ -3035,6 +3058,39 @@ mod tests {
         .unwrap()
     }
 
+    #[test]
+    fn test_covering_excludes_legacy_part_id_column() {
+        // Same legacy `__ivf_part_id` guard as the other quantizer storages:
+        // pre-#3606 `num_partitions=1` builds left it in the storage file, and
+        // it must never be classified as a covering column -- otherwise search
+        // emits a column the exec's declared schema (from
+        // `IndexMetadata.covering_fields`, empty for those indexes) lacks.
+        use crate::vector::PART_ID_COLUMN;
+        use arrow_array::UInt32Array;
+        use arrow_schema::{DataType, Field, Schema};
+
+        let code_dim = 64;
+        let codes = make_test_codes(10, code_dim);
+        let metadata = make_test_metadata(codes.value_length() as usize * 8);
+        let base = make_test_batch(codes);
+
+        let mut fields: Vec<_> = base.schema().fields().iter().cloned().collect();
+        fields.push(Arc::new(Field::new(PART_ID_COLUMN, DataType::UInt32, true)));
+        let mut columns = base.columns().to_vec();
+        columns.push(Arc::new(UInt32Array::from_iter_values(
+            (0..base.num_rows()).map(|_| 0),
+        )) as ArrayRef);
+        let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
+
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        assert!(
+            storage.covering_field_indices().is_empty(),
+            "legacy __ivf_part_id must not be classified as a covering column"
+        );
+    }
+
     fn make_test_ex_codes(num_vectors: usize, code_dim: usize, num_bits: u8) -> FixedSizeListArray {
         let ex_bits = rabit_ex_bits(num_bits).unwrap();
         let ex_code_bytes = rabit_ex_code_bytes(code_dim, ex_bits).unwrap();