From 89d17fccc7d5160471bb46d42d062295e7af491a Mon Sep 17 00:00:00 2001 From: XYZhan Date: Fri, 11 Sep 2026 10:21:35 -0400 Subject: [PATCH 1/5] feat(format): record the requested IVF partition count on a vector index A vector index segment records its build parameters so an engine can rebuild it without reading the index files, but the IVF partition count the index was asked for is not among them. An index rebuilt from its details is auto-sized instead, so one created with an explicit count comes back a different shape. `num_partitions` records that count. Absent means none was requested and the partitioning was derived from the data, which a rebuild derives again. Present means a rebuild starts from the recorded count, and may still train fewer partitions when the data cannot support them. --- docs/src/format/index/vector/index.md | 32 ++++++++++++++++++++++++++ protos/index.proto | 9 ++++++++ rust/lance/src/index/vector/details.rs | 5 ++++ 3 files changed, 46 insertions(+) diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 91d73a23a16..6c1ee4e2d61 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -53,6 +53,38 @@ The Lance vector index format has gone through 3 versions so far. This document currently only records version 3 which is the latest version. The specific version of the vector index is recorded in the `index_version` field of the generic [index metadata](../index.md#loading-an-index). +### Index Details + +An index segment records its build parameters in the `index_details` field of the +generic [index metadata](../index.md#loading-an-index), as a `VectorIndexDetails` +message. These are the parameters an engine needs to rebuild the index without +reading the index files: the distance metric, the quantization scheme, and the +partitioning the index was asked for. + +`num_partitions` records the IVF partition count the index was asked to train. It +is absent when no count was requested, in which case the partitioning was derived +from the data and an engine rebuilding the index derives it again. When present, a +rebuild starts from that count, and may still train fewer partitions when the data +cannot support them. A segment that covers no fragments carries these details with +no index files, so a recorded count is the only statement of the partitioning the +index is to be built with once its column holds enough vectors to train. + +`target_partition_size` records the target number of vectors per partition, and is +0 when unset. An engine that is given neither value derives the partitioning from +the data alone. + +`runtime_hints` carries optional build preferences that do not affect index +structure, keyed by reverse-DNS name. Unrecognized keys must be silently ignored. + +
+ Full protobuf definition + +```protobuf +%%% proto.message.VectorIndexDetails %%% +``` + +
+ ## Storage Layout (V3) Each vector index is stored as 2 regular Lance files - index file and auxiliary file. diff --git a/protos/index.proto b/protos/index.proto index 8284b986bb5..ce4700cad87 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -229,6 +229,15 @@ message VectorIndexDetails { * Unrecognized keys must be silently ignored by all runtimes. */ map runtime_hints = 9; + + /* The number of IVF partitions this index was asked to train. + * + * Absent when no count was requested, in which case the partitioning was + * derived from the data and a rebuild derives it again. Present when a caller + * named a count: a rebuild starts from that count, and may still train fewer + * partitions when the data cannot support them. + */ + optional uint32 num_partitions = 10; } // Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes. diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 836da47adfa..cc1f9048bc7 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -162,6 +162,7 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { hnsw_index_config, compression, runtime_hints, + num_partitions: None, }; prost_types::Any::from_msg(&details).unwrap() } @@ -564,6 +565,7 @@ fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result Date: Fri, 11 Sep 2026 11:12:12 -0400 Subject: [PATCH 2/5] feat(format): name the field as a request and state its precedence A count that was asked for is not the count an index holds, and the name should say so: `target_num_partitions` pairs with `target_partition_size`, the other way of asking for a partitioning. States which of the two wins when both are set. --- docs/src/format/index/vector/index.md | 20 +++++++++++--------- protos/index.proto | 11 +++++++---- rust/lance/src/index/vector/details.rs | 10 +++++----- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 6c1ee4e2d61..a328d0199c5 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -61,17 +61,19 @@ message. These are the parameters an engine needs to rebuild the index without reading the index files: the distance metric, the quantization scheme, and the partitioning the index was asked for. -`num_partitions` records the IVF partition count the index was asked to train. It -is absent when no count was requested, in which case the partitioning was derived -from the data and an engine rebuilding the index derives it again. When present, a -rebuild starts from that count, and may still train fewer partitions when the data -cannot support them. A segment that covers no fragments carries these details with -no index files, so a recorded count is the only statement of the partitioning the -index is to be built with once its column holds enough vectors to train. +`target_num_partitions` records the IVF partition count the index was asked to +train. It is a request, not a description of the index that was built: a build +trains fewer partitions when the data cannot support the count asked for, so the +number of partitions an index holds is read from the index itself and not from +here. It is absent when no count was requested, in which case the partitioning was +derived from the data and an engine rebuilding the index derives it again. A +segment that covers no fragments carries these details with no index files, so a +recorded count is the only statement of the partitioning the index is to be built +with once its column holds enough vectors to train. `target_partition_size` records the target number of vectors per partition, and is -0 when unset. An engine that is given neither value derives the partitioning from -the data alone. +0 when unset. `target_num_partitions` takes precedence when both are set. An engine +given neither derives the partitioning from the data alone. `runtime_hints` carries optional build preferences that do not affect index structure, keyed by reverse-DNS name. Unrecognized keys must be silently ignored. diff --git a/protos/index.proto b/protos/index.proto index ce4700cad87..4487ddd7edb 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -231,13 +231,16 @@ message VectorIndexDetails { map runtime_hints = 9; /* The number of IVF partitions this index was asked to train. + * + * A request, not a description of the index that was built: a build trains + * fewer partitions when the data cannot support the count asked for, so the + * partitions an index holds are read from the index itself. * * Absent when no count was requested, in which case the partitioning was - * derived from the data and a rebuild derives it again. Present when a caller - * named a count: a rebuild starts from that count, and may still train fewer - * partitions when the data cannot support them. + * derived from the data and a rebuild derives it again. Takes precedence over + * `target_partition_size` when both are set. */ - optional uint32 num_partitions = 10; + optional uint32 target_num_partitions = 10; } // Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes. diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index cc1f9048bc7..6cab24fd286 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -162,7 +162,7 @@ pub fn vector_index_details(params: &VectorIndexParams) -> prost_types::Any { hnsw_index_config, compression, runtime_hints, - num_partitions: None, + target_num_partitions: None, }; prost_types::Any::from_msg(&details).unwrap() } @@ -565,7 +565,7 @@ fn convert_legacy_proto_to_details(proto: &pb::Index) -> Result Date: Fri, 11 Sep 2026 16:20:32 -0400 Subject: [PATCH 3/5] feat(format): define the zero case for the partition count A count a writer cannot represent must not be recorded as a truncated value, and a reader must not derive a partitioning from zero partitions. --- docs/src/format/index/vector/index.md | 4 ++++ protos/index.proto | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index a328d0199c5..7fdf21f11a7 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -71,6 +71,10 @@ segment that covers no fragments carries these details with no index files, so a recorded count is the only statement of the partitioning the index is to be built with once its column holds enough vectors to train. +A recorded count is at least 1. A writer that cannot represent the count it was +given records nothing rather than a truncated value, and a reader treats 0 as no +request, so a partitioning is never derived from a count of zero. + `target_partition_size` records the target number of vectors per partition, and is 0 when unset. `target_num_partitions` takes precedence when both are set. An engine given neither derives the partitioning from the data alone. diff --git a/protos/index.proto b/protos/index.proto index 4487ddd7edb..114ff57fe4e 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -239,6 +239,10 @@ message VectorIndexDetails { * Absent when no count was requested, in which case the partitioning was * derived from the data and a rebuild derives it again. Takes precedence over * `target_partition_size` when both are set. + * + * A recorded count is at least 1, and a writer that cannot represent the + * count it was given records nothing rather than a truncated value. Readers + * treat 0 as no request. */ optional uint32 target_num_partitions = 10; } From d4cd920303148be51145a8ca5097f1cec2da4466 Mon Sep 17 00:00:00 2001 From: XYZhan Date: Fri, 11 Sep 2026 16:28:17 -0400 Subject: [PATCH 4/5] docs(format): state the no-files case as a condition, not a claim --- docs/src/format/index/vector/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 7fdf21f11a7..949d4e8f192 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -66,10 +66,10 @@ train. It is a request, not a description of the index that was built: a build trains fewer partitions when the data cannot support the count asked for, so the number of partitions an index holds is read from the index itself and not from here. It is absent when no count was requested, in which case the partitioning was -derived from the data and an engine rebuilding the index derives it again. A -segment that covers no fragments carries these details with no index files, so a -recorded count is the only statement of the partitioning the index is to be built -with once its column holds enough vectors to train. +derived from the data and an engine rebuilding the index derives it again. Where +a segment covers no fragments and carries no index files, a recorded count is the +only statement of the partitioning the index is to be built with once its column +holds enough vectors to train. A recorded count is at least 1. A writer that cannot represent the count it was given records nothing rather than a truncated value, and a reader treats 0 as no From ec7f7109b0c754f06c2a4967a0e1fe76d1334573 Mon Sep 17 00:00:00 2001 From: XYZhan Date: Fri, 11 Sep 2026 16:52:02 -0400 Subject: [PATCH 5/5] docs(format): state the partition-count contract precisely Absence means no count is recorded, which a writer predating the field also produces, so it cannot establish that the original build sized automatically. The actual count may differ from the request rather than always being smaller, and the request is not the only expression of partitioning intent. --- docs/src/format/index/vector/index.md | 28 +++++++++++++-------------- protos/index.proto | 20 +++++++++---------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 949d4e8f192..3a1b77b6cbd 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -61,23 +61,21 @@ message. These are the parameters an engine needs to rebuild the index without reading the index files: the distance metric, the quantization scheme, and the partitioning the index was asked for. -`target_num_partitions` records the IVF partition count the index was asked to -train. It is a request, not a description of the index that was built: a build -trains fewer partitions when the data cannot support the count asked for, so the -number of partitions an index holds is read from the index itself and not from -here. It is absent when no count was requested, in which case the partitioning was -derived from the data and an engine rebuilding the index derives it again. Where -a segment covers no fragments and carries no index files, a recorded count is the -only statement of the partitioning the index is to be built with once its column -holds enough vectors to train. - -A recorded count is at least 1. A writer that cannot represent the count it was -given records nothing rather than a truncated value, and a reader treats 0 as no -request, so a partitioning is never derived from a count of zero. +`target_num_partitions` records the IVF partition count requested for building or +rebuilding the index. The count an index actually holds may differ from it and is +read from the index files. Where a segment covers no fragments and carries no +index files, this field preserves an explicit partition-count request that has no +built index to read it from. `target_partition_size` records the target number of vectors per partition, and is -0 when unset. `target_num_partitions` takes precedence when both are set. An engine -given neither derives the partitioning from the data alone. +0 when unset. A positive `target_num_partitions` takes precedence over it. Absent +or zero means no explicit count is available: a rebuild uses +`target_partition_size` when that is positive, and the engine's own sizing +otherwise. Absence does not imply the original build used automatic sizing, since +a writer that predates this field records no count either way. + +Writers record only positive, representable counts, and omit a count they cannot +represent rather than truncating it. `runtime_hints` carries optional build preferences that do not affect index structure, keyed by reverse-DNS name. Unrecognized keys must be silently ignored. diff --git a/protos/index.proto b/protos/index.proto index 114ff57fe4e..e13ca934340 100644 --- a/protos/index.proto +++ b/protos/index.proto @@ -230,19 +230,19 @@ message VectorIndexDetails { */ map runtime_hints = 9; - /* The number of IVF partitions this index was asked to train. + /* Requested IVF partition count for building or rebuilding this index. * - * A request, not a description of the index that was built: a build trains - * fewer partitions when the data cannot support the count asked for, so the - * partitions an index holds are read from the index itself. + * The actual partition count may differ from this target and is read from the + * index files. * - * Absent when no count was requested, in which case the partitioning was - * derived from the data and a rebuild derives it again. Takes precedence over - * `target_partition_size` when both are set. + * A positive value takes precedence over `target_partition_size`. Absent or + * zero means no explicit count is available: a rebuild uses + * `target_partition_size` when positive, and the engine's own sizing + * otherwise. Absence does not imply the original build used automatic sizing, + * since a writer that predates this field records no count either way. * - * A recorded count is at least 1, and a writer that cannot represent the - * count it was given records nothing rather than a truncated value. Readers - * treat 0 as no request. + * Writers record only positive, representable counts, and omit a count they + * cannot represent rather than truncating it. */ optional uint32 target_num_partitions = 10; }