Add commit-existing-index-segments C API

Cargo.toml

@@ -18,14 +18,14 @@ rust-version = "1.91.0"
 crate-type = ["cdylib", "staticlib", "rlib"]
 
 [dependencies]
-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
+lance = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] }
+lance-core = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-index = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-io = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-table = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] }
 datafusion = { version = "54.0.0", default-features = false }
 arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] }
 arrow-array = "58.0.0"
@@ -47,9 +47,9 @@ snafu = "0.9"
 uuid = { version = "1", features = ["v4"] }
 
 [dev-dependencies]
-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+lance = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] }
+lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
+lance-file = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 arrow-array = "58.0.0"
 arrow-schema = "58.0.0"

include/lance/lance.h

@@ -1710,6 +1710,58 @@ int32_t lance_index_segment_metadata_fragment_ids(
 /** Free parsed segment metadata. NULL-safe. */
 void lance_index_segment_metadata_free(LanceIndexSegmentMetadata* metadata);
 
+/**
+ * Commit previously built uncommitted index segments as one logical index.
+ *
+ * `segment_metadata_bytes[i]` must point to
+ * `segment_metadata_lens[i]` bytes of protobuf-encoded IndexMetadata produced
+ * by lance_index_segment_builder_execute_uncommitted() (typically built on
+ * distributed workers). All segments are registered under `index_name` on
+ * `column` in a single commit, so the dataset version increases by exactly
+ * one on success.
+ *
+ * The segment set is validated by the Lance core and rejected with
+ * LANCE_ERR_INVALID_ARGUMENT when it is empty, contains duplicate segment
+ * UUIDs, or has overlapping fragment coverage. All segments must share one
+ * index type, and the commit fails if `column` does not exist. Every segment
+ * must declare `column` as its keyed field — that is, have been built for
+ * `column` — or the commit fails with LANCE_ERR_INVALID_ARGUMENT.
+ * Vector segments that will coexist (incoming segments and retained existing
+ * segments) must have compatible distance metrics, dimensions, sub-index
+ * types, and quantizer kinds. Independently trained IVF centroids and PQ
+ * codebooks may differ. Incompatible segments are rejected with
+ * LANCE_ERR_INVALID_ARGUMENT without changing the dataset version or index.
+ *
+ * Replacement is automatic and coverage-driven — there is no replace flag:
+ * existing same-name segments of the same index type whose fragment coverage
+ * is fully covered by the incoming set are replaced, while existing segments
+ * covering disjoint fragments are retained as additional deltas of the
+ * logical index. A commit that would orphan fragments from an existing
+ * segment (partial overlap) is rejected. A commit whose index type differs
+ * from the existing same-name index replaces that index entirely, and
+ * therefore requires the incoming segments to cover every current fragment;
+ * a partial-coverage type change is rejected with LANCE_ERR_INVALID_ARGUMENT.
+ * Vector compatibility is checked after selecting replacements, so a complete
+ * replacement may change the metric without conflicting with removed segments.
+ *
+ * @param dataset    Open dataset (mutated; same handle remains valid).
+ * @param index_name Logical index name; must not be NULL or empty.
+ * @param column     Indexed column; must not be NULL or empty.
+ * @param segment_metadata_bytes Array of pointers to encoded IndexMetadata.
+ * @param segment_metadata_lens  Array of byte lengths, parallel to
+ *                               segment_metadata_bytes.
+ * @param segment_count          Number of segments; must be > 0.
+ * @return 0 on success, -1 on error.
+ */
+int32_t lance_dataset_commit_index_segments(
+    LanceDataset* dataset,
+    const char* index_name,
+    const char* column,
+    const uint8_t* const* segment_metadata_bytes,
+    const size_t* segment_metadata_lens,
+    size_t segment_count
+);
+
 /** Drop an index by name. Returns -1 (NOT_FOUND) if no such index. */
 int32_t lance_dataset_drop_index(LanceDataset* dataset, const char* name);
 

include/lance/lance.hpp

@@ -936,6 +936,41 @@ class Dataset {
         return out;
     }
 
+    /// Commit previously built uncommitted index segments as one logical
+    /// index under `index_name` on `column`. Each entry of
+    /// `segment_metadata` is the protobuf-encoded IndexMetadata produced by
+    /// `IndexSegmentBuilder::execute_uncommitted()`. The commit is a single
+    /// dataset version bump. Every segment must have been built for `column`.
+    /// Coexisting vector segments, including retained existing segments, must
+    /// have compatible metrics, dimensions, sub-index types, and quantizer
+    /// kinds; independently trained IVF centroids and PQ codebooks may differ.
+    /// Replacement of existing same-name segments is automatic and
+    /// coverage-driven: fully covered segments are replaced, disjoint ones
+    /// are retained as deltas, and partial overlap is rejected. A commit
+    /// whose index type differs from the existing same-name index replaces
+    /// that index entirely, so it must cover every current fragment; a
+    /// partial-coverage type change is rejected.
+    /// Fully replaced vector segments do not constrain the new metric.
+    /// Throws lance::Error on validation failures (empty set, duplicate
+    /// segment UUIDs, overlapping fragment coverage, unknown or mismatched
+    /// column, incompatible vector segments), leaving the version and index
+    /// unchanged.
+    void commit_index_segments(
+        const std::string& index_name,
+        const std::string& column,
+        const std::vector<std::vector<uint8_t>>& segment_metadata) {
+        std::vector<const uint8_t*> bytes(segment_metadata.size());
+        std::vector<size_t> lens(segment_metadata.size());
+        for (size_t i = 0; i < segment_metadata.size(); ++i) {
+            bytes[i] = segment_metadata[i].data();
+            lens[i] = segment_metadata[i].size();
+        }
+        if (lance_dataset_commit_index_segments(
+                handle_.get(), index_name.c_str(), column.c_str(), bytes.data(),
+                lens.data(), segment_metadata.size()) != 0)
+            check_error();
+    }
+
     /// Access the underlying C handle (does not transfer ownership).
     const LanceDataset* c_handle() const { return handle_.get(); }