feat(format): feature flag for a fragment reuse index on stable row id tables

docs/src/format/index/system/frag_reuse.md

@@ -202,6 +202,13 @@ trimmed. Cleanup must retain intermediate transitions still needed to translate
 old addresses. External mapping files can be deleted only when no retained
 dataset version references them.
 
+## Stable Row IDs
+
+Older readers and writers did not expect both stable row IDs and a Fragment Reuse Index but are
+not prevented from opening tables that have both. These older systems could silently corrupt such
+tables. A table version that has both sets `FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS` in the reader and
+writer [feature flags](../../table/versioning.md#current-feature-flags) to guard against this.
+
 ## Impacts
 
 ### Conflict Resolution

docs/src/format/table/versioning.md

@@ -32,8 +32,9 @@ they should return an "unsupported" error on any read or write operation.
 | 64       | `FLAG_UNSTABLE_DATA_OVERLAY_FILES` | Yes          | Yes             | Fragments may carry data overlay files. Unstable: release builds reject it unless explicitly opted in.      |
 | 128      | `FLAG_COVERED_INDEX_METADATA`   | Yes             | Yes             | Some index declares covering columns (`IndexMetadata.covering_fields`), so `fields` means keyed columns followed by carried ones. An implementation without this flag selects an index by membership of `fields` and would answer a query on a merely-carried column with an index keyed on a different one. |
 | 256      | `FLAG_MIXED_DATA_FILE_VERSIONS` | Yes             | Yes             | The snapshot may reference recognized V2 data files with different exact versions. Both bits must be set and remain set on later versions. |
+| 512      | `FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS` | Yes       | Yes             | The table uses stable row IDs and carries a [Fragment Reuse Index](../index/system/frag_reuse.md). |
 | 1024     | `FLAG_FRAGMENT_REUSE_INDEX`     | Yes             | Yes             | The fragment reuse index records tagged transitions (`IndexMetadata.index_version >= 1`). Readers must translate row addresses through them; writers must preserve them. An implementation without this flag would decode the details as the legacy format and silently drop the transitions when it next rewrites the fragment reuse index. See [FRI index versions](../index/system/frag_reuse.md#fri-index-versions). |
 
 </div>
 
-Flag bit 512 is reserved. Flags with bit values 2048 and above are unknown; unknown flags cause implementations to reject the dataset with an "unsupported" error. The paired mixed-version reader and writer bits must either both be set or both be clear; a half-set manifest is invalid.
+Flags with bit values 2048 and above are unknown; unknown flags cause implementations to reject the dataset with an "unsupported" error. The paired mixed-version reader and writer bits must either both be set or both be clear; a half-set manifest is invalid.

protos/table.proto

@@ -134,6 +134,7 @@ message Manifest {
    *   exact versions. Readers and writers must use each DataFile's version
    *   instead of treating data_format.version as a snapshot-wide identity.
    *   This bit is paired in the reader and writer feature words and is one-way.
+   * * 1 << 9: the table uses stable row ids and carries a fragment reuse index.
    */
   uint64 reader_feature_flags = 9;
 

rust/lance-table/src/feature_flags.rs

@@ -54,14 +54,24 @@ pub const FLAG_COVERED_INDEX_METADATA: u64 = 1 << 7;
 /// versions. Readers and writers must both understand the per-file version
 /// contract before either can safely access the dataset.
 pub const FLAG_MIXED_DATA_FILE_VERSIONS: u64 = 1 << 8;
+/// The table uses stable row ids and carries a fragment reuse index.
+///
+/// Reserved ahead of its implementation. This build treats the bit as unknown
+/// (see `supported_flags_when`), so a build that knows the flag but not the
+/// handling behind it cannot open such a table.
+pub const FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS: u64 = 1 << 9;
 /// The first bit that is unknown as a feature flag
-pub const FLAG_UNKNOWN: u64 = 1 << 9;
+pub const FLAG_UNKNOWN: u64 = 1 << 10;
 
 const _: () = assert!(FLAG_COVERED_INDEX_METADATA < FLAG_UNKNOWN);
 // The fence needs a bit the current released build already refuses, which means
 // at or above the boundary that build shipped with (bit 7).
 const _: () = assert!(FLAG_COVERED_INDEX_METADATA >= 1 << 7);
 const _: () = assert!(FLAG_MIXED_DATA_FILE_VERSIONS < FLAG_UNKNOWN);
+// Same fence for the stable-row-id fragment-reuse bit: the released build's
+// boundary is bit 8, so anything at or above it is refused there.
+const _: () = assert!(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS >= 1 << 8);
+const _: () = assert!(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS < FLAG_UNKNOWN);
 
 /// Tagged FRI requires a reader that interprets its mappings and a writer that
 /// preserves them during maintenance. Legacy-only FRI does not set this bit.
@@ -197,6 +207,8 @@ fn supported_flags_when(overlay_enabled: bool) -> u64 {
         FLAG_UNSTABLE_DATA_OVERLAY_FILES,
         overlay_enabled,
     );
+    // Reserved, not implemented: see the flag's doc comment.
+    mark_supported(&mut supported, FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS, false);
     supported
 }
 
@@ -299,6 +311,42 @@ mod tests {
     use super::*;
     use crate::format::BasePath;
 
+    /// Reserved ahead of its implementation: refused for reading and writing
+    /// until the handling lands, so a build from the gap cannot open the table.
+    #[test]
+    fn test_frag_reuse_with_stable_row_ids_flag_is_reserved_not_supported() {
+        use crate::format::{DataStorageFormat, Manifest};
+        use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
+        use lance_core::datatypes::Schema;
+        use std::collections::HashMap;
+        use std::sync::Arc;
+
+        assert!(!can_read_dataset(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS));
+        assert!(!can_write_dataset(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS));
+
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            arrow_schema::DataType::Int64,
+            false,
+        )]);
+        let mut manifest = Manifest::new(
+            Schema::try_from(&arrow_schema).unwrap(),
+            Arc::new(vec![]),
+            DataStorageFormat::default(),
+            HashMap::new(),
+        );
+        manifest.reader_feature_flags = FLAG_STABLE_ROW_IDS | FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS;
+        manifest.writer_feature_flags = FLAG_STABLE_ROW_IDS | FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS;
+        assert!(matches!(
+            ensure_can_read_manifest(&manifest).unwrap_err(),
+            Error::NotSupported { .. }
+        ));
+        assert!(matches!(
+            ensure_can_write_manifest(&manifest).unwrap_err(),
+            Error::NotSupported { .. }
+        ));
+    }
+
     #[test]
     fn test_read_check() {
         assert!(can_read_dataset(0));