From 0c87706f04b9f205788f12e46b84228935260bbc Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Wed, 9 Sep 2026 21:15:02 -0700 Subject: [PATCH] feat(format): feature flag for a fragment reuse index on stable row id tables Reserve reader/writer feature flag 1 << 9 for a table version that uses stable row ids and carries a fragment reuse index. The index records how physical row addresses moved; entries of an index that stores stable row ids must not be remapped through it, so builds without domain-aware index loading must refuse such a table. This build defines the bit but does not support it yet; support lands with the implementation. Part of #8977 --- docs/src/format/index/system/frag_reuse.md | 7 +++ docs/src/format/table/versioning.md | 3 +- protos/table.proto | 1 + rust/lance-table/src/feature_flags.rs | 50 +++++++++++++++++++++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/src/format/index/system/frag_reuse.md b/docs/src/format/index/system/frag_reuse.md index 742740b2f1a..1179234f13a 100644 --- a/docs/src/format/index/system/frag_reuse.md +++ b/docs/src/format/index/system/frag_reuse.md @@ -202,6 +202,13 @@ trimmed. Cleanup must retain intermediate transitions still needed to translate old addresses. External mapping files can be deleted only when no retained dataset version references them. +## Stable Row IDs + +Older readers and writers did not expect both stable row IDs and a Fragment Reuse Index but are +not prevented from opening tables that have both. These older systems could silently corrupt such +tables. A table version that has both sets `FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS` in the reader and +writer [feature flags](../../table/versioning.md#current-feature-flags) to guard against this. + ## Impacts ### Conflict Resolution diff --git a/docs/src/format/table/versioning.md b/docs/src/format/table/versioning.md index 504fc98d3d8..4b12e2d2de6 100644 --- a/docs/src/format/table/versioning.md +++ b/docs/src/format/table/versioning.md @@ -32,8 +32,9 @@ they should return an "unsupported" error on any read or write operation. | 64 | `FLAG_UNSTABLE_DATA_OVERLAY_FILES` | Yes | Yes | Fragments may carry data overlay files. Unstable: release builds reject it unless explicitly opted in. | | 128 | `FLAG_COVERED_INDEX_METADATA` | Yes | Yes | Some index declares covering columns (`IndexMetadata.covering_fields`), so `fields` means keyed columns followed by carried ones. An implementation without this flag selects an index by membership of `fields` and would answer a query on a merely-carried column with an index keyed on a different one. | | 256 | `FLAG_MIXED_DATA_FILE_VERSIONS` | Yes | Yes | The snapshot may reference recognized V2 data files with different exact versions. Both bits must be set and remain set on later versions. | +| 512 | `FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS` | Yes | Yes | The table uses stable row IDs and carries a [Fragment Reuse Index](../index/system/frag_reuse.md). | | 1024 | `FLAG_FRAGMENT_REUSE_INDEX` | Yes | Yes | The fragment reuse index records tagged transitions (`IndexMetadata.index_version >= 1`). Readers must translate row addresses through them; writers must preserve them. An implementation without this flag would decode the details as the legacy format and silently drop the transitions when it next rewrites the fragment reuse index. See [FRI index versions](../index/system/frag_reuse.md#fri-index-versions). | -Flag bit 512 is reserved. Flags with bit values 2048 and above are unknown; unknown flags cause implementations to reject the dataset with an "unsupported" error. The paired mixed-version reader and writer bits must either both be set or both be clear; a half-set manifest is invalid. +Flags with bit values 2048 and above are unknown; unknown flags cause implementations to reject the dataset with an "unsupported" error. The paired mixed-version reader and writer bits must either both be set or both be clear; a half-set manifest is invalid. diff --git a/protos/table.proto b/protos/table.proto index 3e09196802d..a8533c0bb5b 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -134,6 +134,7 @@ message Manifest { * exact versions. Readers and writers must use each DataFile's version * instead of treating data_format.version as a snapshot-wide identity. * This bit is paired in the reader and writer feature words and is one-way. + * * 1 << 9: the table uses stable row ids and carries a fragment reuse index. */ uint64 reader_feature_flags = 9; diff --git a/rust/lance-table/src/feature_flags.rs b/rust/lance-table/src/feature_flags.rs index ca060dc056a..09ff7fdbe69 100644 --- a/rust/lance-table/src/feature_flags.rs +++ b/rust/lance-table/src/feature_flags.rs @@ -54,14 +54,24 @@ pub const FLAG_COVERED_INDEX_METADATA: u64 = 1 << 7; /// versions. Readers and writers must both understand the per-file version /// contract before either can safely access the dataset. pub const FLAG_MIXED_DATA_FILE_VERSIONS: u64 = 1 << 8; +/// The table uses stable row ids and carries a fragment reuse index. +/// +/// Reserved ahead of its implementation. This build treats the bit as unknown +/// (see `supported_flags_when`), so a build that knows the flag but not the +/// handling behind it cannot open such a table. +pub const FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS: u64 = 1 << 9; /// The first bit that is unknown as a feature flag -pub const FLAG_UNKNOWN: u64 = 1 << 9; +pub const FLAG_UNKNOWN: u64 = 1 << 10; const _: () = assert!(FLAG_COVERED_INDEX_METADATA < FLAG_UNKNOWN); // The fence needs a bit the current released build already refuses, which means // at or above the boundary that build shipped with (bit 7). const _: () = assert!(FLAG_COVERED_INDEX_METADATA >= 1 << 7); const _: () = assert!(FLAG_MIXED_DATA_FILE_VERSIONS < FLAG_UNKNOWN); +// Same fence for the stable-row-id fragment-reuse bit: the released build's +// boundary is bit 8, so anything at or above it is refused there. +const _: () = assert!(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS >= 1 << 8); +const _: () = assert!(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS < FLAG_UNKNOWN); /// Tagged FRI requires a reader that interprets its mappings and a writer that /// preserves them during maintenance. Legacy-only FRI does not set this bit. @@ -197,6 +207,8 @@ fn supported_flags_when(overlay_enabled: bool) -> u64 { FLAG_UNSTABLE_DATA_OVERLAY_FILES, overlay_enabled, ); + // Reserved, not implemented: see the flag's doc comment. + mark_supported(&mut supported, FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS, false); supported } @@ -299,6 +311,42 @@ mod tests { use super::*; use crate::format::BasePath; + /// Reserved ahead of its implementation: refused for reading and writing + /// until the handling lands, so a build from the gap cannot open the table. + #[test] + fn test_frag_reuse_with_stable_row_ids_flag_is_reserved_not_supported() { + use crate::format::{DataStorageFormat, Manifest}; + use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use std::collections::HashMap; + use std::sync::Arc; + + assert!(!can_read_dataset(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS)); + assert!(!can_write_dataset(FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS)); + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "id", + arrow_schema::DataType::Int64, + false, + )]); + let mut manifest = Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::default(), + HashMap::new(), + ); + manifest.reader_feature_flags = FLAG_STABLE_ROW_IDS | FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS; + manifest.writer_feature_flags = FLAG_STABLE_ROW_IDS | FLAG_FRAG_REUSE_WITH_STABLE_ROW_IDS; + assert!(matches!( + ensure_can_read_manifest(&manifest).unwrap_err(), + Error::NotSupported { .. } + )); + assert!(matches!( + ensure_can_write_manifest(&manifest).unwrap_err(), + Error::NotSupported { .. } + )); + } + #[test] fn test_read_check() { assert!(can_read_dataset(0));