From 94a0540eb03a82d55eae95794128e62c2e7573d4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:29:27 +0000 Subject: [PATCH 01/15] Initial plan From bceeba02457ac9c5751f68f1a40dfd2b6bf2645d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:36:14 +0000 Subject: [PATCH 02/15] fix: start handling binary offset overflow errors Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/binary.rs | 29 ++++--- .../src/array_encoding/logical/list.rs | 48 ++++++++++-- .../src/array_encoding/physical/binary.rs | 76 ++++++++++++++++--- 3 files changed, 128 insertions(+), 25 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/binary.rs b/rust/lance-encoding/src/array_encoding/logical/binary.rs index 697c1503e0a..46e0ca5d7d6 100644 --- a/rust/lance-encoding/src/array_encoding/logical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/logical/binary.rs @@ -11,7 +11,7 @@ use arrow_array::{ use arrow_schema::DataType; use futures::{FutureExt, future::BoxFuture}; -use lance_core::Result; +use lance_core::{Error, Result}; use log::trace; use crate::{ @@ -152,7 +152,7 @@ pub struct BinaryArrayDecoder { } impl BinaryArrayDecoder { - fn from_list_array(array: &GenericListArray) -> ArrayRef { + fn from_list_array(array: &GenericListArray) -> Result { let values = array .values() .as_primitive::() @@ -160,11 +160,16 @@ impl BinaryArrayDecoder { .inner() .clone(); let offsets = array.offsets().clone(); - Arc::new(GenericByteArray::::new( - offsets, - values, - array.nulls().cloned(), - )) + let array = GenericByteArray::::try_new(offsets, values, array.nulls().cloned()) + .map_err(|err| { + Error::not_supported(format!( + "Could not create array with more than 2GiB of string/binary data in a \ + single batch. Please reduce the batch_size, set LANCE_DEFAULT_BATCH_SIZE \ + to a smaller value, or convert the column to large_string/large_binary. \ + Arrow error: {err}" + )) + })?; + Ok(Arc::new(array)) } } @@ -173,10 +178,12 @@ impl DecodeArrayTask for BinaryArrayDecoder { let data_type = self.data_type; let (arr, _) = self.inner.decode()?; let result = match data_type { - DataType::Binary => Self::from_list_array::(arr.as_list::()), - DataType::LargeBinary => Self::from_list_array::(arr.as_list::()), - DataType::Utf8 => Self::from_list_array::(arr.as_list::()), - DataType::LargeUtf8 => Self::from_list_array::(arr.as_list::()), + DataType::Binary => Self::from_list_array::(arr.as_list::())?, + DataType::LargeBinary => { + Self::from_list_array::(arr.as_list::())? + } + DataType::Utf8 => Self::from_list_array::(arr.as_list::())?, + DataType::LargeUtf8 => Self::from_list_array::(arr.as_list::())?, _ => panic!("Binary decoder does not support this data type"), }; // data_size is only tracked in the v2.1 structural decode path; the v2.0 array diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index d76532ab3bb..f53d7692700 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -637,6 +637,27 @@ struct ListDecodeTask { offset_type: DataType, } +fn oversized_batch_error( + items_field: &Field, + requested_rows: u64, + num_items: u64, +) -> Error { + if items_field.data_type() == &DataType::UInt8 { + Error::not_supported(format!( + "Could not create array with more than 2GiB of string/binary data in a single batch \ + ({requested_rows} rows would require {num_items} bytes). Please reduce the \ + batch_size, set LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert the column \ + to large_string/large_binary." + )) + } else { + Error::not_supported(format!( + "Could not create a list array with more than i32::MAX items in a single batch \ + ({requested_rows} rows would require {num_items} items). Please reduce the \ + batch_size." + )) + } +} + impl DecodeArrayTask for ListDecodeTask { fn decode(self: Box) -> Result<(ArrayRef, u64)> { let items = self @@ -782,11 +803,12 @@ impl LogicalPageDecoder for ListPageDecoder { } } if actual_num_rows < num_rows { - // TODO: We should be able to automatically - // shrink the read batch size if we detect the batches are going to be huge (maybe - // even achieve this with a read_batch_bytes parameter, though some estimation may - // still be required) - return Err(Error::not_supported_source(format!("loading a batch of {} lists would require creating an array with over i32::MAX items and we don't yet support returning smaller than requested batches", num_rows).into())); + let num_items = self.offsets[(self.rows_drained + num_rows) as usize] - item_start; + return Err(oversized_batch_error( + self.items_field.as_ref(), + num_rows, + num_items, + )); } let offsets = self.offsets [self.rows_drained as usize..(self.rows_drained + actual_num_rows + 1) as usize] @@ -837,6 +859,22 @@ impl LogicalPageDecoder for ListPageDecoder { } } +#[cfg(test)] +mod tests { + use arrow_schema::{DataType, Field}; + + use super::oversized_batch_error; + + #[test] + fn oversized_binary_batch_error_is_actionable() { + let error = oversized_batch_error(&Field::new("item", DataType::UInt8, false), 128, i32::MAX as u64 + 1); + assert!(error.to_string().contains("more than 2GiB of string/binary data")); + assert!(error.to_string().contains("batch_size")); + assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); + assert!(error.to_string().contains("large_string/large_binary")); + } +} + struct IndirectlyLoaded { offsets: Arc<[u64]>, validity: BooleanBuffer, diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index 294b295def3..1915d117717 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -30,7 +30,7 @@ use crate::{ use arrow_array::{PrimitiveArray, UInt64Array}; use arrow_schema::DataType; -use lance_core::Result; +use lance_core::{Error, Result}; struct IndicesNormalizer { indices: Vec, @@ -315,14 +315,36 @@ impl PrimitivePageDecoder for BinaryPageDecoder { // Normalize and cast (TODO: could fuse these into one pass for micro-optimization) let target_vec = target_offsets.values(); let start = target_vec[0]; - let offsets_buffer = - match bytes_per_offset { - 4 => ScalarBuffer::from_iter(target_vec.iter().map(|x| (x - start) as i32)) - .into_inner(), - 8 => ScalarBuffer::from_iter(target_vec.iter().map(|x| (x - start) as i64)) - .into_inner(), - _ => panic!("Unsupported offsets type"), - }; + let end = *target_vec.last().unwrap(); + let offsets_buffer = match bytes_per_offset { + 4 => { + let num_bytes = end - start; + if num_bytes > i32::MAX as u64 { + return Err(Error::not_supported(format!( + "Could not create array with more than 2GiB of string/binary data in a \ + single batch ({} rows would require {} bytes). Please reduce the \ + batch_size, set LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert \ + the column to large_string/large_binary.", + num_rows, num_bytes + ))); + } + ScalarBuffer::from( + target_vec + .iter() + .map(|&offset| i32::try_from(offset - start).expect("checked above")) + .collect::>(), + ) + .into_inner() + } + 8 => ScalarBuffer::from( + target_vec + .iter() + .map(|&offset| i64::try_from(offset - start).expect("u64 offsets fit in i64")) + .collect::>(), + ) + .into_inner(), + _ => panic!("Unsupported offsets type"), + }; let bytes_to_skip = self.decoded_indices.value(rows_to_skip as usize); let num_bytes = self @@ -538,6 +560,20 @@ mod tests { use super::*; + #[derive(Debug)] + struct EmptyBytesDecoder; + + impl PrimitivePageDecoder for EmptyBytesDecoder { + fn decode(&self, _rows_to_skip: u64, _num_rows: u64) -> Result { + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value: 8, + data: LanceBuffer::empty(), + num_values: 0, + block_info: BlockInfo::new(), + })) + } + } + #[test] fn test_encode_indices_adjusts_nulls() { // Null entries in string arrays should be adjusted @@ -568,4 +604,26 @@ mod tests { ); assert_eq!(null_adjustment, 7); } + + #[test] + fn test_binary_overflow_error_is_actionable() { + let num_rows = 1; + let start = 100_u64; + let end = start + i32::MAX as u64 + 1; + let decoded_indices = UInt64Array::from(vec![start, end]); + let decoder = BinaryPageDecoder { + decoded_indices, + validity: BooleanBuffer::from_iter([true]), + offsets_type: DataType::Int32, + bytes_decoder: Box::new(EmptyBytesDecoder), + }; + + let error = decoder.decode(0, num_rows).unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. })); + let message = error.to_string(); + assert!(message.contains("more than 2GiB of string/binary data")); + assert!(message.contains("batch_size")); + assert!(message.contains("LANCE_DEFAULT_BATCH_SIZE")); + assert!(message.contains("large_string/large_binary")); + } } From a4efae29d25245052c8a994eaf097deef183628f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:37:05 +0000 Subject: [PATCH 03/15] fix: add actionable overflow messaging Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index f53d7692700..0181b85da52 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -859,22 +859,6 @@ impl LogicalPageDecoder for ListPageDecoder { } } -#[cfg(test)] -mod tests { - use arrow_schema::{DataType, Field}; - - use super::oversized_batch_error; - - #[test] - fn oversized_binary_batch_error_is_actionable() { - let error = oversized_batch_error(&Field::new("item", DataType::UInt8, false), 128, i32::MAX as u64 + 1); - assert!(error.to_string().contains("more than 2GiB of string/binary data")); - assert!(error.to_string().contains("batch_size")); - assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); - assert!(error.to_string().contains("large_string/large_binary")); - } -} - struct IndirectlyLoaded { offsets: Arc<[u64]>, validity: BooleanBuffer, @@ -1319,3 +1303,23 @@ impl FieldEncoder for ListFieldEncoder { .boxed() } } + +#[cfg(test)] +mod tests { + use arrow_schema::{DataType, Field}; + + use super::oversized_batch_error; + + #[test] + fn oversized_binary_batch_error_is_actionable() { + let error = oversized_batch_error( + &Field::new("item", DataType::UInt8, false), + 128, + i32::MAX as u64 + 1, + ); + assert!(error.to_string().contains("more than 2GiB of string/binary data")); + assert!(error.to_string().contains("batch_size")); + assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); + assert!(error.to_string().contains("large_string/large_binary")); + } +} From fdd5e1d96c7501c0e95abafca572ecbab075b775 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:37:51 +0000 Subject: [PATCH 04/15] test: verify offset overflow regression paths Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index 0181b85da52..e651086dd25 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -637,11 +637,7 @@ struct ListDecodeTask { offset_type: DataType, } -fn oversized_batch_error( - items_field: &Field, - requested_rows: u64, - num_items: u64, -) -> Error { +fn oversized_batch_error(items_field: &Field, requested_rows: u64, num_items: u64) -> Error { if items_field.data_type() == &DataType::UInt8 { Error::not_supported(format!( "Could not create array with more than 2GiB of string/binary data in a single batch \ @@ -1317,7 +1313,11 @@ mod tests { 128, i32::MAX as u64 + 1, ); - assert!(error.to_string().contains("more than 2GiB of string/binary data")); + assert!( + error + .to_string() + .contains("more than 2GiB of string/binary data") + ); assert!(error.to_string().contains("batch_size")); assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); assert!(error.to_string().contains("large_string/large_binary")); From c7e55921c8e641b0cad959744c97ac2a22b04e76 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:48:42 +0000 Subject: [PATCH 05/15] fix: tighten binary overflow error handling Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/binary.rs | 58 +++++++++++++++---- .../src/array_encoding/physical/binary.rs | 56 +++++++++++------- 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/binary.rs b/rust/lance-encoding/src/array_encoding/logical/binary.rs index 46e0ca5d7d6..2d95a5b6d35 100644 --- a/rust/lance-encoding/src/array_encoding/logical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/logical/binary.rs @@ -11,7 +11,7 @@ use arrow_array::{ use arrow_schema::DataType; use futures::{FutureExt, future::BoxFuture}; -use lance_core::{Error, Result}; +use lance_core::Result; use log::trace; use crate::{ @@ -160,15 +160,7 @@ impl BinaryArrayDecoder { .inner() .clone(); let offsets = array.offsets().clone(); - let array = GenericByteArray::::try_new(offsets, values, array.nulls().cloned()) - .map_err(|err| { - Error::not_supported(format!( - "Could not create array with more than 2GiB of string/binary data in a \ - single batch. Please reduce the batch_size, set LANCE_DEFAULT_BATCH_SIZE \ - to a smaller value, or convert the column to large_string/large_binary. \ - Arrow error: {err}" - )) - })?; + let array = GenericByteArray::::try_new(offsets, values, array.nulls().cloned())?; Ok(Arc::new(array)) } } @@ -191,3 +183,49 @@ impl DecodeArrayTask for BinaryArrayDecoder { Ok((result, 0)) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ListArray, UInt8Array}; + use arrow_buffer::OffsetBuffer; + use arrow_schema::Field; + + use super::*; + use crate::decoder::DecodeArrayTask; + + struct StubDecodeTask { + array: ArrayRef, + } + + impl DecodeArrayTask for StubDecodeTask { + fn decode(self: Box) -> Result<(ArrayRef, u64)> { + Ok((self.array, 0)) + } + } + + #[test] + fn logical_utf8_decode_preserves_non_overflow_arrow_error() { + let offsets = OffsetBuffer::from_lengths([1_usize]); + let values: ArrayRef = Arc::new(UInt8Array::from(vec![0xFF_u8])); + let list = ListArray::try_new( + Arc::new(Field::new("item", DataType::UInt8, false)), + offsets, + values, + None, + ) + .unwrap(); + let decoder = BinaryArrayDecoder { + inner: Box::new(StubDecodeTask { + array: Arc::new(list), + }), + data_type: DataType::Utf8, + }; + + let error = Box::new(decoder).decode().unwrap_err(); + let message = error.to_string(); + assert!(!message.contains("more than 2GiB of string/binary data")); + assert!(message.to_lowercase().contains("utf")); + } +} diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index 1915d117717..4ae84a5deb9 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -32,6 +32,16 @@ use arrow_array::{PrimitiveArray, UInt64Array}; use arrow_schema::DataType; use lance_core::{Error, Result}; +fn oversized_binary_batch_error(num_rows: u64, num_bytes: u64) -> Error { + Error::not_supported(format!( + "Could not create array with more than 2GiB of string/binary data in a single batch \ + ({} rows would require {} bytes). Please reduce the batch_size, set \ + LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert the column to \ + large_string/large_binary.", + num_rows, num_bytes + )) +} + struct IndicesNormalizer { indices: Vec, validity: BooleanBufferBuilder, @@ -320,29 +330,35 @@ impl PrimitivePageDecoder for BinaryPageDecoder { 4 => { let num_bytes = end - start; if num_bytes > i32::MAX as u64 { - return Err(Error::not_supported(format!( - "Could not create array with more than 2GiB of string/binary data in a \ - single batch ({} rows would require {} bytes). Please reduce the \ - batch_size, set LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert \ - the column to large_string/large_binary.", - num_rows, num_bytes - ))); + return Err(oversized_binary_batch_error(num_rows, num_bytes)); } - ScalarBuffer::from( - target_vec - .iter() - .map(|&offset| i32::try_from(offset - start).expect("checked above")) - .collect::>(), - ) - .into_inner() + let offsets = target_vec + .iter() + .map(|&offset| { + i32::try_from(offset - start) + .map_err(|_| oversized_binary_batch_error(num_rows, num_bytes)) + }) + .collect::>>()?; + ScalarBuffer::from(offsets).into_inner() } - 8 => ScalarBuffer::from( - target_vec + 8 => { + let num_bytes = end - start; + let offsets = target_vec .iter() - .map(|&offset| i64::try_from(offset - start).expect("u64 offsets fit in i64")) - .collect::>(), - ) - .into_inner(), + .map(|&offset| { + i64::try_from(offset - start).map_err(|_| { + Error::not_supported(format!( + "Could not create large_string/large_binary array in a single \ + batch because {} rows would require {} bytes, which exceeds \ + i64::MAX. Please reduce the batch_size or set \ + LANCE_DEFAULT_BATCH_SIZE to a smaller value.", + num_rows, num_bytes + )) + }) + }) + .collect::>>()?; + ScalarBuffer::from(offsets).into_inner() + } _ => panic!("Unsupported offsets type"), }; From 8bc8969b602b1ac340dc468197d2ead6ac72443c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:51:08 +0000 Subject: [PATCH 06/15] fix: clarify overflow row span in errors Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index e651086dd25..9b30c208582 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -637,19 +637,20 @@ struct ListDecodeTask { offset_type: DataType, } -fn oversized_batch_error(items_field: &Field, requested_rows: u64, num_items: u64) -> Error { +fn oversized_batch_error(items_field: &Field, row_range: Range, num_items: u64) -> Error { if items_field.data_type() == &DataType::UInt8 { Error::not_supported(format!( "Could not create array with more than 2GiB of string/binary data in a single batch \ - ({requested_rows} rows would require {num_items} bytes). Please reduce the \ + (rows {}..{} would require {num_items} bytes). Please reduce the \ batch_size, set LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert the column \ - to large_string/large_binary." + to large_string/large_binary.", + row_range.start, row_range.end )) } else { Error::not_supported(format!( "Could not create a list array with more than i32::MAX items in a single batch \ - ({requested_rows} rows would require {num_items} items). Please reduce the \ - batch_size." + (rows {}..{} would require {num_items} items). Please reduce the batch_size.", + row_range.start, row_range.end )) } } @@ -802,7 +803,7 @@ impl LogicalPageDecoder for ListPageDecoder { let num_items = self.offsets[(self.rows_drained + num_rows) as usize] - item_start; return Err(oversized_batch_error( self.items_field.as_ref(), - num_rows, + self.rows_drained..self.rows_drained + num_rows, num_items, )); } @@ -1310,7 +1311,7 @@ mod tests { fn oversized_binary_batch_error_is_actionable() { let error = oversized_batch_error( &Field::new("item", DataType::UInt8, false), - 128, + 32..160, i32::MAX as u64 + 1, ); assert!( @@ -1318,6 +1319,7 @@ mod tests { .to_string() .contains("more than 2GiB of string/binary data") ); + assert!(error.to_string().contains("rows 32..160")); assert!(error.to_string().contains("batch_size")); assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); assert!(error.to_string().contains("large_string/large_binary")); From 5630a0c3a050f024fab8aca7595d294fbe2f9376 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:53:07 +0000 Subject: [PATCH 07/15] test: cover large binary overflow path Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/physical/binary.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index 4ae84a5deb9..d984403a1a8 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -642,4 +642,26 @@ mod tests { assert!(message.contains("LANCE_DEFAULT_BATCH_SIZE")); assert!(message.contains("large_string/large_binary")); } + + #[test] + fn test_large_binary_overflow_error_is_actionable() { + let num_rows = 1; + let start = 100_u64; + let end = start + i64::MAX as u64 + 1; + let decoded_indices = UInt64Array::from(vec![start, end]); + let decoder = BinaryPageDecoder { + decoded_indices, + validity: BooleanBuffer::from_iter([true]), + offsets_type: DataType::Int64, + bytes_decoder: Box::new(EmptyBytesDecoder), + }; + + let error = decoder.decode(0, num_rows).unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. })); + let message = error.to_string(); + assert!(message.contains("large_string/large_binary")); + assert!(message.contains("exceeds i64::MAX")); + assert!(message.contains("batch_size")); + assert!(message.contains("LANCE_DEFAULT_BATCH_SIZE")); + } } From 4becc771fc49646cc7f756147bc2c913081774ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:57:37 +0000 Subject: [PATCH 08/15] test: cover decoder overflow branches Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 30 +++++++++++++- .../src/array_encoding/physical/binary.rs | 40 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index 9b30c208582..f0aa585c66f 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -1303,9 +1303,13 @@ impl FieldEncoder for ListFieldEncoder { #[cfg(test)] mod tests { + use std::sync::Arc; + + use arrow_buffer::BooleanBuffer; use arrow_schema::{DataType, Field}; - use super::oversized_batch_error; + use super::{ListPageDecoder, oversized_batch_error}; + use crate::decoder::LogicalPageDecoder; #[test] fn oversized_binary_batch_error_is_actionable() { @@ -1324,4 +1328,28 @@ mod tests { assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); assert!(error.to_string().contains("large_string/large_binary")); } + + #[test] + fn list_decoder_overflow_reports_row_span() { + let mut decoder = ListPageDecoder { + unloaded: None, + offsets: Arc::<[u64]>::from(vec![0_u64, 1, 2, 3, 3 + i32::MAX as u64 + 1]), + validity: BooleanBuffer::from_iter([true, true, true, true]), + item_decoder: None, + num_rows: 4, + rows_drained: 2, + rows_loaded: 4, + items_field: Arc::new(Field::new("item", DataType::UInt8, false)), + offset_type: DataType::Int32, + data_type: DataType::List(Arc::new(Field::new("item", DataType::UInt8, false))), + }; + + let Err(error) = decoder.drain(2) else { + panic!("expected overflow error"); + }; + let message = error.to_string(); + assert!(message.contains("rows 2..4")); + assert!(message.contains(&(i32::MAX as u64 + 2).to_string())); + assert!(message.contains("batch_size")); + } } diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index d984403a1a8..262877ae9a0 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -590,6 +590,24 @@ mod tests { } } + #[derive(Debug)] + struct BytesDecoder { + bytes: Vec, + } + + impl PrimitivePageDecoder for BytesDecoder { + fn decode(&self, rows_to_skip: u64, num_rows: u64) -> Result { + let start = rows_to_skip as usize; + let end = start + num_rows as usize; + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value: 8, + data: LanceBuffer::from(self.bytes[start..end].to_vec()), + num_values: num_rows, + block_info: BlockInfo::new(), + })) + } + } + #[test] fn test_encode_indices_adjusts_nulls() { // Null entries in string arrays should be adjusted @@ -664,4 +682,26 @@ mod tests { assert!(message.contains("batch_size")); assert!(message.contains("LANCE_DEFAULT_BATCH_SIZE")); } + + #[test] + fn test_large_binary_decode_success_path() { + let decoded_indices = UInt64Array::from(vec![100_u64, 102, 105]); + let mut bytes = vec![0_u8; 100]; + bytes.extend_from_slice(b"abcde"); + let decoder = BinaryPageDecoder { + decoded_indices, + validity: BooleanBuffer::from_iter([true, true]), + offsets_type: DataType::Int64, + bytes_decoder: Box::new(BytesDecoder { bytes }), + }; + + let data = decoder.decode(0, 2).unwrap(); + let variable = data.as_variable_width().unwrap(); + assert_eq!(variable.bits_per_offset, 64); + assert_eq!(variable.data.as_ref(), b"abcde"); + assert_eq!( + variable.offsets.borrow_to_typed_slice::().as_ref(), + &[0_i64, 2, 5] + ); + } } From ac86c8715cf2e1f4520738d865abc74f7f9eef44 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:59:40 +0000 Subject: [PATCH 09/15] fix: report first overflowing row span Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../lance-encoding/src/array_encoding/logical/list.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index f0aa585c66f..4c401efff68 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -800,10 +800,13 @@ impl LogicalPageDecoder for ListPageDecoder { } } if actual_num_rows < num_rows { - let num_items = self.offsets[(self.rows_drained + num_rows) as usize] - item_start; + let failing_rows_start = self.rows_drained + actual_num_rows; + let failing_item_start = self.offsets[failing_rows_start as usize]; + let num_items = + self.offsets[(self.rows_drained + num_rows) as usize] - failing_item_start; return Err(oversized_batch_error( self.items_field.as_ref(), - self.rows_drained..self.rows_drained + num_rows, + failing_rows_start..self.rows_drained + num_rows, num_items, )); } @@ -1348,8 +1351,8 @@ mod tests { panic!("expected overflow error"); }; let message = error.to_string(); - assert!(message.contains("rows 2..4")); - assert!(message.contains(&(i32::MAX as u64 + 2).to_string())); + assert!(message.contains("rows 3..4")); + assert!(message.contains(&(i32::MAX as u64 + 1).to_string())); assert!(message.contains("batch_size")); } } From 06beabcb5565a9d4d9f93703e1c4a750f0f90928 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:01:52 +0000 Subject: [PATCH 10/15] fix: correct overflowing row span calculation Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- rust/lance-encoding/src/array_encoding/logical/list.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index 4c401efff68..09e6c369f2f 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -800,7 +800,11 @@ impl LogicalPageDecoder for ListPageDecoder { } } if actual_num_rows < num_rows { - let failing_rows_start = self.rows_drained + actual_num_rows; + let failing_rows_start = if actual_num_rows == 0 { + self.rows_drained + } else { + self.rows_drained + actual_num_rows - 1 + }; let failing_item_start = self.offsets[failing_rows_start as usize]; let num_items = self.offsets[(self.rows_drained + num_rows) as usize] - failing_item_start; @@ -1351,8 +1355,8 @@ mod tests { panic!("expected overflow error"); }; let message = error.to_string(); - assert!(message.contains("rows 3..4")); - assert!(message.contains(&(i32::MAX as u64 + 1).to_string())); + assert!(message.contains("rows 2..4")); + assert!(message.contains(&(i32::MAX as u64 + 2).to_string())); assert!(message.contains("batch_size")); } } From fbfe9950c39a6b018d3234cea1f93b981b7164af Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:04:11 +0000 Subject: [PATCH 11/15] fix: clarify decodable prefix in overflow errors Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index 09e6c369f2f..c071c83d780 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -637,20 +637,47 @@ struct ListDecodeTask { offset_type: DataType, } -fn oversized_batch_error(items_field: &Field, row_range: Range, num_items: u64) -> Error { +fn oversized_batch_error( + items_field: &Field, + requested_range: Range, + decodable_prefix_end: u64, + num_items: u64, +) -> Error { + let prefix_detail = if decodable_prefix_end > requested_range.start { + format!( + "rows {}..{} fit, but requesting rows {}..{} would require {num_items} {}", + requested_range.start, + decodable_prefix_end, + requested_range.start, + requested_range.end, + if items_field.data_type() == &DataType::UInt8 { + "bytes" + } else { + "items" + } + ) + } else { + format!( + "requesting rows {}..{} would require {num_items} {}", + requested_range.start, + requested_range.end, + if items_field.data_type() == &DataType::UInt8 { + "bytes" + } else { + "items" + } + ) + }; if items_field.data_type() == &DataType::UInt8 { Error::not_supported(format!( "Could not create array with more than 2GiB of string/binary data in a single batch \ - (rows {}..{} would require {num_items} bytes). Please reduce the \ - batch_size, set LANCE_DEFAULT_BATCH_SIZE to a smaller value, or convert the column \ - to large_string/large_binary.", - row_range.start, row_range.end + ({prefix_detail}). Please reduce the batch_size, set LANCE_DEFAULT_BATCH_SIZE to a \ + smaller value, or convert the column to large_string/large_binary." )) } else { Error::not_supported(format!( "Could not create a list array with more than i32::MAX items in a single batch \ - (rows {}..{} would require {num_items} items). Please reduce the batch_size.", - row_range.start, row_range.end + ({prefix_detail}). Please reduce the batch_size." )) } } @@ -800,17 +827,12 @@ impl LogicalPageDecoder for ListPageDecoder { } } if actual_num_rows < num_rows { - let failing_rows_start = if actual_num_rows == 0 { - self.rows_drained - } else { - self.rows_drained + actual_num_rows - 1 - }; - let failing_item_start = self.offsets[failing_rows_start as usize]; - let num_items = - self.offsets[(self.rows_drained + num_rows) as usize] - failing_item_start; + let requested_range = self.rows_drained..self.rows_drained + num_rows; + let num_items = self.offsets[requested_range.end as usize] - item_start; return Err(oversized_batch_error( self.items_field.as_ref(), - failing_rows_start..self.rows_drained + num_rows, + requested_range, + self.rows_drained + actual_num_rows, num_items, )); } @@ -1323,6 +1345,7 @@ mod tests { let error = oversized_batch_error( &Field::new("item", DataType::UInt8, false), 32..160, + 96, i32::MAX as u64 + 1, ); assert!( @@ -1330,7 +1353,8 @@ mod tests { .to_string() .contains("more than 2GiB of string/binary data") ); - assert!(error.to_string().contains("rows 32..160")); + assert!(error.to_string().contains("rows 32..96 fit")); + assert!(error.to_string().contains("requesting rows 32..160")); assert!(error.to_string().contains("batch_size")); assert!(error.to_string().contains("LANCE_DEFAULT_BATCH_SIZE")); assert!(error.to_string().contains("large_string/large_binary")); @@ -1355,7 +1379,8 @@ mod tests { panic!("expected overflow error"); }; let message = error.to_string(); - assert!(message.contains("rows 2..4")); + assert!(message.contains("rows 2..3 fit")); + assert!(message.contains("requesting rows 2..4")); assert!(message.contains(&(i32::MAX as u64 + 2).to_string())); assert!(message.contains("batch_size")); } From 232bc2c3db229c8ba0d29e7dd4509ee4fb506c4e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:06:14 +0000 Subject: [PATCH 12/15] perf: avoid extra offset buffer copy Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/physical/binary.rs | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index 262877ae9a0..f43c91c73cb 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -332,32 +332,21 @@ impl PrimitivePageDecoder for BinaryPageDecoder { if num_bytes > i32::MAX as u64 { return Err(oversized_binary_batch_error(num_rows, num_bytes)); } - let offsets = target_vec - .iter() - .map(|&offset| { - i32::try_from(offset - start) - .map_err(|_| oversized_binary_batch_error(num_rows, num_bytes)) - }) - .collect::>>()?; - ScalarBuffer::from(offsets).into_inner() + ScalarBuffer::from_iter(target_vec.iter().map(|&offset| (offset - start) as i32)) + .into_inner() } 8 => { let num_bytes = end - start; - let offsets = target_vec - .iter() - .map(|&offset| { - i64::try_from(offset - start).map_err(|_| { - Error::not_supported(format!( - "Could not create large_string/large_binary array in a single \ - batch because {} rows would require {} bytes, which exceeds \ - i64::MAX. Please reduce the batch_size or set \ - LANCE_DEFAULT_BATCH_SIZE to a smaller value.", - num_rows, num_bytes - )) - }) - }) - .collect::>>()?; - ScalarBuffer::from(offsets).into_inner() + if num_bytes > i64::MAX as u64 { + return Err(Error::not_supported(format!( + "Could not create large_string/large_binary array in a single batch \ + because {} rows would require {} bytes, which exceeds i64::MAX. Please \ + reduce the batch_size or set LANCE_DEFAULT_BATCH_SIZE to a smaller value.", + num_rows, num_bytes + ))); + } + ScalarBuffer::from_iter(target_vec.iter().map(|&offset| (offset - start) as i64)) + .into_inner() } _ => panic!("Unsupported offsets type"), }; From fb0b66cd1c85a1795e86b0384a1cd8ede6fcbe33 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:08:17 +0000 Subject: [PATCH 13/15] test: cover generic list overflow message Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/list.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/lance-encoding/src/array_encoding/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs index c071c83d780..60e32301fdc 100644 --- a/rust/lance-encoding/src/array_encoding/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -1360,6 +1360,21 @@ mod tests { assert!(error.to_string().contains("large_string/large_binary")); } + #[test] + fn oversized_list_batch_error_is_actionable() { + let error = oversized_batch_error( + &Field::new("item", DataType::Int32, false), + 8..32, + 16, + i32::MAX as u64 + 1, + ); + let message = error.to_string(); + assert!(message.contains("list array")); + assert!(message.contains("more than i32::MAX items")); + assert!(message.contains("rows 8..16 fit")); + assert!(message.contains("requesting rows 8..32")); + } + #[test] fn list_decoder_overflow_reports_row_span() { let mut decoder = ListPageDecoder { From 7083af4605f7ca2533372d0e53b21e6e43e2bb4e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:10:32 +0000 Subject: [PATCH 14/15] refactor: share overflow size calculation Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/physical/binary.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs index f43c91c73cb..8f413e2bdf3 100644 --- a/rust/lance-encoding/src/array_encoding/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -42,6 +42,15 @@ fn oversized_binary_batch_error(num_rows: u64, num_bytes: u64) -> Error { )) } +fn oversized_large_binary_batch_error(num_rows: u64, num_bytes: u64) -> Error { + Error::not_supported(format!( + "Could not create large_string/large_binary array in a single batch because {} rows \ + would require {} bytes, which exceeds i64::MAX. Please reduce the batch_size or set \ + LANCE_DEFAULT_BATCH_SIZE to a smaller value.", + num_rows, num_bytes + )) +} + struct IndicesNormalizer { indices: Vec, validity: BooleanBufferBuilder, @@ -326,9 +335,9 @@ impl PrimitivePageDecoder for BinaryPageDecoder { let target_vec = target_offsets.values(); let start = target_vec[0]; let end = *target_vec.last().unwrap(); + let num_bytes = end - start; let offsets_buffer = match bytes_per_offset { 4 => { - let num_bytes = end - start; if num_bytes > i32::MAX as u64 { return Err(oversized_binary_batch_error(num_rows, num_bytes)); } @@ -336,14 +345,8 @@ impl PrimitivePageDecoder for BinaryPageDecoder { .into_inner() } 8 => { - let num_bytes = end - start; if num_bytes > i64::MAX as u64 { - return Err(Error::not_supported(format!( - "Could not create large_string/large_binary array in a single batch \ - because {} rows would require {} bytes, which exceeds i64::MAX. Please \ - reduce the batch_size or set LANCE_DEFAULT_BATCH_SIZE to a smaller value.", - num_rows, num_bytes - ))); + return Err(oversized_large_binary_batch_error(num_rows, num_bytes)); } ScalarBuffer::from_iter(target_vec.iter().map(|&offset| (offset - start) as i64)) .into_inner() From f448c0c03f26a6b79cf64477dcb44a581346e9e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:38:48 +0000 Subject: [PATCH 15/15] Apply remaining changes Co-authored-by: wjones127 <5488879+wjones127@users.noreply.github.com> --- .../src/array_encoding/logical/binary.rs | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/rust/lance-encoding/src/array_encoding/logical/binary.rs b/rust/lance-encoding/src/array_encoding/logical/binary.rs index 2d95a5b6d35..1bb900a6785 100644 --- a/rust/lance-encoding/src/array_encoding/logical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/logical/binary.rs @@ -11,7 +11,7 @@ use arrow_array::{ use arrow_schema::DataType; use futures::{FutureExt, future::BoxFuture}; -use lance_core::Result; +use lance_core::{Error, Result}; use log::trace; use crate::{ @@ -176,7 +176,11 @@ impl DecodeArrayTask for BinaryArrayDecoder { } DataType::Utf8 => Self::from_list_array::(arr.as_list::())?, DataType::LargeUtf8 => Self::from_list_array::(arr.as_list::())?, - _ => panic!("Binary decoder does not support this data type"), + other => { + return Err(Error::internal(format!( + "Binary decoder does not support data type {other}" + ))); + } }; // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. @@ -205,17 +209,21 @@ mod tests { } } - #[test] - fn logical_utf8_decode_preserves_non_overflow_arrow_error() { + fn make_single_byte_list(value: u8) -> ListArray { let offsets = OffsetBuffer::from_lengths([1_usize]); - let values: ArrayRef = Arc::new(UInt8Array::from(vec![0xFF_u8])); - let list = ListArray::try_new( + let values: ArrayRef = Arc::new(UInt8Array::from(vec![value])); + ListArray::try_new( Arc::new(Field::new("item", DataType::UInt8, false)), offsets, values, None, ) - .unwrap(); + .unwrap() + } + + #[test] + fn logical_utf8_decode_preserves_non_overflow_arrow_error() { + let list = make_single_byte_list(0xFF_u8); let decoder = BinaryArrayDecoder { inner: Box::new(StubDecodeTask { array: Arc::new(list), @@ -228,4 +236,23 @@ mod tests { assert!(!message.contains("more than 2GiB of string/binary data")); assert!(message.to_lowercase().contains("utf")); } + + #[test] + fn logical_binary_decode_returns_internal_error_for_unsupported_type() { + let list = make_single_byte_list(b'x'); + let decoder = BinaryArrayDecoder { + inner: Box::new(StubDecodeTask { + array: Arc::new(list), + }), + data_type: DataType::Int32, + }; + + let error = Box::new(decoder).decode().unwrap_err(); + assert!(matches!(error, Error::Internal { .. })); + assert!( + error + .to_string() + .contains("Binary decoder does not support data type Int32") + ); + } }