From 11f4099d0dd52f0738dfb0efb00ff5d9b127863a Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Thu, 17 Sep 2026 14:45:00 -0400 Subject: [PATCH 1/2] . --- .../functions-nested/benches/array_has.rs | 61 +++++++----- datafusion/functions-nested/src/array_has.rs | 92 ++++++++++++++++++- 2 files changed, 123 insertions(+), 30 deletions(-) diff --git a/datafusion/functions-nested/benches/array_has.rs b/datafusion/functions-nested/benches/array_has.rs index 1a64f1cc4b160..c55a86dd5d4b6 100644 --- a/datafusion/functions-nested/benches/array_has.rs +++ b/datafusion/functions-nested/benches/array_has.rs @@ -913,31 +913,42 @@ fn bench_array_has_any_scalar(c: &mut Criterion) { Field::new("second", list_type_i64.clone(), false).into(), ]; - for &scalar_size in &scalar_sizes { - let scalar_arg = create_int64_scalar_list(scalar_size, array_size as i64); - let args = vec![ - ColumnarValue::Array(first_arr_i64.clone()), - ColumnarValue::Scalar(scalar_arg), - ]; - group.bench_with_input( - BenchmarkId::new("i64_no_match", scalar_size), - &scalar_size, - |b, _| { - let udf = ArrayHasAny::new(); - b.iter(|| { - black_box( - udf.invoke_with_args(ScalarFunctionArgs { - args: args.clone(), - arg_fields: arg_fields_i64.clone(), - number_rows: NUM_ROWS, - return_field: return_field.clone(), - config_options: config_options.clone(), - }) - .unwrap(), - ) - }) - }, - ); + // The same number of visible rows, sliced from the middle of a backing + // array that is 100 times larger. + let backing_rows = NUM_ROWS * 100; + let sliced_arr_i64 = create_int64_list_array(backing_rows, array_size, NULL_DENSITY) + .slice(backing_rows / 2, NUM_ROWS); + + for (name, arr) in [ + ("i64_no_match", &first_arr_i64), + ("i64_no_match_sliced", &sliced_arr_i64), + ] { + for &scalar_size in &scalar_sizes { + let scalar_arg = create_int64_scalar_list(scalar_size, array_size as i64); + let args = vec![ + ColumnarValue::Array(arr.clone()), + ColumnarValue::Scalar(scalar_arg), + ]; + group.bench_with_input( + BenchmarkId::new(name, scalar_size), + &scalar_size, + |b, _| { + let udf = ArrayHasAny::new(); + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: args.clone(), + arg_fields: arg_fields_i64.clone(), + number_rows: NUM_ROWS, + return_field: return_field.clone(), + config_options: config_options.clone(), + }) + .unwrap(), + ) + }) + }, + ); + } } // String benchmarks diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 156f9477f5491..7e867ec34da64 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -966,10 +966,17 @@ fn array_has_any_with_scalar_general( }; let col_list: ArrayWrapper = col_arr.as_ref().try_into()?; - let col_rows = converter.convert_columns(&[Arc::clone(col_list.values())])?; let col_offsets: Vec = col_list.offsets().collect(); let col_nulls = col_list.nulls(); + // For efficiency with sliced arrays, only convert the visible elements, + // not the entire underlying buffer. Indices into `col_rows` are therefore + // relative to `elem_start`. + let elem_start = col_offsets[0]; + let elem_end = col_offsets[col_list.len()]; + let visible_values = col_list.values().slice(elem_start, elem_end - elem_start); + let col_rows = converter.convert_columns(&[visible_values])?; + let mut result = BooleanBufferBuilder::new(col_list.len()); let num_scalar = scalar_rows.num_rows(); @@ -984,8 +991,8 @@ fn array_has_any_with_scalar_general( result.append(false); continue; } - let start = col_offsets[i]; - let end = col_offsets[i + 1]; + let start = col_offsets[i] - elem_start; + let end = col_offsets[i + 1] - elem_start; let found = (start..end).any(|j| scalar_set.contains(col_rows.row(j).as_ref())); result.append(found); @@ -997,8 +1004,8 @@ fn array_has_any_with_scalar_general( result.append(false); continue; } - let start = col_offsets[i]; - let end = col_offsets[i + 1]; + let start = col_offsets[i] - elem_start; + let end = col_offsets[i + 1] - elem_start; let found = (start..end) .any(|j| (0..num_scalar).any(|k| col_rows.row(j) == scalar_rows.row(k))); result.append(found); @@ -1683,4 +1690,79 @@ mod tests { vec![Some(true), Some(false)] ); } + + /// Invoke `array_has_any` with an array haystack and a scalar list of + /// `Int32` elements. + fn invoke_array_has_any_scalar(haystack: ArrayRef, scalar: Vec) -> ArrayRef { + let num_rows = haystack.len(); + let haystack_type = haystack.data_type().clone(); + let scalar = SingleRowListArrayBuilder::new(Arc::new(Int32Array::from(scalar))) + .build_list_scalar(); + let scalar_type = scalar.data_type(); + ArrayHasAny::new() + .invoke_with_args(ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(haystack), + ColumnarValue::Scalar(scalar), + ], + arg_fields: vec![ + Arc::new(Field::new("haystack", haystack_type, true)), + Arc::new(Field::new("scalar", scalar_type, true)), + ], + number_rows: num_rows, + return_field: Arc::new(Field::new("return", DataType::Boolean, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap() + .into_array(num_rows) + .unwrap() + } + + #[test] + fn test_array_has_any_scalar_sliced() { + // The scalar path only row-converts the visible elements of a sliced + // haystack, so its per-row element ranges must be relative to the + // start of the visible range. 1 and 70 appear only in sliced-away rows. + let full = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(10), None, Some(30)]), + None, + Some(vec![]), + Some(vec![Some(50), Some(60)]), + Some(vec![Some(70)]), + ]); + let sliced: ArrayRef = Arc::new(full.slice(1, 4)); + + // At most `SCALAR_SMALL_THRESHOLD` scalar elements: linear scan. + let result = invoke_array_has_any_scalar(Arc::clone(&sliced), vec![1, 70, 60]); + assert_eq!( + result.as_boolean().iter().collect::>(), + vec![Some(false), None, Some(false), Some(true)] + ); + let result = invoke_array_has_any_scalar(Arc::clone(&sliced), vec![30]); + assert_eq!( + result.as_boolean().iter().collect::>(), + vec![Some(true), None, Some(false), Some(false)] + ); + + // More than `SCALAR_SMALL_THRESHOLD` scalar elements: HashSet lookup. + let large_scalar = vec![1, 70, 60, 101, 102, 103, 104, 105, 106, 107]; + let result = invoke_array_has_any_scalar(sliced, large_scalar); + assert_eq!( + result.as_boolean().iter().collect::>(), + vec![Some(false), None, Some(false), Some(true)] + ); + + // Sliced FixedSizeList (width 2; rows 1..=2 of + // [[1,2],[11,12],[21,22],[31,32]] visible). + let field = Arc::new(Field::new("item", DataType::Int32, true)); + let fsl_values = Arc::new(Int32Array::from(vec![1, 2, 11, 12, 21, 22, 31, 32])); + let fsl: ArrayRef = + Arc::new(FixedSizeListArray::new(field, 2, fsl_values, None).slice(1, 2)); + let result = invoke_array_has_any_scalar(fsl, vec![1, 22, 31]); + assert_eq!( + result.as_boolean().iter().collect::>(), + vec![Some(false), Some(true)] + ); + } } From 7b200f3e7800b949e8b1a8a8a9470fec7a37a6d0 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Fri, 18 Sep 2026 10:26:37 -0400 Subject: [PATCH 2/2] Extend test, per review comment --- datafusion/functions-nested/src/array_has.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 7e867ec34da64..3a1505d7f3fff 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -1223,8 +1223,8 @@ mod tests { use arrow::datatypes::Int32Type; use arrow::{ array::{ - Array, ArrayRef, AsArray, FixedSizeListArray, Int32Array, ListArray, - create_array, + Array, ArrayRef, AsArray, FixedSizeListArray, Int32Array, LargeListArray, + ListArray, create_array, }, buffer::OffsetBuffer, datatypes::{DataType, Field, Schema}, @@ -1764,5 +1764,19 @@ mod tests { result.as_boolean().iter().collect::>(), vec![Some(false), Some(true)] ); + + // Sliced LargeList; 1 and 70 appear only in sliced-away rows. + let large = LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(10), None, Some(30)]), + Some(vec![Some(50), Some(60)]), + Some(vec![Some(70)]), + ]); + let large: ArrayRef = Arc::new(large.slice(1, 2)); + let result = invoke_array_has_any_scalar(large, vec![1, 70, 60]); + assert_eq!( + result.as_boolean().iter().collect::>(), + vec![Some(false), Some(true)] + ); } }