diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 7275c4b674d20..89f42761535e3 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -245,6 +245,10 @@ name = "math_query_sql" harness = false name = "filter_query_sql" +[[bench]] +harness = false +name = "in_list_rewrite" + [[bench]] harness = false name = "struct_query_sql" diff --git a/datafusion/core/benches/in_list_rewrite.rs b/datafusion/core/benches/in_list_rewrite.rs new file mode 100644 index 0000000000000..078c8f5e7db4b --- /dev/null +++ b/datafusion/core/benches/in_list_rewrite.rs @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks short SQL `IN` lists using the default optimizer. Comparing the +//! same cases across commits measures changes to the chosen evaluation strategy. +//! Criterion times expression evaluation, with planning and table setup outside +//! the measured loop. Input types and expected Boolean results are checked first. +//! +//! All types cover misses, balanced matches, and uniform or skewed +//! first-item hits. Floating-point and fixed-size-binary types also cover +//! nulls and small batches; a misaligned binary batch exercises alignment copies. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray}; +use arrow::buffer::{Buffer, MutableBuffer}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use datafusion::prelude::SessionContext; +use datafusion_common::ScalarValue; +use datafusion_expr::LogicalPlan; +use datafusion_physical_expr::PhysicalExpr; +use rand::prelude::*; +use tokio::runtime::Runtime; + +const MISS_VALUE_BASE: usize = 10_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + Int32, + Int64, + UInt64, + Float32, + Float64, + Utf8, + Utf8View, + FixedSizeBinary(i32), +} + +impl ValueKind { + const ALL: [Self; 12] = [ + Self::Int32, + Self::Int64, + Self::UInt64, + Self::Float32, + Self::Float64, + Self::Utf8, + Self::Utf8View, + Self::FixedSizeBinary(1), + Self::FixedSizeBinary(2), + Self::FixedSizeBinary(4), + Self::FixedSizeBinary(8), + Self::FixedSizeBinary(16), + ]; + + fn name(self) -> String { + match self { + Self::Int32 => "int32", + Self::Int64 => "int64", + Self::UInt64 => "uint64", + Self::Float32 => "float32", + Self::Float64 => "float64", + Self::Utf8 => "utf8", + Self::Utf8View => "utf8view_inline", + Self::FixedSizeBinary(width) => return format!("fixed_size_binary_{width}"), + } + .to_owned() + } + + fn seed_tag(self) -> u64 { + match self { + Self::Int32 => 1, + Self::Float64 => 2, + Self::Utf8 => 3, + Self::Utf8View => 4, + Self::FixedSizeBinary(8) => 5, + Self::Int64 => 6, + Self::Float32 => 7, + Self::UInt64 => 8, + Self::FixedSizeBinary(width) => 20 + width as u64, + } + } + + // The same typed values build batches and bind SQL parameters, avoiding + // implicit SQL literal widening for Float32 and UInt64. + fn scalar(self, value: Option) -> ScalarValue { + match self { + Self::Int32 => ScalarValue::Int32(value.map(|v| v as i32)), + Self::Int64 => ScalarValue::Int64(value.map(|v| v as i64)), + Self::UInt64 => ScalarValue::UInt64(value.map(|v| v as u64)), + Self::Float32 => ScalarValue::Float32(value.map(|v| v as f32)), + Self::Float64 => ScalarValue::Float64(value.map(|v| v as f64)), + Self::Utf8 => ScalarValue::Utf8(value.map(string_value)), + Self::Utf8View => ScalarValue::Utf8View(value.map(string_value)), + Self::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + width, + value.map(|v| { + // Narrow misses stay disjoint from the hit values 0, 1, and 2. + let v = if width == 1 && v >= MISS_VALUE_BASE { + 4 + (v - MISS_VALUE_BASE) % 252 + } else { + v + }; + (v as u128).to_be_bytes()[16 - width as usize..].to_vec() + }), + ), + } + } + + fn data_type(self) -> DataType { + self.scalar(Some(0)).data_type() + } + + fn make_array(self, values: &[Option]) -> ArrayRef { + let array = + ScalarValue::iter_to_array(values.iter().map(|&v| self.scalar(v))).unwrap(); + if let Self::FixedSizeBinary(width) = self { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + binary.values().as_ptr().align_offset(width as usize), + 0, + "builder-created binary buffer must be aligned" + ); + } + array + } +} + +// Eight-byte strings use Arrow's inline byte-view representation. +fn string_value(value: usize) -> String { + format!("v{value:07}") +} + +#[derive(Clone, Copy)] +enum Profile { + Miss, + Balanced, + AllFirstHit, + SkewedHit, + Nullable, + SmallBatch, + SingleMiss, + SingleHit, + ListNull, + Unaligned, +} + +impl Profile { + const ALL: [Self; 10] = [ + Self::Miss, + Self::Balanced, + Self::AllFirstHit, + Self::SkewedHit, + Self::Nullable, + Self::SmallBatch, + Self::SingleMiss, + Self::SingleHit, + Self::ListNull, + Self::Unaligned, + ]; + + fn name(self) -> &'static str { + match self { + Self::Miss => "miss", + Self::Balanced => "balanced", + Self::AllFirstHit => "all_first_hit", + Self::SkewedHit => "skewed_hit", + Self::Nullable => "balanced_nullable", + Self::SmallBatch => "balanced_small_batch", + Self::SingleMiss => "single_row_miss", + Self::SingleHit => "single_row_hit", + Self::ListNull => "list_null_balanced", + Self::Unaligned => "balanced_unaligned", + } + } + + fn batch_size(self) -> usize { + match self { + Self::SmallBatch => 64, + Self::SingleMiss | Self::SingleHit => 1, + _ => 8192, + } + } + + fn null_percent(self) -> usize { + if matches!(self, Self::Nullable) { + 20 + } else { + 0 + } + } + + fn match_percent(self) -> usize { + match self { + Self::Miss | Self::SingleMiss => 0, + Self::AllFirstHit | Self::SingleHit => 100, + Self::SkewedHit => 99, + _ => 50, + } + } + + fn permits_kind(self, kind: ValueKind) -> bool { + match self { + Self::Unaligned => kind == ValueKind::FixedSizeBinary(16), + Self::Miss | Self::Balanced | Self::AllFirstHit | Self::SkewedHit => true, + _ => matches!( + kind, + ValueKind::Float32 | ValueKind::Float64 | ValueKind::FixedSizeBinary(_) + ), + } + } +} + +// Project the predicate so optimization preserves its Boolean result, including +// NULLs, rather than simplifying it based on WHERE semantics. +fn plan_in_list( + ctx: &SessionContext, + runtime: &Runtime, + kind: ValueKind, + batch: &RecordBatch, + list_len: usize, + negated: bool, + list_has_null: bool, +) -> Arc { + ctx.deregister_table(kind.name()).unwrap(); + ctx.register_batch(kind.name(), batch.clone()).unwrap(); + let params = (0..list_len) + .map(|i| kind.scalar((!list_has_null || i + 1 < list_len).then_some(i))) + .collect::>(); + assert!( + params + .iter() + .all(|value| value.data_type() == kind.data_type()) + ); + let placeholders = (1..=list_len).map(|i| format!("${i}")).collect::>(); + let sql = format!( + "SELECT a {}IN ({}) AS matches FROM {}", + if negated { "NOT " } else { "" }, + placeholders.join(", "), + kind.name() + ); + let plan = runtime + .block_on(ctx.sql(&sql)) + .unwrap() + .with_param_values(params) + .unwrap() + .into_optimized_plan() + .unwrap(); + let LogicalPlan::Projection(projection) = plan else { + panic!("SQL must project the predicate"); + }; + assert_eq!(projection.expr.len(), 1); + assert_eq!( + projection.input.schema().field(0).data_type(), + &kind.data_type() + ); + let expr = ctx + .create_physical_expr( + projection.expr[0].clone().unalias(), + projection.input.schema(), + ) + .unwrap(); + let schema = batch.schema(); + assert_eq!(expr.data_type(schema.as_ref()).unwrap(), DataType::Boolean); + // Check types without requiring either IN or comparison-chain expressions. + let mut nodes = vec![Arc::clone(&expr)]; + while let Some(node) = nodes.pop() { + let data_type = node.data_type(schema.as_ref()).unwrap(); + assert!(data_type == DataType::Boolean || data_type == kind.data_type()); + nodes.extend(node.children().into_iter().cloned()); + } + expr +} + +fn make_batch( + kind: ValueKind, + profile: Profile, + profile_index: usize, + list_len: usize, + negated: bool, +) -> (RecordBatch, BooleanArray) { + let null_count = profile.batch_size() * profile.null_percent() / 100; + let non_null_count = profile.batch_size() - null_count; + let match_count = non_null_count * profile.match_percent() / 100; + let list_has_null = matches!(profile, Profile::ListNull); + let first_hits = matches!( + profile, + Profile::AllFirstHit | Profile::SkewedHit | Profile::SingleHit + ); + let mut values = Vec::with_capacity(profile.batch_size()); + values.extend(std::iter::repeat_n(None, null_count)); + values.extend((0..match_count).map(|i| { + Some(if first_hits { + 0 + } else { + i % (list_len - usize::from(list_has_null)) + }) + })); + values.extend((0..non_null_count - match_count).map(|i| Some(MISS_VALUE_BASE + i))); + let seed = 0x1A11_1575_5EED_u64 + ^ (kind.seed_tag() << 48) + ^ ((profile_index as u64) << 32) + ^ list_len as u64; + values.shuffle(&mut StdRng::seed_from_u64(seed)); + let expected = values + .iter() + .map(|value| { + value.and_then(|v| { + if v < list_len - usize::from(list_has_null) { + Some(!negated) + } else if list_has_null { + None + } else { + Some(negated) + } + }) + }) + .collect::(); + let mut array = kind.make_array(&values); + if matches!(profile, Profile::Unaligned) { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + let mut bytes = MutableBuffer::with_capacity(binary.values().len() + 1); + bytes.push(0_u8); + bytes.extend_from_slice(binary.values()); + let buffer = Buffer::from(bytes).slice(1); + assert_ne!(buffer.as_ptr().align_offset(16), 0); + array = Arc::new(FixedSizeBinaryArray::new( + 16, + buffer, + binary.nulls().cloned(), + )); + } + let schema = Arc::new(Schema::new(vec![Field::new("a", kind.data_type(), true)])); + (RecordBatch::try_new(schema, vec![array]).unwrap(), expected) +} + +fn criterion_benchmark(c: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + let ctx = SessionContext::new(); + for kind in ValueKind::ALL { + for (profile_index, profile) in Profile::ALL.into_iter().enumerate() { + if !profile.permits_kind(kind) { + continue; + } + let mut group = c.benchmark_group(format!( + "in_list_rewrite/{}/{}/batch={}/nulls={}%/match={}%", + kind.name(), + profile.name(), + profile.batch_size(), + profile.null_percent(), + profile.match_percent() + )); + group.throughput(Throughput::Elements(profile.batch_size() as u64)); + for list_len in [2, 3] { + for negated in [false, true] { + let (batch, expected) = + make_batch(kind, profile, profile_index, list_len, negated); + let expr = plan_in_list( + &ctx, + &runtime, + kind, + &batch, + list_len, + negated, + matches!(profile, Profile::ListNull), + ); + let output = expr + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(output.as_boolean(), &expected); + let case = format!( + "{}/list={list_len}", + if negated { "not_in" } else { "in" } + ); + group.bench_function(case, |b| { + b.iter(|| black_box(expr.evaluate(black_box(&batch)).unwrap())) + }); + } + } + group.finish(); + } + } +} + +criterion_group! { + name = benches; + config = Criterion::default() + .warm_up_time(Duration::from_millis(100)) + .measurement_time(Duration::from_millis(500)); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 506692f0dfa6d..a4766da61db9d 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -117,6 +117,10 @@ pub struct ExprSimplifier { max_simplifier_cycles: u32, } +/// Largest `IN` list considered for expansion into binary comparisons. +/// +/// The final decision also accounts for the measured performance of the column +/// representation and whether the list values are literals. pub const THRESHOLD_INLINE_INLIST: usize = 3; pub const DEFAULT_MAX_SIMPLIFIER_CYCLES: u32 = 3; @@ -199,7 +203,8 @@ impl ExprSimplifier { let mut simplifier = Simplifier::new(&self.info); let config_options = Some(Arc::clone(self.info.config_options())); let mut const_evaluator = ConstEvaluator::try_new(config_options)?; - let mut shorten_in_list_simplifier = ShortenInListSimplifier::new(); + let mut shorten_in_list_simplifier = + ShortenInListSimplifier::new(self.info.schema()); let guarantees_map: HashMap<&Expr, &NullableInterval> = self.guarantees.iter().map(|(k, v)| (k, v)).collect(); @@ -5032,6 +5037,8 @@ mod tests { }; let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?; let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?; + let retains_short_lists = + matches!(data_type, DataType::Float32 | DataType::Float64); let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)])); let batch = RecordBatch::try_new(schema, vec![values])?; let schema = batch.schema().to_dfschema_ref()?; @@ -5056,7 +5063,7 @@ mod tests { (left(true).or(right(true)), vec![false, false]), ] { let simplified = simplifier.simplify(expr.clone())?; - if list_len > THRESHOLD_INLINE_INLIST { + if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists { // Floating-point literals must bypass structural set rewrites. assert_eq!(simplified, expr); } else { @@ -5125,6 +5132,99 @@ mod tests { } } + #[test] + fn simplify_short_inlist_measured_representations() { + for data_type in [ + DataType::Float32, + DataType::Float64, + DataType::FixedSizeBinary(8), + DataType::Float16, + DataType::Int32, + DataType::Int64, + DataType::UInt64, + DataType::FixedSizeBinary(1), + DataType::FixedSizeBinary(2), + DataType::FixedSizeBinary(3), + DataType::FixedSizeBinary(4), + DataType::FixedSizeBinary(16), + DataType::Utf8View, + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Float64)), + ] { + let values = (1..=4) + .map(|value| { + lit(match &data_type { + DataType::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + *width, + Some(vec![value; *width as usize]), + ), + DataType::Utf8View => { + ScalarValue::Utf8View(Some(value.to_string())) + } + _ => ScalarValue::Int32(Some(i32::from(value))) + .cast_to(&data_type) + .unwrap(), + }) + }) + .collect::>(); + let retain_short = matches!( + data_type, + DataType::Int32 + | DataType::Int64 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) + ); + let schema = Schema::new(vec![ + Field::new("value", data_type.clone(), true), + Field::new("other", data_type, true), + ]) + .to_dfschema_ref() + .unwrap(); + let simplifier = ExprSimplifier::new( + SimplifyContext::builder().with_schema(schema).build(), + ); + for list_len in 1..=4 { + for negated in [false, true] { + let list = values[..list_len].to_vec(); + let expr = in_list(col("value"), list.clone(), negated); + let expected = if list_len > THRESHOLD_INLINE_INLIST + || list_len > 1 && retain_short + { + expr.clone() + } else { + let comparisons = list.into_iter().map(|value| { + if negated { + col("value").not_eq(value) + } else { + col("value").eq(value) + } + }); + comparisons + .reduce(|left, right| { + if negated { + left.and(right) + } else { + left.or(right) + } + }) + .unwrap() + }; + assert_eq!(simplifier.simplify(expr).unwrap(), expected); + } + } + + // A column-valued list keeps the comparison path even for a + // representation whose static lists use membership filters. + let expr = + in_list(col("value"), vec![values[0].clone(), col("other")], false); + let expected = col("value") + .eq(values[0].clone()) + .or(col("value").eq(col("other"))); + assert_eq!(simplifier.simplify(expr).unwrap(), expected); + } + } + #[test] fn simplify_inlist_set_operation_propagates_nullability_errors() { let info = SimplifyContext::builder() diff --git a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs index 17112d4f0ae24..4d0df21a56f74 100644 --- a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs @@ -19,24 +19,48 @@ use super::THRESHOLD_INLINE_INLIST; -use datafusion_common::Result; +use arrow::datatypes::DataType; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; -use datafusion_expr::Expr; +use datafusion_common::{DFSchema, Result}; use datafusion_expr::expr::InList; +use datafusion_expr::{Expr, ExprSchemable}; -pub(super) struct ShortenInListSimplifier {} +pub(super) struct ShortenInListSimplifier<'a> { + schema: &'a DFSchema, +} + +impl<'a> ShortenInListSimplifier<'a> { + pub(super) fn new(schema: &'a DFSchema) -> Self { + Self { schema } + } -impl ShortenInListSimplifier { - pub(super) fn new() -> Self { - Self {} + /// The paired `in_list_rewrite` benchmark favors membership for these + /// representations. For integers, prefer the gains on mixed and miss-heavy + /// batches despite the comparison chain's advantage on first-item matches. + fn should_retain_short_inlist(&self, expr: &Expr, list: &[Expr]) -> bool { + let Ok(data_type) = expr.get_type(self.schema) else { + return false; + }; + matches!( + data_type, + DataType::Int32 + | DataType::Int64 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) + ) && list.iter().all(|expr| { + expr.as_literal() + .is_some_and(|value| value.data_type() == data_type) + }) } } -impl TreeNodeRewriter for ShortenInListSimplifier { +impl TreeNodeRewriter for ShortenInListSimplifier<'_> { type Node = Expr; fn f_up(&mut self, expr: Expr) -> Result> { - // if expr is a single column reference: + // Rewrite eligible short lists to left-deep comparison chains: // expr IN (A, B, ...) --> (expr = A) OR (expr = B) OR (expr = C) if let Expr::InList(InList { ref expr, @@ -52,6 +76,7 @@ impl TreeNodeRewriter for ShortenInListSimplifier { list.len() == 1 || list.len() <= THRESHOLD_INLINE_INLIST && expr.try_as_col().is_some() + && !self.should_retain_short_inlist(expr, list) ) { let first_val = list[0].clone(); diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt index a17ce11a0bff0..7e4d100b74237 100644 --- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt +++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt @@ -45,9 +45,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, 3); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] # Verify result correctness @@ -63,9 +63,9 @@ explain select * from t1 right join t2 on t1.a = t2.x where t1.a in (1, 2); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -80,9 +80,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t1.a in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -97,9 +97,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, null); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(NULL)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(NULL)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -168,9 +168,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t1.a in (1, 2, 3); ---- logical_plan 01)Left Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -185,7 +185,7 @@ query TT explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) or t2.x is null; ---- logical_plan -01)Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x IS NULL +01)Filter: t2.x IN ([Int32(1), Int32(2)]) OR t2.x IS NULL 02)--Left Join: t1.a = t2.x 03)----TableScan: t1 projection=[a, b, c] 04)----TableScan: t2 projection=[x, y, z] @@ -459,9 +459,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t2.x in (1, 2); ---- logical_plan 01)Right Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -512,9 +512,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: (t2.x = Int32(1) OR t2.x = Int32(2)) AND t2.y >= Int32(50) AND t2.y <= Int32(250) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) AND t2.y >= Int32(50) AND t2.y <= Int32(250) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort diff --git a/datafusion/sqllogictest/test_files/in_list.slt b/datafusion/sqllogictest/test_files/in_list.slt index 3ff47afb643fa..60f3b5d1a1e28 100644 --- a/datafusion/sqllogictest/test_files/in_list.slt +++ b/datafusion/sqllogictest/test_files/in_list.slt @@ -20,9 +20,11 @@ # # This file focuses on the IN operator and its various specializations # -# Note that "short" IN LISTS do not go through the InList implementation at all, -# instead they are rewritten into a series of OR expressions. See: -# https://github.com/apache/datafusion/blob/ed37b6c9555bc278130dc774ed833b8c0bd29bfa/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs#L39-L88 +# One-item IN lists are rewritten to a comparison. Two- and three-item literal +# lists on Int32, Int64, UInt64, Float32, Float64, or FixedSizeBinary columns +# with widths 1, 2, 4, 8, or 16 retain their static InList filters. Other short +# lists on columns are rewritten to OR/AND expressions. See: +# https://github.com/apache/datafusion/blob/main/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs ########## @@ -211,6 +213,66 @@ CREATE TABLE in_list_ints_nullable ( ('no_match', 7, 7, 7, 7, 7, 7, 7, 7), ('nulls', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +# Measured integer representations retain two- and three-item literal lists. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match false true false true false true +nulls NULL NULL NULL NULL NULL NULL + +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(NULL, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(NULL, 'Int32'), arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(NULL, 'Int64'), arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(NULL, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(NULL, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(NULL, 'UInt64'), arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match NULL NULL NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list keeps the comparison path for these representations. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), i16), + i32 NOT IN (arrow_cast(3, 'Int32'), i16), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + u64 IN (arrow_cast(3, 'UInt64'), u32), + u64 NOT IN (arrow_cast(3, 'UInt64'), u32) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match true false true false true false +nulls NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')) AS i32_in_two, + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')) AS i64_not_in_three, + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) AS u64_in_three +FROM in_list_ints_nullable; +---- +logical_plan +01)Projection: in_list_ints_nullable.i32 IN ([Int32(3), Int32(11)]) AS i32_in_two, in_list_ints_nullable.i64 NOT IN ([Int64(3), Int64(4), Int64(11)]) AS i64_not_in_three, in_list_ints_nullable.u64 IN ([UInt64(3), UInt64(4), UInt64(11)]) AS u64_in_three +02)--TableScan: in_list_ints_nullable projection=[i32, i64, u64] +physical_plan +01)ProjectionExec: expr=[i32@0 IN (SET) ([3, 11]) as i32_in_two, i64@1 NOT IN (SET) ([3, 4, 11]) as i64_not_in_three, u64@2 IN (SET) ([3, 4, 11]) as u64_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Null input values return NULL when the IN list has no nulls. query TBBBBBBBB SELECT @@ -308,6 +370,79 @@ FROM (VALUES ('nulls', NULL) ) AS t(label, value); +# Short Float32 and Float64 lists retain membership filters for IN and NOT IN. +query TBBBBBBBB +SELECT + label, + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TBBBBBBBB +SELECT + label, + f64 IN (3.0, 11.0), + f64 NOT IN (3.0, 11.0), + f64 IN (3.0, 4.0, 11.0), + f64 NOT IN (3.0, 4.0, 11.0), + f64 IN (NULL, 11.0), + f64 NOT IN (NULL, 11.0), + f64 IN (NULL, 3.0, 11.0), + f64 NOT IN (NULL, 3.0, 11.0) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +# A list containing a column is evaluated through comparisons. +query TBBBB +SELECT label, + f32 IN (arrow_cast(3.0, 'Float32'), f16), + f32 NOT IN (arrow_cast(3.0, 'Float32'), f16), + f64 IN (3.0, f32), f64 NOT IN (3.0, f32) +FROM in_list_floats +ORDER BY label +---- +match true false true false +no_match true false true false +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS in_two, + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f32 IN ([Float32(3), Float32(11)]) AS in_two, in_list_floats.f32 NOT IN ([Float32(3), Float32(4), Float32(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f32] +physical_plan +01)ProjectionExec: expr=[f32@0 IN (SET) ([3, 11]) as in_two, f32@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +EXPLAIN SELECT f64 IN (3.0, 11.0) AS in_two, f64 NOT IN (3.0, 4.0, 11.0) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f64 IN ([Float64(3), Float64(11)]) AS in_two, in_list_floats.f64 NOT IN ([Float64(3), Float64(4), Float64(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f64] +physical_plan +01)ProjectionExec: expr=[f64@0 IN (SET) ([3, 11]) as in_two, f64@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Five element IN lists cover the specialized Float16/32/64 paths. query TTBBB SELECT @@ -616,6 +751,82 @@ LIMIT 1 ---- FixedSizeBinary(1) FixedSizeBinary(2) FixedSizeBinary(3) FixedSizeBinary(4) FixedSizeBinary(8) FixedSizeBinary(16) +# Short lists at the width endpoints cover both list lengths and typed nulls. +query TBBBBBB +SELECT + label, + fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'04', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 IN (arrow_cast(NULL, 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true true false false +no_match false true NULL false true NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list retains the comparison path. +query TBBBB +SELECT l.label, + l.fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16), + l.fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16) +FROM in_list_fsb l CROSS JOIN in_list_fsb r +WHERE r.label = 'match' +ORDER BY l.label +---- +match true false true false +no_match false true false true +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')) AS in_two, + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb1 IN ([FixedSizeBinary(1, "3"), FixedSizeBinary(1, "11")]) AS in_two, in_list_fsb.fsb16 NOT IN ([FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb1, fsb16] +physical_plan +01)ProjectionExec: expr=[fsb1@0 IN (SET) ([03, 0B]) as in_two, fsb16@1 NOT IN (SET) ([00000000000000000000..., 00000000000000000000..., 00000000000000000000...]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Short eight-byte lists retain membership filters, including list nulls. +query TBBBBBBBB +SELECT + label, + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS in_two, + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb8 IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS in_two, in_list_fsb.fsb8 NOT IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,4"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb8] +physical_plan +01)ProjectionExec: expr=[fsb8@0 IN (SET) ([0000000000000003, 000000000000000B]) as in_two, fsb8@0 NOT IN (SET) ([0000000000000003, 0000000000000004, 000000000000000B]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Eight element lists, all matching or none matching query TBBBBBB SELECT diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index a15f4a3b82410..f2763a8ff18bc 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -257,7 +257,7 @@ logical_plan 01)Sort: t1.t1_id ASC NULLS LAST, t2.t2_id ASC NULLS LAST 02)--Inner Join: Filter: t1.t1_id <= t2.t2_id 03)----SubqueryAlias: t1 -04)------Filter: join_t1.t1_id = Int32(11) OR join_t1.t1_id = Int32(44) +04)------Filter: join_t1.t1_id IN ([Int32(11), Int32(44)]) 05)--------TableScan: join_t1 projection=[t1_id] 06)----SubqueryAlias: t2 07)------Projection: join_t2.t2_id @@ -268,7 +268,7 @@ physical_plan 02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=LtEq, join_type=Inner, on=(t1_id <= t2_id) 04)------SortExec: expr=[t1_id@0 DESC], preserve_partitioning=[false] -05)--------FilterExec: t1_id@0 = 11 OR t1_id@0 = 44 +05)--------FilterExec: t1_id@0 IN (SET) ([11, 44]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] 07)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 08)--------FilterExec: t2_name@1 != y, projection=[t2_id@0] diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 463cd1799e857..85b15f7a38e76 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -1932,7 +1932,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20, pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20]), pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] 03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TT @@ -1946,7 +1946,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- Plan with Metrics 01)HashJoinExec: mode=Partitionedmetrics=[output_rows=2,] -02)--DataSourceExec: file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20metrics=[output_rows=2,] +02)--DataSourceExec: file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20])metrics=[output_rows=2,] 03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] query III diff --git a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt index 196da416a2037..c09f2c986e7c0 100644 --- a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt +++ b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt @@ -229,10 +229,10 @@ query TT EXPLAIN SELECT * FROM t_int WHERE column1 IN ('5', '325'); ---- logical_plan -01)Filter: t_int.column1 = Int64(5) OR t_int.column1 = Int64(325) +01)Filter: t_int.column1 IN ([Int64(5), Int64(325)]) 02)--TableScan: t_int projection=[column1] physical_plan -01)FilterExec: column1@0 = 5 OR column1@0 = 325 +01)FilterExec: column1@0 IN (SET) ([5, 325]) 02)--DataSourceExec: partitions=1, partition_sizes=[1] # Error on invalid string in IN list diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 115a010330103..e89316fae2117 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -365,14 +365,14 @@ EXPLAIN SELECT id, name FROM t1 WHERE id = 1 UNION SELECT id, name FROM t1 WHERE ---- logical_plan 01)Aggregate: groupBy=[[id, name]], aggr=[[]] -02)--Filter: t1.id = Int32(1) OR t1.id = Int32(2) +02)--Filter: t1.id IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[id, name] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Regression: schema recomputation must preserve the unqualified UNION @@ -415,14 +415,14 @@ SELECT x.id FROM t1 AS x WHERE x.id = 2 logical_plan 01)Aggregate: groupBy=[[id]], aggr=[[]] 02)--SubqueryAlias: x -03)----Filter: t1.id = Int32(1) OR t1.id = Int32(2) +03)----Filter: t1.id IN ([Int32(1), Int32(2)]) 04)------TableScan: t1 projection=[id] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Matching computed projections remain eligible for the rewrite and must diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index a6b64b98de0e3..4093562b55dcd 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -5397,7 +5397,7 @@ logical_plan 01)Sort: t1.c1 ASC NULLS LAST, t1.c2 ASC NULLS LAST, rank ASC NULLS LAST 02)--Projection: t1.c1, t1.c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rank 03)----WindowAggr: windowExpr=[[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] -04)------Filter: t1.c1 = Int32(2) OR t1.c1 = Int32(3) +04)------Filter: t1.c1 IN ([Int32(2), Int32(3)]) 05)--------TableScan: t1 projection=[c1, c2] physical_plan 01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank@2 ASC NULLS LAST] @@ -5405,7 +5405,7 @@ physical_plan 03)----BoundedWindowAggExec: wdw=[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 04)------SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST], preserve_partitioning=[true] 05)--------RepartitionExec: partitioning=Hash([c1@0], 2), input_partitions=2 -06)----------FilterExec: c1@0 = 2 OR c1@0 = 3 +06)----------FilterExec: c1@0 IN (SET) ([2, 3]) 07)------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 08)--------------DataSourceExec: partitions=1, partition_sizes=[1]