fix: normalize signed zero in nested IN-list values

Author: geoffreyclaude

datafusion/common/src/utils/mod.rs

@@ -1544,9 +1544,24 @@ pub fn has_float_leaf(data_type: &DataType) -> bool {
 }
 
 /// Replace `-0.0` with `+0.0` in `Float16`, `Float32`, or `Float64` scalar
-/// values, including dictionary-wrapped floats. Other variants are returned
-/// unchanged. See [`normalize_float_zero`] for context.
+/// values, including floats inside nested and encoded values. Other variants
+/// are returned unchanged. See [`normalize_float_zero`] for context.
 pub fn normalize_float_zero_scalar(scalar: ScalarValue) -> ScalarValue {
+    fn normalize_array<A: Array + Clone + 'static>(array: Arc<A>) -> Arc<A> {
+        let original = Arc::clone(&array) as ArrayRef;
+        let normalized = normalize_float_zero(&original);
+        if Arc::ptr_eq(&original, &normalized) {
+            // Preserve the original scalar array when no values changed.
+            return array;
+        }
+
+        let normalized_array = normalized
+            .as_any()
+            .downcast_ref::<A>()
+            .expect("float-zero normalization preserves the array type");
+        Arc::new(normalized_array.clone())
+    }
+
     match scalar {
         ScalarValue::Float32(Some(v)) if v.to_bits() << 1 == 0 => {
             ScalarValue::Float32(Some(0.0))
@@ -1557,10 +1572,29 @@ pub fn normalize_float_zero_scalar(scalar: ScalarValue) -> ScalarValue {
         ScalarValue::Float16(Some(v)) if v.to_bits() << 1 == 0 => {
             ScalarValue::Float16(Some(half::f16::from_bits(0)))
         }
+        ScalarValue::FixedSizeList(array) => {
+            ScalarValue::FixedSizeList(normalize_array(array))
+        }
+        ScalarValue::List(array) => ScalarValue::List(normalize_array(array)),
+        ScalarValue::LargeList(array) => ScalarValue::LargeList(normalize_array(array)),
+        ScalarValue::ListView(array) => ScalarValue::ListView(normalize_array(array)),
+        ScalarValue::LargeListView(array) => {
+            ScalarValue::LargeListView(normalize_array(array))
+        }
+        ScalarValue::Struct(array) => ScalarValue::Struct(normalize_array(array)),
+        ScalarValue::Map(array) => ScalarValue::Map(normalize_array(array)),
+        ScalarValue::Union(Some((type_id, mut value)), fields, mode) => {
+            *value = normalize_float_zero_scalar(*value);
+            ScalarValue::Union(Some((type_id, value)), fields, mode)
+        }
         ScalarValue::Dictionary(key, mut value) => {
             *value = normalize_float_zero_scalar(*value);
             ScalarValue::Dictionary(key, value)
         }
+        ScalarValue::RunEndEncoded(run_ends, values, mut value) => {
+            *value = normalize_float_zero_scalar(*value);
+            ScalarValue::RunEndEncoded(run_ends, values, value)
+        }
         other => other,
     }
 }
@@ -1723,6 +1757,77 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn normalize_float_zero_scalar_preserves_union_and_run_metadata() -> Result<()> {
+        use arrow::datatypes::{UnionFields, UnionMode};
+
+        let run_ends = Arc::new(Field::new("ends", DataType::Int32, false));
+        let values = Arc::new(Field::new("samples", DataType::Float64, true));
+        let fields = UnionFields::try_new(
+            [7, 42],
+            [
+                Field::new(
+                    "runs",
+                    DataType::RunEndEncoded(Arc::clone(&run_ends), Arc::clone(&values)),
+                    true,
+                ),
+                Field::new("other", DataType::Int32, true),
+            ],
+        )?;
+        let nan = f64::from_bits(0x7ff8_0000_0000_0001);
+        for mode in [UnionMode::Sparse, UnionMode::Dense] {
+            let wrap = |value| {
+                ScalarValue::Union(
+                    Some((
+                        7,
+                        Box::new(ScalarValue::RunEndEncoded(
+                            Arc::clone(&run_ends),
+                            Arc::clone(&values),
+                            Box::new(ScalarValue::Float64(value)),
+                        )),
+                    )),
+                    fields.clone(),
+                    mode,
+                )
+            };
+            for (value, expected) in [
+                (Some(-0.0), Some(0.0)),
+                (Some(nan), Some(nan)),
+                (None, None),
+            ] {
+                assert_eq!(normalize_float_zero_scalar(wrap(value)), wrap(expected));
+            }
+            let null = ScalarValue::Union(None, fields.clone(), mode);
+            assert_eq!(normalize_float_zero_scalar(null.clone()), null);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn normalize_float_zero_scalar_preserves_sliced_list() {
+        let nan = f64::from_bits(0x7ff8_0000_0000_0001);
+        let list = Arc::new(
+            ListArray::from_iter_primitive::<Float64Type, _, _>([
+                Some(vec![Some(99.0)]),
+                Some(vec![Some(-0.0), Some(nan), None]),
+                Some(vec![Some(42.0)]),
+            ])
+            .slice(1, 1),
+        );
+        let ScalarValue::List(normalized) =
+            normalize_float_zero_scalar(ScalarValue::List(list))
+        else {
+            panic!("normalization must preserve the scalar variant");
+        };
+        assert_eq!(normalized.len(), 1);
+        let child = normalized.value(0);
+        let child = child.as_primitive::<Float64Type>();
+        assert_eq!(child.len(), 3);
+        assert_eq!(child.value(0).to_bits(), 0.0_f64.to_bits());
+        assert_eq!(child.value(1).to_bits(), nan.to_bits());
+        assert!(child.is_null(2));
+    }
+
     #[test]
     fn test_bisect_linear_left_and_right() -> Result<()> {
         let arrays: Vec<ArrayRef> = vec![

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -30,6 +30,7 @@ use std::sync::LazyLock;
 
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::nested_struct::has_one_of_more_common_fields;
+use datafusion_common::utils::has_float_leaf;
 use datafusion_common::{
     DFSchema, DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_err,
 };
@@ -2318,8 +2319,9 @@ fn are_inlist_and_eq_and_match_neg(
 fn inlists_have_set_comparable_literals(left: &Expr, right: &Expr) -> bool {
     match (left, right) {
         (Expr::InList(l), Expr::InList(r)) => l.list.iter().chain(&r.list).all(|item| {
-            item.as_literal()
-                .is_some_and(|value| !value.is_null() && !value.data_type().is_floating())
+            item.as_literal().is_some_and(|value| {
+                !value.is_null() && !has_float_leaf(&value.data_type())
+            })
         }),
         _ => false,
     }

datafusion/physical-expr/src/expressions/in_list.rs

@@ -48,7 +48,7 @@ mod static_filter;
 mod strategy;
 
 use static_filter::StaticFilterRef;
-use strategy::{dictionary_value_type, instantiate_static_filter};
+use strategy::instantiate_static_filter;
 
 /// InList
 pub struct InListExpr {
@@ -85,15 +85,10 @@ fn supports_arrow_eq(dt: &DataType) -> bool {
 
 fn normalize_in_list_float_zero_value(value: ColumnarValue) -> ColumnarValue {
     match value {
-        ColumnarValue::Array(array)
-            if dictionary_value_type(array.data_type()).is_floating() =>
-        {
-            ColumnarValue::Array(normalize_float_zero(&array))
-        }
+        ColumnarValue::Array(array) => ColumnarValue::Array(normalize_float_zero(&array)),
         ColumnarValue::Scalar(scalar) => {
             ColumnarValue::Scalar(normalize_float_zero_scalar(scalar))
         }
-        value => value,
     }
 }
 

datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs

@@ -15,13 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow::array::{Array, ArrayRef, BooleanArray, make_comparator};
+use arrow::array::{Array, ArrayRef, BooleanArray, make_array, make_comparator};
 use arrow::buffer::{BooleanBuffer, NullBuffer};
 use arrow::compute::SortOptions;
 use arrow::datatypes::DataType;
 use arrow::util::bit_iterator::BitIndexIterator;
 use datafusion_common::Result;
 use datafusion_common::hash_utils::{RandomState, with_hashes};
+use datafusion_common::utils::{has_float_leaf, normalize_float_zero};
 use hashbrown::HashTable;
 
 use super::result::build_in_list_result;
@@ -53,6 +54,8 @@ impl ArrayStaticFilter {
             });
         }
 
+        // Hashing treats both signed zeros alike; the comparator must do so too.
+        let in_array = normalize_float_zero(&in_array);
         let state = RandomState::default();
         let table = Self::build_haystack_table(&in_array, &state)?;
 
@@ -138,6 +141,11 @@ impl StaticFilter for ArrayStaticFilter {
             ));
         }
 
-        self.find_needles_in_haystack(v, negated)
+        if has_float_leaf(v.data_type()) {
+            let normalized = normalize_float_zero(&make_array(v.to_data()));
+            self.find_needles_in_haystack(normalized.as_ref(), negated)
+        } else {
+            self.find_needles_in_haystack(v, negated)
+        }
     }
 }

datafusion/sqllogictest/test_files/negative_zero.slt

@@ -290,13 +290,74 @@ true false true
 statement ok
 CREATE TABLE nested_zeros(id INT, a DOUBLE[]) AS VALUES (1, [0.0]), (2, [-0.0]);
 
-query II
-SELECT l.id, r.id FROM nested_zeros l JOIN nested_zeros r ON l.a = r.a ORDER BY l.id, r.id;
+# Column-valued list items exercise dynamic array comparisons in both directions.
+query IIBB
+SELECT l.id, r.id,
+       l.a IN (r.a, [1.0], [2.0], [3.0]),
+       l.a NOT IN (r.a, [1.0], [2.0], [3.0])
+FROM nested_zeros l JOIN nested_zeros r ON l.a = r.a
+ORDER BY l.id, r.id;
 ----
-1 1
-1 2
-2 1
-2 2
+1 1 true false
+1 2 true false
+2 1 true false
+2 2 true false
+
+# Short lists become equality comparisons; four-item constant lists use a static filter.
+query IBBBB
+SELECT id,
+       a IN ([0.0]),
+       a IN ([0.0], [1.0], [2.0], [3.0]),
+       a IN ([-0.0], [1.0], [2.0], [3.0]),
+       a NOT IN ([-0.0], [1.0], [2.0], [3.0])
+FROM nested_zeros ORDER BY id;
+----
+1 true true true false
+2 true true true false
+
+# The nonmatching column-derived item keeps scalar RHS comparisons dynamic.
+# Also check scalar LHS values against array-valued list items.
+query IBBBB
+SELECT id,
+       a IN ([-0.0], [1.0], [2.0], [CAST(id AS DOUBLE)]),
+       a NOT IN ([0.0], [1.0], [2.0], [CAST(id AS DOUBLE)]),
+       [-0.0] IN (a, [1.0], [2.0], [3.0]),
+       [0.0] NOT IN (a, [1.0], [2.0], [3.0])
+FROM nested_zeros ORDER BY id;
+----
+1 true false true false
+2 true false true false
+
+# Structural literal equality must not drive IN intersection or difference for nested floats.
+query IBB
+SELECT id,
+       a IN ([0.0], [1.0], [2.0], [3.0])
+         AND a IN ([-0.0], [4.0], [5.0], [6.0]),
+       a IN ([0.0], [1.0], [2.0], [3.0])
+         AND a NOT IN ([-0.0], [4.0], [5.0], [6.0])
+FROM nested_zeros ORDER BY id;
+----
+1 true false
+2 true false
+
+statement ok
+INSERT INTO nested_zeros VALUES (3, NULL), (4, [NULL]), (5, [9.0]);
+
+# Matches win over an outer NULL list item; nested NULL elements remain comparable.
+# A missing match or a NULL input produces NULL, for both static and dynamic lists.
+query IBBBB
+SELECT id,
+       a IN ([-0.0], [NULL], [1.0], NULL),
+       a NOT IN ([-0.0], [NULL], [1.0], NULL),
+       a IN ([-0.0], [NULL], [CAST(id AS DOUBLE)], NULL),
+       a NOT IN ([-0.0], [NULL], [CAST(id AS DOUBLE)], NULL)
+FROM nested_zeros ORDER BY id;
+----
+1 true false true false
+2 true false true false
+3 NULL NULL NULL NULL
+4 true false true false
+5 NULL NULL NULL NULL
 
 statement ok
 DROP TABLE nested_zeros;