Skip to content

Commit 964f925

Browse files
[branch-56] IN LIST: treat signed zeros as equal (backport #25186) (#27204)
## Which issue does this PR close? - part of #24461 - Backport of #25186 to `branch-56` - Related to #27121. Co-authored-by: Geoffrey Claude <[email protected]>
1 parent 3b69898 commit 964f925

7 files changed

Lines changed: 495 additions & 46 deletions

File tree

‎datafusion/common/src/float_semantics.rs‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,12 @@ pub fn has_float_leaf(data_type: &DataType) -> bool {
189189
#[cfg(test)]
190190
mod tests {
191191
use super::*;
192-
use arrow::array::{Float16Array, Float32Array, Float64Array, Int32Array, ListArray};
193-
use arrow::datatypes::Int32Type;
192+
use crate::{Result, ScalarValue};
193+
use arrow::array::{
194+
DictionaryArray, Float16Array, Float32Array, Float64Array, Int8Array, Int32Array,
195+
ListArray,
196+
};
197+
use arrow::datatypes::{Field, Int8Type, Int32Type};
194198

195199
#[test]
196200
fn normalize_float_folds_negative_zero() {
@@ -295,4 +299,100 @@ mod tests {
295299
let array: ArrayRef = Arc::new(list);
296300
assert!(Arc::ptr_eq(&normalize_floats(&array), &array));
297301
}
302+
303+
#[test]
304+
fn normalize_floats_in_dictionary_arrays_and_scalars() -> Result<()> {
305+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
306+
let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);
307+
let array: ArrayRef = Arc::new(DictionaryArray::try_new(
308+
keys.clone(),
309+
Arc::new(Float64Array::from(vec![-0.0, nan, 1.0])),
310+
)?);
311+
312+
let normalized = normalize_floats(&array);
313+
let dictionary = normalized.as_dictionary::<Int8Type>();
314+
assert_eq!(dictionary.keys(), &keys);
315+
let values = dictionary.values().as_primitive::<Float64Type>();
316+
assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits());
317+
assert_eq!(values.value(1).to_bits(), nan.to_bits());
318+
assert_eq!(values.value(2), 1.0);
319+
320+
assert!(Arc::ptr_eq(&normalize_floats(&normalized), &normalized));
321+
322+
assert_eq!(
323+
ScalarValue::try_from_array(&array, 0)?.normalize(),
324+
ScalarValue::try_from_array(&normalized, 0)?
325+
);
326+
327+
Ok(())
328+
}
329+
330+
#[test]
331+
fn normalize_preserves_union_and_run_metadata() -> Result<()> {
332+
use arrow::datatypes::{UnionFields, UnionMode};
333+
334+
let run_ends = Arc::new(Field::new("ends", DataType::Int32, false));
335+
let values = Arc::new(Field::new("samples", DataType::Float64, true));
336+
let fields = UnionFields::try_new(
337+
[7, 42],
338+
[
339+
Field::new(
340+
"runs",
341+
DataType::RunEndEncoded(Arc::clone(&run_ends), Arc::clone(&values)),
342+
true,
343+
),
344+
Field::new("other", DataType::Int32, true),
345+
],
346+
)?;
347+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
348+
for mode in [UnionMode::Sparse, UnionMode::Dense] {
349+
let wrap = |value| {
350+
ScalarValue::Union(
351+
Some((
352+
7,
353+
Box::new(ScalarValue::RunEndEncoded(
354+
Arc::clone(&run_ends),
355+
Arc::clone(&values),
356+
Box::new(ScalarValue::Float64(value)),
357+
)),
358+
)),
359+
fields.clone(),
360+
mode,
361+
)
362+
};
363+
for (value, expected) in [
364+
(Some(-0.0), Some(0.0)),
365+
(Some(nan), Some(nan)),
366+
(None, None),
367+
] {
368+
assert_eq!(wrap(value).normalize(), wrap(expected));
369+
}
370+
let null = ScalarValue::Union(None, fields.clone(), mode);
371+
assert_eq!(null.clone().normalize(), null);
372+
}
373+
Ok(())
374+
}
375+
376+
#[test]
377+
fn normalize_preserves_sliced_list() {
378+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
379+
let list = Arc::new(
380+
ListArray::from_iter_primitive::<Float64Type, _, _>([
381+
Some(vec![Some(99.0)]),
382+
Some(vec![Some(-0.0), Some(nan), None]),
383+
Some(vec![Some(42.0)]),
384+
])
385+
.slice(1, 1),
386+
);
387+
let ScalarValue::List(normalized) = ScalarValue::List(list).normalize() else {
388+
panic!("normalization must preserve the scalar variant");
389+
};
390+
assert_eq!(normalized.len(), 1);
391+
let child = normalized.value(0);
392+
let child = child.as_primitive::<Float64Type>();
393+
assert_eq!(child.len(), 3);
394+
assert_eq!(child.value(0).to_bits(), 0.0_f64.to_bits());
395+
assert_eq!(child.value(1).to_bits(), nan.to_bits());
396+
assert!(child.is_null(2));
397+
}
298398
}

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ use arrow::array::{
6262
Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
6363
TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
6464
TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
65-
downcast_run_array, make_comparator, new_empty_array, new_null_array,
65+
downcast_array, downcast_run_array, make_comparator, new_empty_array, new_null_array,
6666
};
6767
use arrow::buffer::{BooleanBuffer, ScalarBuffer};
6868
use arrow::compute::kernels::cast::{CastOptions, cast_with_options};
@@ -2762,24 +2762,43 @@ impl ScalarValue {
27622762
)
27632763
}
27642764

2765-
/// Applies [`normalize_float`] to a `Float16`, `Float32` or `Float64`
2766-
/// value, so `-0.0` becomes `0.0`. Other values are returned as is.
2765+
/// Applies DataFusion's floating-point equality semantics to this value,
2766+
/// including floats inside nested and encoded values. Negative zero becomes
2767+
/// positive zero; nulls, NaN payloads and other values are preserved.
27672768
///
2768-
/// [`normalize_float`]: crate::float_semantics::normalize_float
2769-
pub fn normalize(self) -> Self {
2770-
use crate::float_semantics::normalize_float;
2771-
match self {
2772-
ScalarValue::Float16(Some(v)) => {
2773-
ScalarValue::Float16(Some(normalize_float(v)))
2774-
}
2775-
ScalarValue::Float32(Some(v)) => {
2776-
ScalarValue::Float32(Some(normalize_float(v)))
2769+
/// See [`crate::float_semantics`] for the shared normalization rules.
2770+
pub fn normalize(mut self) -> Self {
2771+
use crate::float_semantics::{normalize_float, normalize_floats};
2772+
2773+
fn normalize_array<A: Array + From<ArrayData> + 'static>(array: &mut Arc<A>) {
2774+
let original: ArrayRef = Arc::clone(array) as ArrayRef;
2775+
let normalized = normalize_floats(&original);
2776+
if !Arc::ptr_eq(&original, &normalized) {
2777+
*array = Arc::new(downcast_array(normalized.as_ref()));
27772778
}
2778-
ScalarValue::Float64(Some(v)) => {
2779-
ScalarValue::Float64(Some(normalize_float(v)))
2779+
}
2780+
2781+
fn normalize(scalar: &mut ScalarValue) {
2782+
match scalar {
2783+
ScalarValue::Float16(Some(value)) => *value = normalize_float(*value),
2784+
ScalarValue::Float32(Some(value)) => *value = normalize_float(*value),
2785+
ScalarValue::Float64(Some(value)) => *value = normalize_float(*value),
2786+
ScalarValue::FixedSizeList(array) => normalize_array(array),
2787+
ScalarValue::List(array) => normalize_array(array),
2788+
ScalarValue::LargeList(array) => normalize_array(array),
2789+
ScalarValue::ListView(array) => normalize_array(array),
2790+
ScalarValue::LargeListView(array) => normalize_array(array),
2791+
ScalarValue::Struct(array) => normalize_array(array),
2792+
ScalarValue::Map(array) => normalize_array(array),
2793+
ScalarValue::Union(Some((_, value)), _, _)
2794+
| ScalarValue::Dictionary(_, value)
2795+
| ScalarValue::RunEndEncoded(_, _, value) => normalize(value),
2796+
_ => {}
27802797
}
2781-
other => other,
27822798
}
2799+
2800+
normalize(&mut self);
2801+
self
27832802
}
27842803

27852804
/// whether this value is null or not.

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use std::sync::Arc;
1616
use std::sync::LazyLock;
1717

1818
use datafusion_common::config::ConfigOptions;
19+
use datafusion_common::float_semantics::has_float_leaf;
1920
use datafusion_common::nested_struct::has_one_of_more_common_fields;
2021
use datafusion_common::{
2122
DFSchema, DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_err,
@@ -2234,8 +2235,9 @@ fn are_inlist_and_eq_and_match_neg(
22342235
fn inlists_have_set_comparable_literals(left: &Expr, right: &Expr) -> bool {
22352236
match (left, right) {
22362237
(Expr::InList(l), Expr::InList(r)) => l.list.iter().chain(&r.list).all(|item| {
2237-
item.as_literal()
2238-
.is_some_and(|value| !value.is_null() && !value.data_type().is_floating())
2238+
item.as_literal().is_some_and(|value| {
2239+
!value.is_null() && !has_float_leaf(&value.data_type())
2240+
})
22392241
}),
22402242
_ => false,
22412243
}

‎datafusion/physical-expr/src/expressions/in_list.rs‎

Lines changed: 88 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use arrow::compute::kernels::boolean::{not, or_kleene};
1919
use arrow::compute::kernels::cmp::eq as arrow_eq;
2020
use arrow::datatypes::*;
2121

22+
use datafusion_common::float_semantics::normalize_floats;
2223
use datafusion_common::tree_node::TreeNode;
2324
use datafusion_common::{
2425
DFSchema, Result, ScalarValue, assert_or_internal_err, exec_err,
@@ -71,6 +72,13 @@ fn supports_arrow_eq(dt: &DataType) -> bool {
7172
}
7273
}
7374

75+
fn normalize_in_list_value(value: ColumnarValue) -> ColumnarValue {
76+
match value {
77+
ColumnarValue::Array(array) => ColumnarValue::Array(normalize_floats(&array)),
78+
ColumnarValue::Scalar(scalar) => ColumnarValue::Scalar(scalar.normalize()),
79+
}
80+
}
81+
7482
/// Evaluates the list of expressions into an array, flattening any dictionaries
7583
fn evaluate_list(
7684
list: &[Arc<dyn PhysicalExpr>],
@@ -368,12 +376,14 @@ impl PhysicalExpr for InListExpr {
368376
// Use Arrow's vectorized eq kernel for types it supports (primitive,
369377
// boolean, string, binary, dictionary), falling back to row-by-row
370378
// comparator for unsupported types (nested, RunEndEncoded, etc.).
371-
let value = value.into_array(num_rows)?;
379+
// Normalize the left side once for the whole list. Doing this
380+
// outside `compare_one` avoids rescanning it for every item.
381+
let value = normalize_in_list_value(value).into_array(num_rows)?;
372382
let lhs_supports_arrow_eq = supports_arrow_eq(value.data_type());
373383

374384
// Helper: compare value against a single list expression
375385
let compare_one = |expr: &Arc<dyn PhysicalExpr>| -> Result<BooleanArray> {
376-
match expr.evaluate(batch)? {
386+
match normalize_in_list_value(expr.evaluate(batch)?) {
377387
ColumnarValue::Array(array) => {
378388
if lhs_supports_arrow_eq
379389
&& supports_arrow_eq(array.data_type())
@@ -1287,20 +1297,28 @@ mod tests {
12871297
Arc::new(schema.clone()),
12881298
vec![Arc::new(Int32Array::from(vec![0, 0, 0]))],
12891299
)?;
1290-
let needle = lit(ScalarValue::Dictionary(
1291-
Box::new(DataType::Int16),
1292-
Box::new(ScalarValue::Dictionary(
1293-
Box::new(DataType::Int8),
1294-
Box::new(ScalarValue::Int32(Some(2))),
1295-
)),
1296-
));
1300+
for (value, list) in [
1301+
(ScalarValue::Int32(Some(2)), vec![lit(1_i32), lit(2_i32)]),
1302+
(
1303+
ScalarValue::Float64(Some(-0.0)),
1304+
vec![lit(0.0_f64), lit(1.0_f64)],
1305+
),
1306+
] {
1307+
let needle = lit(ScalarValue::Dictionary(
1308+
Box::new(DataType::Int16),
1309+
Box::new(ScalarValue::Dictionary(
1310+
Box::new(DataType::Int8),
1311+
Box::new(value),
1312+
)),
1313+
));
12971314

1298-
let expr = in_list(needle, vec![lit(1_i32), lit(2_i32)], &false, &schema)?;
1299-
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
1300-
assert_eq!(
1301-
as_boolean_array(&result),
1302-
&BooleanArray::from(vec![true, true, true])
1303-
);
1315+
let expr = in_list(needle, list, &false, &schema)?;
1316+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
1317+
assert_eq!(
1318+
as_boolean_array(&result),
1319+
&BooleanArray::from(vec![true, true, true])
1320+
);
1321+
}
13041322

13051323
Ok(())
13061324
}
@@ -3391,6 +3409,44 @@ mod tests {
33913409
Ok(())
33923410
}
33933411

3412+
#[test]
3413+
fn test_in_list_with_columns_float_signed_zero() -> Result<()> {
3414+
use arrow::compute::cast;
3415+
3416+
for (data_type, dictionary) in [
3417+
(DataType::Float32, false),
3418+
(DataType::Float64, false),
3419+
(DataType::Float64, true),
3420+
] {
3421+
let mut left = cast(&Float64Array::from(vec![0.0, -0.0, 1.0]), &data_type)?;
3422+
let mut right = cast(&Float64Array::from(vec![-0.0, 0.0, 2.0]), &data_type)?;
3423+
if dictionary {
3424+
left = wrap_in_dict(left);
3425+
right = wrap_in_dict(right);
3426+
}
3427+
let scalar = lit(ScalarValue::try_from_array(left.as_ref(), 1)?);
3428+
let batch = RecordBatch::try_from_iter([("a", left), ("b", right)])?;
3429+
let schema = batch.schema();
3430+
3431+
// Exercise array and scalar values on both sides, including scalar
3432+
// normalization before the left-hand side is broadcast.
3433+
for (left, right) in [
3434+
(col("a", &schema)?, col("b", &schema)?),
3435+
(col("a", &schema)?, Arc::clone(&scalar)),
3436+
(scalar, col("a", &schema)?),
3437+
] {
3438+
let expr = make_in_list_with_columns(left, vec![right], false);
3439+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
3440+
assert_eq!(
3441+
as_boolean_array(&result),
3442+
&BooleanArray::from(vec![true, true, false]),
3443+
"{data_type:?}, dictionary={dictionary}: {expr}"
3444+
);
3445+
}
3446+
}
3447+
Ok(())
3448+
}
3449+
33943450
/// Tests that short-circuit evaluation produces correct results.
33953451
/// When all rows match after the first list item, remaining items
33963452
/// should be skipped without affecting correctness.
@@ -3916,6 +3972,23 @@ mod tests {
39163972
Ok(())
39173973
}
39183974

3975+
#[test]
3976+
fn test_try_new_from_array_dict_haystack_float64_signed_zero() -> Result<()> {
3977+
// One value beyond the branchless limit selects the hash-set strategy.
3978+
let list_len =
3979+
<Float64Type as branchless_filter::BranchlessFilterType>::MAX_LIST_LEN + 1;
3980+
let haystack = make_f64_dict_array(vec![Some(-0.0); list_len]);
3981+
let needles: ArrayRef = Arc::new(Float64Array::from(vec![0.0]));
3982+
for needles in [Arc::clone(&needles), wrap_in_dict(needles)] {
3983+
assert_eq!(
3984+
eval_in_list_from_array(needles, Arc::clone(&haystack))?,
3985+
BooleanArray::from(vec![true])
3986+
);
3987+
}
3988+
3989+
Ok(())
3990+
}
3991+
39193992
#[test]
39203993
fn test_try_new_from_array_type_mismatch_rejects() -> Result<()> {
39213994
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);

‎datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
33
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
44

5-
use arrow::array::{Array, ArrayRef, BooleanArray, make_comparator};
5+
use arrow::array::{Array, ArrayRef, BooleanArray, make_array, make_comparator};
66
use arrow::buffer::{BooleanBuffer, NullBuffer};
77
use arrow::compute::SortOptions;
88
use arrow::datatypes::DataType;
99
use arrow::util::bit_iterator::BitIndexIterator;
1010
use datafusion_common::Result;
11+
use datafusion_common::float_semantics::{has_float_leaf, normalize_floats};
1112
use datafusion_common::hash_utils::{RandomState, with_hashes};
1213
use hashbrown::HashTable;
1314

@@ -40,6 +41,8 @@ impl ArrayStaticFilter {
4041
});
4142
}
4243

44+
// Hashing treats both signed zeros alike; the comparator must do so too.
45+
let in_array = normalize_floats(&in_array);
4346
let state = RandomState::default();
4447
let table = Self::build_haystack_table(&in_array, &state)?;
4548

@@ -125,6 +128,11 @@ impl StaticFilter for ArrayStaticFilter {
125128
));
126129
}
127130

128-
self.find_needles_in_haystack(v, negated)
131+
if has_float_leaf(v.data_type()) {
132+
let normalized = normalize_floats(&make_array(v.to_data()));
133+
self.find_needles_in_haystack(normalized.as_ref(), negated)
134+
} else {
135+
self.find_needles_in_haystack(v, negated)
136+
}
129137
}
130138
}

0 commit comments

Comments
 (0)