Skip to content

Commit 4a60604

Browse files
refactor: use shared float semantics for IN-list equality
Normalize dynamic and generic IN operands through normalize_floats and recursive ScalarValue::normalize. Keep primitive static filters on constructor-time signed-zero expansion and use the shared negative-zero predicate before selecting their lookup strategy. Move normalization regressions to the shared module and cover nested scalar self-membership and encoded scalar needles.
1 parent c2a47cf commit 4a60604

8 files changed

Lines changed: 190 additions & 151 deletions

File tree

‎datafusion/common/src/float_semantics.rs‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,12 @@ pub fn has_float_leaf(data_type: &DataType) -> bool {
189189
#[cfg(test)]
190190
mod tests {
191191
use super::*;
192-
use arrow::array::{Float16Array, Float32Array, Float64Array, Int32Array, ListArray};
193-
use arrow::datatypes::Int32Type;
192+
use crate::{Result, ScalarValue};
193+
use arrow::array::{
194+
DictionaryArray, Float16Array, Float32Array, Float64Array, Int8Array, Int32Array,
195+
ListArray,
196+
};
197+
use arrow::datatypes::{Field, Int8Type, Int32Type};
194198

195199
#[test]
196200
fn normalize_float_folds_negative_zero() {
@@ -295,4 +299,100 @@ mod tests {
295299
let array: ArrayRef = Arc::new(list);
296300
assert!(Arc::ptr_eq(&normalize_floats(&array), &array));
297301
}
302+
303+
#[test]
304+
fn normalize_floats_in_dictionary_arrays_and_scalars() -> Result<()> {
305+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
306+
let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);
307+
let array: ArrayRef = Arc::new(DictionaryArray::try_new(
308+
keys.clone(),
309+
Arc::new(Float64Array::from(vec![-0.0, nan, 1.0])),
310+
)?);
311+
312+
let normalized = normalize_floats(&array);
313+
let dictionary = normalized.as_dictionary::<Int8Type>();
314+
assert_eq!(dictionary.keys(), &keys);
315+
let values = dictionary.values().as_primitive::<Float64Type>();
316+
assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits());
317+
assert_eq!(values.value(1).to_bits(), nan.to_bits());
318+
assert_eq!(values.value(2), 1.0);
319+
320+
assert!(Arc::ptr_eq(&normalize_floats(&normalized), &normalized));
321+
322+
assert_eq!(
323+
ScalarValue::try_from_array(&array, 0)?.normalize(),
324+
ScalarValue::try_from_array(&normalized, 0)?
325+
);
326+
327+
Ok(())
328+
}
329+
330+
#[test]
331+
fn normalize_preserves_union_and_run_metadata() -> Result<()> {
332+
use arrow::datatypes::{UnionFields, UnionMode};
333+
334+
let run_ends = Arc::new(Field::new("ends", DataType::Int32, false));
335+
let values = Arc::new(Field::new("samples", DataType::Float64, true));
336+
let fields = UnionFields::try_new(
337+
[7, 42],
338+
[
339+
Field::new(
340+
"runs",
341+
DataType::RunEndEncoded(Arc::clone(&run_ends), Arc::clone(&values)),
342+
true,
343+
),
344+
Field::new("other", DataType::Int32, true),
345+
],
346+
)?;
347+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
348+
for mode in [UnionMode::Sparse, UnionMode::Dense] {
349+
let wrap = |value| {
350+
ScalarValue::Union(
351+
Some((
352+
7,
353+
Box::new(ScalarValue::RunEndEncoded(
354+
Arc::clone(&run_ends),
355+
Arc::clone(&values),
356+
Box::new(ScalarValue::Float64(value)),
357+
)),
358+
)),
359+
fields.clone(),
360+
mode,
361+
)
362+
};
363+
for (value, expected) in [
364+
(Some(-0.0), Some(0.0)),
365+
(Some(nan), Some(nan)),
366+
(None, None),
367+
] {
368+
assert_eq!(wrap(value).normalize(), wrap(expected));
369+
}
370+
let null = ScalarValue::Union(None, fields.clone(), mode);
371+
assert_eq!(null.clone().normalize(), null);
372+
}
373+
Ok(())
374+
}
375+
376+
#[test]
377+
fn normalize_preserves_sliced_list() {
378+
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
379+
let list = Arc::new(
380+
ListArray::from_iter_primitive::<Float64Type, _, _>([
381+
Some(vec![Some(99.0)]),
382+
Some(vec![Some(-0.0), Some(nan), None]),
383+
Some(vec![Some(42.0)]),
384+
])
385+
.slice(1, 1),
386+
);
387+
let ScalarValue::List(normalized) = ScalarValue::List(list).normalize() else {
388+
panic!("normalization must preserve the scalar variant");
389+
};
390+
assert_eq!(normalized.len(), 1);
391+
let child = normalized.value(0);
392+
let child = child.as_primitive::<Float64Type>();
393+
assert_eq!(child.len(), 3);
394+
assert_eq!(child.value(0).to_bits(), 0.0_f64.to_bits());
395+
assert_eq!(child.value(1).to_bits(), nan.to_bits());
396+
assert!(child.is_null(2));
397+
}
298398
}

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ use arrow::array::{
6262
Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
6363
TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
6464
TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
65-
downcast_run_array, make_comparator, new_empty_array, new_null_array,
65+
downcast_array, downcast_run_array, make_comparator, new_empty_array, new_null_array,
6666
};
6767
use arrow::buffer::{BooleanBuffer, ScalarBuffer};
6868
use arrow::compute::kernels::cast::{CastOptions, cast_with_options};
@@ -2762,24 +2762,43 @@ impl ScalarValue {
27622762
)
27632763
}
27642764

2765-
/// Applies [`normalize_float`] to a `Float16`, `Float32` or `Float64`
2766-
/// value, so `-0.0` becomes `0.0`. Other values are returned as is.
2765+
/// Applies DataFusion's floating-point equality semantics to this value,
2766+
/// including floats inside nested and encoded values. Negative zero becomes
2767+
/// positive zero; nulls, NaN payloads and other values are preserved.
27672768
///
2768-
/// [`normalize_float`]: crate::float_semantics::normalize_float
2769-
pub fn normalize(self) -> Self {
2770-
use crate::float_semantics::normalize_float;
2771-
match self {
2772-
ScalarValue::Float16(Some(v)) => {
2773-
ScalarValue::Float16(Some(normalize_float(v)))
2774-
}
2775-
ScalarValue::Float32(Some(v)) => {
2776-
ScalarValue::Float32(Some(normalize_float(v)))
2769+
/// See [`crate::float_semantics`] for the shared normalization rules.
2770+
pub fn normalize(mut self) -> Self {
2771+
use crate::float_semantics::{normalize_float, normalize_floats};
2772+
2773+
fn normalize_array<A: Array + From<ArrayData> + 'static>(array: &mut Arc<A>) {
2774+
let original: ArrayRef = Arc::clone(array) as ArrayRef;
2775+
let normalized = normalize_floats(&original);
2776+
if !Arc::ptr_eq(&original, &normalized) {
2777+
*array = Arc::new(downcast_array(normalized.as_ref()));
27772778
}
2778-
ScalarValue::Float64(Some(v)) => {
2779-
ScalarValue::Float64(Some(normalize_float(v)))
2779+
}
2780+
2781+
fn normalize(scalar: &mut ScalarValue) {
2782+
match scalar {
2783+
ScalarValue::Float16(Some(value)) => *value = normalize_float(*value),
2784+
ScalarValue::Float32(Some(value)) => *value = normalize_float(*value),
2785+
ScalarValue::Float64(Some(value)) => *value = normalize_float(*value),
2786+
ScalarValue::FixedSizeList(array) => normalize_array(array),
2787+
ScalarValue::List(array) => normalize_array(array),
2788+
ScalarValue::LargeList(array) => normalize_array(array),
2789+
ScalarValue::ListView(array) => normalize_array(array),
2790+
ScalarValue::LargeListView(array) => normalize_array(array),
2791+
ScalarValue::Struct(array) => normalize_array(array),
2792+
ScalarValue::Map(array) => normalize_array(array),
2793+
ScalarValue::Union(Some((_, value)), _, _)
2794+
| ScalarValue::Dictionary(_, value)
2795+
| ScalarValue::RunEndEncoded(_, _, value) => normalize(value),
2796+
_ => {}
27802797
}
2781-
other => other,
27822798
}
2799+
2800+
normalize(&mut self);
2801+
self
27832802
}
27842803

27852804
/// whether this value is null or not.

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 4 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ use crate::error::{
1717
};
1818
use crate::{Result, ScalarValue};
1919
use arrow::array::{
20-
Array, ArrayData, ArrayRef, FixedSizeListArray, LargeListArray, ListArray,
21-
OffsetSizeTrait, cast::AsArray, downcast_array,
20+
Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, OffsetSizeTrait,
21+
cast::AsArray,
2222
};
2323
use arrow::array::{
2424
ArrowPrimitiveType, BooleanArray, Datum, GenericListArray, Int32Array, Int64Array,
@@ -1491,8 +1491,8 @@ mod tests {
14911491
use super::*;
14921492
use crate::ScalarValue::Null;
14931493
use arrow::{
1494-
array::{DictionaryArray, Float64Array, Int8Array, Int32Array, NullArray},
1495-
datatypes::{Float64Type, Int8Type, Int32Type},
1494+
array::{Float64Array, Int32Array, NullArray},
1495+
datatypes::Int32Type,
14961496
};
14971497
#[cfg(feature = "sql")]
14981498
use sqlparser::ast::Ident;
@@ -1586,104 +1586,6 @@ mod tests {
15861586
}
15871587
}
15881588

1589-
#[test]
1590-
fn normalize_float_zero_in_dictionary_arrays_and_scalars() -> Result<()> {
1591-
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
1592-
let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);
1593-
let array: ArrayRef = Arc::new(DictionaryArray::try_new(
1594-
keys.clone(),
1595-
Arc::new(Float64Array::from(vec![-0.0, nan, 1.0])),
1596-
)?);
1597-
1598-
let normalized = normalize_float_zero(&array);
1599-
let dictionary = normalized.as_dictionary::<Int8Type>();
1600-
assert_eq!(dictionary.keys(), &keys);
1601-
let values = dictionary.values().as_primitive::<Float64Type>();
1602-
assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits());
1603-
assert_eq!(values.value(1).to_bits(), nan.to_bits());
1604-
assert_eq!(values.value(2), 1.0);
1605-
1606-
assert!(Arc::ptr_eq(&normalize_float_zero(&normalized), &normalized));
1607-
1608-
assert_eq!(
1609-
normalize_float_zero_scalar(ScalarValue::try_from_array(&array, 0)?),
1610-
ScalarValue::try_from_array(&normalized, 0)?
1611-
);
1612-
1613-
Ok(())
1614-
}
1615-
1616-
#[test]
1617-
fn normalize_float_zero_scalar_preserves_union_and_run_metadata() -> Result<()> {
1618-
use arrow::datatypes::{UnionFields, UnionMode};
1619-
1620-
let run_ends = Arc::new(Field::new("ends", DataType::Int32, false));
1621-
let values = Arc::new(Field::new("samples", DataType::Float64, true));
1622-
let fields = UnionFields::try_new(
1623-
[7, 42],
1624-
[
1625-
Field::new(
1626-
"runs",
1627-
DataType::RunEndEncoded(Arc::clone(&run_ends), Arc::clone(&values)),
1628-
true,
1629-
),
1630-
Field::new("other", DataType::Int32, true),
1631-
],
1632-
)?;
1633-
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
1634-
for mode in [UnionMode::Sparse, UnionMode::Dense] {
1635-
let wrap = |value| {
1636-
ScalarValue::Union(
1637-
Some((
1638-
7,
1639-
Box::new(ScalarValue::RunEndEncoded(
1640-
Arc::clone(&run_ends),
1641-
Arc::clone(&values),
1642-
Box::new(ScalarValue::Float64(value)),
1643-
)),
1644-
)),
1645-
fields.clone(),
1646-
mode,
1647-
)
1648-
};
1649-
for (value, expected) in [
1650-
(Some(-0.0), Some(0.0)),
1651-
(Some(nan), Some(nan)),
1652-
(None, None),
1653-
] {
1654-
assert_eq!(normalize_float_zero_scalar(wrap(value)), wrap(expected));
1655-
}
1656-
let null = ScalarValue::Union(None, fields.clone(), mode);
1657-
assert_eq!(normalize_float_zero_scalar(null.clone()), null);
1658-
}
1659-
Ok(())
1660-
}
1661-
1662-
#[test]
1663-
fn normalize_float_zero_scalar_preserves_sliced_list() {
1664-
let nan = f64::from_bits(0x7ff8_0000_0000_0001);
1665-
let list = Arc::new(
1666-
ListArray::from_iter_primitive::<Float64Type, _, _>([
1667-
Some(vec![Some(99.0)]),
1668-
Some(vec![Some(-0.0), Some(nan), None]),
1669-
Some(vec![Some(42.0)]),
1670-
])
1671-
.slice(1, 1),
1672-
);
1673-
let ScalarValue::List(normalized) =
1674-
normalize_float_zero_scalar(ScalarValue::List(list))
1675-
else {
1676-
panic!("normalization must preserve the scalar variant");
1677-
};
1678-
assert_eq!(normalized.len(), 1);
1679-
let child = normalized.value(0);
1680-
let child = child.as_primitive::<Float64Type>();
1681-
assert_eq!(child.len(), 3);
1682-
assert_eq!(child.value(0).to_bits(), 0.0_f64.to_bits());
1683-
assert_eq!(child.value(1).to_bits(), nan.to_bits());
1684-
assert!(child.is_null(2));
1685-
}
1686-
16871589
#[test]
16881590
fn test_bisect_linear_left_and_right() -> Result<()> {
16891591
let arrays: Vec<ArrayRef> = vec![

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ use std::sync::Arc;
1616
use std::sync::LazyLock;
1717

1818
use datafusion_common::config::ConfigOptions;
19+
use datafusion_common::float_semantics::has_float_leaf;
1920
use datafusion_common::nested_struct::has_one_of_more_common_fields;
20-
use datafusion_common::utils::has_float_leaf;
2121
use datafusion_common::{
2222
DFSchema, DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_err,
2323
};

0 commit comments

Comments
 (0)