Skip to content

Commit ed46339

Browse files
committed
fix: treat -0.0 and +0.0 as equal in comparisons and IN list (apache#22490)
1 parent a786471 commit ed46339

6 files changed

Lines changed: 532 additions & 7 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/physical-expr-common/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ chrono = { workspace = true }
5252
datafusion-common = { workspace = true }
5353
datafusion-expr-common = { workspace = true }
5454
datafusion-proto-models = { workspace = true, optional = true }
55+
half = { workspace = true }
5556
hashbrown = { workspace = true }
5657
indexmap = { workspace = true }
5758
itertools = { workspace = true }

‎datafusion/physical-expr-common/src/datum.rs‎

Lines changed: 343 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,23 @@
1616
// under the License.
1717

1818
use arrow::array::BooleanArray;
19-
use arrow::array::{ArrayRef, Datum, make_comparator};
19+
use arrow::array::{Array, ArrayRef, AsArray, Datum, make_comparator};
2020
use arrow::buffer::{BooleanBuffer, NullBuffer};
21+
use arrow::compute::kernels::arity::unary;
2122
use arrow::compute::kernels::cmp::{
2223
distinct, eq, gt, gt_eq, lt, lt_eq, neq, not_distinct,
2324
};
2425
use arrow::compute::{SortOptions, ilike, like, nilike, nlike};
26+
use arrow::datatypes::{
27+
DataType, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type,
28+
};
29+
use arrow::downcast_dictionary_array;
2530
use arrow::error::ArrowError;
2631
use datafusion_common::{Result, ScalarValue};
2732
use datafusion_common::{arrow_datafusion_err, assert_or_internal_err, internal_err};
2833
use datafusion_expr_common::columnar_value::ColumnarValue;
2934
use datafusion_expr_common::operator::Operator;
35+
use half::f16;
3036
use std::sync::Arc;
3137

3238
/// Applies a binary [`Datum`] kernel `f` to `lhs` and `rhs`
@@ -84,7 +90,147 @@ pub fn apply_cmp(
8490
}
8591
};
8692

87-
apply(lhs, rhs, |l, r| Ok(Arc::new(f(l, r)?)))
93+
let lhs = normalize_neg_zero(lhs);
94+
let rhs = normalize_neg_zero(rhs);
95+
apply(&lhs, &rhs, |l, r| Ok(Arc::new(f(l, r)?)))
96+
}
97+
}
98+
99+
/// Replace `-0.0` with `+0.0` on float inputs so that comparison operators
100+
/// follow IEEE 754 default semantics (where `-0.0 == +0.0`).
101+
///
102+
/// arrow-rs' comparison kernels intentionally use totalOrder semantics, which
103+
/// treats `-0.0` as strictly less than `+0.0` and not equal to it
104+
///
105+
/// See [`normalize_neg_zero_array`] / [`normalize_neg_zero_scalar`] for the
106+
/// per-variant behavior, including dictionary- and run-end-encoded arrays.
107+
pub fn normalize_neg_zero(value: &ColumnarValue) -> ColumnarValue {
108+
match value {
109+
ColumnarValue::Array(array) => {
110+
ColumnarValue::Array(normalize_neg_zero_array(array))
111+
}
112+
ColumnarValue::Scalar(scalar) => {
113+
ColumnarValue::Scalar(normalize_neg_zero_scalar(scalar))
114+
}
115+
}
116+
}
117+
118+
/// Array variant of [`normalize_neg_zero`]. Returns the input unchanged for
119+
/// arrays that don't contain `-0.0` (no allocation) and for arrays whose
120+
/// (transitive) value type is not floating-point.
121+
///
122+
/// Dictionary- and run-end-encoded arrays are peeled to their inner values
123+
/// and rebuilt with normalized values when needed; the keys/run-ends are
124+
/// preserved.
125+
pub fn normalize_neg_zero_array(array: &ArrayRef) -> ArrayRef {
126+
if !data_type_contains_float(array.data_type()) {
127+
return Arc::clone(array);
128+
}
129+
130+
match array.data_type() {
131+
DataType::Float16 => {
132+
let arr = array.as_primitive::<Float16Type>();
133+
if arr
134+
.values()
135+
.iter()
136+
.any(|v| v.is_sign_negative() && *v == f16::ZERO)
137+
{
138+
Arc::new(unary::<Float16Type, _, Float16Type>(arr, |x| {
139+
if x == f16::ZERO { f16::ZERO } else { x }
140+
}))
141+
} else {
142+
Arc::clone(array)
143+
}
144+
}
145+
DataType::Float32 => {
146+
let arr = array.as_primitive::<Float32Type>();
147+
if arr
148+
.values()
149+
.iter()
150+
.any(|v| v.is_sign_negative() && *v == 0.0)
151+
{
152+
Arc::new(unary::<Float32Type, _, Float32Type>(arr, |x| {
153+
if x == 0.0 { 0.0 } else { x }
154+
}))
155+
} else {
156+
Arc::clone(array)
157+
}
158+
}
159+
DataType::Float64 => {
160+
let arr = array.as_primitive::<Float64Type>();
161+
if arr
162+
.values()
163+
.iter()
164+
.any(|v| v.is_sign_negative() && *v == 0.0)
165+
{
166+
Arc::new(unary::<Float64Type, _, Float64Type>(arr, |x| {
167+
if x == 0.0 { 0.0 } else { x }
168+
}))
169+
} else {
170+
Arc::clone(array)
171+
}
172+
}
173+
DataType::Dictionary(_, _) => {
174+
let dyn_array: &dyn Array = array.as_ref();
175+
downcast_dictionary_array!(
176+
dyn_array => {
177+
let inner = dyn_array.values();
178+
let normalized = normalize_neg_zero_array(inner);
179+
if Arc::ptr_eq(&normalized, inner) {
180+
Arc::clone(array)
181+
} else {
182+
Arc::new(dyn_array.with_values(normalized))
183+
}
184+
}
185+
_ => unreachable!("data_type matched Dictionary"),
186+
)
187+
}
188+
DataType::RunEndEncoded(run_ends_field, _) => match run_ends_field.data_type() {
189+
DataType::Int16 => normalize_neg_zero_ree::<Int16Type>(array),
190+
DataType::Int32 => normalize_neg_zero_ree::<Int32Type>(array),
191+
DataType::Int64 => normalize_neg_zero_ree::<Int64Type>(array),
192+
_ => Arc::clone(array),
193+
},
194+
_ => Arc::clone(array),
195+
}
196+
}
197+
198+
/// Returns `true` if `dt` is, or transitively wraps, a floating-point type.
199+
/// Used to short-circuit [`normalize_neg_zero_array`] for non-float inputs.
200+
fn data_type_contains_float(dt: &DataType) -> bool {
201+
match dt {
202+
DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
203+
DataType::Dictionary(_, value_type) => data_type_contains_float(value_type),
204+
DataType::RunEndEncoded(_, values_field) => {
205+
data_type_contains_float(values_field.data_type())
206+
}
207+
_ => false,
208+
}
209+
}
210+
211+
fn normalize_neg_zero_ree<R: arrow::datatypes::RunEndIndexType>(
212+
array: &ArrayRef,
213+
) -> ArrayRef {
214+
let ree = array.as_ref().as_run::<R>();
215+
let inner = ree.values();
216+
let normalized = normalize_neg_zero_array(inner);
217+
if Arc::ptr_eq(&normalized, inner) {
218+
Arc::clone(array)
219+
} else {
220+
Arc::new(ree.with_values(normalized))
221+
}
222+
}
223+
224+
/// Scalar variant of [`normalize_neg_zero`]. Returns the input unchanged for
225+
/// non-float scalars and for non-zero float values.
226+
pub fn normalize_neg_zero_scalar(scalar: &ScalarValue) -> ScalarValue {
227+
match scalar {
228+
ScalarValue::Float16(Some(v)) if *v == f16::ZERO => {
229+
ScalarValue::Float16(Some(f16::ZERO))
230+
}
231+
ScalarValue::Float32(Some(v)) if *v == 0.0 => ScalarValue::Float32(Some(0.0)),
232+
ScalarValue::Float64(Some(v)) if *v == 0.0 => ScalarValue::Float64(Some(0.0)),
233+
_ => scalar.clone(),
88234
}
89235
}
90236

@@ -205,3 +351,198 @@ pub fn compare_op_for_nested(
205351
Ok(BooleanArray::new(values, nulls))
206352
}
207353
}
354+
355+
#[cfg(test)]
356+
mod tests {
357+
use super::*;
358+
use arrow::array::{
359+
DictionaryArray, Float16Array, Float32Array, Float64Array, Int32Array, RunArray,
360+
};
361+
362+
#[test]
363+
fn data_type_contains_float_detects_nested() {
364+
use arrow::datatypes::Field;
365+
366+
assert!(data_type_contains_float(&DataType::Float32));
367+
assert!(!data_type_contains_float(&DataType::Int64));
368+
369+
let dict_of_float =
370+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Float64));
371+
assert!(data_type_contains_float(&dict_of_float));
372+
373+
let dict_of_int =
374+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int64));
375+
assert!(!data_type_contains_float(&dict_of_int));
376+
377+
let ree_of_float = DataType::RunEndEncoded(
378+
Arc::new(Field::new("run_ends", DataType::Int32, false)),
379+
Arc::new(Field::new("values", DataType::Float64, true)),
380+
);
381+
assert!(data_type_contains_float(&ree_of_float));
382+
}
383+
384+
fn has_neg_zero_f64(array: &dyn Array) -> bool {
385+
array
386+
.as_primitive::<Float64Type>()
387+
.values()
388+
.iter()
389+
.any(|v| v.is_sign_negative() && *v == 0.0)
390+
}
391+
392+
#[test]
393+
fn normalize_float64_rewrites_neg_zero() {
394+
let array: ArrayRef = Arc::new(Float64Array::from(vec![-0.0, 0.0, 1.5, -2.0]));
395+
let normalized = normalize_neg_zero_array(&array);
396+
397+
assert!(!Arc::ptr_eq(&normalized, &array));
398+
assert!(!has_neg_zero_f64(normalized.as_ref()));
399+
let values = normalized.as_primitive::<Float64Type>().values();
400+
assert_eq!(values[2], 1.5);
401+
assert_eq!(values[3], -2.0);
402+
}
403+
404+
#[test]
405+
fn normalize_float64_passthrough_when_no_neg_zero() {
406+
let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0, 1.0, 2.0]));
407+
let normalized = normalize_neg_zero_array(&array);
408+
assert!(Arc::ptr_eq(&normalized, &array));
409+
}
410+
411+
#[test]
412+
fn normalize_dict_of_float64_peels_and_rewrites() {
413+
let values: ArrayRef = Arc::new(Float64Array::from(vec![-0.0, 1.0, 2.0]));
414+
let keys = Int32Array::from(vec![0, 1, 0, 2]);
415+
let array: ArrayRef =
416+
Arc::new(DictionaryArray::<Int32Type>::try_new(keys, values).unwrap());
417+
418+
let normalized = normalize_neg_zero_array(&array);
419+
assert!(!Arc::ptr_eq(&normalized, &array));
420+
421+
let dict = normalized
422+
.as_ref()
423+
.as_any()
424+
.downcast_ref::<DictionaryArray<Int32Type>>()
425+
.unwrap();
426+
assert!(!has_neg_zero_f64(dict.values().as_ref()));
427+
}
428+
429+
#[test]
430+
fn normalize_dict_of_non_float_passes_through() {
431+
let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30]));
432+
let keys = Int32Array::from(vec![0, 1, 2, 0]);
433+
let array: ArrayRef =
434+
Arc::new(DictionaryArray::<Int32Type>::try_new(keys, values).unwrap());
435+
436+
let normalized = normalize_neg_zero_array(&array);
437+
assert!(Arc::ptr_eq(&normalized, &array));
438+
}
439+
440+
#[test]
441+
fn normalize_ree_of_float32_peels_and_rewrites() {
442+
let run_ends = Int32Array::from(vec![2, 3, 5]);
443+
let values: ArrayRef = Arc::new(Float32Array::from(vec![-0.0, 1.0, 2.0]));
444+
let array: ArrayRef =
445+
Arc::new(RunArray::<Int32Type>::try_new(&run_ends, values.as_ref()).unwrap());
446+
447+
let normalized = normalize_neg_zero_array(&array);
448+
assert!(!Arc::ptr_eq(&normalized, &array));
449+
450+
let ree = normalized.as_ref().as_run::<Int32Type>();
451+
let inner = ree.values().as_primitive::<Float32Type>();
452+
assert!(
453+
!inner
454+
.values()
455+
.iter()
456+
.any(|v| v.is_sign_negative() && *v == 0.0)
457+
);
458+
}
459+
460+
#[test]
461+
fn normalize_float16_rewrites_neg_zero() {
462+
let array: ArrayRef = Arc::new(Float16Array::from(vec![
463+
f16::NEG_ZERO,
464+
f16::ZERO,
465+
f16::from_f32(1.5),
466+
]));
467+
let normalized = normalize_neg_zero_array(&array);
468+
469+
assert!(!Arc::ptr_eq(&normalized, &array));
470+
let values = normalized.as_primitive::<Float16Type>().values();
471+
assert!(
472+
!values
473+
.iter()
474+
.any(|v| v.is_sign_negative() && *v == f16::ZERO)
475+
);
476+
assert_eq!(values[2], f16::from_f32(1.5));
477+
}
478+
479+
#[test]
480+
fn normalize_nested_dict_recurses() {
481+
// Dictionary<Int32, Dictionary<Int32, Float64>> — exercises the
482+
// recursive peel through two layers of dictionary encoding.
483+
let values: ArrayRef = Arc::new(Float64Array::from(vec![-0.0, 1.0]));
484+
let inner_keys = Int32Array::from(vec![0, 1, 0]);
485+
let inner_dict: ArrayRef =
486+
Arc::new(DictionaryArray::<Int32Type>::try_new(inner_keys, values).unwrap());
487+
let outer_keys = Int32Array::from(vec![0, 1, 2]);
488+
let array: ArrayRef = Arc::new(
489+
DictionaryArray::<Int32Type>::try_new(outer_keys, inner_dict).unwrap(),
490+
);
491+
492+
let normalized = normalize_neg_zero_array(&array);
493+
assert!(!Arc::ptr_eq(&normalized, &array));
494+
495+
let outer = normalized
496+
.as_ref()
497+
.as_any()
498+
.downcast_ref::<DictionaryArray<Int32Type>>()
499+
.unwrap();
500+
let inner = outer
501+
.values()
502+
.as_any()
503+
.downcast_ref::<DictionaryArray<Int32Type>>()
504+
.unwrap();
505+
assert!(!has_neg_zero_f64(inner.values().as_ref()));
506+
}
507+
508+
#[test]
509+
fn normalize_scalar_float64() {
510+
assert_eq!(
511+
normalize_neg_zero_scalar(&ScalarValue::Float64(Some(-0.0))),
512+
ScalarValue::Float64(Some(0.0))
513+
);
514+
assert_eq!(
515+
normalize_neg_zero_scalar(&ScalarValue::Float64(Some(0.0))),
516+
ScalarValue::Float64(Some(0.0))
517+
);
518+
// Non-zero values are unchanged, including the sign of negative numbers.
519+
assert_eq!(
520+
normalize_neg_zero_scalar(&ScalarValue::Float64(Some(-1.5))),
521+
ScalarValue::Float64(Some(-1.5))
522+
);
523+
assert_eq!(
524+
normalize_neg_zero_scalar(&ScalarValue::Float64(None)),
525+
ScalarValue::Float64(None)
526+
);
527+
}
528+
529+
#[test]
530+
fn normalize_scalar_float32_and_float16() {
531+
assert_eq!(
532+
normalize_neg_zero_scalar(&ScalarValue::Float32(Some(-0.0))),
533+
ScalarValue::Float32(Some(0.0))
534+
);
535+
assert_eq!(
536+
normalize_neg_zero_scalar(&ScalarValue::Float16(Some(f16::NEG_ZERO))),
537+
ScalarValue::Float16(Some(f16::ZERO))
538+
);
539+
}
540+
541+
#[test]
542+
fn normalize_scalar_non_float_passthrough() {
543+
let s = ScalarValue::Int32(Some(0));
544+
assert_eq!(normalize_neg_zero_scalar(&s), s);
545+
let s = ScalarValue::Utf8(Some("hello".into()));
546+
assert_eq!(normalize_neg_zero_scalar(&s), s);
547+
}
548+
}

0 commit comments

Comments
 (0)