Skip to content

Commit 1a564ec

Browse files
committed
feat: Support IEEE 754 for SQL ops
1 parent 4387243 commit 1a564ec

4 files changed

Lines changed: 38 additions & 26 deletions

File tree

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,8 @@ impl PartialEq for ScalarValue {
505505
(Boolean(_), _) => false,
506506
(Float32(v1), Float32(v2)) => match (v1, v2) {
507507
(Some(f1), Some(f2)) => {
508-
*f1 == 0.0 && *f2 == 0.0 || f1.to_bits() == f2.to_bits()
508+
let (b1, b2) = (f1.to_bits(), f2.to_bits());
509+
((b1 << 1) == 0 && (b2 << 1) == 0) || b1 == b2
509510
}
510511
_ => v1.eq(v2),
511512
},
@@ -520,7 +521,8 @@ impl PartialEq for ScalarValue {
520521
(Float16(_), _) => false,
521522
(Float64(v1), Float64(v2)) => match (v1, v2) {
522523
(Some(f1), Some(f2)) => {
523-
*f1 == 0.0 && *f2 == 0.0 || f1.to_bits() == f2.to_bits()
524+
let (b1, b2) = (f1.to_bits(), f2.to_bits());
525+
((b1 << 1) == 0 && (b2 << 1) == 0) || b1 == b2
524526
}
525527
_ => v1.eq(v2),
526528
},
@@ -667,11 +669,13 @@ impl PartialOrd for ScalarValue {
667669
(Boolean(v1), Boolean(v2)) => v1.partial_cmp(v2),
668670
(Boolean(_), _) => None,
669671
(Float32(v1), Float32(v2)) => match (v1, v2) {
670-
(Some(a), Some(b)) => Some(if *a == 0.0 && *b == 0.0 {
671-
Ordering::Equal
672-
} else {
673-
a.total_cmp(b)
674-
}),
672+
(Some(a), Some(b)) => {
673+
Some(if a.to_bits() << 1 == 0 && b.to_bits() << 1 == 0 {
674+
Ordering::Equal
675+
} else {
676+
a.total_cmp(b)
677+
})
678+
}
675679
_ => v1.partial_cmp(v2),
676680
},
677681
(Float16(v1), Float16(v2)) => match (v1, v2) {
@@ -687,11 +691,13 @@ impl PartialOrd for ScalarValue {
687691
(Float32(_), _) => None,
688692
(Float16(_), _) => None,
689693
(Float64(v1), Float64(v2)) => match (v1, v2) {
690-
(Some(a), Some(b)) => Some(if *a == 0.0 && *b == 0.0 {
691-
Ordering::Equal
692-
} else {
693-
a.total_cmp(b)
694-
}),
694+
(Some(a), Some(b)) => {
695+
Some(if a.to_bits() << 1 == 0 && b.to_bits() << 1 == 0 {
696+
Ordering::Equal
697+
} else {
698+
a.total_cmp(b)
699+
})
700+
}
695701
_ => v1.partial_cmp(v2),
696702
},
697703
(Float64(_), _) => None,

‎datafusion/functions-nested/src/set_ops.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -362,16 +362,17 @@ fn generic_set_lists<OffsetSize: OffsetSizeTrait>(
362362
// elements between the first and last offset are referenced.
363363
let l_first = l.offsets()[0].as_usize();
364364
let l_len = l.offsets()[l.len()].as_usize() - l_first;
365-
let rows_l = converter.convert_columns(&[l_values_norm.slice(l_first, l_len)])?;
365+
let l_values = l_values_norm.slice(l_first, l_len);
366+
let rows_l = converter.convert_columns(&[Arc::clone(&l_values)])?;
366367

367368
let r_first = r.offsets()[0].as_usize();
368369
let r_len = r.offsets()[r.len()].as_usize() - r_first;
369-
let rows_r = converter.convert_columns(&[r_values_norm.slice(r_first, r_len)])?;
370-
371-
// Combine the *sliced* value arrays so 0-based indices from the row
372-
// converter map directly into the concatenated array.
373-
let l_values = l_values_norm.slice(l_first, l_len);
374370
let r_values = r_values_norm.slice(r_first, r_len);
371+
let rows_r = converter.convert_columns(&[Arc::clone(&r_values)])?;
372+
373+
// Indices from the row converter are 0-based in the per-side slice;
374+
// concatenating those same slices lets indices map directly into the
375+
// combined values array.
375376
let combined_values = concat(&[l_values.as_ref(), r_values.as_ref()])?;
376377
let r_offset = l_len;
377378

‎datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,7 @@ macro_rules! hash_float {
7676
$(impl HashValue for $t {
7777
#[cfg(not(feature = "force_hash_collisions"))]
7878
fn hash(&self, state: &RandomState) -> u64 {
79-
let bits = self.to_bits();
80-
// +0.0 and -0.0 differ only in the sign bit but compare equal
81-
// under IEEE 754; normalize -0.0 → +0.0 so the hash agrees
82-
// with the equality check used by `GroupValuesPrimitive`.
83-
let bits = if bits << 1 == 0 { 0 } else { bits };
84-
state.hash_one(bits)
79+
state.hash_one(self.canonicalize().to_bits())
8580
}
8681

8782
#[cfg(feature = "force_hash_collisions")]

‎datafusion/physical-plan/src/joins/utils.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2196,8 +2196,18 @@ fn eq_dyn_null(
21962196
return Ok(compare_op_for_nested(op, &left, &right)?);
21972197
}
21982198
// Arrow's `eq` / `not_distinct` use IEEE 754 totalOrder semantics for
2199-
// floats, so `-0.0` and `+0.0` would compare unequal. Normalize the
2200-
// operands so SQL semantics (`+0.0 == -0.0`) hold; no-op for non-floats.
2199+
// floats, so `-0.0` and `+0.0` would compare unequal. Normalize float
2200+
// operands first; non-float types dispatch directly to avoid the
2201+
// `make_array(to_data())` round-trip.
2202+
if !matches!(
2203+
left.data_type(),
2204+
DataType::Float16 | DataType::Float32 | DataType::Float64
2205+
) {
2206+
return match null_equality {
2207+
NullEquality::NullEqualsNothing => eq(&left, &right),
2208+
NullEquality::NullEqualsNull => not_distinct(&left, &right),
2209+
};
2210+
}
22012211
let left_arr: ArrayRef = make_array(left.to_data());
22022212
let right_arr: ArrayRef = make_array(right.to_data());
22032213
let left_norm = normalize_float_zero(&left_arr);

0 commit comments

Comments
 (0)