Skip to content

Commit 15eaa9f

Browse files
authored
fix: expand float fuzz coverage and correct discovered inconsistencies (apache#24433)
## Which issue does this PR close? - Closes apache#24431. - Closes apache#24432. ## Rationale for this change Two floating-point edge cases produced inconsistent results: - `ScalarValue::eq_array` used IEEE equality for floats, while `ScalarValue` equality uses bit representations. An identical NaN therefore compared unequal, while `+0.0` and `-0.0` compared equal. - Grouped floating-point `MIN` and `MAX` used partial ordering. With NaNs or signed zeros, results could depend on input order, batching, or partitioning and disagree with the non-grouped accumulators. Both issues were exposed by generating special floating-point values in fuzz data and adding fuzz coverage for `ScalarValue` array conversions. ## What changes are included in this PR? - Generate `+0.0`, `-0.0`, positive and negative infinities, and NaNs with varied signs and payloads in random Float32 and Float64 test data. - Add fuzz coverage checking consistency and round trips among `ScalarValue::try_from_array`, `eq_array`, `to_array`, and `iter_to_array` across supported array types and sliced arrays. - Make Float16, Float32, and Float64 `ScalarValue::eq_array` comparisons use the same bitwise equality as `ScalarValue::PartialEq`. - Make grouped Float16, Float32, and Float64 `MIN`/`MAX` use total ordering and total-order extrema as their initial accumulator values. ## Are these changes tested? Yes. The existing `scalar_eq_array` test now covers NaNs and signed zeros. New deterministic grouped-accumulator tests cover positive and negative NaNs, signed zeros, and the total-order sentinel values. The scalar conversion and aggregation fuzz tests cover the same paths across randomized arrays and execution configurations. ## Are there any user-facing changes? Yes. `ScalarValue::eq_array` now uses bitwise float equality, and grouped floating-point `MIN` and `MAX` now return deterministic results consistent with DataFusion's existing total-order semantics. There are no API signature changes.
1 parent 0eecdfc commit 15eaa9f

5 files changed

Lines changed: 260 additions & 13 deletions

File tree

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,6 +1217,17 @@ macro_rules! eq_array_primitive {
12171217
}};
12181218
}
12191219

1220+
macro_rules! eq_array_float {
1221+
($array:expr, $index:expr, $array_cast:ident, $VALUE:expr) => {{
1222+
let array = $array_cast($array)?;
1223+
let is_valid = array.is_valid($index);
1224+
Ok::<bool, DataFusionError>(match $VALUE {
1225+
Some(val) => is_valid && array.value($index).to_bits() == val.to_bits(),
1226+
None => !is_valid,
1227+
})
1228+
}};
1229+
}
1230+
12201231
impl ScalarValue {
12211232
/// Create a [`Result<ScalarValue>`] with the provided value and datatype
12221233
///
@@ -4574,13 +4585,13 @@ impl ScalarValue {
45744585
eq_array_primitive!(array, index, as_boolean_array, val)?
45754586
}
45764587
ScalarValue::Float16(val) => {
4577-
eq_array_primitive!(array, index, as_float16_array, val)?
4588+
eq_array_float!(array, index, as_float16_array, val)?
45784589
}
45794590
ScalarValue::Float32(val) => {
4580-
eq_array_primitive!(array, index, as_float32_array, val)?
4591+
eq_array_float!(array, index, as_float32_array, val)?
45814592
}
45824593
ScalarValue::Float64(val) => {
4583-
eq_array_primitive!(array, index, as_float64_array, val)?
4594+
eq_array_float!(array, index, as_float64_array, val)?
45844595
}
45854596
ScalarValue::Int8(val) => {
45864597
eq_array_primitive!(array, index, as_int8_array, val)?
@@ -7874,7 +7885,7 @@ mod tests {
78747885
}
78757886

78767887
let bool_vals = [Some(true), None, Some(false)];
7877-
let f32_vals = [Some(-1.0), None, Some(1.0)];
7888+
let f32_vals = [Some(-0.0), Some(0.0), Some(f32::NAN), None, Some(1.0)];
78787889
let f64_vals = make_typed_vec!(f32_vals, f64);
78797890

78807891
let i8_vals = [Some(-1), None, Some(1)];

‎datafusion/core/tests/fuzz_cases/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,5 @@ mod window_fuzz;
4141
// Utility modules
4242
mod once_exec;
4343
mod record_batch_generator;
44+
mod scalar_value_fuzz;
4445
mod spilling_fuzz_in_memory_constrained_env;
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use arrow::array::ArrayRef;
19+
use datafusion_common::ScalarValue;
20+
use rand::random;
21+
22+
use super::record_batch_generator::{RecordBatchGenerator, get_supported_types_columns};
23+
24+
const NUM_SEEDS: usize = 32;
25+
26+
#[test]
27+
fn scalar_value_eq_array_consistency() {
28+
for_each_array(|context, array| {
29+
for (index, expected) in extract_scalars(array, context).iter().enumerate() {
30+
assert!(
31+
expected
32+
.eq_array(array, index)
33+
.unwrap_or_else(|e| panic!("eq_array failed for {context}: {e}")),
34+
"eq_array disagreed with try_from_array at index {index} for {context}"
35+
);
36+
}
37+
});
38+
}
39+
40+
#[test]
41+
fn scalar_value_iter_to_array_roundtrip() {
42+
for_each_array(|context, array| {
43+
let expected = extract_scalars(array, context);
44+
let rebuilt = ScalarValue::iter_to_array(expected.iter().cloned())
45+
.unwrap_or_else(|e| panic!("iter_to_array failed for {context}: {e}"));
46+
47+
assert_eq!(
48+
rebuilt.data_type(),
49+
array.data_type(),
50+
"iter_to_array changed the type for {context}"
51+
);
52+
assert_eq!(
53+
rebuilt.len(),
54+
array.len(),
55+
"iter_to_array changed the length for {context}"
56+
);
57+
58+
let actual = extract_scalars(&rebuilt, context);
59+
assert_eq!(
60+
actual, expected,
61+
"iter_to_array changed values for {context}"
62+
);
63+
});
64+
}
65+
66+
#[test]
67+
fn scalar_value_to_array_roundtrip() {
68+
for_each_array(|context, array| {
69+
for (index, expected) in extract_scalars(array, context).iter().enumerate() {
70+
let singleton = expected
71+
.to_array()
72+
.unwrap_or_else(|e| panic!("to_array failed for {context}: {e}"));
73+
assert_eq!(
74+
singleton.len(),
75+
1,
76+
"to_array returned the wrong length at index {index} for {context}"
77+
);
78+
79+
let actual = ScalarValue::try_from_array(&singleton, 0).unwrap_or_else(|e| {
80+
panic!(
81+
"try_from_array failed on to_array output at index {index} for {context}: {e}"
82+
)
83+
});
84+
assert_eq!(
85+
&actual, expected,
86+
"to_array roundtrip changed index {index} for {context}"
87+
);
88+
}
89+
});
90+
}
91+
92+
fn for_each_array(check: impl Fn(&str, &ArrayRef)) {
93+
for _ in 0..NUM_SEEDS {
94+
let seed = random();
95+
let columns = get_supported_types_columns(seed);
96+
let batch = RecordBatchGenerator::new(1, 128, columns)
97+
.with_seed(seed)
98+
.generate()
99+
.unwrap_or_else(|e| panic!("failed to generate batch for seed {seed}: {e}"));
100+
101+
for (field, array) in batch.schema().fields().iter().zip(batch.columns()) {
102+
let check_array = |array: &ArrayRef| {
103+
let context = format!(
104+
"seed={seed}, column={}, type={}, len={}, offset={}",
105+
field.name(),
106+
array.data_type(),
107+
array.len(),
108+
array.offset()
109+
);
110+
check(&context, array);
111+
};
112+
113+
check_array(array);
114+
115+
// Also exercise arrays with non-zero offsets.
116+
if array.len() > 2 {
117+
check_array(&array.slice(1, array.len() - 2));
118+
}
119+
}
120+
}
121+
}
122+
123+
fn extract_scalars(array: &ArrayRef, context: &str) -> Vec<ScalarValue> {
124+
(0..array.len())
125+
.map(|index| {
126+
ScalarValue::try_from_array(array, index).unwrap_or_else(|e| {
127+
panic!("try_from_array failed at index {index} for {context}: {e}")
128+
})
129+
})
130+
.collect()
131+
}

‎datafusion/functions-aggregate/src/min_max.rs‎

Lines changed: 86 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,17 @@ macro_rules! primitive_max_accumulator {
130130
.with_starting_value($NATIVE::MIN),
131131
))
132132
}};
133+
($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{
134+
Ok(Box::new(
135+
PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new($DATA_TYPE, |cur, new| {
136+
if new.total_cmp(cur) == Ordering::Greater {
137+
*cur = new
138+
}
139+
})
140+
// Use the total-order minimum so negative NaNs replace the sentinel.
141+
.with_starting_value($NATIVE::from_bits($BITS::MAX)),
142+
))
143+
}};
133144
}
134145

135146
/// Creates a [`PrimitiveGroupsAccumulator`] for computing `MIN`
@@ -153,6 +164,17 @@ macro_rules! primitive_min_accumulator {
153164
.with_starting_value($NATIVE::MAX),
154165
))
155166
}};
167+
($DATA_TYPE:ident, $NATIVE:ident, $PRIMTYPE:ident, total, $BITS:ident) => {{
168+
Ok(Box::new(
169+
PrimitiveGroupsAccumulator::<$PRIMTYPE, _>::new(&$DATA_TYPE, |cur, new| {
170+
if new.total_cmp(cur) == Ordering::Less {
171+
*cur = new
172+
}
173+
})
174+
// Use the total-order maximum so positive NaNs replace the sentinel.
175+
.with_starting_value($NATIVE::from_bits($BITS::MAX >> 1)),
176+
))
177+
}};
156178
}
157179

158180
trait FromColumnStatistics {
@@ -272,13 +294,13 @@ impl AggregateUDFImpl for Max {
272294
UInt32 => primitive_max_accumulator!(data_type, u32, UInt32Type),
273295
UInt64 => primitive_max_accumulator!(data_type, u64, UInt64Type),
274296
Float16 => {
275-
primitive_max_accumulator!(data_type, f16, Float16Type)
297+
primitive_max_accumulator!(data_type, f16, Float16Type, total, u16)
276298
}
277299
Float32 => {
278-
primitive_max_accumulator!(data_type, f32, Float32Type)
300+
primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)
279301
}
280302
Float64 => {
281-
primitive_max_accumulator!(data_type, f64, Float64Type)
303+
primitive_max_accumulator!(data_type, f64, Float64Type, total, u64)
282304
}
283305
Date32 => primitive_max_accumulator!(data_type, i32, Date32Type),
284306
Date64 => primitive_max_accumulator!(data_type, i64, Date64Type),
@@ -566,13 +588,13 @@ impl AggregateUDFImpl for Min {
566588
UInt32 => primitive_min_accumulator!(data_type, u32, UInt32Type),
567589
UInt64 => primitive_min_accumulator!(data_type, u64, UInt64Type),
568590
Float16 => {
569-
primitive_min_accumulator!(data_type, f16, Float16Type)
591+
primitive_min_accumulator!(data_type, f16, Float16Type, total, u16)
570592
}
571593
Float32 => {
572-
primitive_min_accumulator!(data_type, f32, Float32Type)
594+
primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)
573595
}
574596
Float64 => {
575-
primitive_min_accumulator!(data_type, f64, Float64Type)
597+
primitive_min_accumulator!(data_type, f64, Float64Type, total, u64)
576598
}
577599
Date32 => primitive_min_accumulator!(data_type, i32, Date32Type),
578600
Date64 => primitive_min_accumulator!(data_type, i64, Date64Type),
@@ -1014,7 +1036,7 @@ mod tests {
10141036
use super::*;
10151037
use arrow::{
10161038
array::{
1017-
Array, DictionaryArray, Float32Array, Int8Array, Int32Array,
1039+
Array, AsArray, DictionaryArray, Float32Array, Int8Array, Int32Array,
10181040
IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,
10191041
PrimitiveArray, StringArray,
10201042
},
@@ -1023,8 +1045,65 @@ mod tests {
10231045
IntervalUnit, IntervalYearMonthType,
10241046
},
10251047
};
1048+
use datafusion_expr::EmitTo;
10261049
use std::sync::Arc;
10271050

1051+
#[test]
1052+
fn grouped_float_min_max_total_order() -> Result<()> {
1053+
fn grouped_float32_min() -> Result<Box<dyn GroupsAccumulator>> {
1054+
let data_type = &DataType::Float32;
1055+
primitive_min_accumulator!(data_type, f32, Float32Type, total, u32)
1056+
}
1057+
1058+
fn grouped_float32_max() -> Result<Box<dyn GroupsAccumulator>> {
1059+
let data_type = &DataType::Float32;
1060+
primitive_max_accumulator!(data_type, f32, Float32Type, total, u32)
1061+
}
1062+
1063+
fn evaluate_grouped_float32(
1064+
mut accumulator: Box<dyn GroupsAccumulator>,
1065+
values: Vec<f32>,
1066+
) -> f32 {
1067+
let group_indices = vec![0; values.len()];
1068+
let values = Arc::new(Float32Array::from(values)) as ArrayRef;
1069+
accumulator
1070+
.update_batch(&[values], &group_indices, None, 1)
1071+
.unwrap();
1072+
accumulator
1073+
.evaluate(EmitTo::All)
1074+
.unwrap()
1075+
.as_primitive::<Float32Type>()
1076+
.value(0)
1077+
}
1078+
1079+
let positive_nan = f32::NAN;
1080+
let negative_nan = f32::from_bits(f32::NAN.to_bits() | (1 << 31));
1081+
1082+
let min_cases = [
1083+
(vec![positive_nan, 1.0], 1.0),
1084+
(vec![1.0, negative_nan], negative_nan),
1085+
(vec![0.0, -0.0], -0.0),
1086+
(vec![positive_nan], positive_nan),
1087+
];
1088+
for (values, expected) in min_cases {
1089+
let actual = evaluate_grouped_float32(grouped_float32_min()?, values);
1090+
assert_eq!(actual.to_bits(), expected.to_bits());
1091+
}
1092+
1093+
let max_cases = [
1094+
(vec![positive_nan, 1.0], positive_nan),
1095+
(vec![1.0, negative_nan], 1.0),
1096+
(vec![-0.0, 0.0], 0.0),
1097+
(vec![negative_nan], negative_nan),
1098+
];
1099+
for (values, expected) in max_cases {
1100+
let actual = evaluate_grouped_float32(grouped_float32_max()?, values);
1101+
assert_eq!(actual.to_bits(), expected.to_bits());
1102+
}
1103+
1104+
Ok(())
1105+
}
1106+
10281107
#[test]
10291108
fn interval_min_max() {
10301109
// IntervalYearMonth

‎test-utils/src/array_gen/random_data.rs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,31 @@ macro_rules! basic_random_data {
5151
};
5252
}
5353

54+
macro_rules! float_random_data {
55+
($ARROW_TYPE:ty, $NATIVE:ty) => {
56+
impl RandomNativeData for $ARROW_TYPE {
57+
fn generate_random_native_data(rng: &mut StdRng) -> Self::Native {
58+
match rng.random_range(0..100) {
59+
0 => 0.0,
60+
1 => -0.0,
61+
2 => <$NATIVE>::INFINITY,
62+
3 => <$NATIVE>::NEG_INFINITY,
63+
4 => {
64+
// Generate a NaN with a uniformly selected sign and nonzero payload.
65+
let sign_mask = (-0.0 as $NATIVE).to_bits();
66+
let exponent_mask = <$NATIVE>::INFINITY.to_bits();
67+
let payload_mask = !(sign_mask | exponent_mask);
68+
let sign = if rng.random() { sign_mask } else { 0 };
69+
let payload = rng.random_range(1..=payload_mask);
70+
<$NATIVE>::from_bits(sign | exponent_mask | payload)
71+
}
72+
_ => rng.random(),
73+
}
74+
}
75+
}
76+
};
77+
}
78+
5479
basic_random_data!(Int8Type);
5580
basic_random_data!(Int16Type);
5681
basic_random_data!(Int32Type);
@@ -59,8 +84,8 @@ basic_random_data!(UInt8Type);
5984
basic_random_data!(UInt16Type);
6085
basic_random_data!(UInt32Type);
6186
basic_random_data!(UInt64Type);
62-
basic_random_data!(Float32Type);
63-
basic_random_data!(Float64Type);
87+
float_random_data!(Float32Type, f32);
88+
float_random_data!(Float64Type, f64);
6489
basic_random_data!(Date32Type);
6590
basic_random_data!(Time32SecondType);
6691
basic_random_data!(Time32MillisecondType);

0 commit comments

Comments
 (0)