Skip to content

Commit bdd2e13

Browse files
authored
fix: canonicalize NaN in flat arrays_overlap float keys (#5376)
1 parent a28ac34 commit bdd2e13

3 files changed

Lines changed: 217 additions & 6 deletions

File tree

‎native/spark-expr/src/array_funcs/arrays_overlap.rs‎

Lines changed: 139 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -272,10 +272,9 @@ fn range_has_null(nulls: Option<&NullBuffer>, range: Range<usize>) -> bool {
272272
nulls.is_some_and(|n| n.null_count() > 0 && n.slice(range.start, range.len()).null_count() > 0)
273273
}
274274

275-
/// Projects a native value onto a hashable key whose equality matches the Arrow compare kernels:
276-
/// the value itself for integral types, the bit pattern for floats. Arrow orders floats by total
277-
/// order rather than IEEE semantics (NaN equals NaN, and 0.0 does not equal -0.0), which is
278-
/// exactly bit equality.
275+
/// Projects a native value onto a Spark-compatible hashable key. Floating-point keys canonicalize
276+
/// every NaN representation while preserving the distinct bit patterns of positive and negative
277+
/// zero.
279278
trait OverlapKey: Copy {
280279
type Key: Hash + Eq + Copy;
281280

@@ -299,15 +298,23 @@ impl OverlapKey for f32 {
299298
type Key = u32;
300299

301300
fn overlap_key(self) -> u32 {
302-
self.to_bits()
301+
if self.is_nan() {
302+
f32::NAN.to_bits()
303+
} else {
304+
self.to_bits()
305+
}
303306
}
304307
}
305308

306309
impl OverlapKey for f64 {
307310
type Key = u64;
308311

309312
fn overlap_key(self) -> u64 {
310-
self.to_bits()
313+
if self.is_nan() {
314+
f64::NAN.to_bits()
315+
} else {
316+
self.to_bits()
317+
}
311318
}
312319
}
313320

@@ -554,6 +561,132 @@ mod tests {
554561
Ok(())
555562
}
556563

564+
#[test]
565+
fn test_flat_float32_nan_payloads_and_signed_zero() -> Result<()> {
566+
let positive_nan = f32::from_bits(0x7fc0_0001);
567+
let negative_nan = f32::from_bits(0xffc0_0002);
568+
let signaling_nan = f32::from_bits(0x7f80_0001);
569+
570+
let hash_nan_left = (1..=16)
571+
.map(|value| Some(value as f32))
572+
.chain([Some(positive_nan)])
573+
.collect::<Vec<_>>();
574+
let hash_nan_right = (17..=32)
575+
.map(|value| Some(value as f32))
576+
.chain([Some(negative_nan)])
577+
.collect::<Vec<_>>();
578+
let hash_zero_left = (1..=16)
579+
.map(|value| Some(value as f32))
580+
.chain([Some(0.0)])
581+
.collect::<Vec<_>>();
582+
let hash_zero_right = (17..=32)
583+
.map(|value| Some(value as f32))
584+
.chain([Some(-0.0)])
585+
.collect::<Vec<_>>();
586+
587+
let left = ListArray::from_iter_primitive::<Float32Type, _, _>([
588+
Some(vec![Some(positive_nan)]),
589+
Some(vec![Some(negative_nan)]),
590+
Some(vec![Some(signaling_nan)]),
591+
Some(vec![Some(0.0)]),
592+
Some(vec![Some(-0.0)]),
593+
Some(vec![Some(positive_nan), None]),
594+
Some(vec![Some(0.0), None]),
595+
Some(hash_nan_left),
596+
Some(hash_zero_left),
597+
]);
598+
let right = ListArray::from_iter_primitive::<Float32Type, _, _>([
599+
Some(vec![Some(f32::NAN)]),
600+
Some(vec![Some(positive_nan)]),
601+
Some(vec![Some(negative_nan)]),
602+
Some(vec![Some(-0.0)]),
603+
Some(vec![Some(0.0)]),
604+
Some(vec![Some(negative_nan)]),
605+
Some(vec![Some(-0.0)]),
606+
Some(hash_nan_right),
607+
Some(hash_zero_right),
608+
]);
609+
610+
let result = arrays_overlap_list::<i32>(&left, &right)?;
611+
let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
612+
let expected = BooleanArray::from(vec![
613+
Some(true),
614+
Some(true),
615+
Some(true),
616+
Some(false),
617+
Some(false),
618+
Some(true),
619+
None,
620+
Some(true),
621+
Some(false),
622+
]);
623+
assert_eq!(result, &expected);
624+
Ok(())
625+
}
626+
627+
#[test]
628+
fn test_flat_float64_nan_payloads_and_signed_zero() -> Result<()> {
629+
let positive_nan = f64::from_bits(0x7ff8_0000_0000_0001);
630+
let negative_nan = f64::from_bits(0xfff8_0000_0000_0002);
631+
let signaling_nan = f64::from_bits(0x7ff0_0000_0000_0001);
632+
633+
let hash_nan_left = (1..=16)
634+
.map(|value| Some(value as f64))
635+
.chain([Some(positive_nan)])
636+
.collect::<Vec<_>>();
637+
let hash_nan_right = (17..=32)
638+
.map(|value| Some(value as f64))
639+
.chain([Some(negative_nan)])
640+
.collect::<Vec<_>>();
641+
let hash_zero_left = (1..=16)
642+
.map(|value| Some(value as f64))
643+
.chain([Some(0.0)])
644+
.collect::<Vec<_>>();
645+
let hash_zero_right = (17..=32)
646+
.map(|value| Some(value as f64))
647+
.chain([Some(-0.0)])
648+
.collect::<Vec<_>>();
649+
650+
let left = ListArray::from_iter_primitive::<Float64Type, _, _>([
651+
Some(vec![Some(positive_nan)]),
652+
Some(vec![Some(negative_nan)]),
653+
Some(vec![Some(signaling_nan)]),
654+
Some(vec![Some(0.0)]),
655+
Some(vec![Some(-0.0)]),
656+
Some(vec![Some(positive_nan), None]),
657+
Some(vec![Some(0.0), None]),
658+
Some(hash_nan_left),
659+
Some(hash_zero_left),
660+
]);
661+
let right = ListArray::from_iter_primitive::<Float64Type, _, _>([
662+
Some(vec![Some(f64::NAN)]),
663+
Some(vec![Some(positive_nan)]),
664+
Some(vec![Some(negative_nan)]),
665+
Some(vec![Some(-0.0)]),
666+
Some(vec![Some(0.0)]),
667+
Some(vec![Some(negative_nan)]),
668+
Some(vec![Some(-0.0)]),
669+
Some(hash_nan_right),
670+
Some(hash_zero_right),
671+
]);
672+
673+
let result = arrays_overlap_list::<i32>(&left, &right)?;
674+
let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
675+
let expected = BooleanArray::from(vec![
676+
Some(true),
677+
Some(true),
678+
Some(true),
679+
Some(false),
680+
Some(false),
681+
Some(true),
682+
None,
683+
Some(true),
684+
Some(false),
685+
]);
686+
assert_eq!(result, &expected);
687+
Ok(())
688+
}
689+
557690
#[test]
558691
fn test_null_only_overlap() -> Result<()> {
559692
// [1, NULL] vs [NULL, 2] => null (no definite overlap, but nulls present)

‎spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,57 @@ INSERT INTO test_overlap_dbl VALUES (array(1.0, 2.0), array(2.0, 3.0)), (array(1
122122
query
123123
SELECT a, b, arrays_overlap(a, b) FROM test_overlap_dbl
124124

125+
-- Flat FLOAT/DOUBLE NaNs must compare equal regardless of their sign or payload.
126+
-- Parquet canonicalizes NaNs, so negate a scanned column to produce a noncanonical
127+
-- NaN at execution time. String casts preserve the sign of the zero controls.
128+
statement
129+
CREATE TABLE test_overlap_floating_point(
130+
id int, fl float, fr float, dl double, dr double
131+
) USING parquet
132+
133+
statement
134+
INSERT INTO test_overlap_floating_point VALUES
135+
(0, CAST('NaN' AS FLOAT), CAST('NaN' AS FLOAT), CAST('NaN' AS DOUBLE), CAST('NaN' AS DOUBLE)),
136+
(1, CAST('0.0' AS FLOAT), CAST('-0.0' AS FLOAT), CAST('0.0' AS DOUBLE), CAST('-0.0' AS DOUBLE)),
137+
(2, CAST('-0.0' AS FLOAT), CAST('0.0' AS FLOAT), CAST('-0.0' AS DOUBLE), CAST('0.0' AS DOUBLE)),
138+
(3, CAST('0.0' AS FLOAT), CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE), CAST('0.0' AS DOUBLE)),
139+
(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE), CAST('-0.0' AS DOUBLE)),
140+
(5, CAST(1 AS FLOAT), CAST(2 AS FLOAT), CAST(1 AS DOUBLE), CAST(2 AS DOUBLE)),
141+
(6, CAST('NaN' AS FLOAT), CAST(1 AS FLOAT), CAST('NaN' AS DOUBLE), CAST(1 AS DOUBLE)),
142+
(7, CAST(1 AS FLOAT), CAST('NaN' AS FLOAT), CAST(1 AS DOUBLE), CAST('NaN' AS DOUBLE))
143+
144+
-- Short arrays, reversed operands, and direct signed-zero controls. Spark 4.2+
145+
-- normalizes signed zeros before arrays_overlap (SPARK-54918); comparing with the
146+
-- running Spark version checks the appropriate behavior without fixed expectations.
147+
query
148+
SELECT id,
149+
arrays_overlap(array(fl), array(-fr)) AS float_short,
150+
arrays_overlap(array(dl), array(-dr)) AS double_short,
151+
arrays_overlap(array(-fr), array(fl)) AS float_reversed,
152+
arrays_overlap(array(-dr), array(dl)) AS double_reversed,
153+
arrays_overlap(array(fl), array(fr)) AS float_direct,
154+
arrays_overlap(array(dl), array(dr)) AS double_direct
155+
FROM test_overlap_floating_point
156+
157+
-- Seventeen elements on each side exceed the native budget of 256 comparisons
158+
-- and exercise the hash-probe path, including signed-zero and nonmatching rows.
159+
query
160+
SELECT id,
161+
arrays_overlap(array_repeat(fl, 17), array_repeat(-fr, 17)) AS float_long,
162+
arrays_overlap(array_repeat(dl, 17), array_repeat(-dr, 17)) AS double_long,
163+
arrays_overlap(array_repeat(-fr, 17), array_repeat(fl, 17)) AS float_long_reversed,
164+
arrays_overlap(array_repeat(-dr, 17), array_repeat(dl, 17)) AS double_long_reversed
165+
FROM test_overlap_floating_point
166+
167+
-- A definite NaN match wins over NULL; a nonmatch with NULL remains unknown.
168+
query
169+
SELECT id,
170+
arrays_overlap(array(fl, CAST(NULL AS FLOAT)), array(-fr)),
171+
arrays_overlap(array(dl, CAST(NULL AS DOUBLE)), array(-dr)),
172+
arrays_overlap(array(-fr), array(fl, CAST(NULL AS FLOAT))),
173+
arrays_overlap(array(-dr), array(dl, CAST(NULL AS DOUBLE)))
174+
FROM test_overlap_floating_point
175+
125176
-- boolean arrays
126177
query
127178
SELECT arrays_overlap(array(true, false), array(false)) FROM test_overlap_dbl

‎spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,33 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
560560
}
561561
}
562562

563+
test("arrays_overlap - runtime NaN representations") {
564+
val floatNaN = java.lang.Float.intBitsToFloat(0x7fc01234 | Int.MinValue)
565+
val doubleNaN = java.lang.Double.longBitsToDouble(0x7ff8000000001234L | Long.MinValue)
566+
567+
withParquetTable(
568+
Seq((floatNaN, doubleNaN)),
569+
"floating_point_overlap",
570+
withDictionary = false) {
571+
// The behavioral cases live in arrays_overlap.sql. SQL equality cannot distinguish NaN
572+
// representations, so verify here that Parquet canonicalizes the inputs and that native
573+
// runtime negation produces noncanonical NaNs after the scan.
574+
val query = sql("SELECT _1, -_1, _2, -_2 FROM floating_point_overlap")
575+
checkSparkAnswerAndOperator(query)
576+
val row = query.head()
577+
val canonicalFloatNaNBits = java.lang.Float.floatToRawIntBits(Float.NaN)
578+
val canonicalDoubleNaNBits = java.lang.Double.doubleToRawLongBits(Double.NaN)
579+
assert(java.lang.Float.floatToRawIntBits(row.getFloat(0)) == canonicalFloatNaNBits)
580+
assert(
581+
java.lang.Float.floatToRawIntBits(row.getFloat(1)) ==
582+
(canonicalFloatNaNBits | Int.MinValue))
583+
assert(java.lang.Double.doubleToRawLongBits(row.getDouble(2)) == canonicalDoubleNaNBits)
584+
assert(
585+
java.lang.Double.doubleToRawLongBits(row.getDouble(3)) ==
586+
(canonicalDoubleNaNBits | Long.MinValue))
587+
}
588+
}
589+
563590
test("arrays_overlap - null handling behavior verification") {
564591
withSQLConf(
565592
"spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {

0 commit comments

Comments
 (0)