@@ -272,10 +272,9 @@ fn range_has_null(nulls: Option<&NullBuffer>, range: Range<usize>) -> bool {
272272 nulls. is_some_and ( |n| n. null_count ( ) > 0 && n. slice ( range. start , range. len ( ) ) . null_count ( ) > 0 )
273273}
274274
275- /// Projects a native value onto a hashable key whose equality matches the Arrow compare kernels:
276- /// the value itself for integral types, the bit pattern for floats. Arrow orders floats by total
277- /// order rather than IEEE semantics (NaN equals NaN, and 0.0 does not equal -0.0), which is
278- /// exactly bit equality.
275+ /// Projects a native value onto a Spark-compatible hashable key. Floating-point keys canonicalize
276+ /// every NaN representation while preserving the distinct bit patterns of positive and negative
277+ /// zero.
279278trait OverlapKey : Copy {
280279 type Key : Hash + Eq + Copy ;
281280
@@ -299,15 +298,23 @@ impl OverlapKey for f32 {
299298 type Key = u32 ;
300299
301300 fn overlap_key ( self ) -> u32 {
302- self . to_bits ( )
301+ if self . is_nan ( ) {
302+ f32:: NAN . to_bits ( )
303+ } else {
304+ self . to_bits ( )
305+ }
303306 }
304307}
305308
306309impl OverlapKey for f64 {
307310 type Key = u64 ;
308311
309312 fn overlap_key ( self ) -> u64 {
310- self . to_bits ( )
313+ if self . is_nan ( ) {
314+ f64:: NAN . to_bits ( )
315+ } else {
316+ self . to_bits ( )
317+ }
311318 }
312319}
313320
@@ -554,6 +561,132 @@ mod tests {
554561 Ok ( ( ) )
555562 }
556563
564+ #[ test]
565+ fn test_flat_float32_nan_payloads_and_signed_zero ( ) -> Result < ( ) > {
566+ let positive_nan = f32:: from_bits ( 0x7fc0_0001 ) ;
567+ let negative_nan = f32:: from_bits ( 0xffc0_0002 ) ;
568+ let signaling_nan = f32:: from_bits ( 0x7f80_0001 ) ;
569+
570+ let hash_nan_left = ( 1 ..=16 )
571+ . map ( |value| Some ( value as f32 ) )
572+ . chain ( [ Some ( positive_nan) ] )
573+ . collect :: < Vec < _ > > ( ) ;
574+ let hash_nan_right = ( 17 ..=32 )
575+ . map ( |value| Some ( value as f32 ) )
576+ . chain ( [ Some ( negative_nan) ] )
577+ . collect :: < Vec < _ > > ( ) ;
578+ let hash_zero_left = ( 1 ..=16 )
579+ . map ( |value| Some ( value as f32 ) )
580+ . chain ( [ Some ( 0.0 ) ] )
581+ . collect :: < Vec < _ > > ( ) ;
582+ let hash_zero_right = ( 17 ..=32 )
583+ . map ( |value| Some ( value as f32 ) )
584+ . chain ( [ Some ( -0.0 ) ] )
585+ . collect :: < Vec < _ > > ( ) ;
586+
587+ let left = ListArray :: from_iter_primitive :: < Float32Type , _ , _ > ( [
588+ Some ( vec ! [ Some ( positive_nan) ] ) ,
589+ Some ( vec ! [ Some ( negative_nan) ] ) ,
590+ Some ( vec ! [ Some ( signaling_nan) ] ) ,
591+ Some ( vec ! [ Some ( 0.0 ) ] ) ,
592+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
593+ Some ( vec ! [ Some ( positive_nan) , None ] ) ,
594+ Some ( vec ! [ Some ( 0.0 ) , None ] ) ,
595+ Some ( hash_nan_left) ,
596+ Some ( hash_zero_left) ,
597+ ] ) ;
598+ let right = ListArray :: from_iter_primitive :: < Float32Type , _ , _ > ( [
599+ Some ( vec ! [ Some ( f32 :: NAN ) ] ) ,
600+ Some ( vec ! [ Some ( positive_nan) ] ) ,
601+ Some ( vec ! [ Some ( negative_nan) ] ) ,
602+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
603+ Some ( vec ! [ Some ( 0.0 ) ] ) ,
604+ Some ( vec ! [ Some ( negative_nan) ] ) ,
605+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
606+ Some ( hash_nan_right) ,
607+ Some ( hash_zero_right) ,
608+ ] ) ;
609+
610+ let result = arrays_overlap_list :: < i32 > ( & left, & right) ?;
611+ let result = result. as_any ( ) . downcast_ref :: < BooleanArray > ( ) . unwrap ( ) ;
612+ let expected = BooleanArray :: from ( vec ! [
613+ Some ( true ) ,
614+ Some ( true ) ,
615+ Some ( true ) ,
616+ Some ( false ) ,
617+ Some ( false ) ,
618+ Some ( true ) ,
619+ None ,
620+ Some ( true ) ,
621+ Some ( false ) ,
622+ ] ) ;
623+ assert_eq ! ( result, & expected) ;
624+ Ok ( ( ) )
625+ }
626+
627+ #[ test]
628+ fn test_flat_float64_nan_payloads_and_signed_zero ( ) -> Result < ( ) > {
629+ let positive_nan = f64:: from_bits ( 0x7ff8_0000_0000_0001 ) ;
630+ let negative_nan = f64:: from_bits ( 0xfff8_0000_0000_0002 ) ;
631+ let signaling_nan = f64:: from_bits ( 0x7ff0_0000_0000_0001 ) ;
632+
633+ let hash_nan_left = ( 1 ..=16 )
634+ . map ( |value| Some ( value as f64 ) )
635+ . chain ( [ Some ( positive_nan) ] )
636+ . collect :: < Vec < _ > > ( ) ;
637+ let hash_nan_right = ( 17 ..=32 )
638+ . map ( |value| Some ( value as f64 ) )
639+ . chain ( [ Some ( negative_nan) ] )
640+ . collect :: < Vec < _ > > ( ) ;
641+ let hash_zero_left = ( 1 ..=16 )
642+ . map ( |value| Some ( value as f64 ) )
643+ . chain ( [ Some ( 0.0 ) ] )
644+ . collect :: < Vec < _ > > ( ) ;
645+ let hash_zero_right = ( 17 ..=32 )
646+ . map ( |value| Some ( value as f64 ) )
647+ . chain ( [ Some ( -0.0 ) ] )
648+ . collect :: < Vec < _ > > ( ) ;
649+
650+ let left = ListArray :: from_iter_primitive :: < Float64Type , _ , _ > ( [
651+ Some ( vec ! [ Some ( positive_nan) ] ) ,
652+ Some ( vec ! [ Some ( negative_nan) ] ) ,
653+ Some ( vec ! [ Some ( signaling_nan) ] ) ,
654+ Some ( vec ! [ Some ( 0.0 ) ] ) ,
655+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
656+ Some ( vec ! [ Some ( positive_nan) , None ] ) ,
657+ Some ( vec ! [ Some ( 0.0 ) , None ] ) ,
658+ Some ( hash_nan_left) ,
659+ Some ( hash_zero_left) ,
660+ ] ) ;
661+ let right = ListArray :: from_iter_primitive :: < Float64Type , _ , _ > ( [
662+ Some ( vec ! [ Some ( f64 :: NAN ) ] ) ,
663+ Some ( vec ! [ Some ( positive_nan) ] ) ,
664+ Some ( vec ! [ Some ( negative_nan) ] ) ,
665+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
666+ Some ( vec ! [ Some ( 0.0 ) ] ) ,
667+ Some ( vec ! [ Some ( negative_nan) ] ) ,
668+ Some ( vec ! [ Some ( -0.0 ) ] ) ,
669+ Some ( hash_nan_right) ,
670+ Some ( hash_zero_right) ,
671+ ] ) ;
672+
673+ let result = arrays_overlap_list :: < i32 > ( & left, & right) ?;
674+ let result = result. as_any ( ) . downcast_ref :: < BooleanArray > ( ) . unwrap ( ) ;
675+ let expected = BooleanArray :: from ( vec ! [
676+ Some ( true ) ,
677+ Some ( true ) ,
678+ Some ( true ) ,
679+ Some ( false ) ,
680+ Some ( false ) ,
681+ Some ( true ) ,
682+ None ,
683+ Some ( true ) ,
684+ Some ( false ) ,
685+ ] ) ;
686+ assert_eq ! ( result, & expected) ;
687+ Ok ( ( ) )
688+ }
689+
557690 #[ test]
558691 fn test_null_only_overlap ( ) -> Result < ( ) > {
559692 // [1, NULL] vs [NULL, 2] => null (no definite overlap, but nulls present)
0 commit comments