@@ -77,8 +77,13 @@ impl SparkPivotFirst {
7777 let mut pivot_index = HashMap :: with_capacity ( pivot_values. len ( ) ) ;
7878 // Spark's PivotFirst uses the FIRST occurrence's index (HashMap/TreeMap semantics), so
7979 // when duplicates are somehow present we mirror that by only inserting the first one.
80+ // `pivot_key` can fold two distinct pivot values (`0.0` and `-0.0`) onto one key, and can
81+ // drop one entirely (NaN), so the index is not necessarily the same length as the slot
82+ // vector - the slot count is always `pivot_values.len()`.
8083 for ( i, v) in pivot_values. iter ( ) . enumerate ( ) {
81- pivot_index. entry ( v. clone ( ) ) . or_insert ( i) ;
84+ if let Some ( key) = pivot_key ( v. clone ( ) ) {
85+ pivot_index. entry ( key) . or_insert ( i) ;
86+ }
8287 }
8388 Self {
8489 signature : Signature :: user_defined ( Immutable ) ,
@@ -89,6 +94,44 @@ impl SparkPivotFirst {
8994 }
9095}
9196
97+ /// Rewrite a pivot column value into the key Spark would match it on, or `None` when Spark can
98+ /// never match it.
99+ ///
100+ /// Spark's `PivotFirst` looks pivot values up in a Scala `HashMap[Any, Int]`, so matching goes
101+ /// through `BoxesRunTime.equals` / `Statics.anyHash` on the boxed Catalyst value rather than
102+ /// through `ScalarValue`'s own equality. The two disagree on floats in opposite directions:
103+ ///
104+ /// * `-0.0` and `0.0` are one key for Spark (`-0.0 == 0.0` numerically, and `doubleHash` folds
105+ /// both onto the hash of `0L`), while `ScalarValue` keeps them apart.
106+ /// * `NaN` matches nothing for Spark, not even another `NaN`, because Scala's `==` on `Double`
107+ /// is IEEE. `ScalarValue` treats `NaN` as equal to itself.
108+ ///
109+ /// Nulls are left alone: a null pivot column value does match a null entry in the pivot list,
110+ /// which is what Spark's `pivotIndex.getOrElse(null, -1)` does.
111+ fn pivot_key ( v : ScalarValue ) -> Option < ScalarValue > {
112+ match v {
113+ ScalarValue :: Float32 ( Some ( f) ) => {
114+ if f. is_nan ( ) {
115+ None
116+ } else if f == 0.0 {
117+ Some ( ScalarValue :: Float32 ( Some ( 0.0 ) ) )
118+ } else {
119+ Some ( ScalarValue :: Float32 ( Some ( f) ) )
120+ }
121+ }
122+ ScalarValue :: Float64 ( Some ( f) ) => {
123+ if f. is_nan ( ) {
124+ None
125+ } else if f == 0.0 {
126+ Some ( ScalarValue :: Float64 ( Some ( 0.0 ) ) )
127+ } else {
128+ Some ( ScalarValue :: Float64 ( Some ( f) ) )
129+ }
130+ }
131+ other => Some ( other) ,
132+ }
133+ }
134+
92135impl AggregateUDFImpl for SparkPivotFirst {
93136 fn name ( & self ) -> & str {
94137 "pivot_first"
@@ -124,6 +167,7 @@ impl AggregateUDFImpl for SparkPivotFirst {
124167 Ok ( Box :: new ( PivotFirstAccumulator :: new (
125168 self . value_type . clone ( ) ,
126169 Arc :: clone ( & self . pivot_index ) ,
170+ self . pivot_values . len ( ) ,
127171 ) ) )
128172 }
129173}
@@ -138,8 +182,15 @@ struct PivotFirstAccumulator {
138182}
139183
140184impl PivotFirstAccumulator {
141- fn new ( value_type : DataType , pivot_index : Arc < HashMap < ScalarValue , usize > > ) -> Self {
142- let slots = vec ! [ None ; pivot_index. len( ) ] ;
185+ /// `num_slots` is the pivot list's length, which is what `state_fields` declares. It can
186+ /// exceed `pivot_index.len()` when pivot values collide under `pivot_key` or are unmatchable
187+ /// (NaN); those slots exist in the output and stay null.
188+ fn new (
189+ value_type : DataType ,
190+ pivot_index : Arc < HashMap < ScalarValue , usize > > ,
191+ num_slots : usize ,
192+ ) -> Self {
193+ let slots = vec ! [ None ; num_slots] ;
143194 Self {
144195 value_type,
145196 pivot_index,
@@ -177,7 +228,11 @@ impl Accumulator for PivotFirstAccumulator {
177228 // `PivotFirst.update` never writes for a null value, so a mid-batch null does not
178229 // clobber an earlier non-null.
179230 let pivot_scalar = ScalarValue :: try_from_array ( pivot_arr, row) ?;
180- if let Some ( & slot_idx) = self . pivot_index . get ( & pivot_scalar) {
231+ let Some ( key) = pivot_key ( pivot_scalar) else {
232+ // A NaN pivot value matches no slot in Spark, so the row is ignored.
233+ continue ;
234+ } ;
235+ if let Some ( & slot_idx) = self . pivot_index . get ( & key) {
181236 if !value_arr. is_null ( row) {
182237 self . slots [ slot_idx] = Some ( ScalarValue :: try_from_array ( value_arr, row) ?) ;
183238 }
@@ -236,7 +291,7 @@ impl Accumulator for PivotFirstAccumulator {
236291#[ cfg( test) ]
237292mod tests {
238293 use super :: * ;
239- use arrow:: array:: { Int32Array , StringArray } ;
294+ use arrow:: array:: { Float32Array , Float64Array , Int32Array , StringArray } ;
240295
241296 fn scalar ( v : i32 ) -> ScalarValue {
242297 ScalarValue :: Int32 ( Some ( v) )
@@ -252,7 +307,18 @@ mod tests {
252307 . enumerate ( )
253308 . map ( |( i, v) | ( v. clone ( ) , i) )
254309 . collect ( ) ;
255- PivotFirstAccumulator :: new ( DataType :: Int32 , Arc :: new ( pivot_index) )
310+ PivotFirstAccumulator :: new ( DataType :: Int32 , Arc :: new ( pivot_index) , pivot_values. len ( ) )
311+ }
312+
313+ /// Build an accumulator the way the UDAF does, so the tests exercise `pivot_key` on both the
314+ /// index-building and the per-row lookup side.
315+ fn acc_for ( value_type : DataType , pivot_values : Vec < ScalarValue > ) -> PivotFirstAccumulator {
316+ let udaf = SparkPivotFirst :: new ( value_type. clone ( ) , pivot_values. clone ( ) ) ;
317+ PivotFirstAccumulator :: new (
318+ value_type,
319+ Arc :: clone ( & udaf. pivot_index ) ,
320+ pivot_values. len ( ) ,
321+ )
256322 }
257323
258324 #[ test]
@@ -346,4 +412,86 @@ mod tests {
346412 vec![ Some ( scalar( 1 ) ) , Some ( scalar( 2 ) ) , Some ( scalar( 3 ) ) ]
347413 ) ;
348414 }
415+
416+ #[ test]
417+ fn signed_zero_pivot_values_are_one_key ( ) {
418+ // Spark keys the pivot list on the boxed Catalyst value, where `-0.0 == 0.0`, so a
419+ // `-0.0` pivot column matches rows carrying either sign of zero. `ScalarValue` alone
420+ // keeps them distinct, which returned NULL for the whole column.
421+ //
422+ // Only positive zero appears on the input side on purpose: adding a `-0.0` row would
423+ // make the assertion hold either way, because the later write wins the slot regardless
424+ // of whether the `0.0` row matched.
425+ let mut acc = acc_for (
426+ DataType :: Int32 ,
427+ vec ! [
428+ ScalarValue :: Float64 ( Some ( -0.0 ) ) ,
429+ ScalarValue :: Float64 ( Some ( 1.0 ) ) ,
430+ ] ,
431+ ) ;
432+ let pivots: ArrayRef = Arc :: new ( Float64Array :: from ( vec ! [ 0.0 , 1.0 ] ) ) ;
433+ let values: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 10 , 30 ] ) ) ;
434+ acc. update_batch ( & [ pivots, values] ) . unwrap ( ) ;
435+ assert_eq ! ( acc. slots, vec![ Some ( scalar( 10 ) ) , Some ( scalar( 30 ) ) ] ) ;
436+ }
437+
438+ #[ test]
439+ fn signed_zero_matches_in_both_directions ( ) {
440+ // The mirror of the above: a `0.0` entry in the pivot list has to catch a `-0.0` row.
441+ let mut acc = acc_for ( DataType :: Int32 , vec ! [ ScalarValue :: Float64 ( Some ( 0.0 ) ) ] ) ;
442+ let pivots: ArrayRef = Arc :: new ( Float64Array :: from ( vec ! [ -0.0 ] ) ) ;
443+ let values: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 42 ] ) ) ;
444+ acc. update_batch ( & [ pivots, values] ) . unwrap ( ) ;
445+ assert_eq ! ( acc. slots, vec![ Some ( scalar( 42 ) ) ] ) ;
446+ }
447+
448+ #[ test]
449+ fn nan_pivot_value_matches_nothing ( ) {
450+ // Scala's `==` on Double is IEEE, so `pivotIndex.getOrElse(NaN, -1)` never hits even
451+ // when NaN is in the pivot list. `ScalarValue` treats NaN as equal to itself, which
452+ // populated the column instead of leaving it NULL.
453+ let mut acc = acc_for (
454+ DataType :: Int32 ,
455+ vec ! [
456+ ScalarValue :: Float64 ( Some ( f64 :: NAN ) ) ,
457+ ScalarValue :: Float64 ( Some ( 2.0 ) ) ,
458+ ] ,
459+ ) ;
460+ let pivots: ArrayRef = Arc :: new ( Float64Array :: from ( vec ! [ f64 :: NAN , 2.0 ] ) ) ;
461+ let values: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 10 , 20 ] ) ) ;
462+ acc. update_batch ( & [ pivots, values] ) . unwrap ( ) ;
463+ // The NaN slot still exists (the output column is declared) but stays null.
464+ assert_eq ! ( acc. slots, vec![ None , Some ( scalar( 20 ) ) ] ) ;
465+ }
466+
467+ #[ test]
468+ fn unmatchable_pivot_value_keeps_its_state_slot ( ) {
469+ // A NaN entry is dropped from the index, so the slot vector must be sized from the pivot
470+ // list rather than from the index, or `state()` would return fewer columns than
471+ // `state_fields` declares and the shuffle exchange would reject the batch.
472+ let mut acc = acc_for (
473+ DataType :: Int32 ,
474+ vec ! [
475+ ScalarValue :: Float64 ( Some ( f64 :: NAN ) ) ,
476+ ScalarValue :: Float64 ( Some ( 0.0 ) ) ,
477+ ScalarValue :: Float64 ( Some ( -0.0 ) ) ,
478+ ] ,
479+ ) ;
480+ assert_eq ! ( acc. state( ) . unwrap( ) . len( ) , 3 ) ;
481+ }
482+
483+ #[ test]
484+ fn float32_signed_zero_and_nan_follow_the_same_rules ( ) {
485+ let mut acc = acc_for (
486+ DataType :: Int32 ,
487+ vec ! [
488+ ScalarValue :: Float32 ( Some ( -0.0 ) ) ,
489+ ScalarValue :: Float32 ( Some ( f32 :: NAN ) ) ,
490+ ] ,
491+ ) ;
492+ let pivots: ArrayRef = Arc :: new ( Float32Array :: from ( vec ! [ 0.0 , f32 :: NAN ] ) ) ;
493+ let values: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 10 , 20 ] ) ) ;
494+ acc. update_batch ( & [ pivots, values] ) . unwrap ( ) ;
495+ assert_eq ! ( acc. slots, vec![ Some ( scalar( 10 ) ) , None ] ) ;
496+ }
349497}
0 commit comments