Skip to content

Commit bae9225

Browse files
committed
fix: match Spark's float pivot key semantics in PivotFirst
1 parent 7e2d0b6 commit bae9225

3 files changed

Lines changed: 255 additions & 6 deletions

File tree

‎native/spark-expr/src/agg_funcs/pivot_first.rs‎

Lines changed: 154 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,13 @@ impl SparkPivotFirst {
7777
let mut pivot_index = HashMap::with_capacity(pivot_values.len());
7878
// Spark's PivotFirst uses the FIRST occurrence's index (HashMap/TreeMap semantics), so
7979
// when duplicates are somehow present we mirror that by only inserting the first one.
80+
// `pivot_key` can fold two distinct pivot values (`0.0` and `-0.0`) onto one key, and can
81+
// drop one entirely (NaN), so the index is not necessarily the same length as the slot
82+
// vector - the slot count is always `pivot_values.len()`.
8083
for (i, v) in pivot_values.iter().enumerate() {
81-
pivot_index.entry(v.clone()).or_insert(i);
84+
if let Some(key) = pivot_key(v.clone()) {
85+
pivot_index.entry(key).or_insert(i);
86+
}
8287
}
8388
Self {
8489
signature: Signature::user_defined(Immutable),
@@ -89,6 +94,44 @@ impl SparkPivotFirst {
8994
}
9095
}
9196

97+
/// Rewrite a pivot column value into the key Spark would match it on, or `None` when Spark can
98+
/// never match it.
99+
///
100+
/// Spark's `PivotFirst` looks pivot values up in a Scala `HashMap[Any, Int]`, so matching goes
101+
/// through `BoxesRunTime.equals` / `Statics.anyHash` on the boxed Catalyst value rather than
102+
/// through `ScalarValue`'s own equality. The two disagree on floats in opposite directions:
103+
///
104+
/// * `-0.0` and `0.0` are one key for Spark (`-0.0 == 0.0` numerically, and `doubleHash` folds
105+
/// both onto the hash of `0L`), while `ScalarValue` keeps them apart.
106+
/// * `NaN` matches nothing for Spark, not even another `NaN`, because Scala's `==` on `Double`
107+
/// is IEEE. `ScalarValue` treats `NaN` as equal to itself.
108+
///
109+
/// Nulls are left alone: a null pivot column value does match a null entry in the pivot list,
110+
/// which is what Spark's `pivotIndex.getOrElse(null, -1)` does.
111+
fn pivot_key(v: ScalarValue) -> Option<ScalarValue> {
112+
match v {
113+
ScalarValue::Float32(Some(f)) => {
114+
if f.is_nan() {
115+
None
116+
} else if f == 0.0 {
117+
Some(ScalarValue::Float32(Some(0.0)))
118+
} else {
119+
Some(ScalarValue::Float32(Some(f)))
120+
}
121+
}
122+
ScalarValue::Float64(Some(f)) => {
123+
if f.is_nan() {
124+
None
125+
} else if f == 0.0 {
126+
Some(ScalarValue::Float64(Some(0.0)))
127+
} else {
128+
Some(ScalarValue::Float64(Some(f)))
129+
}
130+
}
131+
other => Some(other),
132+
}
133+
}
134+
92135
impl AggregateUDFImpl for SparkPivotFirst {
93136
fn name(&self) -> &str {
94137
"pivot_first"
@@ -124,6 +167,7 @@ impl AggregateUDFImpl for SparkPivotFirst {
124167
Ok(Box::new(PivotFirstAccumulator::new(
125168
self.value_type.clone(),
126169
Arc::clone(&self.pivot_index),
170+
self.pivot_values.len(),
127171
)))
128172
}
129173
}
@@ -138,8 +182,15 @@ struct PivotFirstAccumulator {
138182
}
139183

140184
impl PivotFirstAccumulator {
141-
fn new(value_type: DataType, pivot_index: Arc<HashMap<ScalarValue, usize>>) -> Self {
142-
let slots = vec![None; pivot_index.len()];
185+
/// `num_slots` is the pivot list's length, which is what `state_fields` declares. It can
186+
/// exceed `pivot_index.len()` when pivot values collide under `pivot_key` or are unmatchable
187+
/// (NaN); those slots exist in the output and stay null.
188+
fn new(
189+
value_type: DataType,
190+
pivot_index: Arc<HashMap<ScalarValue, usize>>,
191+
num_slots: usize,
192+
) -> Self {
193+
let slots = vec![None; num_slots];
143194
Self {
144195
value_type,
145196
pivot_index,
@@ -177,7 +228,11 @@ impl Accumulator for PivotFirstAccumulator {
177228
// `PivotFirst.update` never writes for a null value, so a mid-batch null does not
178229
// clobber an earlier non-null.
179230
let pivot_scalar = ScalarValue::try_from_array(pivot_arr, row)?;
180-
if let Some(&slot_idx) = self.pivot_index.get(&pivot_scalar) {
231+
let Some(key) = pivot_key(pivot_scalar) else {
232+
// A NaN pivot value matches no slot in Spark, so the row is ignored.
233+
continue;
234+
};
235+
if let Some(&slot_idx) = self.pivot_index.get(&key) {
181236
if !value_arr.is_null(row) {
182237
self.slots[slot_idx] = Some(ScalarValue::try_from_array(value_arr, row)?);
183238
}
@@ -236,7 +291,7 @@ impl Accumulator for PivotFirstAccumulator {
236291
#[cfg(test)]
237292
mod tests {
238293
use super::*;
239-
use arrow::array::{Int32Array, StringArray};
294+
use arrow::array::{Float32Array, Float64Array, Int32Array, StringArray};
240295

241296
fn scalar(v: i32) -> ScalarValue {
242297
ScalarValue::Int32(Some(v))
@@ -252,7 +307,18 @@ mod tests {
252307
.enumerate()
253308
.map(|(i, v)| (v.clone(), i))
254309
.collect();
255-
PivotFirstAccumulator::new(DataType::Int32, Arc::new(pivot_index))
310+
PivotFirstAccumulator::new(DataType::Int32, Arc::new(pivot_index), pivot_values.len())
311+
}
312+
313+
/// Build an accumulator the way the UDAF does, so the tests exercise `pivot_key` on both the
314+
/// index-building and the per-row lookup side.
315+
fn acc_for(value_type: DataType, pivot_values: Vec<ScalarValue>) -> PivotFirstAccumulator {
316+
let udaf = SparkPivotFirst::new(value_type.clone(), pivot_values.clone());
317+
PivotFirstAccumulator::new(
318+
value_type,
319+
Arc::clone(&udaf.pivot_index),
320+
pivot_values.len(),
321+
)
256322
}
257323

258324
#[test]
@@ -346,4 +412,86 @@ mod tests {
346412
vec![Some(scalar(1)), Some(scalar(2)), Some(scalar(3))]
347413
);
348414
}
415+
416+
#[test]
417+
fn signed_zero_pivot_values_are_one_key() {
418+
// Spark keys the pivot list on the boxed Catalyst value, where `-0.0 == 0.0`, so a
419+
// `-0.0` pivot column matches rows carrying either sign of zero. `ScalarValue` alone
420+
// keeps them distinct, which returned NULL for the whole column.
421+
//
422+
// Only positive zero appears on the input side on purpose: adding a `-0.0` row would
423+
// make the assertion hold either way, because the later write wins the slot regardless
424+
// of whether the `0.0` row matched.
425+
let mut acc = acc_for(
426+
DataType::Int32,
427+
vec![
428+
ScalarValue::Float64(Some(-0.0)),
429+
ScalarValue::Float64(Some(1.0)),
430+
],
431+
);
432+
let pivots: ArrayRef = Arc::new(Float64Array::from(vec![0.0, 1.0]));
433+
let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 30]));
434+
acc.update_batch(&[pivots, values]).unwrap();
435+
assert_eq!(acc.slots, vec![Some(scalar(10)), Some(scalar(30))]);
436+
}
437+
438+
#[test]
439+
fn signed_zero_matches_in_both_directions() {
440+
// The mirror of the above: a `0.0` entry in the pivot list has to catch a `-0.0` row.
441+
let mut acc = acc_for(DataType::Int32, vec![ScalarValue::Float64(Some(0.0))]);
442+
let pivots: ArrayRef = Arc::new(Float64Array::from(vec![-0.0]));
443+
let values: ArrayRef = Arc::new(Int32Array::from(vec![42]));
444+
acc.update_batch(&[pivots, values]).unwrap();
445+
assert_eq!(acc.slots, vec![Some(scalar(42))]);
446+
}
447+
448+
#[test]
449+
fn nan_pivot_value_matches_nothing() {
450+
// Scala's `==` on Double is IEEE, so `pivotIndex.getOrElse(NaN, -1)` never hits even
451+
// when NaN is in the pivot list. `ScalarValue` treats NaN as equal to itself, which
452+
// populated the column instead of leaving it NULL.
453+
let mut acc = acc_for(
454+
DataType::Int32,
455+
vec![
456+
ScalarValue::Float64(Some(f64::NAN)),
457+
ScalarValue::Float64(Some(2.0)),
458+
],
459+
);
460+
let pivots: ArrayRef = Arc::new(Float64Array::from(vec![f64::NAN, 2.0]));
461+
let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20]));
462+
acc.update_batch(&[pivots, values]).unwrap();
463+
// The NaN slot still exists (the output column is declared) but stays null.
464+
assert_eq!(acc.slots, vec![None, Some(scalar(20))]);
465+
}
466+
467+
#[test]
468+
fn unmatchable_pivot_value_keeps_its_state_slot() {
469+
// A NaN entry is dropped from the index, so the slot vector must be sized from the pivot
470+
// list rather than from the index, or `state()` would return fewer columns than
471+
// `state_fields` declares and the shuffle exchange would reject the batch.
472+
let mut acc = acc_for(
473+
DataType::Int32,
474+
vec![
475+
ScalarValue::Float64(Some(f64::NAN)),
476+
ScalarValue::Float64(Some(0.0)),
477+
ScalarValue::Float64(Some(-0.0)),
478+
],
479+
);
480+
assert_eq!(acc.state().unwrap().len(), 3);
481+
}
482+
483+
#[test]
484+
fn float32_signed_zero_and_nan_follow_the_same_rules() {
485+
let mut acc = acc_for(
486+
DataType::Int32,
487+
vec![
488+
ScalarValue::Float32(Some(-0.0)),
489+
ScalarValue::Float32(Some(f32::NAN)),
490+
],
491+
);
492+
let pivots: ArrayRef = Arc::new(Float32Array::from(vec![0.0, f32::NAN]));
493+
let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20]));
494+
acc.update_batch(&[pivots, values]).unwrap();
495+
assert_eq!(acc.slots, vec![Some(scalar(10)), None]);
496+
}
349497
}

‎spark/src/test/resources/sql-tests/expressions/aggregate/PivotFirst.sql‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,3 +319,62 @@ SELECT * FROM pf_sales_datepivot
319319
PIVOT (sum(earnings)
320320
FOR d IN (DATE '2024-01-01', DATE '2024-06-15'))
321321
ORDER BY region
322+
323+
-- ============================================================
324+
-- Non-string pivot column: double, including the two values whose
325+
-- key matching differs between Spark and a plain hash of the native
326+
-- scalar. Spark keys the pivot list on the boxed Catalyst value, so
327+
-- -0.0 and 0.0 are the same key while NaN matches nothing at all.
328+
--
329+
-- The zero rows are written as -0.0D / 0.0D rather than
330+
-- CAST(-0.0 AS DOUBLE), because the cast constant-folds to +0.0 and
331+
-- would make the case vacuous.
332+
-- ============================================================
333+
334+
statement
335+
CREATE TABLE pf_sales_dblpivot (region string, k double, earnings int) USING parquet
336+
337+
statement
338+
INSERT INTO pf_sales_dblpivot VALUES
339+
('NA', 0.0D, 15000),
340+
('NA', -0.0D, 48000),
341+
('NA', double('NaN'), 11000),
342+
('NA', 1.5D, 20000),
343+
('EU', 0.0D, 30000),
344+
('EU', double('NaN'), 12000),
345+
('EU', 1.5D, 40000)
346+
347+
-- -0.0 in the pivot list has to collect both signs of zero (63000 / 30000),
348+
-- and the NaN column has to stay NULL for every group.
349+
query spark_answer_only
350+
SELECT * FROM pf_sales_dblpivot
351+
PIVOT (sum(earnings)
352+
FOR k IN (-0.0D, double('NaN'), 1.5D))
353+
ORDER BY region
354+
355+
-- The mirror: a 0.0 entry in the pivot list must also catch the -0.0 rows.
356+
query spark_answer_only
357+
SELECT * FROM pf_sales_dblpivot
358+
PIVOT (sum(earnings)
359+
FOR k IN (0.0D, 1.5D))
360+
ORDER BY region
361+
362+
-- ============================================================
363+
-- Non-string pivot column: float. Same two rules.
364+
-- ============================================================
365+
366+
statement
367+
CREATE TABLE pf_sales_fltpivot (region string, k float, earnings int) USING parquet
368+
369+
statement
370+
INSERT INTO pf_sales_fltpivot VALUES
371+
('NA', 0.0F, 15000),
372+
('NA', -0.0F, 48000),
373+
('NA', float('NaN'), 11000),
374+
('EU', -0.0F, 30000)
375+
376+
query spark_answer_only
377+
SELECT * FROM pf_sales_fltpivot
378+
PIVOT (sum(earnings)
379+
FOR k IN (-0.0F, float('NaN')))
380+
ORDER BY region

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2490,4 +2490,46 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
24902490
}
24912491
}
24922492

2493+
// A double pivot column exercises the one place where Spark's key matching is neither
2494+
// IEEE nor `ScalarValue`'s: the pivot list is a Scala `HashMap[Any, Int]`, so `-0.0` and
2495+
// `0.0` are the same key while NaN matches nothing, not even another NaN. Both directions
2496+
// have to hold on the native path, and it has to be the native path that produced them.
2497+
test("PivotFirst matches Spark's float key semantics natively") {
2498+
withTempDir { dir =>
2499+
val path = new Path(dir.toURI.toString, "fkeys")
2500+
spark
2501+
.sql("""SELECT * FROM VALUES
2502+
| ('NA', 0.0D, 15000),
2503+
| ('NA', -0.0D, 48000),
2504+
| ('NA', double('NaN'), 11000),
2505+
| ('NA', 1.5D, 20000),
2506+
| ('EU', -0.0D, 30000),
2507+
| ('EU', double('NaN'), 12000)
2508+
|AS t(region, k, earnings)""".stripMargin)
2509+
.write
2510+
.parquet(path.toUri.toString)
2511+
withParquetTable(path.toUri.toString, "fkeys") {
2512+
// `-0.0D` rather than `CAST(-0.0 AS DOUBLE)`: the cast constant-folds to +0.0, which
2513+
// would make the signed-zero half of this test vacuous.
2514+
Seq("-0.0D", "0.0D").foreach { zero =>
2515+
val df = spark.sql(s"""SELECT * FROM fkeys
2516+
| PIVOT (sum(earnings) FOR k IN ($zero, double('NaN'), 1.5D))
2517+
| ORDER BY region""".stripMargin)
2518+
val plan = df.queryExecution.executedPlan
2519+
val pivotFirstAggs = collectWithSubqueries(plan) {
2520+
case a: CometHashAggregateExec
2521+
if a.aggregateExpressions.exists(_.aggregateFunction
2522+
.isInstanceOf[
2523+
org.apache.spark.sql.catalyst.expressions.aggregate.PivotFirst]) =>
2524+
a
2525+
}
2526+
assert(
2527+
pivotFirstAggs.nonEmpty,
2528+
s"Expected a CometHashAggregateExec containing PivotFirst for $zero in:\n$plan")
2529+
checkSparkAnswer(df)
2530+
}
2531+
}
2532+
}
2533+
}
2534+
24932535
}

0 commit comments

Comments
 (0)