Skip to content

Commit d89c385

Browse files
committed
test: cover regression aggregates merging fractional constants
1 parent a16b7e1 commit d89c385

2 files changed

Lines changed: 88 additions & 0 deletions

File tree

‎native/spark-expr/src/agg_funcs/regr.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -721,6 +721,51 @@ mod tests {
721721
}
722722
}
723723

724+
#[test]
725+
fn merge_fractional_constant_partials() {
726+
// #6423: the first merge into an empty final buffer used to round the
727+
// constant mean away from 0.1. Later merges then produced nonzero M2.
728+
for (kind, constant_dependent, expected) in [
729+
(RegrType::Slope, false, None),
730+
(RegrType::Intercept, false, None),
731+
(RegrType::R2, false, None),
732+
(RegrType::R2, true, Some(1.0)),
733+
(RegrType::SXX, false, Some(0.0)),
734+
(RegrType::SYY, true, Some(0.0)),
735+
(RegrType::SXY, false, Some(0.0)),
736+
] {
737+
for reverse in [false, true] {
738+
let mut merged = acc(kind);
739+
let starts = if reverse { [3, 0] } else { [0, 3] };
740+
for start in starts {
741+
let varying = (start..start + 3).map(|v| Some(v as f64)).collect();
742+
let constant = vec![Some(0.1); 3];
743+
let values = match kind {
744+
// Spark's RegrReplacement duplicates the selected column.
745+
RegrType::SXX | RegrType::SYY => cols(constant.clone(), constant),
746+
_ if constant_dependent => cols(constant, varying),
747+
_ => cols(varying, constant),
748+
};
749+
let mut partial = acc(kind);
750+
partial.update_batch(&values).unwrap();
751+
let state = partial
752+
.state()
753+
.unwrap()
754+
.iter()
755+
.map(|v| v.to_array_of_size(1).unwrap())
756+
.collect::<Vec<_>>();
757+
merged.merge_batch(&state).unwrap();
758+
}
759+
// A tolerance would hide the tiny nonzero moments behind the bug.
760+
assert_eq!(
761+
merged.evaluate().unwrap(),
762+
ScalarValue::Float64(expected),
763+
"{kind:?}, constant_dependent={constant_dependent}, reverse={reverse}"
764+
);
765+
}
766+
}
767+
}
768+
724769
#[test]
725770
fn merge_matches_single_batch() {
726771
let (y, x) = perfect_line();

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2425,6 +2425,49 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
24252425
}
24262426
}
24272427

2428+
test("statistical aggregates merge fractional constants across partitions") {
2429+
// https://github.com/apache/datafusion-comet/issues/6423
2430+
withSQLConf(
2431+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
2432+
SQLConf.SHUFFLE_PARTITIONS.key -> "1",
2433+
SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1048576",
2434+
SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "1048576",
2435+
CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
2436+
CometConf.COMET_SHUFFLE_MODE.key -> "native") {
2437+
val swapped = RegrSparkVersions.r2DegenerateCasesSwapped(spark.version)
2438+
val constantX: java.lang.Double = if (swapped) null else 1.0
2439+
val constantY: java.lang.Double = if (swapped) 1.0 else null
2440+
for (reverse <- Seq(false, true)) {
2441+
withTempPath { path =>
2442+
val starts = if (reverse) Seq(3, 0) else Seq(0, 3)
2443+
for (start <- starts) {
2444+
(start until start + 3)
2445+
.map(y => (0, y.toDouble, 0.1))
2446+
.toDF("g", "y", "x")
2447+
.coalesce(1)
2448+
.write
2449+
.mode("append")
2450+
.parquet(path.getCanonicalPath)
2451+
}
2452+
withParquetTable(path.getCanonicalPath, "fractional_constants") {
2453+
assert(spark.table("fractional_constants").rdd.getNumPartitions == 2)
2454+
for (groupBy <- Seq("", " GROUP BY g")) {
2455+
val query = "SELECT regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), " +
2456+
"regr_sxx(y, x), regr_sxy(y, x), regr_r2(x, y), regr_syy(x, y) " +
2457+
"FROM fractional_constants" + groupBy
2458+
val (_, cometPlan) = checkSparkAnswerAndOperator(query)
2459+
val aggregates = cometPlan.collect { case a: CometHashAggregateExec => a }
2460+
assert(aggregates.exists(_.modes.contains(Partial)))
2461+
assert(aggregates.exists(_.modes.contains(Final)))
2462+
// Keep the degenerate-case results exact. A tolerance can hide nonzero M2.
2463+
checkAnswer(sql(query), Seq(Row(null, null, constantX, 0.0, 0.0, constantY, 0.0)))
2464+
}
2465+
}
2466+
}
2467+
}
2468+
}
2469+
}
2470+
24282471
test("var_pop and var_samp") {
24292472
withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") {
24302473
Seq("native", "jvm").foreach { cometShuffleMode =>

0 commit comments

Comments
 (0)