Skip to content

Commit 0cc2c7f

Browse files
test: tighten corr ANSI coverage per review
Use expect_error_class in corr.sql so the ANSI queries also check native operators, exception class and SQLSTATE. Keep only the ANSI branch of the Scala test, whose LEGACY checks repeat the fractional-constants test, and drop a Rust test loop over null_on_divide_by_zero that never reaches the count == 1 branch.
1 parent 0c79b31 commit 0cc2c7f

3 files changed

Lines changed: 50 additions & 74 deletions

File tree

‎native/spark-expr/src/agg_funcs/correlation.rs‎

Lines changed: 33 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -677,54 +677,42 @@ mod groups_tests {
677677
]
678678
};
679679
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
680-
for null_on_divide_by_zero in [false, true] {
681-
let mut scalar = CorrelationAccumulator::try_new(
682-
null_on_divide_by_zero,
683-
zero_divisor(eval_mode),
684-
)
680+
// n is 6 everywhere here, so null_on_divide_by_zero, which only the
681+
// count == 1 branch reads, does not matter.
682+
let mut scalar =
683+
CorrelationAccumulator::try_new(true, zero_divisor(eval_mode)).unwrap();
684+
let mut grouped = CorrelationGroupsAccumulator::new(true, zero_divisor(eval_mode));
685+
scalar.update_batch(&input(0..6)).unwrap();
686+
grouped
687+
.update_batch(&input(0..6), &[0; 6], None, 1)
685688
.unwrap();
686-
let mut grouped = CorrelationGroupsAccumulator::new(
687-
null_on_divide_by_zero,
688-
zero_divisor(eval_mode),
689-
);
690-
scalar.update_batch(&input(0..6)).unwrap();
691-
grouped
692-
.update_batch(&input(0..6), &[0; 6], None, 1)
693-
.unwrap();
694-
let mut scalars = vec![scalar];
695-
let mut groups = vec![grouped];
696-
// Merging partials in either order must keep xMk exactly zero.
697-
for ranges in [[0..3, 3..6], [3..6, 0..3]] {
698-
let mut scalar = CorrelationAccumulator::try_new(
699-
null_on_divide_by_zero,
700-
zero_divisor(eval_mode),
701-
)
702-
.unwrap();
703-
let mut grouped = CorrelationGroupsAccumulator::new(
704-
null_on_divide_by_zero,
705-
zero_divisor(eval_mode),
706-
);
707-
for range in ranges {
708-
let state = partial_state(&input(range));
709-
scalar.merge_batch(&state).unwrap();
710-
grouped.merge_batch(&state, &[0], 1).unwrap();
711-
}
712-
scalars.push(scalar);
713-
groups.push(grouped);
689+
let mut scalars = vec![scalar];
690+
let mut groups = vec![grouped];
691+
// Merging partials in either order must keep xMk exactly zero.
692+
for ranges in [[0..3, 3..6], [3..6, 0..3]] {
693+
let mut scalar =
694+
CorrelationAccumulator::try_new(true, zero_divisor(eval_mode)).unwrap();
695+
let mut grouped = CorrelationGroupsAccumulator::new(true, zero_divisor(eval_mode));
696+
for range in ranges {
697+
let state = partial_state(&input(range));
698+
scalar.merge_batch(&state).unwrap();
699+
grouped.merge_batch(&state, &[0], 1).unwrap();
714700
}
715-
for mut scalar in scalars {
716-
if eval_mode == EvalMode::Ansi {
717-
assert_divide_by_zero(scalar.evaluate());
718-
} else {
719-
assert_eq!(scalar.evaluate().unwrap(), ScalarValue::Float64(None));
720-
}
701+
scalars.push(scalar);
702+
groups.push(grouped);
703+
}
704+
for mut scalar in scalars {
705+
if eval_mode == EvalMode::Ansi {
706+
assert_divide_by_zero(scalar.evaluate());
707+
} else {
708+
assert_eq!(scalar.evaluate().unwrap(), ScalarValue::Float64(None));
721709
}
722-
for mut grouped in groups {
723-
if eval_mode == EvalMode::Ansi {
724-
assert_divide_by_zero(grouped.evaluate(EmitTo::All));
725-
} else {
726-
assert_eq!(evaluate(&mut grouped), vec![None]);
727-
}
710+
}
711+
for mut grouped in groups {
712+
if eval_mode == EvalMode::Ansi {
713+
assert_divide_by_zero(grouped.evaluate(EmitTo::All));
714+
} else {
715+
assert_eq!(evaluate(&mut grouped), vec![None]);
728716
}
729717
}
730718
}

‎spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ SELECT grp, corr(y, x), corr(x, y) FROM test_corr_const GROUP BY grp ORDER BY gr
5555
statement
5656
SET spark.sql.ansi.enabled=true
5757

58-
query expect_error(DIVIDE_BY_ZERO)
58+
query expect_error_class(DIVIDE_BY_ZERO)
5959
SELECT corr(y, x) FROM test_corr_const WHERE grp = 'a'
6060

61-
query expect_error(DIVIDE_BY_ZERO)
61+
query expect_error_class(DIVIDE_BY_ZERO)
6262
SELECT grp, corr(x, y) FROM test_corr_const GROUP BY grp
6363

6464
-- Spark checks for zero and one rows before dividing, even in ANSI mode

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 15 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3415,37 +3415,25 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
34153415
}
34163416
withParquetTable(path.getCanonicalPath, "corr_constant") {
34173417
assert(spark.table("corr_constant").rdd.getNumPartitions == numPartitions)
3418+
// The LEGACY results on this data are covered by "statistical aggregates merge
3419+
// fractional constants across partitions".
34183420
for {
3419-
ansi <- Seq(false, true)
34203421
groupBy <- Seq("", " GROUP BY g")
3422+
args <- Seq("y, x", "x, y", "x, x")
34213423
} {
3422-
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
3423-
for (args <- Seq("y, x", "x, y", "x, x")) {
3424-
val query = s"SELECT corr($args) FROM corr_constant" + groupBy
3425-
val df = sql(query)
3426-
val aggregates = stripAQEPlan(df.queryExecution.executedPlan).collect {
3427-
case a: CometHashAggregateExec => a
3428-
}
3429-
assert(aggregates.size == 2)
3430-
assert(aggregates.exists(_.modes.contains(Partial)))
3431-
assert(aggregates.exists(_.modes.contains(Final)))
3432-
// Spark divides ck by sqrt(xMk * yMk) = 0.
3433-
if (ansi) {
3434-
val error = checkSparkError(df, "DIVIDE_BY_ZERO")
3435-
assert(error.getSqlState == "22012")
3436-
checkQueryContextMatchesSpark(error, query)
3437-
} else {
3438-
checkSparkAnswerAndOperator(df)
3439-
checkCometAnswer(df, Seq(Row(null)))
3440-
}
3424+
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
3425+
val query = s"SELECT corr($args) FROM corr_constant" + groupBy
3426+
val df = sql(query)
3427+
val aggregates = stripAQEPlan(df.queryExecution.executedPlan).collect {
3428+
case a: CometHashAggregateExec => a
34413429
}
3442-
3443-
// The other statistical aggregates have no zero divisor here.
3444-
val statsQuery = "SELECT covar_pop(y, x), covar_samp(y, x), var_pop(x), " +
3445-
"var_samp(x), stddev_pop(x), stddev_samp(x), corr(y, y) " +
3446-
"FROM corr_constant" + groupBy
3447-
checkSparkAnswerAndOperator(statsQuery)
3448-
checkAnswer(sql(statsQuery), Seq(Row(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0)))
3430+
assert(aggregates.size == 2)
3431+
assert(aggregates.exists(_.modes.contains(Partial)))
3432+
assert(aggregates.exists(_.modes.contains(Final)))
3433+
// Spark divides ck by sqrt(xMk * yMk) = 0.
3434+
val error = checkSparkError(df, "DIVIDE_BY_ZERO")
3435+
assert(error.getSqlState == "22012")
3436+
checkQueryContextMatchesSpark(error, query)
34493437
}
34503438
}
34513439
}

0 commit comments

Comments
 (0)