@@ -2425,6 +2425,49 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
24252425 }
24262426 }
24272427
2428+ test(" statistical aggregates merge fractional constants across partitions" ) {
2429+ // https://github.com/apache/datafusion-comet/issues/6423
2430+ withSQLConf(
2431+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
2432+ SQLConf .SHUFFLE_PARTITIONS .key -> " 1" ,
2433+ SQLConf .FILES_MAX_PARTITION_BYTES .key -> " 1048576" ,
2434+ SQLConf .FILES_OPEN_COST_IN_BYTES .key -> " 1048576" ,
2435+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
2436+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ) {
2437+ val swapped = RegrSparkVersions .r2DegenerateCasesSwapped(spark.version)
2438+ val constantX : java.lang.Double = if (swapped) null else 1.0
2439+ val constantY : java.lang.Double = if (swapped) 1.0 else null
2440+ for (reverse <- Seq (false , true )) {
2441+ withTempPath { path =>
2442+ val starts = if (reverse) Seq (3 , 0 ) else Seq (0 , 3 )
2443+ for (start <- starts) {
2444+ (start until start + 3 )
2445+ .map(y => (0 , y.toDouble, 0.1 ))
2446+ .toDF(" g" , " y" , " x" )
2447+ .coalesce(1 )
2448+ .write
2449+ .mode(" append" )
2450+ .parquet(path.getCanonicalPath)
2451+ }
2452+ withParquetTable(path.getCanonicalPath, " fractional_constants" ) {
2453+ assert(spark.table(" fractional_constants" ).rdd.getNumPartitions == 2 )
2454+ for (groupBy <- Seq (" " , " GROUP BY g" )) {
2455+ val query = " SELECT regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), " +
2456+ " regr_sxx(y, x), regr_sxy(y, x), regr_r2(x, y), regr_syy(x, y) " +
2457+ " FROM fractional_constants" + groupBy
2458+ val (_, cometPlan) = checkSparkAnswerAndOperator(query)
2459+ val aggregates = cometPlan.collect { case a : CometHashAggregateExec => a }
2460+ assert(aggregates.exists(_.modes.contains(Partial )))
2461+ assert(aggregates.exists(_.modes.contains(Final )))
2462+ // Keep the degenerate-case results exact. A tolerance can hide nonzero M2.
2463+ checkAnswer(sql(query), Seq (Row (null , null , constantX, 0.0 , 0.0 , constantY, 0.0 )))
2464+ }
2465+ }
2466+ }
2467+ }
2468+ }
2469+ }
2470+
24282471 test(" var_pop and var_samp" ) {
24292472 withSQLConf(CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ) {
24302473 Seq (" native" , " jvm" ).foreach { cometShuffleMode =>
0 commit comments