@@ -37,8 +37,9 @@ import org.apache.spark.{SparkConf, SparkException}
3737import org .apache .spark .CometDriverPlugin
3838import org .apache .spark .sql .{CometTestBase , DataFrame , Observation , QueryTest , Row }
3939import org .apache .spark .sql .catalyst .expressions .{And , Attribute , AttributeReference , EqualTo , Expression , GreaterThan , GreaterThanOrEqual , LessThan , Literal }
40+ import org .apache .spark .sql .catalyst .optimizer .{BuildLeft , BuildRight , BuildSide }
4041import org .apache .spark .sql .columnar .{CachedBatch , SimpleMetricsCachedBatch }
41- import org .apache .spark .sql .comet .{CometBroadcastHashJoinExec , CometHashJoinExec , CometInMemoryTableScanExec , CometSortAggregateExec , CometSortExec , CometSortMergeJoinExec }
42+ import org .apache .spark .sql .comet .{CometBroadcastHashJoinExec , CometHashAggregateExec , CometHashJoinExec , CometInMemoryTableScanExec , CometSortAggregateExec , CometSortExec , CometSortMergeJoinExec }
4243import org .apache .spark .sql .comet .execution .arrow .{ArrowCachedBatchSerializer , CometCachedBatchHelper }
4344import org .apache .spark .sql .comet .execution .shuffle .CometCelebornShuffleManager
4445import org .apache .spark .sql .comet .util .Utils
@@ -3292,9 +3293,10 @@ class CometInMemoryCacheSuite extends CometTestBase {
32923293 test(" hash joins over a sorted cache keep the streamed side's order" ) {
32933294 // CometInMemoryTableScan reports the cached plan's sort order, so Spark drops the sort it
32943295 // would otherwise plan above a join that keeps that order. DataFusion's hash join keeps the
3295- // unmatched probe rows of a left outer join built on the right in place only when the probe
3296- // input declares an order, which the native scan over the cache does not, so the native join
3297- // declares the cache's order on its probe input. An inner join never moves probe rows.
3296+ // unmatched probe rows of a left outer join built on the right, or of a right outer join built
3297+ // on the left, in place only when the probe input declares an order, which the native scan
3298+ // over the cache does not, so the native join declares the cache's order on its probe input.
3299+ // An inner join never moves probe rows.
32983300 withSQLConf(
32993301 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
33003302 SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
@@ -3308,11 +3310,21 @@ class CometInMemoryCacheSuite extends CometTestBase {
33083310 streamed.count()
33093311 val build = spark.table(" small" )
33103312
3311- def query (hint : String , joinType : String ): DataFrame =
3312- streamed
3313- .join(build.hint(hint), streamed(" _1" ) === build(" _1" ), joinType)
3313+ def query (hint : String , joinType : String ): DataFrame = {
3314+ val joined = if (joinType == " right_outer" ) {
3315+ build.hint(hint).join(streamed, build(" _1" ) === streamed(" _1" ), joinType)
3316+ } else {
3317+ streamed.join(build.hint(hint), streamed(" _1" ) === build(" _1" ), joinType)
3318+ }
3319+ joined
33143320 .select(streamed(" _1" ).as(" k" ), streamed(" _2" ).as(" v" ), build(" _2" ).as(" w" ))
33153321 .sortWithinPartitions(" k" )
3322+ }
3323+
3324+ def joins (plan : SparkPlan ): Seq [(Class [_], BuildSide )] = collect(plan) {
3325+ case j : CometHashJoinExec => (j.getClass, j.buildSide)
3326+ case j : CometBroadcastHashJoinExec => (j.getClass, j.buildSide)
3327+ }
33163328
33173329 def sortedPartitions (df : DataFrame ): Array [Boolean ] =
33183330 df.queryExecution.toRdd
@@ -3335,19 +3347,24 @@ class CometInMemoryCacheSuite extends CometTestBase {
33353347 " shuffle_hash" -> classOf [CometHashJoinExec ],
33363348 " broadcast" -> classOf [CometBroadcastHashJoinExec ]).foreach {
33373349 case (hint, joinClass) =>
3338- val outer = query(hint, " left_outer" )
3339- val outerPlan = outer.queryExecution.executedPlan
3340- assert(collect(outerPlan) { case j if j.getClass == joinClass => j }.size == 1 )
3341- assert(collect(outerPlan) { case s : CometInMemoryTableScanExec => s }.size == 1 )
3342- assert(sorts(outerPlan).isEmpty, outerPlan)
3343- assert(outer.collect().toSet == outerRows.toSet)
3344- assert(sortedPartitions(outer).forall(identity), outerPlan)
3350+ Seq (" left_outer" -> BuildRight , " right_outer" -> BuildLeft ).foreach {
3351+ case (joinType, buildSide) =>
3352+ val outer = query(hint, joinType)
3353+ val outerPlan = outer.queryExecution.executedPlan
3354+ assert(joins(outerPlan) == Seq ((joinClass, buildSide)), outerPlan)
3355+ assert(collect(outerPlan) { case s : CometInMemoryTableScanExec =>
3356+ s
3357+ }.size == 1 )
3358+ assert(sorts(outerPlan).isEmpty, outerPlan)
3359+ QueryTest .sameRows(outerRows, outer.collect().toSeq).foreach(fail(_))
3360+ assert(sortedPartitions(outer).forall(identity), outerPlan)
3361+ }
33453362
33463363 val inner = query(hint, " inner" )
33473364 val innerPlan = inner.queryExecution.executedPlan
3348- assert(collect (innerPlan) { case j if j.getClass == joinClass => j }.size == 1 )
3365+ assert(joins (innerPlan) == Seq (( joinClass, BuildRight )), innerPlan )
33493366 assert(sorts(innerPlan).isEmpty, innerPlan)
3350- assert( inner.collect().toSet == innerRows.toSet )
3367+ QueryTest .sameRows(innerRows, inner.collect().toSeq).foreach(fail(_) )
33513368 assert(sortedPartitions(inner).forall(identity), innerPlan)
33523369 }
33533370 } finally {
@@ -3357,4 +3374,59 @@ class CometInMemoryCacheSuite extends CometTestBase {
33573374 }
33583375 }
33593376 }
3377+
3378+ test(" an aggregate on the key above a hash join over a sorted cache groups each key once" ) {
3379+ // The order the join declares on its probe input reaches past the join, so the aggregate
3380+ // above it in the same native plan groups in sorted, streaming mode. That is right while
3381+ // Spark's reported order holds, and a wrong order would split a key across groups, which only
3382+ // a comparison of the rows as a multiset catches. Small batches make each group span several.
3383+ withSQLConf(
3384+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
3385+ SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
3386+ SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ,
3387+ SQLConf .COLUMN_BATCH_SIZE .key -> " 8" ,
3388+ CometConf .COMET_BATCH_SIZE .key -> " 8" ,
3389+ CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " true" ) {
3390+ withParquetTable((0 until 2000 ).map(i => ((i % 40 ) * 1000 , i)), " big" ) {
3391+ withParquetTable((0 until 40 by 2 ).map(m => (m * 1000 , m)), " small" ) {
3392+ try {
3393+ val streamed =
3394+ spark.table(" big" ).repartition(2 , $" _1" ).sortWithinPartitions(" _1" ).cache()
3395+ streamed.count()
3396+ val build = spark.table(" small" )
3397+
3398+ // Fifty rows per key, and a match in the build side for every even key.
3399+ val expected = (0 until 40 ).map { m =>
3400+ Row (m * 1000 , 50L , 50L * m + 49000L , if (m % 2 == 0 ) 50L else 0L )
3401+ }
3402+
3403+ for {
3404+ hint <- Seq (" shuffle_hash" , " broadcast" )
3405+ joinType <- Seq (" left_outer" , " right_outer" )
3406+ } {
3407+ val joined = if (joinType == " right_outer" ) {
3408+ build.hint(hint).join(streamed, build(" _1" ) === streamed(" _1" ), joinType)
3409+ } else {
3410+ streamed.join(build.hint(hint), streamed(" _1" ) === build(" _1" ), joinType)
3411+ }
3412+ val df = joined
3413+ .groupBy(streamed(" _1" ))
3414+ .agg(count(lit(1 )), sum(streamed(" _2" )), count(build(" _2" )))
3415+ val plan = df.queryExecution.executedPlan
3416+ assert(
3417+ collect(plan) { case j @ (_ : CometHashJoinExec | _ : CometBroadcastHashJoinExec ) =>
3418+ j
3419+ }.size == 1 ,
3420+ plan)
3421+ assert(collect(plan) { case a : CometHashAggregateExec => a }.size == 2 , plan)
3422+ assert(collect(plan) { case s : CometInMemoryTableScanExec => s }.size == 1 , plan)
3423+ QueryTest .sameRows(expected, df.collect().toSeq).foreach(fail(_))
3424+ }
3425+ } finally {
3426+ spark.catalog.clearCache()
3427+ }
3428+ }
3429+ }
3430+ }
3431+ }
33603432}
0 commit comments