@@ -3488,70 +3488,87 @@ class CometInMemoryCacheSuite extends CometTestBase {
34883488 // The order the join declares on its probe input reaches past the join, so the aggregate
34893489 // above it in the same native plan groups in sorted, streaming mode. That is right while
34903490 // Spark's reported order holds, and a wrong order would split a key across groups, which only
3491- // a comparison of the rows as a multiset catches. Small batches make each group span several.
3491+ // a comparison of the rows as a multiset catches. With 40 keys and 8-row batches each group
3492+ // spans several batches. With 1000 keys a batch holds four keys, alternately matched and
3493+ // unmatched, so a join that moved its unmatched rows would also change the order in which the
3494+ // aggregate first sees the keys, and its output would not come back sorted.
34923495 withSQLConf(
34933496 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
34943497 SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
34953498 SQLConf .SHUFFLE_PARTITIONS .key -> " 2" ,
34963499 SQLConf .COLUMN_BATCH_SIZE .key -> " 8" ,
34973500 CometConf .COMET_BATCH_SIZE .key -> " 8" ,
34983501 CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " true" ) {
3499- withParquetTable((0 until 2000 ).map(i => ((i % 40 ) * 1000 , i)), " big" ) {
3500- withParquetTable((0 until 40 by 2 ).map(m => (m * 1000 , m)), " small" ) {
3501- try {
3502- val streamed =
3503- spark.table(" big" ).repartition(2 , $" _1" ).sortWithinPartitions(" _1" ).cache()
3504- streamed.count()
3505- val build = spark.table(" small" )
3506-
3507- // Fifty rows per key, and a match in the build side for every even key.
3508- val expected = (0 until 40 ).map { m =>
3509- Row (m * 1000 , 50L , 50L * m + 49000L , if (m % 2 == 0 ) 50L else 0L )
3510- }
3511-
3512- for {
3513- (hint, joinClass) <- Seq (
3514- " shuffle_hash" -> classOf [CometHashJoinExec ],
3515- " broadcast" -> classOf [CometBroadcastHashJoinExec ])
3516- (joinType, buildSide) <- Seq (" left_outer" -> BuildRight , " right_outer" -> BuildLeft )
3517- } {
3518- val joined = if (joinType == " right_outer" ) {
3519- build.hint(hint).join(streamed, build(" _1" ) === streamed(" _1" ), joinType)
3520- } else {
3521- streamed.join(build.hint(hint), streamed(" _1" ) === build(" _1" ), joinType)
3502+ Seq (40 , 1000 ).foreach { keys =>
3503+ val data = (0 until 2000 ).map(i => ((i % keys) * 1000 , i))
3504+ withParquetTable(data, " big" ) {
3505+ withParquetTable((0 until keys by 2 ).map(m => (m * 1000 , m)), " small" ) {
3506+ try {
3507+ val streamed =
3508+ spark.table(" big" ).repartition(2 , $" _1" ).sortWithinPartitions(" _1" ).cache()
3509+ streamed.count()
3510+ val build = spark.table(" small" )
3511+
3512+ // A match in the build side for every even key.
3513+ val expected = data.groupBy(_._1).toSeq.map { case (k, rows) =>
3514+ Row (
3515+ k,
3516+ rows.size.toLong,
3517+ rows.map(_._2.toLong).sum,
3518+ if ((k / 1000 ) % 2 == 0 ) {
3519+ rows.size.toLong
3520+ } else 0L )
35223521 }
3523- val df = joined
3524- .groupBy(streamed(" _1" ))
3525- .agg(count(lit(1 )), sum(streamed(" _2" )), count(build(" _2" )))
3526- val plan = df.queryExecution.executedPlan
3527- val joins : Seq [(SparkPlan , BuildSide )] = collect(plan) {
3528- case j : CometHashJoinExec => (j, j.buildSide)
3529- case j : CometBroadcastHashJoinExec => (j, j.buildSide)
3522+
3523+ for {
3524+ (hint, joinClass) <- Seq (
3525+ " shuffle_hash" -> classOf [CometHashJoinExec ],
3526+ " broadcast" -> classOf [CometBroadcastHashJoinExec ])
3527+ (joinType, buildSide) <- Seq (
3528+ " left_outer" -> BuildRight ,
3529+ " right_outer" -> BuildLeft )
3530+ } {
3531+ val joined = if (joinType == " right_outer" ) {
3532+ build.hint(hint).join(streamed, build(" _1" ) === streamed(" _1" ), joinType)
3533+ } else {
3534+ streamed.join(build.hint(hint), streamed(" _1" ) === build(" _1" ), joinType)
3535+ }
3536+ val df = joined
3537+ .groupBy(streamed(" _1" ))
3538+ .agg(count(lit(1 )), sum(streamed(" _2" )), count(build(" _2" )))
3539+ val plan = df.queryExecution.executedPlan
3540+ val joins : Seq [(SparkPlan , BuildSide )] = collect(plan) {
3541+ case j : CometHashJoinExec => (j, j.buildSide)
3542+ case j : CometBroadcastHashJoinExec => (j, j.buildSide)
3543+ }
3544+ assert(
3545+ joins.map { case (j, side) => (j.getClass, side) } == Seq (
3546+ (joinClass, buildSide)),
3547+ plan)
3548+ // The serde sent the order, which is what lets the aggregates above run in
3549+ // sorted mode, and a sorted aggregate emits its groups in key order.
3550+ assert(
3551+ collect(plan) {
3552+ case j : CometHashJoinExec => j.nativeOp
3553+ case j : CometBroadcastHashJoinExec => j.nativeOp
3554+ }.forall(_.getHashJoin.getStreamedSortOrdersCount > 0 ),
3555+ plan)
3556+ assert(sortedPartitions(df).forall(identity), plan)
3557+ // A final aggregate after a shuffle would merge a key a partial aggregate split,
3558+ // so both aggregates have to sit above the join with no exchange in between.
3559+ def aboveJoin (p : SparkPlan ): Boolean = p.find(_ eq joins.head._1).isDefined
3560+ assert(
3561+ collect(plan) { case a : CometHashAggregateExec => a }.count(aboveJoin) == 2 ,
3562+ plan)
3563+ assert(
3564+ ! collect(plan) { case e : ShuffleExchangeLike => e }.exists(aboveJoin),
3565+ plan)
3566+ assert(collect(plan) { case s : CometInMemoryTableScanExec => s }.size == 1 , plan)
3567+ QueryTest .sameRows(expected, df.collect().toSeq).foreach(fail(_))
35303568 }
3531- assert(
3532- joins.map { case (j, side) => (j.getClass, side) } == Seq ((joinClass, buildSide)),
3533- plan)
3534- // The serde sent the order, which is what lets the aggregates above run in sorted
3535- // mode, and a sorted aggregate emits its groups in key order.
3536- assert(
3537- collect(plan) {
3538- case j : CometHashJoinExec => j.nativeOp
3539- case j : CometBroadcastHashJoinExec => j.nativeOp
3540- }.forall(_.getHashJoin.getStreamedSortOrdersCount > 0 ),
3541- plan)
3542- assert(sortedPartitions(df).forall(identity), plan)
3543- // A final aggregate after a shuffle would merge a key a partial aggregate split, so
3544- // both aggregates have to sit above the join with no exchange in between.
3545- def aboveJoin (p : SparkPlan ): Boolean = p.find(_ eq joins.head._1).isDefined
3546- assert(
3547- collect(plan) { case a : CometHashAggregateExec => a }.count(aboveJoin) == 2 ,
3548- plan)
3549- assert(! collect(plan) { case e : ShuffleExchangeLike => e }.exists(aboveJoin), plan)
3550- assert(collect(plan) { case s : CometInMemoryTableScanExec => s }.size == 1 , plan)
3551- QueryTest .sameRows(expected, df.collect().toSeq).foreach(fail(_))
3569+ } finally {
3570+ spark.catalog.clearCache()
35523571 }
3553- } finally {
3554- spark.catalog.clearCache()
35553572 }
35563573 }
35573574 }
0 commit comments