Skip to content

Commit 8e75130

Browse files
committed
test: make the aggregate test fail when the join's order is not declared
With 40 keys of 50 rows each, a join that moved its unmatched rows still fed the aggregate the keys in first-seen order, so the test passed without the declaration. Add a layout of 1000 keys with two rows each, where a batch holds alternately matched and unmatched keys, and assert that the serde sent the order and that each partition comes back in key order.
1 parent 5adbeb4 commit 8e75130

1 file changed

Lines changed: 71 additions & 54 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 71 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -3488,70 +3488,87 @@ class CometInMemoryCacheSuite extends CometTestBase {
34883488
// The order the join declares on its probe input reaches past the join, so the aggregate
34893489
// above it in the same native plan groups in sorted, streaming mode. That is right while
34903490
// Spark's reported order holds, and a wrong order would split a key across groups, which only
3491-
// a comparison of the rows as a multiset catches. Small batches make each group span several.
3491+
// a comparison of the rows as a multiset catches. With 40 keys and 8-row batches each group
3492+
// spans several batches. With 1000 keys a batch holds four keys, alternately matched and
3493+
// unmatched, so a join that moved its unmatched rows would also change the order in which the
3494+
// aggregate first sees the keys, and its output would not come back sorted.
34923495
withSQLConf(
34933496
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
34943497
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
34953498
SQLConf.SHUFFLE_PARTITIONS.key -> "2",
34963499
SQLConf.COLUMN_BATCH_SIZE.key -> "8",
34973500
CometConf.COMET_BATCH_SIZE.key -> "8",
34983501
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") {
3499-
withParquetTable((0 until 2000).map(i => ((i % 40) * 1000, i)), "big") {
3500-
withParquetTable((0 until 40 by 2).map(m => (m * 1000, m)), "small") {
3501-
try {
3502-
val streamed =
3503-
spark.table("big").repartition(2, $"_1").sortWithinPartitions("_1").cache()
3504-
streamed.count()
3505-
val build = spark.table("small")
3506-
3507-
// Fifty rows per key, and a match in the build side for every even key.
3508-
val expected = (0 until 40).map { m =>
3509-
Row(m * 1000, 50L, 50L * m + 49000L, if (m % 2 == 0) 50L else 0L)
3510-
}
3511-
3512-
for {
3513-
(hint, joinClass) <- Seq(
3514-
"shuffle_hash" -> classOf[CometHashJoinExec],
3515-
"broadcast" -> classOf[CometBroadcastHashJoinExec])
3516-
(joinType, buildSide) <- Seq("left_outer" -> BuildRight, "right_outer" -> BuildLeft)
3517-
} {
3518-
val joined = if (joinType == "right_outer") {
3519-
build.hint(hint).join(streamed, build("_1") === streamed("_1"), joinType)
3520-
} else {
3521-
streamed.join(build.hint(hint), streamed("_1") === build("_1"), joinType)
3502+
Seq(40, 1000).foreach { keys =>
3503+
val data = (0 until 2000).map(i => ((i % keys) * 1000, i))
3504+
withParquetTable(data, "big") {
3505+
withParquetTable((0 until keys by 2).map(m => (m * 1000, m)), "small") {
3506+
try {
3507+
val streamed =
3508+
spark.table("big").repartition(2, $"_1").sortWithinPartitions("_1").cache()
3509+
streamed.count()
3510+
val build = spark.table("small")
3511+
3512+
// A match in the build side for every even key.
3513+
val expected = data.groupBy(_._1).toSeq.map { case (k, rows) =>
3514+
Row(
3515+
k,
3516+
rows.size.toLong,
3517+
rows.map(_._2.toLong).sum,
3518+
if ((k / 1000) % 2 == 0) {
3519+
rows.size.toLong
3520+
} else 0L)
35223521
}
3523-
val df = joined
3524-
.groupBy(streamed("_1"))
3525-
.agg(count(lit(1)), sum(streamed("_2")), count(build("_2")))
3526-
val plan = df.queryExecution.executedPlan
3527-
val joins: Seq[(SparkPlan, BuildSide)] = collect(plan) {
3528-
case j: CometHashJoinExec => (j, j.buildSide)
3529-
case j: CometBroadcastHashJoinExec => (j, j.buildSide)
3522+
3523+
for {
3524+
(hint, joinClass) <- Seq(
3525+
"shuffle_hash" -> classOf[CometHashJoinExec],
3526+
"broadcast" -> classOf[CometBroadcastHashJoinExec])
3527+
(joinType, buildSide) <- Seq(
3528+
"left_outer" -> BuildRight,
3529+
"right_outer" -> BuildLeft)
3530+
} {
3531+
val joined = if (joinType == "right_outer") {
3532+
build.hint(hint).join(streamed, build("_1") === streamed("_1"), joinType)
3533+
} else {
3534+
streamed.join(build.hint(hint), streamed("_1") === build("_1"), joinType)
3535+
}
3536+
val df = joined
3537+
.groupBy(streamed("_1"))
3538+
.agg(count(lit(1)), sum(streamed("_2")), count(build("_2")))
3539+
val plan = df.queryExecution.executedPlan
3540+
val joins: Seq[(SparkPlan, BuildSide)] = collect(plan) {
3541+
case j: CometHashJoinExec => (j, j.buildSide)
3542+
case j: CometBroadcastHashJoinExec => (j, j.buildSide)
3543+
}
3544+
assert(
3545+
joins.map { case (j, side) => (j.getClass, side) } == Seq(
3546+
(joinClass, buildSide)),
3547+
plan)
3548+
// The serde sent the order, which is what lets the aggregates above run in
3549+
// sorted mode, and a sorted aggregate emits its groups in key order.
3550+
assert(
3551+
collect(plan) {
3552+
case j: CometHashJoinExec => j.nativeOp
3553+
case j: CometBroadcastHashJoinExec => j.nativeOp
3554+
}.forall(_.getHashJoin.getStreamedSortOrdersCount > 0),
3555+
plan)
3556+
assert(sortedPartitions(df).forall(identity), plan)
3557+
// A final aggregate after a shuffle would merge a key a partial aggregate split,
3558+
// so both aggregates have to sit above the join with no exchange in between.
3559+
def aboveJoin(p: SparkPlan): Boolean = p.find(_ eq joins.head._1).isDefined
3560+
assert(
3561+
collect(plan) { case a: CometHashAggregateExec => a }.count(aboveJoin) == 2,
3562+
plan)
3563+
assert(
3564+
!collect(plan) { case e: ShuffleExchangeLike => e }.exists(aboveJoin),
3565+
plan)
3566+
assert(collect(plan) { case s: CometInMemoryTableScanExec => s }.size == 1, plan)
3567+
QueryTest.sameRows(expected, df.collect().toSeq).foreach(fail(_))
35303568
}
3531-
assert(
3532-
joins.map { case (j, side) => (j.getClass, side) } == Seq((joinClass, buildSide)),
3533-
plan)
3534-
// The serde sent the order, which is what lets the aggregates above run in sorted
3535-
// mode, and a sorted aggregate emits its groups in key order.
3536-
assert(
3537-
collect(plan) {
3538-
case j: CometHashJoinExec => j.nativeOp
3539-
case j: CometBroadcastHashJoinExec => j.nativeOp
3540-
}.forall(_.getHashJoin.getStreamedSortOrdersCount > 0),
3541-
plan)
3542-
assert(sortedPartitions(df).forall(identity), plan)
3543-
// A final aggregate after a shuffle would merge a key a partial aggregate split, so
3544-
// both aggregates have to sit above the join with no exchange in between.
3545-
def aboveJoin(p: SparkPlan): Boolean = p.find(_ eq joins.head._1).isDefined
3546-
assert(
3547-
collect(plan) { case a: CometHashAggregateExec => a }.count(aboveJoin) == 2,
3548-
plan)
3549-
assert(!collect(plan) { case e: ShuffleExchangeLike => e }.exists(aboveJoin), plan)
3550-
assert(collect(plan) { case s: CometInMemoryTableScanExec => s }.size == 1, plan)
3551-
QueryTest.sameRows(expected, df.collect().toSeq).foreach(fail(_))
3569+
} finally {
3570+
spark.catalog.clearCache()
35523571
}
3553-
} finally {
3554-
spark.catalog.clearCache()
35553572
}
35563573
}
35573574
}

0 commit comments

Comments
 (0)