Skip to content

Commit 42db91b

Browse files
authored
test: expect Spark 3.5's empty LocalTableScan in the single-split DPP join test (apache#6902)
With AQE on, once DPP prunes one shuffled input, AQEPropagateEmptyRelation replaces the join with an empty relation. Spark 4.0 plans that as EmptyRelationExec, which Comet converts to CometEmptyRelationExec, but Spark 3.5 has no EmptyRelation node and plans an empty LocalTableScanExec. Check for the node each version plans. Closes apache#6901.
1 parent c937df0 commit 42db91b

1 file changed

Lines changed: 13 additions & 2 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
3636
import org.apache.spark.sql.catalyst.expressions.DynamicPruningExpression
3737
import org.apache.spark.sql.comet._
3838
import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec}
39-
import org.apache.spark.sql.execution.{InSubqueryExec, ReusedSubqueryExec, SparkPlan, SubqueryAdaptiveBroadcastExec, SubqueryBroadcastExec, SubqueryExec}
39+
import org.apache.spark.sql.execution.{InSubqueryExec, LocalTableScanExec, ReusedSubqueryExec, SparkPlan, SubqueryAdaptiveBroadcastExec, SubqueryBroadcastExec, SubqueryExec}
4040
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, BroadcastQueryStageExec}
4141
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
4242
import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
@@ -5322,10 +5322,21 @@ class CometIcebergNativeSuite
53225322
if (isSpark35Plus && aqeEnabled && scans.isEmpty) {
53235323
// Newer Spark versions shuffle the join inputs, allowing AQE to replace
53245324
// the join and its scans with an empty relation after DPP prunes one input.
5325+
// Spark 4.0+ (SPARK-48466) plans it as EmptyRelationExec, which Comet
5326+
// converts to CometEmptyRelationExec. Spark 3.5 has no EmptyRelation node
5327+
// and plans an empty LocalTableScanExec instead.
53255328
// This checks result compatibility; the SinglePartition regression assertions
53265329
// below target Spark 3.4, where both scans must remain in one native block.
5330+
val emptyRelations = if (isSpark40Plus) {
5331+
collect(plan) { case empty: CometEmptyRelationExec => empty }
5332+
} else {
5333+
collect(plan) {
5334+
case empty: LocalTableScanExec if empty.rows.isEmpty => empty
5335+
case empty: CometLocalTableScanExec if empty.rows.isEmpty => empty
5336+
}
5337+
}
53275338
assert(
5328-
collect(plan) { case empty: CometEmptyRelationExec => empty }.nonEmpty,
5339+
emptyRelations.nonEmpty,
53295340
s"Expected AQE to replace the pruned join with an empty relation:\n$plan")
53305341
} else {
53315342
assert(scans.size == 2, s"Expected two native Iceberg scans:\n$plan")

0 commit comments

Comments
 (0)