Skip to content

Commit aaca944

Browse files
committed
address review: make native cartesian product opt-in (default false)
DataFusion's CrossJoinExec buffers the whole left input and cannot spill, so a large left can OOM where Spark's spillable CartesianProduct succeeds (sunchao's P2). Default the flag to false until the native path can spill; add a regression test that a cross join stays on Spark by default. Part of apache#199.
1 parent 9c9292a commit aaca944

2 files changed

Lines changed: 28 additions & 1 deletion

File tree

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,14 @@ object CometConf extends ShimCometConf {
362362
val COMET_EXEC_SORT_MERGE_JOIN_ENABLED: ConfigEntry[Boolean] =
363363
createExecEnabledConfig("sortMergeJoin", defaultValue = true)
364364
val COMET_EXEC_CARTESIAN_PRODUCT_ENABLED: ConfigEntry[Boolean] =
365-
createExecEnabledConfig("cartesianProduct", defaultValue = true)
365+
createExecEnabledConfig(
366+
"cartesianProduct",
367+
defaultValue = false,
368+
notes = Some(
369+
"Opt-in: DataFusion's CrossJoinExec buffers the entire left input in memory and " +
370+
"cannot spill, so a large left input can exhaust memory where Spark's spillable " +
371+
"CartesianProduct succeeds. Only single-partition, condition-free cross joins are " +
372+
"eligible; everything else falls back to Spark"))
366373
val COMET_EXEC_EXISTENCE_JOIN_ENABLED: ConfigEntry[Boolean] =
367374
conf("spark.comet.exec.existenceJoin.enabled")
368375
.category(CATEGORY_ENABLE_EXEC)

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,26 @@ class CometJoinSuite extends CometTestBase {
188188
}
189189
}
190190

191+
test("cross join stays on Spark by default (native path is opt-in)") {
192+
// Native CrossJoinExec can't spill, so it's opt-in; by default the cross join stays on Spark.
193+
withSQLConf(
194+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
195+
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
196+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
197+
withParquetTable(Seq((0, 10), (1, 11), (2, 12)), "l") {
198+
withParquetTable(Seq((0, 100), (1, 101)), "r") {
199+
val df = sql(
200+
"SELECT * FROM (SELECT /*+ COALESCE(1) */ * FROM l) l " +
201+
"CROSS JOIN (SELECT /*+ COALESCE(1) */ * FROM r) r")
202+
checkSparkAnswer(df)
203+
assert(
204+
!df.queryExecution.executedPlan.exists(_.isInstanceOf[CometCartesianProductExec]),
205+
"cross join must stay on Spark when cartesianProduct.enabled is false")
206+
}
207+
}
208+
}
209+
}
210+
191211
test("join - self join") {
192212
val df1 = testData.select(testData("key")).as("df1")
193213
val df2 = testData.select(testData("key")).as("df2")

0 commit comments

Comments
 (0)