Skip to content

Commit f9a39fe

Browse files
committed
address review: make native cartesian product opt-in (default false)
DataFusion's CrossJoinExec buffers the whole left input and cannot spill, so a large left can OOM where Spark's spillable CartesianProduct succeeds (sunchao's P2). Default the flag to false until the native path can spill; add a regression test that a cross join stays on Spark by default. Part of apache#199.
1 parent 5db5de8 commit f9a39fe

2 files changed

Lines changed: 28 additions & 1 deletion

File tree

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,14 @@ object CometConf extends ShimCometConf {
320320
val COMET_EXEC_SORT_MERGE_JOIN_ENABLED: ConfigEntry[Boolean] =
321321
createExecEnabledConfig("sortMergeJoin", defaultValue = true)
322322
val COMET_EXEC_CARTESIAN_PRODUCT_ENABLED: ConfigEntry[Boolean] =
323-
createExecEnabledConfig("cartesianProduct", defaultValue = true)
323+
createExecEnabledConfig(
324+
"cartesianProduct",
325+
defaultValue = false,
326+
notes = Some(
327+
"Opt-in: DataFusion's CrossJoinExec buffers the entire left input in memory and " +
328+
"cannot spill, so a large left input can exhaust memory where Spark's spillable " +
329+
"CartesianProduct succeeds. Only single-partition, condition-free cross joins are " +
330+
"eligible; everything else falls back to Spark"))
324331
val COMET_EXEC_EXISTENCE_JOIN_ENABLED: ConfigEntry[Boolean] =
325332
conf("spark.comet.exec.existenceJoin.enabled")
326333
.category(CATEGORY_ENABLE_EXEC)

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,26 @@ class CometJoinSuite extends CometTestBase {
184184
}
185185
}
186186

187+
test("cross join stays on Spark by default (native path is opt-in)") {
188+
// Native CrossJoinExec can't spill, so it's opt-in; by default the cross join stays on Spark.
189+
withSQLConf(
190+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
191+
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
192+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
193+
withParquetTable(Seq((0, 10), (1, 11), (2, 12)), "l") {
194+
withParquetTable(Seq((0, 100), (1, 101)), "r") {
195+
val df = sql(
196+
"SELECT * FROM (SELECT /*+ COALESCE(1) */ * FROM l) l " +
197+
"CROSS JOIN (SELECT /*+ COALESCE(1) */ * FROM r) r")
198+
checkSparkAnswer(df)
199+
assert(
200+
!df.queryExecution.executedPlan.exists(_.isInstanceOf[CometCartesianProductExec]),
201+
"cross join must stay on Spark when cartesianProduct.enabled is false")
202+
}
203+
}
204+
}
205+
}
206+
187207
test("join - self join") {
188208
val df1 = testData.select(testData("key")).as("df1")
189209
val df2 = testData.select(testData("key")).as("df2")

0 commit comments

Comments
 (0)