Skip to content

Commit 7067606

Browse files
authored
fix: SortMergeJoin with unsupported key type should fall back to Spark (apache#355)
* fix: SortMergeJoin with unsupported key type should fall back to Spark * Fix * For review * For review
1 parent b326637 commit 7067606

3 files changed

Lines changed: 57 additions & 2 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2104,6 +2104,18 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde {
21042104
expression
21052105
}
21062106

2107+
/**
2108+
* Returns true if given datatype is supported as a key in DataFusion sort merge join.
2109+
*/
2110+
def supportedSortMergeJoinEqualType(dataType: DataType): Boolean = dataType match {
2111+
case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType |
2112+
_: DoubleType | _: StringType | _: DateType | _: DecimalType | _: BooleanType =>
2113+
true
2114+
// `TimestampNTZType` is private in Spark 3.2/3.3.
2115+
case dt if dt.typeName == "timestamp_ntz" => true
2116+
case _ => false
2117+
}
2118+
21072119
/**
21082120
* Convert a Spark plan operator to a protobuf Comet operator.
21092121
*
@@ -2410,6 +2422,20 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde {
24102422
return None
24112423
}
24122424

2425+
// Checks if the join keys are supported by DataFusion SortMergeJoin.
2426+
val errorMsgs = join.leftKeys.flatMap { key =>
2427+
if (!supportedSortMergeJoinEqualType(key.dataType)) {
2428+
Some(s"Unsupported join key type ${key.dataType} on key: ${key.sql}")
2429+
} else {
2430+
None
2431+
}
2432+
}
2433+
2434+
if (errorMsgs.nonEmpty) {
2435+
withInfo(op, errorMsgs.flatten.mkString("\n"))
2436+
return None
2437+
}
2438+
24132439
val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output))
24142440
val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output))
24152441

‎spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,25 @@ class CometJoinSuite extends CometTestBase {
4040
}
4141
}
4242

43+
test("SortMergeJoin with unsupported key type should fall back to Spark") {
44+
withSQLConf(
45+
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "Asia/Kathmandu",
46+
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
47+
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
48+
withTable("t1", "t2") {
49+
sql("CREATE TABLE t1(name STRING, time TIMESTAMP) USING PARQUET")
50+
sql("INSERT OVERWRITE t1 VALUES('a', timestamp'2019-01-01 11:11:11')")
51+
52+
sql("CREATE TABLE t2(name STRING, time TIMESTAMP) USING PARQUET")
53+
sql("INSERT OVERWRITE t2 VALUES('a', timestamp'2019-01-01 11:11:11')")
54+
55+
val df = sql("SELECT * FROM t1 JOIN t2 ON t1.time = t2.time")
56+
val (sparkPlan, cometPlan) = checkSparkAnswer(df)
57+
assert(sparkPlan.canonicalized === cometPlan.canonicalized)
58+
}
59+
}
60+
}
61+
4362
test("Broadcast HashJoin without join filter") {
4463
assume(isSpark34Plus, "ChunkedByteBuffer is not serializable before Spark 3.4+")
4564
withSQLConf(

‎spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,18 +127,28 @@ abstract class CometTestBase
127127
}
128128
}
129129

130-
protected def checkSparkAnswer(query: String): Unit = {
130+
protected def checkSparkAnswer(query: String): (SparkPlan, SparkPlan) = {
131131
checkSparkAnswer(sql(query))
132132
}
133133

134-
protected def checkSparkAnswer(df: => DataFrame): Unit = {
134+
/**
135+
* Check the answer of a Comet SQL query with Spark result.
136+
* @param df
137+
* The DataFrame of the query.
138+
* @return
139+
* A tuple of the SparkPlan of the query and the SparkPlan of the Comet query.
140+
*/
141+
protected def checkSparkAnswer(df: => DataFrame): (SparkPlan, SparkPlan) = {
135142
var expected: Array[Row] = Array.empty
143+
var sparkPlan = null.asInstanceOf[SparkPlan]
136144
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
137145
val dfSpark = Dataset.ofRows(spark, df.logicalPlan)
138146
expected = dfSpark.collect()
147+
sparkPlan = dfSpark.queryExecution.executedPlan
139148
}
140149
val dfComet = Dataset.ofRows(spark, df.logicalPlan)
141150
checkAnswer(dfComet, expected)
151+
(sparkPlan, dfComet.queryExecution.executedPlan)
142152
}
143153

144154
protected def checkSparkAnswerAndOperator(query: String, excludedClasses: Class[_]*): Unit = {

0 commit comments

Comments
 (0)