File tree Expand file tree Collapse file tree
main/scala/org/apache/comet/serde Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -2104,6 +2104,18 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde {
21042104 expression
21052105 }
21062106
2107+ /**
2108+ * Returns true if given datatype is supported as a key in DataFusion sort merge join.
2109+ */
2110+ def supportedSortMergeJoinEqualType (dataType : DataType ): Boolean = dataType match {
2111+ case _ : ByteType | _ : ShortType | _ : IntegerType | _ : LongType | _ : FloatType |
2112+ _ : DoubleType | _ : StringType | _ : DateType | _ : DecimalType | _ : BooleanType =>
2113+ true
2114+ // `TimestampNTZType` is private in Spark 3.2/3.3.
2115+ case dt if dt.typeName == " timestamp_ntz" => true
2116+ case _ => false
2117+ }
2118+
21072119 /**
21082120 * Convert a Spark plan operator to a protobuf Comet operator.
21092121 *
@@ -2410,6 +2422,20 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde {
24102422 return None
24112423 }
24122424
2425+ // Checks if the join keys are supported by DataFusion SortMergeJoin.
2426+ val errorMsgs = join.leftKeys.flatMap { key =>
2427+ if (! supportedSortMergeJoinEqualType(key.dataType)) {
2428+ Some (s " Unsupported join key type ${key.dataType} on key: ${key.sql}" )
2429+ } else {
2430+ None
2431+ }
2432+ }
2433+
2434+ if (errorMsgs.nonEmpty) {
2435+ withInfo(op, errorMsgs.flatten.mkString(" \n " ))
2436+ return None
2437+ }
2438+
24132439 val leftKeys = join.leftKeys.map(exprToProto(_, join.left.output))
24142440 val rightKeys = join.rightKeys.map(exprToProto(_, join.right.output))
24152441
Original file line number Diff line number Diff line change @@ -40,6 +40,25 @@ class CometJoinSuite extends CometTestBase {
4040 }
4141 }
4242
43+ test(" SortMergeJoin with unsupported key type should fall back to Spark" ) {
44+ withSQLConf(
45+ SQLConf .SESSION_LOCAL_TIMEZONE .key -> " Asia/Kathmandu" ,
46+ SQLConf .ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ,
47+ SQLConf .AUTO_BROADCASTJOIN_THRESHOLD .key -> " -1" ) {
48+ withTable(" t1" , " t2" ) {
49+ sql(" CREATE TABLE t1(name STRING, time TIMESTAMP) USING PARQUET" )
50+ sql(" INSERT OVERWRITE t1 VALUES('a', timestamp'2019-01-01 11:11:11')" )
51+
52+ sql(" CREATE TABLE t2(name STRING, time TIMESTAMP) USING PARQUET" )
53+ sql(" INSERT OVERWRITE t2 VALUES('a', timestamp'2019-01-01 11:11:11')" )
54+
55+ val df = sql(" SELECT * FROM t1 JOIN t2 ON t1.time = t2.time" )
56+ val (sparkPlan, cometPlan) = checkSparkAnswer(df)
57+ assert(sparkPlan.canonicalized === cometPlan.canonicalized)
58+ }
59+ }
60+ }
61+
4362 test(" Broadcast HashJoin without join filter" ) {
4463 assume(isSpark34Plus, " ChunkedByteBuffer is not serializable before Spark 3.4+" )
4564 withSQLConf(
Original file line number Diff line number Diff line change @@ -127,18 +127,28 @@ abstract class CometTestBase
127127 }
128128 }
129129
130- protected def checkSparkAnswer (query : String ): Unit = {
130+ protected def checkSparkAnswer (query : String ): ( SparkPlan , SparkPlan ) = {
131131 checkSparkAnswer(sql(query))
132132 }
133133
134- protected def checkSparkAnswer (df : => DataFrame ): Unit = {
134+ /**
135+ * Check the answer of a Comet SQL query with Spark result.
136+ * @param df
137+ * The DataFrame of the query.
138+ * @return
139+ * A tuple of the SparkPlan of the query and the SparkPlan of the Comet query.
140+ */
141+ protected def checkSparkAnswer (df : => DataFrame ): (SparkPlan , SparkPlan ) = {
135142 var expected : Array [Row ] = Array .empty
143+ var sparkPlan = null .asInstanceOf [SparkPlan ]
136144 withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
137145 val dfSpark = Dataset .ofRows(spark, df.logicalPlan)
138146 expected = dfSpark.collect()
147+ sparkPlan = dfSpark.queryExecution.executedPlan
139148 }
140149 val dfComet = Dataset .ofRows(spark, df.logicalPlan)
141150 checkAnswer(dfComet, expected)
151+ (sparkPlan, dfComet.queryExecution.executedPlan)
142152 }
143153
144154 protected def checkSparkAnswerAndOperator (query : String , excludedClasses : Class [_]* ): Unit = {
You can’t perform that action at this time.
0 commit comments