Skip to content

Commit 39e030b

Browse files
authored
fix: Unsupported types for SinglePartition should fallback to Spark (apache#765)
1 parent 65f22a2 commit 39e030b

2 files changed

Lines changed: 16 additions & 3 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2931,8 +2931,8 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
29312931
msg = s"unsupported Spark partitioning expressions: $expressions"
29322932
}
29332933
supported
2934-
case SinglePartition => true
2935-
case RoundRobinPartitioning(_) => true
2934+
case SinglePartition => inputs.forall(attr => supportedDataType(attr.dataType))
2935+
case RoundRobinPartitioning(_) => inputs.forall(attr => supportedDataType(attr.dataType))
29362936
case RangePartitioning(orderings, _) =>
29372937
val supported =
29382938
orderings.map(QueryPlanSerde.exprToProto(_, inputs)).forall(_.isDefined) &&
@@ -2980,7 +2980,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
29802980
msg = s"unsupported Spark partitioning expressions: $expressions"
29812981
}
29822982
supported
2983-
case SinglePartition => true
2983+
case SinglePartition => inputs.forall(attr => supportedDataType(attr.dataType))
29842984
case _ =>
29852985
msg = s"unsupported Spark partitioning: ${partitioning.getClass.getName}"
29862986
false

‎spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,19 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
6666

6767
setupTestData()
6868

69+
test("Unsupported types for SinglePartition should fallback to Spark") {
70+
checkSparkAnswer(spark.sql("""
71+
|SELECT
72+
| AVG(null),
73+
| COUNT(null),
74+
| FIRST(null),
75+
| LAST(null),
76+
| MAX(null),
77+
| MIN(null),
78+
| SUM(null)
79+
""".stripMargin))
80+
}
81+
6982
test("Disable Comet shuffle with AQE coalesce partitions enabled") {
7083
Seq(true, false).foreach { coalescePartitionsEnabled =>
7184
withSQLConf(

0 commit comments

Comments
 (0)