Skip to content

Commit 1bb942a

Browse files
committed
fix: fall back on a collated window key only over a trusted order
Falling back hands the rows to Spark's window, which trusts the order the input reports. Over an order mayReportFalseOrder doubts, such as a native nested loop join's, that grouped the rows wrongly, so the window now stays native there, as on main, and the native window rejects the input. The outer join regression test partitions by k, which Spark reports through the alias a, so it needs the order declared next to the join's.
1 parent ee88fd9 commit 1bb942a

2 files changed

Lines changed: 20 additions & 14 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometWindowExec.scala‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,20 @@ object CometWindowExec extends CometOperatorSerde[WindowExec] {
7575
}
7676
}
7777

78+
// Spark plans no sort below the window when its input already reports the order the window
79+
// needs. That order is relied on, by declaring it natively or by handing the rows to Spark's
80+
// window, only when mayReportFalseOrder trusts it: Spark's window would group the rows of a
81+
// false order wrongly, while the native one rejects an input whose order it cannot see.
82+
val requiredOrdering = op.requiredChildOrdering.head
83+
val trustsInputOrder =
84+
SortOrder.orderingSatisfies(op.child.outputOrdering, requiredOrdering) &&
85+
!CometHashJoinExec.mayReportFalseInputOrder(op.child)
86+
7887
// The native window finds partitions and peers by comparing keys byte by byte, so it would
7988
// tell apart collated strings that Spark treats as equal, as CometWindowGroupLimitExec notes.
8089
val collated = (op.partitionSpec ++ op.orderSpec.map(_.child))
8190
.filter(key => hasNonDefaultStringCollation(key.dataType))
82-
if (collated.nonEmpty) {
91+
if (collated.nonEmpty && trustsInputOrder) {
8392
withFallbackReason(
8493
op,
8594
collated
@@ -94,16 +103,11 @@ object CometWindowExec extends CometOperatorSerde[WindowExec] {
94103

95104
if (windowExprProto.forall(_.isDefined) && partitionExprs.forall(_.isDefined)
96105
&& sortOrders.forall(_.isDefined)) {
97-
// Spark plans no sort below the window when its input already reports the order the window
98-
// needs, but rows from the JVM arrive through a ScanExec that declares none. Send the
99-
// partition keys' order, the only one the native window checks, for the planner to declare
100-
// (see SortedInputExec). Nothing is sent when that order may be false. Falling back would
101-
// not help, since Spark's window trusts the order too.
102-
val requiredOrdering = op.requiredChildOrdering.head
106+
// Rows from the JVM arrive through a ScanExec that declares no order. Send the partition
107+
// keys' order, the only one the native window checks, for the planner to declare (see
108+
// SortedInputExec).
103109
val inputSortOrders =
104-
if (op.partitionSpec.nonEmpty &&
105-
SortOrder.orderingSatisfies(op.child.outputOrdering, requiredOrdering) &&
106-
!CometHashJoinExec.mayReportFalseInputOrder(op.child)) {
110+
if (op.partitionSpec.nonEmpty && trustsInputOrder) {
107111
val partitionKeys = requiredOrdering.take(op.partitionSpec.length)
108112
val orders = partitionKeys.flatMap(exprToProto(_, op.child.output))
109113
if (orders.length < partitionKeys.length) {

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3860,18 +3860,20 @@ class CometInMemoryCacheSuite extends CometTestBase {
38603860
Row(nullable(k), nullable(k), id, keySums(k) * keyCounts(k))
38613861
},
38623862
Seq(Seq("1 Ascending NullsFirst"), Seq("0 Ascending NullsFirst")))
3863-
// The residual condition leaves rows of key 0 unmatched among matched ones, so the key
3864-
// stays together only if the join keeps its unmatched probe rows in place.
3863+
// The join declares the order of a, which Spark reports in place of k, so the window by k
3864+
// needs its order declared next to the join's. The residual condition leaves rows of key 0
3865+
// unmatched among matched ones, so the key stays together only if the join keeps its
3866+
// unmatched probe rows in place.
38653867
val one = spark.range(1).toDF("j").hint("broadcast")
38663868
checkWindows(
38673869
cached
38683870
.join(one, col("k") === col("j") && col("v") % 2 === 0, "left_outer")
3869-
.selectExpr("k", "a", "v", "sum(v) OVER (PARTITION BY a) AS s"),
3871+
.selectExpr("k", "a", "v", "sum(v) OVER (PARTITION BY k) AS s"),
38703872
ids.map { id =>
38713873
val k = key(id)
38723874
Row(nullable(k), nullable(k), id, keySums(k))
38733875
},
3874-
Seq(Seq("1 Ascending NullsFirst")))
3876+
Seq(Seq("0 Ascending NullsFirst")))
38753877
} finally {
38763878
spark.catalog.clearCache()
38773879
}

0 commit comments

Comments
 (0)