Skip to content

Commit 0e49f1c

Browse files
committed
fix: scope a TakeOrderedAndProject's sort keys and projection apart, as Spark evaluates them
3090d85 gave CometTakeOrderedAndProjectExec's sort keys and projection one shared set. Spark's ordering evaluates each sort key on its own, without subexpression elimination, and its projection runs separately over the rows the sort keeps. So a cast in two sort keys, or in a sort key and the projection, raised in Comet where Spark returns the rows. The top-K plans now use an empty set, and the projection its projection list.
1 parent 3090d85 commit 0e49f1c

2 files changed

Lines changed: 34 additions & 17 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -124,22 +124,31 @@ case class CometTakeOrderedAndProjectExec(
124124
SortOrder.orderingSatisfies(child.outputOrdering, sortOrder)
125125

126126
// Also exposed to tests so they can inspect the plan actually used for final execution.
127-
def finalNativePlan(inputPartitions: Int): Option[OperatorOuterClass.Operator] =
128-
withSharedSubexpressions(expressions) {
129-
val selectedLocally = child match {
130-
case local: CometLocalTopKExec =>
131-
inputPartitions == 1 && local.limit == limit && local.sortOrder == sortOrder
132-
case _ => false
133-
}
134-
val selection = if (selectedLocally) {
135-
// The local TopK has already ordered and bounded the only input partition. Spark's
136-
// physical limit includes the offset; the native limit applies that offset once.
137-
CometExecUtils.getLimitNativePlan(child.output, limit, offset)
138-
} else {
139-
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit, offset)
140-
}
127+
def finalNativePlan(inputPartitions: Int): Option[OperatorOuterClass.Operator] = {
128+
val selectedLocally = child match {
129+
case local: CometLocalTopKExec =>
130+
inputPartitions == 1 && local.limit == limit && local.sortOrder == sortOrder
131+
case _ => false
132+
}
133+
val selection = if (selectedLocally) {
134+
// The local TopK has already ordered and bounded the only input partition. Spark's
135+
// physical limit includes the offset; the native limit applies that offset once.
136+
CometExecUtils.getLimitNativePlan(child.output, limit, offset)
137+
} else {
138+
topKNativePlan(offset)
139+
}
140+
// Spark's projection runs on its own over the rows the sort keeps, so only its list can share
141+
// a subexpression (see QueryPlanSerde.canShortCircuitNulls)
142+
withSharedSubexpressions(projectList) {
141143
selection.flatMap(CometExecUtils.getProjectionNativePlan(projectList, child.output, _))
142144
}
145+
}
146+
147+
// Spark's ordering evaluates each sort key on its own, without subexpression elimination
148+
private def topKNativePlan(offset: Int): Option[OperatorOuterClass.Operator] =
149+
withSharedSubexpressions(Seq.empty) {
150+
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit, offset)
151+
}
143152

144153
protected override def doExecuteColumnar(): RDD[ColumnarBatch] = {
145154
val childRDD = child.executeColumnar()
@@ -158,9 +167,7 @@ case class CometTakeOrderedAndProjectExec(
158167
} else {
159168
val numParts = childRDD.getNumPartitions
160169
// Serialize the plan once before mapping to avoid repeated serialization per partition
161-
val topK = withSharedSubexpressions(expressions) {
162-
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit).get
163-
}
170+
val topK = topKNativePlan(offset = 0).get
164171
val serializedTopK = CometExec.serializeNativePlan(topK)
165172
val numOutputCols = child.output.length
166173
val inputSchema = CometUtils.fromAttributes(child.output)

‎spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,16 @@ query expect_native(array_contains)
107107
SELECT array_contains(ai, CAST(s AS INT))
108108
FROM (SELECT * FROM array_null_short_circuit ORDER BY i LIMIT 10)
109109

110+
-- A TakeOrderedAndProject compares rows one sort key at a time and evaluates its projection apart
111+
-- from the keys, so a cast in two sort keys, or in a sort key and the projection, is skipped too
112+
query expect_native(array_contains, array_position)
113+
SELECT i FROM (SELECT * FROM array_null_short_circuit
114+
ORDER BY array_contains(ai, CAST(s AS INT)), array_position(ai, CAST(s AS INT)), i LIMIT 10)
115+
116+
query expect_native(array_contains, array_position)
117+
SELECT array_contains(ai, CAST(s AS INT))
118+
FROM (SELECT * FROM array_null_short_circuit ORDER BY array_position(ai, CAST(s AS INT)), i LIMIT 10)
119+
110120
-- Where the array is not NULL, the argument is evaluated and raises as in Spark
111121
query expect_error(CAST_INVALID_INPUT)
112122
SELECT array_contains(ai, CAST(s || 'x' AS INT)) FROM array_null_short_circuit

0 commit comments

Comments
 (0)