Skip to content

Commit 3090d85

Browse files
committed
fix: find shared subexpressions where shuffle keys and TakeOrderedAndProject are serialized
The native shuffle writer serializes its partitioning keys, and CometTakeOrderedAndProjectExec its sort keys and projection, when the plan runs, outside the operator conversion that sets the shared subexpressions. canShortCircuitNulls took the missing set as an empty one, so a cast that two partitioning keys share was skipped on the rows whose array is NULL, where Spark's projection evaluates it for every row and raises. canShortCircuitNulls now skips no argument outside withSharedSubexpressions. The shuffle writer calls it with the partitioning keys, which Spark evaluates together in one projection, and CometTakeOrderedAndProjectExec with its own expressions.
1 parent d01ae0a commit 3090d85

5 files changed

Lines changed: 86 additions & 29 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1306,6 +1306,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
13061306
* `spark.sql.subexpressionElimination.enabled` says, which only keeps them evaluated for every
13071307
* row where Spark might not. They are found on the first `canShortCircuitNulls` call for the
13081308
* operator, so an operator without an argument to skip never runs `EquivalentExpressions`.
1309+
* `None` outside `withSharedSubexpressions`, where they are unknown.
13091310
*/
13101311
private val sharedSubexpressions = new DynamicVariable[Option[SharedSubexpressions]](None)
13111312

@@ -1317,7 +1318,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
13171318
}
13181319
}
13191320

1320-
/** Runs `f`, which serializes an operator whose expressions are `expressions`. */
1321+
/**
1322+
* Runs `f`, which serializes an operator whose expressions are `expressions`. Code that
1323+
* serializes expressions outside an operator's conversion, such as when the plan runs, calls it
1324+
* with the expressions Spark evaluates together, or no argument is skipped.
1325+
*/
13211326
def withSharedSubexpressions[T](expressions: Seq[Expression])(f: => T): T =
13221327
sharedSubexpressions.withValue(Some(new SharedSubexpressions(expressions)))(f)
13231328

@@ -1326,14 +1331,13 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
13261331
* an earlier argument is NULL, as Spark's evaluation of a null-intolerant `BinaryExpression` or
13271332
* `TernaryExpression` does. Not if one of them holds a subexpression that the operator's
13281333
* expressions share: Spark evaluates that for every row, and raises if it fails on a row whose
1329-
* array is NULL.
1334+
* array is NULL. Nor outside `withSharedSubexpressions`, where what they share is unknown.
13301335
*/
13311336
def canShortCircuitNulls(expr: Expression): Boolean = {
13321337
val later = expr.children.tail
13331338
// Subexpression elimination never shares a leaf, such as a column or a literal
1334-
later.forall(_.isInstanceOf[LeafExpression]) || {
1335-
val shared = sharedSubexpressions.value.map(_.common).getOrElse(Nil)
1336-
!later.exists(_.exists(e => shared.exists(_.semanticEquals(e))))
1339+
later.forall(_.isInstanceOf[LeafExpression]) || sharedSubexpressions.value.exists { s =>
1340+
!later.exists(_.exists(e => s.common.exists(_.semanticEquals(e))))
13371341
}
13381342
}
13391343

‎spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch
3434
import org.apache.comet.{CometConf, ConfigEntry}
3535
import org.apache.comet.CometSparkSessionExtensions.isCometShuffleEnabled
3636
import org.apache.comet.serde.{Compatible, OperatorOuterClass, SupportLevel, Unsupported}
37-
import org.apache.comet.serde.QueryPlanSerde.{exprToProto, supportedSortType}
37+
import org.apache.comet.serde.QueryPlanSerde.{exprToProto, supportedSortType, withSharedSubexpressions}
3838
import org.apache.comet.serde.operator.CometSink
3939

4040
object CometTakeOrderedAndProjectExec extends CometSink[TakeOrderedAndProjectExec] {
@@ -124,21 +124,22 @@ case class CometTakeOrderedAndProjectExec(
124124
SortOrder.orderingSatisfies(child.outputOrdering, sortOrder)
125125

126126
// Also exposed to tests so they can inspect the plan actually used for final execution.
127-
def finalNativePlan(inputPartitions: Int): Option[OperatorOuterClass.Operator] = {
128-
val selectedLocally = child match {
129-
case local: CometLocalTopKExec =>
130-
inputPartitions == 1 && local.limit == limit && local.sortOrder == sortOrder
131-
case _ => false
132-
}
133-
val selection = if (selectedLocally) {
134-
// The local TopK has already ordered and bounded the only input partition. Spark's
135-
// physical limit includes the offset; the native limit applies that offset once.
136-
CometExecUtils.getLimitNativePlan(child.output, limit, offset)
137-
} else {
138-
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit, offset)
127+
def finalNativePlan(inputPartitions: Int): Option[OperatorOuterClass.Operator] =
128+
withSharedSubexpressions(expressions) {
129+
val selectedLocally = child match {
130+
case local: CometLocalTopKExec =>
131+
inputPartitions == 1 && local.limit == limit && local.sortOrder == sortOrder
132+
case _ => false
133+
}
134+
val selection = if (selectedLocally) {
135+
// The local TopK has already ordered and bounded the only input partition. Spark's
136+
// physical limit includes the offset; the native limit applies that offset once.
137+
CometExecUtils.getLimitNativePlan(child.output, limit, offset)
138+
} else {
139+
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit, offset)
140+
}
141+
selection.flatMap(CometExecUtils.getProjectionNativePlan(projectList, child.output, _))
139142
}
140-
selection.flatMap(CometExecUtils.getProjectionNativePlan(projectList, child.output, _))
141-
}
142143

143144
protected override def doExecuteColumnar(): RDD[ColumnarBatch] = {
144145
val childRDD = child.executeColumnar()
@@ -157,10 +158,9 @@ case class CometTakeOrderedAndProjectExec(
157158
} else {
158159
val numParts = childRDD.getNumPartitions
159160
// Serialize the plan once before mapping to avoid repeated serialization per partition
160-
val topK =
161-
CometExecUtils
162-
.getTopKNativePlan(child.output, sortOrder, child, limit)
163-
.get
161+
val topK = withSharedSubexpressions(expressions) {
162+
CometExecUtils.getTopKNativePlan(child.output, sortOrder, child, limit).get
163+
}
164164
val serializedTopK = CometExec.serializeNativePlan(topK)
165165
val numOutputCols = child.output.length
166166
val inputSchema = CometUtils.fromAttributes(child.output)

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -343,8 +343,13 @@ class CometNativeShuffleWriter[K, V](
343343
val partitioning = PartitioningOuterClass.HashPartition.newBuilder()
344344
partitioning.setNumPartitions(effectivePartitionCount)
345345

346-
val partitionExprs = hashPartitioning.expressions
347-
.flatMap(e => QueryPlanSerde.exprToProto(e, outputAttributes))
346+
// Spark evaluates the keys in one projection, whose subexpression elimination sees what
347+
// they share (see QueryPlanSerde.canShortCircuitNulls)
348+
val partitionExprs =
349+
QueryPlanSerde.withSharedSubexpressions(hashPartitioning.expressions) {
350+
hashPartitioning.expressions
351+
.flatMap(e => QueryPlanSerde.exprToProto(e, outputAttributes))
352+
}
348353

349354
if (partitionExprs.length != hashPartitioning.expressions.length) {
350355
throw new UnsupportedOperationException(
@@ -377,8 +382,12 @@ class CometNativeShuffleWriter[K, V](
377382
}
378383

379384
{
380-
val orderingExprs = rangePartitioning.ordering
381-
.flatMap(e => QueryPlanSerde.exprToProto(e, outputAttributes))
385+
// As for hash partitioning, Spark evaluates the keys in one projection
386+
val orderingExprs =
387+
QueryPlanSerde.withSharedSubexpressions(rangePartitioning.ordering.map(_.child)) {
388+
rangePartitioning.ordering
389+
.flatMap(e => QueryPlanSerde.exprToProto(e, outputAttributes))
390+
}
382391
if (orderingExprs.length != rangePartitioning.ordering.length) {
383392
throw new UnsupportedOperationException(
384393
s"Partitioning $rangePartitioning is not supported.")

‎spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,29 @@ query expect_native(array_contains)
8484
SELECT array_contains(ai, CAST(s AS INT)), NOT array_contains(ai, CAST(s AS INT))
8585
FROM array_null_short_circuit
8686

87+
-- A shuffle evaluates its partitioning keys together in one projection, and a TakeOrderedAndProject
88+
-- its projection list, so a cast that two of them share raises on the rows whose array is NULL too.
89+
-- Both are serialized when the plan runs, after the operator's conversion.
90+
query expect_error(CAST_INVALID_INPUT)
91+
SELECT ai, s FROM array_null_short_circuit
92+
DISTRIBUTE BY array_contains(ai, CAST(s AS INT)), array_position(ai, CAST(s AS INT))
93+
94+
query expect_error(CAST_INVALID_INPUT)
95+
SELECT array_contains(ai, CAST(s AS INT)), array_position(ai, CAST(s AS INT))
96+
FROM (SELECT * FROM array_null_short_circuit ORDER BY i LIMIT 10)
97+
98+
-- A cast that only one of them holds is skipped, for hash and range partitioning keys and for a
99+
-- TakeOrderedAndProject
100+
query
101+
SELECT ai, s FROM array_null_short_circuit DISTRIBUTE BY array_contains(ai, CAST(s AS INT))
102+
103+
query
104+
SELECT ai, s FROM array_null_short_circuit ORDER BY array_contains(ai, CAST(s AS INT)), s
105+
106+
query expect_native(array_contains)
107+
SELECT array_contains(ai, CAST(s AS INT))
108+
FROM (SELECT * FROM array_null_short_circuit ORDER BY i LIMIT 10)
109+
87110
-- Where the array is not NULL, the argument is evaluated and raises as in Spark
88111
query expect_error(CAST_INVALID_INPUT)
89112
SELECT array_contains(ai, CAST(s || 'x' AS INT)) FROM array_null_short_circuit

‎spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@ package org.apache.comet.serde
2121

2222
import org.scalatest.funsuite.AnyFunSuite
2323

24+
import org.apache.spark.sql.catalyst.expressions.{ArrayContains, ArrayPosition, AttributeReference, Cast}
2425
import org.apache.spark.sql.types._
2526

2627
import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus}
27-
import org.apache.comet.serde.QueryPlanSerde.supportedDataType
28+
import org.apache.comet.serde.QueryPlanSerde.{canShortCircuitNulls, supportedDataType, withSharedSubexpressions}
2829

2930
class QueryPlanSerdeSuite extends AnyFunSuite {
3031

@@ -100,4 +101,24 @@ class QueryPlanSerdeSuite extends AnyFunSuite {
100101
rejected.foreach(dt => assert(!supports(dt), s"$name should reject $dt"))
101102
}
102103
}
104+
105+
test("canShortCircuitNulls skips an argument only where the shared subexpressions are known") {
106+
val array = AttributeReference("ai", ArrayType(IntegerType))()
107+
val value = AttributeReference("i", IntegerType)()
108+
val cast = Cast(AttributeReference("s", StringType)(), IntegerType)
109+
val contains = ArrayContains(array, cast)
110+
val position = ArrayPosition(array, cast)
111+
112+
// A column is never shared, so it can be skipped anywhere
113+
assert(canShortCircuitNulls(ArrayContains(array, value)))
114+
// Something serialized outside an operator's conversion might share the cast
115+
assert(!canShortCircuitNulls(contains))
116+
withSharedSubexpressions(Seq(contains)) {
117+
assert(canShortCircuitNulls(contains))
118+
}
119+
withSharedSubexpressions(Seq(contains, position)) {
120+
assert(!canShortCircuitNulls(contains))
121+
assert(!canShortCircuitNulls(position))
122+
}
123+
}
103124
}

0 commit comments

Comments
 (0)