Skip to content

Commit 8ec0da7

Browse files
committed
fix: retain native inputs required by generated filters
1 parent a8676cf commit 8ec0da7

2 files changed

Lines changed: 107 additions & 2 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import scala.collection.mutable.ListBuffer
2323
import scala.jdk.CollectionConverters._
2424

2525
import org.apache.spark.sql.SparkSession
26-
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, FloatLiteral, GreaterThan, GreaterThanOrEqual, KnownFloatingPointNormalized, LeafExpression, LessThan, LessThanOrEqual, NamedExpression, Remainder, SortOrder}
26+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, FloatLiteral, GreaterThan, GreaterThanOrEqual, IsNotNull, KnownFloatingPointNormalized, LeafExpression, LessThan, LessThanOrEqual, NamedExpression, PredicateHelper, Remainder, SortOrder}
2727
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, Final, Partial, PartialMerge}
2828
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
2929
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
@@ -175,6 +175,7 @@ object CometExecRule {
175175
case class CometExecRule(session: SparkSession)
176176
extends Rule[SparkPlan]
177177
with CometTypeShim
178+
with PredicateHelper
178179
with ShimSubqueryBroadcast {
179180

180181
private lazy val showTransformations = CometConf.COMET_EXPLAIN_TRANSFORMATIONS.get()
@@ -1125,6 +1126,29 @@ case class CometExecRule(session: SparkSession)
11251126
AttributeSet(QueryPlanSerde.eagerlyEvaluatedChildren(expr).flatMap(eagerReferences))
11261127
}
11271128

1129+
def eagerFilterInputs(filter: FilterExec): AttributeSet = {
1130+
// GeneratePredicateHelper loads every referenced projected value before evaluating each
1131+
// predicate, but can skip later conjuncts. It moves null-intolerant IsNotNull checks ahead
1132+
// of predicates that reference those inputs, including inferred checks for attributes
1133+
// inside compound null checks. Match the first check Spark actually emits.
1134+
val (notNullPredicates, otherPredicates) =
1135+
splitConjunctivePredicates(filter.condition).partition {
1136+
case IsNotNull(child) =>
1137+
isNullIntolerant(child) && child.references.subsetOf(filter.child.outputSet)
1138+
case _ => false
1139+
}
1140+
val nullCheckedInputs = AttributeSet(notNullPredicates.flatMap(_.references))
1141+
otherPredicates.headOption match {
1142+
case Some(predicate) =>
1143+
predicate.references.find(nullCheckedInputs.contains) match {
1144+
case Some(firstNullCheckedInput) => AttributeSet(Seq(firstNullCheckedInput))
1145+
case None => predicate.references
1146+
}
1147+
case None =>
1148+
notNullPredicates.headOption.map(_.references).getOrElse(AttributeSet.empty)
1149+
}
1150+
}
1151+
11281152
def protect(
11291153
node: SparkPlan,
11301154
belowLimit: Boolean,
@@ -1176,6 +1200,8 @@ case class CometExecRule(session: SparkSession)
11761200
expr.deterministic && deferredInputs.contains(expr.toAttribute))
11771201
project.inputSet -- project.usedInputs --
11781202
AttributeSet(eagerOutputs.flatMap(eagerReferences))
1203+
case filter: FilterExec if supportsWholeStage(filter) =>
1204+
filter.inputSet -- filter.usedInputs -- eagerFilterInputs(filter)
11791205
case codegen: CodegenSupport if !materializesInput && supportsWholeStage(original) =>
11801206
original.inputSet -- codegen.usedInputs
11811207
case _ => AttributeSet.empty

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.scalatest.PrivateMethodTester._
2929
import org.apache.logging.log4j.Level
3030
import org.apache.spark.sql._
3131
import org.apache.spark.sql.catalyst.FunctionIdentifier
32-
import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute, AttributeReference, DateAdd, Expression, ExpressionInfo, GreaterThan, In, InSet, KnownFloatingPointNormalized, LessThan, Literal, NextDay, Not, SortOrder}
32+
import org.apache.spark.sql.catalyst.expressions.{Alias, And, Ascending, Attribute, AttributeReference, DateAdd, DateDiff, Expression, ExpressionInfo, GreaterThan, In, InSet, IsNotNull, KnownFloatingPointNormalized, LessThan, Literal, NextDay, Not, Or, SortOrder}
3333
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, BloomFilterAggregate, Final, Min, Partial, PartialMerge}
3434
import org.apache.spark.sql.catalyst.optimizer.{BuildRight, NormalizeNaNAndZero}
3535
import org.apache.spark.sql.catalyst.plans.{Inner, LeftSemi}
@@ -2290,6 +2290,85 @@ class CometExecRuleSuite extends CometTestBase {
22902290
}
22912291
}
22922292

2293+
test("ANSI next_day filter inputs follow generated predicate order") {
2294+
withAnsiNextDayProject { original =>
2295+
def project: ProjectExec = ProjectExec(
2296+
original.projectList :+
2297+
Alias(Literal.create(null, DataTypes.DateType), "nullable_date")(),
2298+
original.child)
2299+
val date = Literal.create(java.sql.Date.valueOf("2020-01-01"), DataTypes.DateType)
2300+
val predicates: Seq[(ProjectExec => Expression, Boolean)] = Seq(
2301+
(p => GreaterThan(p.output(1), date), true),
2302+
(p => GreaterThan(p.output(2), p.output(1)), true),
2303+
(p => And(GreaterThan(p.output(1), date), LessThan(p.output.head, Literal(2))), true),
2304+
(p => And(LessThan(p.output.head, Literal(2)), GreaterThan(p.output(1), date)), false),
2305+
// A single predicate loads all referenced projected values before its expression code.
2306+
(p => Or(LessThan(p.output.head, Literal(2)), GreaterThan(p.output(1), date)), true),
2307+
(p => And(IsNotNull(p.output(1)), GreaterThan(p.output(1), date)), true),
2308+
(p => And(IsNotNull(p.output(1)), IsNotNull(p.output(2))), true),
2309+
(p => And(IsNotNull(p.output(2)), IsNotNull(p.output(1))), false),
2310+
// Spark moves this null check ahead of the comparison, even though it appears last.
2311+
(p => And(GreaterThan(p.output(1), p.output(2)), IsNotNull(p.output(2))), false),
2312+
// A compound null check also causes Spark to inject checks for its input attributes.
2313+
(
2314+
p =>
2315+
And(
2316+
GreaterThan(p.output(1), p.output(2)),
2317+
IsNotNull(DateAdd(p.output(2), p.output.head))),
2318+
false),
2319+
(p => IsNotNull(DateDiff(p.output(2), p.output(1))), true))
2320+
for ((predicate, native) <- predicates) {
2321+
val child = project
2322+
val condition = predicate(child)
2323+
val transformed = applyCometExecRule(FilterExec(condition, child))
2324+
withClue(condition.sql) {
2325+
assert(countOperators(transformed, classOf[CometProjectExec]) == (if (native) 1 else 0))
2326+
assert(countOperators(transformed, classOf[ProjectExec]) == (if (native) 0 else 1))
2327+
}
2328+
}
2329+
}
2330+
}
2331+
2332+
test("ANSI next_day required by a filter stays native and preserves errors") {
2333+
withSQLConf(
2334+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
2335+
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true",
2336+
SQLConf.ANSI_ENABLED.key -> "true",
2337+
CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") {
2338+
val date = java.sql.Date.valueOf("2024-01-01")
2339+
val preserve = CometConf.COMET_EXEC_PRESERVE_EVALUATION_MASKS_ENABLED.key
2340+
def query(predicate: String): String =
2341+
s"""SELECT x, r FROM
2342+
|(SELECT next_day(_1, _2) AS x, rand(1L) AS r FROM filter_next_day)
2343+
|WHERE $predicate""".stripMargin
2344+
2345+
withParquetTable(Seq((date, "Monday")), "filter_next_day") {
2346+
for (enabled <- Seq(false, true)) {
2347+
withSQLConf(preserve -> enabled.toString) {
2348+
val (_, plan) = checkSparkAnswerAndImpl(
2349+
sql(query("x >= DATE '2020-01-01'")),
2350+
native = Seq("next_day"))
2351+
assert(collect(plan) { case p: CometProjectExec => p }.size == 1, plan.toString)
2352+
assert(collect(plan) { case f: CometFilterExec => f }.size == 1, plan.toString)
2353+
}
2354+
}
2355+
}
2356+
withParquetTable(Seq((date, "NOT_A_DAY")), "filter_next_day") {
2357+
// The required projected value fails even when the date comparison would reject it.
2358+
val required = sql(query("x < DATE '2020-01-01'"))
2359+
assertExpressionImpl(required.queryExecution.executedPlan, Seq("next_day"), Seq.empty)
2360+
val errorClass = if (isSpark40Plus) "ILLEGAL_DAY_OF_WEEK" else "_LEGACY_ERROR_TEMP_2000"
2361+
val error = checkSparkError(required, errorClass)
2362+
assert(error.getMessage.contains("Illegal input for day of week"))
2363+
// An earlier conjunct can still reject the row before evaluating next_day.
2364+
checkSparkAnswerAndFallbackReason(
2365+
query("r < 0 AND x < DATE '2020-01-01'"),
2366+
"next_day requires Spark evaluation in a deferred projection")
2367+
()
2368+
}
2369+
}
2370+
}
2371+
22932372
test("strict Project inputs stay native unless their output is deferred") {
22942373
withAnsiNextDayProject { project =>
22952374
def strictProject: ProjectExec = ProjectExec(

0 commit comments

Comments
 (0)