@@ -29,7 +29,7 @@ import org.scalatest.PrivateMethodTester._
2929import org .apache .logging .log4j .Level
3030import org .apache .spark .sql ._
3131import org .apache .spark .sql .catalyst .FunctionIdentifier
32- import org .apache .spark .sql .catalyst .expressions .{Alias , Ascending , Attribute , AttributeReference , DateAdd , Expression , ExpressionInfo , GreaterThan , In , InSet , KnownFloatingPointNormalized , LessThan , Literal , NextDay , Not , SortOrder }
32+ import org .apache .spark .sql .catalyst .expressions .{Alias , And , Ascending , Attribute , AttributeReference , DateAdd , DateDiff , Expression , ExpressionInfo , GreaterThan , In , InSet , IsNotNull , KnownFloatingPointNormalized , LessThan , Literal , NextDay , Not , Or , SortOrder }
3333import org .apache .spark .sql .catalyst .expressions .aggregate .{AggregateExpression , BloomFilterAggregate , Final , Min , Partial , PartialMerge }
3434import org .apache .spark .sql .catalyst .optimizer .{BuildRight , NormalizeNaNAndZero }
3535import org .apache .spark .sql .catalyst .plans .{Inner , LeftSemi }
@@ -2290,6 +2290,85 @@ class CometExecRuleSuite extends CometTestBase {
22902290 }
22912291 }
22922292
2293+ test(" ANSI next_day filter inputs follow generated predicate order" ) {
2294+ withAnsiNextDayProject { original =>
2295+ def project : ProjectExec = ProjectExec (
2296+ original.projectList :+
2297+ Alias (Literal .create(null , DataTypes .DateType ), " nullable_date" )(),
2298+ original.child)
2299+ val date = Literal .create(java.sql.Date .valueOf(" 2020-01-01" ), DataTypes .DateType )
2300+ val predicates : Seq [(ProjectExec => Expression , Boolean )] = Seq (
2301+ (p => GreaterThan (p.output(1 ), date), true ),
2302+ (p => GreaterThan (p.output(2 ), p.output(1 )), true ),
2303+ (p => And (GreaterThan (p.output(1 ), date), LessThan (p.output.head, Literal (2 ))), true ),
2304+ (p => And (LessThan (p.output.head, Literal (2 )), GreaterThan (p.output(1 ), date)), false ),
2305+ // A single predicate loads all referenced projected values before its expression code.
2306+ (p => Or (LessThan (p.output.head, Literal (2 )), GreaterThan (p.output(1 ), date)), true ),
2307+ (p => And (IsNotNull (p.output(1 )), GreaterThan (p.output(1 ), date)), true ),
2308+ (p => And (IsNotNull (p.output(1 )), IsNotNull (p.output(2 ))), true ),
2309+ (p => And (IsNotNull (p.output(2 )), IsNotNull (p.output(1 ))), false ),
2310+ // Spark moves this null check ahead of the comparison, even though it appears last.
2311+ (p => And (GreaterThan (p.output(1 ), p.output(2 )), IsNotNull (p.output(2 ))), false ),
2312+ // A compound null check also causes Spark to inject checks for its input attributes.
2313+ (
2314+ p =>
2315+ And (
2316+ GreaterThan (p.output(1 ), p.output(2 )),
2317+ IsNotNull (DateAdd (p.output(2 ), p.output.head))),
2318+ false ),
2319+ (p => IsNotNull (DateDiff (p.output(2 ), p.output(1 ))), true ))
2320+ for ((predicate, native) <- predicates) {
2321+ val child = project
2322+ val condition = predicate(child)
2323+ val transformed = applyCometExecRule(FilterExec (condition, child))
2324+ withClue(condition.sql) {
2325+ assert(countOperators(transformed, classOf [CometProjectExec ]) == (if (native) 1 else 0 ))
2326+ assert(countOperators(transformed, classOf [ProjectExec ]) == (if (native) 0 else 1 ))
2327+ }
2328+ }
2329+ }
2330+ }
2331+
2332+ test(" ANSI next_day required by a filter stays native and preserves errors" ) {
2333+ withSQLConf(
2334+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
2335+ SQLConf .WHOLESTAGE_CODEGEN_ENABLED .key -> " true" ,
2336+ SQLConf .ANSI_ENABLED .key -> " true" ,
2337+ CometConf .COMET_SCALA_UDF_CODEGEN_ENABLED .key -> " true" ) {
2338+ val date = java.sql.Date .valueOf(" 2024-01-01" )
2339+ val preserve = CometConf .COMET_EXEC_PRESERVE_EVALUATION_MASKS_ENABLED .key
2340+ def query (predicate : String ): String =
2341+ s """ SELECT x, r FROM
2342+ |(SELECT next_day(_1, _2) AS x, rand(1L) AS r FROM filter_next_day)
2343+ |WHERE $predicate""" .stripMargin
2344+
2345+ withParquetTable(Seq ((date, " Monday" )), " filter_next_day" ) {
2346+ for (enabled <- Seq (false , true )) {
2347+ withSQLConf(preserve -> enabled.toString) {
2348+ val (_, plan) = checkSparkAnswerAndImpl(
2349+ sql(query(" x >= DATE '2020-01-01'" )),
2350+ native = Seq (" next_day" ))
2351+ assert(collect(plan) { case p : CometProjectExec => p }.size == 1 , plan.toString)
2352+ assert(collect(plan) { case f : CometFilterExec => f }.size == 1 , plan.toString)
2353+ }
2354+ }
2355+ }
2356+ withParquetTable(Seq ((date, " NOT_A_DAY" )), " filter_next_day" ) {
2357+ // The required projected value fails even when the date comparison would reject it.
2358+ val required = sql(query(" x < DATE '2020-01-01'" ))
2359+ assertExpressionImpl(required.queryExecution.executedPlan, Seq (" next_day" ), Seq .empty)
2360+ val errorClass = if (isSpark40Plus) " ILLEGAL_DAY_OF_WEEK" else " _LEGACY_ERROR_TEMP_2000"
2361+ val error = checkSparkError(required, errorClass)
2362+ assert(error.getMessage.contains(" Illegal input for day of week" ))
2363+ // An earlier conjunct can still reject the row before evaluating next_day.
2364+ checkSparkAnswerAndFallbackReason(
2365+ query(" r < 0 AND x < DATE '2020-01-01'" ),
2366+ " next_day requires Spark evaluation in a deferred projection" )
2367+ ()
2368+ }
2369+ }
2370+ }
2371+
22932372 test(" strict Project inputs stay native unless their output is deferred" ) {
22942373 withAnsiNextDayProject { project =>
22952374 def strictProject : ProjectExec = ProjectExec (
0 commit comments