@@ -23,8 +23,9 @@ import scala.util.Random
2323
2424import org .apache .spark .sql ._
2525import org .apache .spark .sql .catalyst .FunctionIdentifier
26- import org .apache .spark .sql .catalyst .expressions .{AttributeReference , Expression , ExpressionInfo , In , Literal , Not }
26+ import org .apache .spark .sql .catalyst .expressions .{AttributeReference , Expression , ExpressionInfo , In , InSet , KnownFloatingPointNormalized , Literal , Not }
2727import org .apache .spark .sql .catalyst .expressions .aggregate .BloomFilterAggregate
28+ import org .apache .spark .sql .catalyst .optimizer .NormalizeNaNAndZero
2829import org .apache .spark .sql .comet ._
2930import org .apache .spark .sql .comet .execution .shuffle .CometShuffleExchangeExec
3031import org .apache .spark .sql .execution ._
@@ -113,6 +114,69 @@ class CometExecRuleSuite extends CometTestBase {
113114 }
114115
115116 for (dataType <- Seq (FloatType , DoubleType )) {
117+ test(s " floating ${dataType.sql} IN serialization preserves prunable finite literal lists " ) {
118+ withSQLConf(" spark.sql.legacy.nullInEmptyListBehavior" -> " false" ) {
119+ val value = AttributeReference (" value" , dataType)()
120+ val other = AttributeReference (" other" , dataType)()
121+ def literal (v : Double ): Literal = dataType match {
122+ case FloatType => Literal (v.toFloat)
123+ case DoubleType => Literal (v)
124+ }
125+ val ordinary = Seq (1.0d , 3.0d ).map(literal)
126+ val nullLiteral = Literal .create(null , dataType)
127+ val lists : Seq [(Seq [Expression ], Boolean )] = Seq (
128+ ordinary -> false ,
129+ (ordinary :+ nullLiteral) -> false ,
130+ Seq (nullLiteral) -> false ,
131+ Seq (value, other) -> true ) ++
132+ Seq (Double .NaN , 0.0d , - 0.0d , Double .PositiveInfinity , Double .NegativeInfinity )
133+ .map(v => (ordinary :+ literal(v)) -> true ) ++
134+ (if (isSpark35Plus) Seq (Seq .empty[Expression ] -> false ) else Nil )
135+ for ((list, needsNormalization) <- lists;
136+ asSet <- Seq (false , true ) if ! asSet || list.forall(_.isInstanceOf [Literal ]);
137+ negate <- Seq (false , true );
138+ alreadyNormalized <- Seq (false , true )) {
139+ withClue(s " list= $list, asSet= $asSet, negate= $negate, normalized= $alreadyNormalized: " ) {
140+ val needle = if (alreadyNormalized) {
141+ KnownFloatingPointNormalized (NormalizeNaNAndZero (value))
142+ } else {
143+ value
144+ }
145+ val in = if (asSet) {
146+ InSet (needle, list.collect { case l : Literal => l.value }.toSet)
147+ } else {
148+ In (needle, list)
149+ }
150+ val result = QueryPlanSerde
151+ .exprToProto(if (negate) Not (in) else in, Seq (value, other))
152+ .get
153+ // NOT InSet uses a separate Not node; NOT In is fused into the membership node.
154+ val serialized = if (result.hasNot) result.getNot.getChild else result
155+ assert(serialized.hasIn)
156+ assert(result.hasNot == (asSet && negate))
157+ assert(serialized.getIn.getNegated == (negate && ! asSet))
158+ val serializedValue = serialized.getIn.getInValue
159+ if (needsNormalization || alreadyNormalized) {
160+ assert(serializedValue.hasNormalizeNanAndZero)
161+ assert(serializedValue.getNormalizeNanAndZero.getChild.hasBound)
162+ } else {
163+ assert(serializedValue.hasBound)
164+ }
165+ assert(serialized.getIn.getListsCount == list.size)
166+ for (i <- list.indices) {
167+ val candidate = serialized.getIn.getLists(i)
168+ if (list(i).isInstanceOf [Literal ]) {
169+ assert(candidate.hasLiteral)
170+ } else {
171+ assert(candidate.hasNormalizeNanAndZero)
172+ assert(candidate.getNormalizeNanAndZero.getChild.hasBound)
173+ }
174+ }
175+ }
176+ }
177+ }
178+ }
179+
116180 test(
117181 s " floating ${dataType.sql} IN serialization retains normalized operand fallback reasons " ) {
118182 val expressionNames = Seq (" Literal" , " KnownFloatingPointNormalized" )
@@ -127,8 +191,8 @@ class CometExecRuleSuite extends CometTestBase {
127191 val value = AttributeReference (" value" , dataType)()
128192 val other = AttributeReference (" other" , dataType)()
129193 val literals = dataType match {
130- case FloatType => Seq (Literal (1 .0f ), Literal (3.0f ))
131- case DoubleType => Seq (Literal (1 .0d ), Literal (3.0d ))
194+ case FloatType => Seq (Literal (0 .0f ), Literal (3.0f ))
195+ case DoubleType => Seq (Literal (0 .0d ), Literal (3.0d ))
132196 }
133197 // Exercise temporary literals and normalizers in both the value and the list.
134198 val in =
@@ -179,7 +243,7 @@ class CometExecRuleSuite extends CometTestBase {
179243 val predicate = if (literalValue) {
180244 s " CAST(1 AS ${dataType.sql}) IN ( $column, - $column) "
181245 } else {
182- s " $column IN (CAST(1 AS ${dataType.sql}), CAST(3 AS ${dataType.sql})) "
246+ s " $column IN (CAST(0 AS ${dataType.sql}), CAST(3 AS ${dataType.sql})) "
183247 }
184248 val expression = if (negate) s " NOT ( $predicate) " else predicate
185249 val df = sql(s " SELECT $expression AS hit FROM range(0, 4, 1, 1) " )
0 commit comments