Skip to content

Commit d48800c

Browse files
committed
fix: preserve pruning for finite floating IN lists
1 parent 9dd5c04 commit d48800c

2 files changed

Lines changed: 79 additions & 8 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/predicates.scala‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -386,10 +386,17 @@ object ComparisonUtils {
386386
inputs: Seq[Attribute],
387387
binding: Boolean,
388388
negate: Boolean): Option[Expr] = {
389-
// Spark treats all NaNs as equal and both signs of zero as equal in IN and InSet too.
390-
// Normalize both sides, including the fused NOT IN path that calls this method directly.
391-
val normalizedValue = normalizeInOperand(value)
392-
val normalizedList = list.map(normalizeInOperand)
389+
// NaNs and either sign of zero cannot match a finite nonzero literal. Leave such lists
390+
// unwrapped so native Parquet scans can still prune using the column's statistics.
391+
val needsNormalization = !list.forall {
392+
case Literal(null, _) => true
393+
case Literal(v: Float, FloatType) => java.lang.Float.isFinite(v) && v != 0.0f
394+
case Literal(v: Double, DoubleType) => java.lang.Double.isFinite(v) && v != 0.0d
395+
case _ => false
396+
}
397+
// Otherwise normalize both sides for Spark's NaN/zero equality, including fused NOT IN.
398+
val normalizedValue = if (needsNormalization) normalizeInOperand(value) else value
399+
val normalizedList = if (needsNormalization) list.map(normalizeInOperand) else list
393400
val valueExpr = exprToProtoInternal(normalizedValue, inputs, binding)
394401
val listExprs = normalizedList.map(exprToProtoInternal(_, inputs, binding))
395402
if (valueExpr.isDefined && listExprs.forall(_.isDefined)) {

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ import scala.util.Random
2323

2424
import org.apache.spark.sql._
2525
import org.apache.spark.sql.catalyst.FunctionIdentifier
26-
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionInfo, In, Literal, Not}
26+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionInfo, In, InSet, KnownFloatingPointNormalized, Literal, Not}
2727
import org.apache.spark.sql.catalyst.expressions.aggregate.BloomFilterAggregate
28+
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
2829
import org.apache.spark.sql.comet._
2930
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
3031
import org.apache.spark.sql.execution._
@@ -113,6 +114,69 @@ class CometExecRuleSuite extends CometTestBase {
113114
}
114115

115116
for (dataType <- Seq(FloatType, DoubleType)) {
117+
test(s"floating ${dataType.sql} IN serialization preserves prunable finite literal lists") {
118+
withSQLConf("spark.sql.legacy.nullInEmptyListBehavior" -> "false") {
119+
val value = AttributeReference("value", dataType)()
120+
val other = AttributeReference("other", dataType)()
121+
def literal(v: Double): Literal = dataType match {
122+
case FloatType => Literal(v.toFloat)
123+
case DoubleType => Literal(v)
124+
}
125+
val ordinary = Seq(1.0d, 3.0d).map(literal)
126+
val nullLiteral = Literal.create(null, dataType)
127+
val lists: Seq[(Seq[Expression], Boolean)] = Seq(
128+
ordinary -> false,
129+
(ordinary :+ nullLiteral) -> false,
130+
Seq(nullLiteral) -> false,
131+
Seq(value, other) -> true) ++
132+
Seq(Double.NaN, 0.0d, -0.0d, Double.PositiveInfinity, Double.NegativeInfinity)
133+
.map(v => (ordinary :+ literal(v)) -> true) ++
134+
(if (isSpark35Plus) Seq(Seq.empty[Expression] -> false) else Nil)
135+
for ((list, needsNormalization) <- lists;
136+
asSet <- Seq(false, true) if !asSet || list.forall(_.isInstanceOf[Literal]);
137+
negate <- Seq(false, true);
138+
alreadyNormalized <- Seq(false, true)) {
139+
withClue(s"list=$list, asSet=$asSet, negate=$negate, normalized=$alreadyNormalized: ") {
140+
val needle = if (alreadyNormalized) {
141+
KnownFloatingPointNormalized(NormalizeNaNAndZero(value))
142+
} else {
143+
value
144+
}
145+
val in = if (asSet) {
146+
InSet(needle, list.collect { case l: Literal => l.value }.toSet)
147+
} else {
148+
In(needle, list)
149+
}
150+
val result = QueryPlanSerde
151+
.exprToProto(if (negate) Not(in) else in, Seq(value, other))
152+
.get
153+
// NOT InSet uses a separate Not node; NOT In is fused into the membership node.
154+
val serialized = if (result.hasNot) result.getNot.getChild else result
155+
assert(serialized.hasIn)
156+
assert(result.hasNot == (asSet && negate))
157+
assert(serialized.getIn.getNegated == (negate && !asSet))
158+
val serializedValue = serialized.getIn.getInValue
159+
if (needsNormalization || alreadyNormalized) {
160+
assert(serializedValue.hasNormalizeNanAndZero)
161+
assert(serializedValue.getNormalizeNanAndZero.getChild.hasBound)
162+
} else {
163+
assert(serializedValue.hasBound)
164+
}
165+
assert(serialized.getIn.getListsCount == list.size)
166+
for (i <- list.indices) {
167+
val candidate = serialized.getIn.getLists(i)
168+
if (list(i).isInstanceOf[Literal]) {
169+
assert(candidate.hasLiteral)
170+
} else {
171+
assert(candidate.hasNormalizeNanAndZero)
172+
assert(candidate.getNormalizeNanAndZero.getChild.hasBound)
173+
}
174+
}
175+
}
176+
}
177+
}
178+
}
179+
116180
test(
117181
s"floating ${dataType.sql} IN serialization retains normalized operand fallback reasons") {
118182
val expressionNames = Seq("Literal", "KnownFloatingPointNormalized")
@@ -127,8 +191,8 @@ class CometExecRuleSuite extends CometTestBase {
127191
val value = AttributeReference("value", dataType)()
128192
val other = AttributeReference("other", dataType)()
129193
val literals = dataType match {
130-
case FloatType => Seq(Literal(1.0f), Literal(3.0f))
131-
case DoubleType => Seq(Literal(1.0d), Literal(3.0d))
194+
case FloatType => Seq(Literal(0.0f), Literal(3.0f))
195+
case DoubleType => Seq(Literal(0.0d), Literal(3.0d))
132196
}
133197
// Exercise temporary literals and normalizers in both the value and the list.
134198
val in =
@@ -179,7 +243,7 @@ class CometExecRuleSuite extends CometTestBase {
179243
val predicate = if (literalValue) {
180244
s"CAST(1 AS ${dataType.sql}) IN ($column, -$column)"
181245
} else {
182-
s"$column IN (CAST(1 AS ${dataType.sql}), CAST(3 AS ${dataType.sql}))"
246+
s"$column IN (CAST(0 AS ${dataType.sql}), CAST(3 AS ${dataType.sql}))"
183247
}
184248
val expression = if (negate) s"NOT ($predicate)" else predicate
185249
val df = sql(s"SELECT $expression AS hit FROM range(0, 4, 1, 1)")

0 commit comments

Comments
 (0)