fix

wangyum · wangyum · commit 4c21e43ca34c · 2020-12-29T15:10:59.000+08:00
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -510,8 +510,9 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
         val (h, t) = branches.span(_._1 != TrueLiteral)
         CaseWhen( h :+ t.head, None)
 
-      case e @ CaseWhen(branches, Some(elseValue))
-          if branches.forall(_._2.semanticEquals(elseValue)) =>
+      case e @ CaseWhen(branches, elseOpt)
+          if branches.forall(_._2.semanticEquals(elseOpt.getOrElse(Literal(null, e.dataType)))) =>
+        val elseValue = elseOpt.getOrElse(Literal(null, e.dataType))
         // For non-deterministic conditions with side effect, we can not remove it, or change
         // the ordering. As a result, we try to remove the deterministic conditions from the tail.
         var hitNonDeterministicCond = false
@@ -527,10 +528,6 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
         } else {
           e.copy(branches = branches.take(i).map(branch => (branch._1, elseValue)))
         }
-
-      case e @ CaseWhen(branches, None)
-          if branches.forall(_._2.semanticEquals(Literal(null, e.dataType))) =>
-        Literal(null, e.dataType)
     }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushFoldableIntoBranchesSuite.scala
@@ -260,13 +260,12 @@ class PushFoldableIntoBranchesSuite
   }
 
   test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
-    Seq(a, LessThan(Rand(1), Literal(0.5))).foreach { condition =>
-      assertEquivalent(
-        EqualTo(CaseWhen(Seq((condition, Literal.create(null, IntegerType)))), Literal(2)),
-        Literal.create(null, BooleanType))
-      assertEquivalent(
-        EqualTo(CaseWhen(Seq((condition, Literal("str")))).cast(IntegerType), Literal(2)),
-        Literal.create(null, BooleanType))
-    }
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((a, Literal.create(null, IntegerType)))), Literal(2)),
+      Literal.create(null, BooleanType))
+    assertEquivalent(
+      EqualTo(CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal("str")))).cast(IntegerType),
+        Literal(2)),
+      CaseWhen(Seq((LessThan(Rand(1), Literal(0.5)), Literal.create(null, BooleanType)))))
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
@@ -237,10 +237,12 @@ class SimplifyConditionalSuite extends PlanTest with ExpressionEvalHelper with P
   }
 
   test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {
-    Seq(GreaterThan('a, 1), GreaterThan(Rand(0), 1)).foreach { condition =>
-      assertEquivalent(
-        CaseWhen((condition, Literal.create(null, IntegerType)) :: Nil, None),
-        Literal.create(null, IntegerType))
-    }
+    assertEquivalent(
+      CaseWhen((GreaterThan('a, 1), Literal.create(null, IntegerType)) :: Nil, None),
+      Literal.create(null, IntegerType))
+
+    assertEquivalent(
+      CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None),
+      CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None))
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -510,8 +510,9 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {`
`510`	`510`	`val (h, t) = branches.span(_._1 != TrueLiteral)`
`511`	`511`	`CaseWhen( h :+ t.head, None)`
`512`	`512`
`513`		`- case e @ CaseWhen(branches, Some(elseValue))`
`514`		`- if branches.forall(_._2.semanticEquals(elseValue)) =>`
	`513`	`+ case e @ CaseWhen(branches, elseOpt)`
	`514`	`+ if branches.forall(_._2.semanticEquals(elseOpt.getOrElse(Literal(null, e.dataType)))) =>`
	`515`	`+ val elseValue = elseOpt.getOrElse(Literal(null, e.dataType))`
`515`	`516`	`// For non-deterministic conditions with side effect, we can not remove it, or change`
`516`	`517`	`// the ordering. As a result, we try to remove the deterministic conditions from the tail.`
`517`	`518`	`var hitNonDeterministicCond = false`
`@@ -527,10 +528,6 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {`
`527`	`528`	`} else {`
`528`	`529`	`e.copy(branches = branches.take(i).map(branch => (branch._1, elseValue)))`
`529`	`530`	`}`
`530`		`-`
`531`		`- case e @ CaseWhen(branches, None)`
`532`		`- if branches.forall(_._2.semanticEquals(Literal(null, e.dataType))) =>`
`533`		`- Literal(null, e.dataType)`
`534`	`531`	`}`
`535`	`532`	`}`
`536`	`533`	`}`
Original file line number	Diff line number	Diff line change
`@@ -237,10 +237,12 @@ class SimplifyConditionalSuite extends PlanTest with ExpressionEvalHelper with P`
`237`	`237`	`}`
`238`	`238`
`239`	`239`	`test("SPARK-33847: Remove the CaseWhen if elseValue is empty and other outputs are null") {`
`240`		`- Seq(GreaterThan('a, 1), GreaterThan(Rand(0), 1)).foreach { condition =>`
`241`		`- assertEquivalent(`
`242`		`- CaseWhen((condition, Literal.create(null, IntegerType)) :: Nil, None),`
`243`		`- Literal.create(null, IntegerType))`
`244`		`- }`
	`240`	`+ assertEquivalent(`
	`241`	`+ CaseWhen((GreaterThan('a, 1), Literal.create(null, IntegerType)) :: Nil, None),`
	`242`	`+ Literal.create(null, IntegerType))`
	`243`	`+`
	`244`	`+ assertEquivalent(`
	`245`	`+ CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None),`
	`246`	`+ CaseWhen((GreaterThan(Rand(0), 0.5), Literal.create(null, IntegerType)) :: Nil, None))`
`245`	`247`	`}`
`246`	`248`	`}`