Fix

maropu · maropu · commit fca22b767fdd · 2017-10-04T10:48:35.000+09:00
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
@@ -392,9 +392,11 @@ case class WholeStageCodegenExec(child: SparkPlan) extends UnaryExecNode with Co
 
     // Check if compiled code has a too large function
     if (maxCodeSize > sqlContext.conf.hugeMethodLimit) {
-      logWarning(s"Found too long generated codes: the bytecode size was $maxCodeSize and " +
-        s"this value went over the limit ${sqlContext.conf.hugeMethodLimit}. To avoid this, " +
-        s"you can the limit ${SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.key} higher:\n$treeString")
+      logWarning(s"Found too long generated codes and JIT optimization might not work: " +
+        s"the bytecode size was $maxCodeSize, this value went over the limit " +
+        s"${sqlContext.conf.hugeMethodLimit}, and the whole-stage codegen was disable " +
+        s"for this plan. To avoid this, you can set the limit " +
+        s"${SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.key} higher:\n$treeString")
       return child.execute()
     }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -151,7 +151,7 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
     }
   }
 
-  def genGroupByCodeGenContext(caseNum: Int): (CodegenContext, CodeAndComment) = {
+  def genGroupByCodeGenContext(caseNum: Int): CodeAndComment = {
     val caseExp = (1 to caseNum).map { i =>
       s"case when id > $i and id <= ${i + 1} then 1 else 0 end as v$i"
     }.toList
@@ -176,14 +176,14 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
     })
 
     assert(wholeStageCodeGenExec.isDefined)
-    wholeStageCodeGenExec.get.asInstanceOf[WholeStageCodegenExec].doCodeGen()
+    wholeStageCodeGenExec.get.asInstanceOf[WholeStageCodegenExec].doCodeGen()._2
   }
 
   test("SPARK-21871 check if we can get large code size when compiling too long functions") {
-    val (_, codeWithShortFunctions) = genGroupByCodeGenContext(3)
+    val codeWithShortFunctions = genGroupByCodeGenContext(3)
     val (_, maxCodeSize1) = CodeGenerator.compile(codeWithShortFunctions)
     assert(maxCodeSize1 < SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.defaultValue.get)
-    val (_, codeWithLongFunctions) = genGroupByCodeGenContext(20)
+    val codeWithLongFunctions = genGroupByCodeGenContext(20)
     val (_, maxCodeSize2) = CodeGenerator.compile(codeWithLongFunctions)
     assert(maxCodeSize2 > SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.defaultValue.get)
   }

Original file line number	Diff line number	Diff line change
`@@ -151,7 +151,7 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {`
`151`	`151`	`}`
`152`	`152`	`}`
`153`	`153`
`154`		`- def genGroupByCodeGenContext(caseNum: Int): (CodegenContext, CodeAndComment) = {`
	`154`	`+ def genGroupByCodeGenContext(caseNum: Int): CodeAndComment = {`
`155`	`155`	`val caseExp = (1 to caseNum).map { i =>`
`156`	`156`	`s"case when id > $i and id <= ${i + 1} then 1 else 0 end as v$i"`
`157`	`157`	`}.toList`
`@@ -176,14 +176,14 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {`
`176`	`176`	`})`
`177`	`177`
`178`	`178`	`assert(wholeStageCodeGenExec.isDefined)`
`179`		`- wholeStageCodeGenExec.get.asInstanceOf[WholeStageCodegenExec].doCodeGen()`
	`179`	`+ wholeStageCodeGenExec.get.asInstanceOf[WholeStageCodegenExec].doCodeGen()._2`
`180`	`180`	`}`
`181`	`181`
`182`	`182`	`test("SPARK-21871 check if we can get large code size when compiling too long functions") {`
`183`		`- val (_, codeWithShortFunctions) = genGroupByCodeGenContext(3)`
	`183`	`+ val codeWithShortFunctions = genGroupByCodeGenContext(3)`
`184`	`184`	`val (_, maxCodeSize1) = CodeGenerator.compile(codeWithShortFunctions)`
`185`	`185`	`assert(maxCodeSize1 < SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.defaultValue.get)`
`186`		`- val (_, codeWithLongFunctions) = genGroupByCodeGenContext(20)`
	`186`	`+ val codeWithLongFunctions = genGroupByCodeGenContext(20)`
`187`	`187`	`val (_, maxCodeSize2) = CodeGenerator.compile(codeWithLongFunctions)`
`188`	`188`	`assert(maxCodeSize2 > SQLConf.WHOLESTAGE_HUGE_METHOD_LIMIT.defaultValue.get)`
`189`	`189`	`}`