Skip to content

Commit 1ec2d6e

Browse files
marmbrusmarmbrus
committed
Address comments
1 parent 033abc6 commit 1ec2d6e

File tree

8 files changed

+58
-12
lines changed

8 files changed

+58
-12
lines changed

‎pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@
113113
<sbt.project.name>spark</sbt.project.name>
114114
<scala.version>2.10.4</scala.version>
115115
<scala.binary.version>2.10</scala.binary.version>
116+
<scala.macros.version>2.0.1</scala.macros.version>
116117
<mesos.version>0.18.1</mesos.version>
117118
<mesos.classifier>shaded-protobuf</mesos.classifier>
118119
<akka.group>org.spark-project.akka</akka.group>
@@ -818,6 +819,15 @@
818819
<javacArg>-target</javacArg>
819820
<javacArg>${java.version}</javacArg>
820821
</javacArgs>
822+
<!-- The following plugin is required to use quasiquotes in Scala 2.10 and is used
823+
by Spark SQL for code generation. -->
824+
<compilerPlugins>
825+
<compilerPlugin>
826+
<groupId>org.scalamacros</groupId>
827+
<artifactId>paradise_${scala.version}</artifactId>
828+
<version>${scala.macros.version}</version>
829+
</compilerPlugin>
830+
</compilerPlugins>
821831
</configuration>
822832
</plugin>
823833
<plugin>

‎project/SparkBuild.scala‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,7 @@ object SparkBuild extends PomBuild {
187187

188188
object Catalyst {
189189
lazy val settings = Seq(
190-
addCompilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full),
191-
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v),
192-
libraryDependencies += "org.scalamacros" %% "quasiquotes" % "2.0.1")
190+
addCompilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full))
193191
}
194192

195193
object SQL {

‎sql/catalyst/pom.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,19 @@
3636
</properties>
3737

3838
<dependencies>
39+
<dependency>
40+
<groupId>org.scala-lang</groupId>
41+
<artifactId>scala-compiler</artifactId>
42+
</dependency>
3943
<dependency>
4044
<groupId>org.scala-lang</groupId>
4145
<artifactId>scala-reflect</artifactId>
4246
</dependency>
47+
<dependency>
48+
<groupId>org.scalamacros</groupId>
49+
<artifactId>quasiquotes_${scala.binary.version}</artifactId>
50+
<version>${scala.macros.version}</version>
51+
</dependency>
4352
<dependency>
4453
<groupId>org.apache.spark</groupId>
4554
<artifactId>spark-core_${scala.binary.version}</artifactId>

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,10 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
142142
evaluateAs(expressions._1.dataType)(f)
143143

144144
def evaluateAs(resultType: DataType)(f: (TermName, TermName) => Tree): Seq[Tree] = {
145-
// Right now some timestamp tests fail if we enforce this...
146-
if (expressions._1.dataType != expressions._2.dataType)
145+
// TODO: Right now some timestamp tests fail if we enforce this...
146+
if (expressions._1.dataType != expressions._2.dataType) {
147147
log.warn(s"${expressions._1.dataType} != ${expressions._2.dataType}")
148+
}
148149

149150
val eval1 = expressionEvaluator(expressions._1)
150151
val eval2 = expressionEvaluator(expressions._2)

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
1-
package org.apache.spark.sql
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
217

18+
package org.apache.spark.sql
319

420
package object catalyst {
521
/**
@@ -8,4 +24,4 @@ package object catalyst {
824
* 2.10.* builds. See SI-6240 for more details.
925
*/
1026
protected[catalyst] object ScalaReflectionLock
11-
}
27+
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,21 @@ object PhysicalOperation extends PredicateHelper {
104104
}
105105
}
106106

107+
/**
108+
* Matches a logical aggregation that can be performed on distributed data in two steps. The first
109+
* operates on the data in each partition performing partial aggregation for each group. The second
110+
* occurs after the shuffle and completes the aggregation.
111+
*
112+
* This pattern will only match if all aggregate expressions can be computed partially and will
113+
* return the rewritten aggregation expressions for both phases.
114+
*
115+
* The returned values for this match are as follows:
116+
* - Grouping attributes for the final aggregation.
117+
* - Aggregates for the final aggregation.
118+
* - Grouping expressions for the partial aggregation.
119+
* - Partial aggregate expressions.
120+
* - Input to the aggregation.
121+
*/
107122
object PartialAggregation {
108123
type ReturnType =
109124
(Seq[Attribute], Seq[NamedExpression], Seq[Expression], Seq[NamedExpression], LogicalPlan)

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,10 +154,7 @@ abstract class NumericType extends NativeType with PrimitiveType {
154154
}
155155

156156
object NumericType {
157-
def unapply(a: Expression): Boolean = a match {
158-
case e: Expression if e.dataType.isInstanceOf[NumericType] => true
159-
case _ => false
160-
}
157+
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
161158
}
162159

163160
/** Matcher for any expressions that evaluate to [[IntegralType]]s */

‎sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ trait SQLConf {
3838
private[spark] def numShufflePartitions: Int = get("spark.sql.shuffle.partitions", "200").toInt
3939

4040
/**
41-
* When set to true, Spark SQL will use the scala compiler at runtime to generate custom bytecode
41+
* When set to true, Spark SQL will use the Scala compiler at runtime to generate custom bytecode
4242
* that evaluates expressions found in queries. In general this custom code runs much faster
4343
* than interpreted evaluation, but there are significant start-up costs due to compilation.
4444
* As a result codegen is only benificial when queries run for a long time, or when the same

0 commit comments

Comments
 (0)