Skip to content

Commit 3891bf2

Browse files
leahmcguireleahmcguire
committed
synced with apache spark and resolved merge conflict
2 parents d9477ed + cfff397 commit 3891bf2

File tree

924 files changed

+64616
-16776
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

924 files changed

+64616
-16776
lines changed

‎.rat-excludes‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
target
2+
cache
23
.gitignore
34
.gitattributes
45
.project
@@ -18,6 +19,7 @@ fairscheduler.xml.template
1819
spark-defaults.conf.template
1920
log4j.properties
2021
log4j.properties.template
22+
metrics.properties
2123
metrics.properties.template
2224
slaves
2325
slaves.template

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ To build Spark and its example programs, run:
2626

2727
(You do not need to do this if you downloaded a pre-built package.)
2828
More detailed documentation is available from the project site, at
29-
["Building Spark with Maven"](http://spark.apache.org/docs/latest/building-spark.html).
29+
["Building Spark"](http://spark.apache.org/docs/latest/building-spark.html).
3030

3131
## Interactive Scala Shell
3232

‎assembly/pom.xml‎

Lines changed: 10 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,9 @@
3636
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
3737
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
3838
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>
39-
<deb.pkg.name>spark</deb.pkg.name>
40-
<deb.install.path>/usr/share/spark</deb.install.path>
41-
<deb.user>root</deb.user>
42-
<deb.bin.filemode>744</deb.bin.filemode>
4339
</properties>
4440

4541
<dependencies>
46-
<!-- Promote Guava to compile scope in this module so it's included while shading. -->
47-
<dependency>
48-
<groupId>com.google.guava</groupId>
49-
<artifactId>guava</artifactId>
50-
<scope>compile</scope>
51-
</dependency>
5242
<dependency>
5343
<groupId>org.apache.spark</groupId>
5444
<artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -124,6 +114,16 @@
124114
<exclude>META-INF/*.RSA</exclude>
125115
</excludes>
126116
</filter>
117+
<filter>
118+
<!-- Exclude libgfortran, libgcc for license issues -->
119+
<artifact>org.jblas:jblas</artifact>
120+
<excludes>
121+
<!-- Linux amd64 is OK; not statically linked -->
122+
<exclude>lib/static/Linux/i386/**</exclude>
123+
<exclude>lib/static/Mac OS X/**</exclude>
124+
<exclude>lib/static/Windows/**</exclude>
125+
</excludes>
126+
</filter>
127127
</filters>
128128
</configuration>
129129
<executions>
@@ -133,20 +133,6 @@
133133
<goal>shade</goal>
134134
</goals>
135135
<configuration>
136-
<relocations>
137-
<relocation>
138-
<pattern>com.google</pattern>
139-
<shadedPattern>org.spark-project.guava</shadedPattern>
140-
<includes>
141-
<include>com.google.common.**</include>
142-
</includes>
143-
<excludes>
144-
<exclude>com/google/common/base/Absent*</exclude>
145-
<exclude>com/google/common/base/Optional*</exclude>
146-
<exclude>com/google/common/base/Present*</exclude>
147-
</excludes>
148-
</relocation>
149-
</relocations>
150136
<transformers>
151137
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
152138
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
@@ -237,113 +223,6 @@
237223
</plugins>
238224
</build>
239225
</profile>
240-
<profile>
241-
<id>deb</id>
242-
<build>
243-
<plugins>
244-
<plugin>
245-
<groupId>org.codehaus.mojo</groupId>
246-
<artifactId>buildnumber-maven-plugin</artifactId>
247-
<version>1.2</version>
248-
<executions>
249-
<execution>
250-
<phase>validate</phase>
251-
<goals>
252-
<goal>create</goal>
253-
</goals>
254-
<configuration>
255-
<shortRevisionLength>8</shortRevisionLength>
256-
</configuration>
257-
</execution>
258-
</executions>
259-
</plugin>
260-
<plugin>
261-
<groupId>org.vafer</groupId>
262-
<artifactId>jdeb</artifactId>
263-
<version>0.11</version>
264-
<executions>
265-
<execution>
266-
<phase>package</phase>
267-
<goals>
268-
<goal>jdeb</goal>
269-
</goals>
270-
<configuration>
271-
<deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
272-
<attach>false</attach>
273-
<compression>gzip</compression>
274-
<dataSet>
275-
<data>
276-
<src>${spark.jar}</src>
277-
<type>file</type>
278-
<mapper>
279-
<type>perm</type>
280-
<user>${deb.user}</user>
281-
<group>${deb.user}</group>
282-
<prefix>${deb.install.path}/jars</prefix>
283-
</mapper>
284-
</data>
285-
<data>
286-
<src>${basedir}/src/deb/RELEASE</src>
287-
<type>file</type>
288-
<mapper>
289-
<type>perm</type>
290-
<user>${deb.user}</user>
291-
<group>${deb.user}</group>
292-
<prefix>${deb.install.path}</prefix>
293-
</mapper>
294-
</data>
295-
<data>
296-
<src>${basedir}/../conf</src>
297-
<type>directory</type>
298-
<mapper>
299-
<type>perm</type>
300-
<user>${deb.user}</user>
301-
<group>${deb.user}</group>
302-
<prefix>${deb.install.path}/conf</prefix>
303-
<filemode>744</filemode>
304-
</mapper>
305-
</data>
306-
<data>
307-
<src>${basedir}/../bin</src>
308-
<type>directory</type>
309-
<mapper>
310-
<type>perm</type>
311-
<user>${deb.user}</user>
312-
<group>${deb.user}</group>
313-
<prefix>${deb.install.path}/bin</prefix>
314-
<filemode>${deb.bin.filemode}</filemode>
315-
</mapper>
316-
</data>
317-
<data>
318-
<src>${basedir}/../sbin</src>
319-
<type>directory</type>
320-
<mapper>
321-
<type>perm</type>
322-
<user>${deb.user}</user>
323-
<group>${deb.user}</group>
324-
<prefix>${deb.install.path}/sbin</prefix>
325-
<filemode>744</filemode>
326-
</mapper>
327-
</data>
328-
<data>
329-
<src>${basedir}/../python</src>
330-
<type>directory</type>
331-
<mapper>
332-
<type>perm</type>
333-
<user>${deb.user}</user>
334-
<group>${deb.user}</group>
335-
<prefix>${deb.install.path}/python</prefix>
336-
<filemode>744</filemode>
337-
</mapper>
338-
</data>
339-
</dataSet>
340-
</configuration>
341-
</execution>
342-
</executions>
343-
</plugin>
344-
</plugins>
345-
</build>
346-
</profile>
347226
<profile>
348227
<id>kinesis-asl</id>
349228
<dependencies>

‎assembly/src/deb/RELEASE‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎assembly/src/deb/control/control‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎bin/compute-classpath.sh‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ fi
5050
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
5151
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
5252
"classes ahead of assembly." >&2
53+
# Spark classes
5354
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
54-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
5555
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
5656
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
5757
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
@@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then
6363
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
6464
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
6565
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
66+
# Jars for shaded deps in their original form (copied here during build)
67+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
6668
fi
6769

6870
# Use spark-assembly jar from either RELEASE or assembly directory
@@ -72,22 +74,25 @@ else
7274
assembly_folder="$ASSEMBLY_DIR"
7375
fi
7476

75-
num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar$" | wc -l)"
76-
if [ "$num_jars" -eq "0" ]; then
77-
echo "Failed to find Spark assembly in $assembly_folder"
78-
echo "You need to build Spark before running this program."
79-
exit 1
80-
fi
77+
num_jars=0
78+
79+
for f in "${assembly_folder}"/spark-assembly*hadoop*.jar; do
80+
if [[ ! -e "$f" ]]; then
81+
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
82+
echo "You need to build Spark before running this program." 1>&2
83+
exit 1
84+
fi
85+
ASSEMBLY_JAR="$f"
86+
num_jars=$((num_jars+1))
87+
done
88+
8189
if [ "$num_jars" -gt "1" ]; then
82-
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar$")
83-
echo "Found multiple Spark assembly jars in $assembly_folder:"
84-
echo "$jars_list"
85-
echo "Please remove all but one jar."
90+
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
91+
ls "${assembly_folder}"/spark-assembly*hadoop*.jar 1>&2
92+
echo "Please remove all but one jar." 1>&2
8693
exit 1
8794
fi
8895

89-
ASSEMBLY_JAR="$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)"
90-
9196
# Verify that versions of java used to build the jars and run Spark are compatible
9297
jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1)
9398
if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then

‎bin/run-example‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,32 @@ else
3535
fi
3636

3737
if [ -f "$FWDIR/RELEASE" ]; then
38-
export SPARK_EXAMPLES_JAR="`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`"
39-
elif [ -e "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
40-
export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar`"
38+
JAR_PATH="${FWDIR}/lib"
39+
else
40+
JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}"
4141
fi
4242

43-
if [[ -z "$SPARK_EXAMPLES_JAR" ]]; then
44-
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
45-
echo "You need to build Spark before running this program" 1>&2
43+
JAR_COUNT=0
44+
45+
for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do
46+
if [[ ! -e "$f" ]]; then
47+
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
48+
echo "You need to build Spark before running this program" 1>&2
49+
exit 1
50+
fi
51+
SPARK_EXAMPLES_JAR="$f"
52+
JAR_COUNT=$((JAR_COUNT+1))
53+
done
54+
55+
if [ "$JAR_COUNT" -gt "1" ]; then
56+
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
57+
ls "${JAR_PATH}"/spark-examples-*hadoop*.jar 1>&2
58+
echo "Please remove all but one jar." 1>&2
4659
exit 1
4760
fi
4861

62+
export SPARK_EXAMPLES_JAR
63+
4964
EXAMPLE_MASTER=${MASTER:-"local[*]"}
5065

5166
if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then

‎bin/spark-class‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2929

3030
# Export this as SPARK_HOME
3131
export SPARK_HOME="$FWDIR"
32+
export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}"
3233

3334
. "$FWDIR"/bin/load-spark-env.sh
3435

@@ -71,6 +72,8 @@ case "$1" in
7172
'org.apache.spark.executor.MesosExecutorBackend')
7273
OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_EXECUTOR_OPTS"
7374
OUR_JAVA_MEM=${SPARK_EXECUTOR_MEMORY:-$DEFAULT_MEM}
75+
export PYTHONPATH="$FWDIR/python:$PYTHONPATH"
76+
export PYTHONPATH="$FWDIR/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH"
7477
;;
7578

7679
# Spark submit uses SPARK_JAVA_OPTS + SPARK_SUBMIT_OPTS +
@@ -118,8 +121,8 @@ fi
118121
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
119122

120123
# Load extra JAVA_OPTS from conf/java-opts, if it exists
121-
if [ -e "$FWDIR/conf/java-opts" ] ; then
122-
JAVA_OPTS="$JAVA_OPTS `cat "$FWDIR"/conf/java-opts`"
124+
if [ -e "$SPARK_CONF_DIR/java-opts" ] ; then
125+
JAVA_OPTS="$JAVA_OPTS `cat "$SPARK_CONF_DIR"/java-opts`"
123126
fi
124127

125128
# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!
@@ -148,7 +151,7 @@ fi
148151
if [[ "$1" =~ org.apache.spark.tools.* ]]; then
149152
if test -z "$SPARK_TOOLS_JAR"; then
150153
echo "Failed to find Spark Tools Jar in $FWDIR/tools/target/scala-$SPARK_SCALA_VERSION/" 1>&2
151-
echo "You need to build Spark before running $1." 1>&2
154+
echo "You need to run \"build/sbt tools/package\" before running $1." 1>&2
152155
exit 1
153156
fi
154157
CLASSPATH="$CLASSPATH:$SPARK_TOOLS_JAR"

‎bin/spark-shell.cmd‎

100755100644
File mode changed.

‎bin/spark-submit2.cmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ set ORIG_ARGS=%*
2525
rem Reset the values of all variables used
2626
set SPARK_SUBMIT_DEPLOY_MODE=client
2727

28-
if not defined %SPARK_CONF_DIR% (
28+
if [%SPARK_CONF_DIR%] == [] (
2929
set SPARK_CONF_DIR=%SPARK_HOME%\conf
3030
)
3131
set SPARK_SUBMIT_PROPERTIES_FILE=%SPARK_CONF_DIR%\spark-defaults.conf

0 commit comments

Comments
 (0)