Skip to content

Commit 77f35d7

Browse files
EugenCepoiEugenCepoi
committed
SPARK-2058: Overriding SPARK_HOME/conf with SPARK_CONF_DIR
1 parent d112a6c commit 77f35d7

File tree

5 files changed

+64
-20
lines changed

5 files changed

+64
-20
lines changed

‎bin/compute-classpath.cmd‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,13 @@ rem Load environment variables from conf\spark-env.cmd, if it exists
3636
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
3737

3838
rem Build up classpath
39-
set CLASSPATH=%SPARK_CLASSPATH%;%SPARK_SUBMIT_CLASSPATH%;%FWDIR%conf
39+
set CLASSPATH=%SPARK_CLASSPATH%;%SPARK_SUBMIT_CLASSPATH%
40+
41+
if "x%SPARK_CONF_DIR%"!="x" (
42+
set CLASSPATH=%CLASSPATH%;%SPARK_CONF_DIR%
43+
) else (
44+
set CLASSPATH=%%CLASSPATH%;%FWDIR%conf
45+
)
4046

4147
if exist "%FWDIR%RELEASE" (
4248
for %%d in ("%FWDIR%lib\spark-assembly*.jar") do (

‎bin/compute-classpath.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,14 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2727

2828
. "$FWDIR"/bin/load-spark-env.sh
2929

30+
CLASSPATH="$SPARK_CLASSPATH:$SPARK_SUBMIT_CLASSPATH"
31+
3032
# Build up classpath
31-
CLASSPATH="$SPARK_CLASSPATH:$SPARK_SUBMIT_CLASSPATH:$FWDIR/conf"
33+
if [ -n "$SPARK_CONF_DIR" ]; then
34+
CLASSPATH="$CLASSPATH:$SPARK_CONF_DIR"
35+
else
36+
CLASSPATH="$CLASSPATH:$FWDIR/conf"
37+
fi
3238

3339
ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION"
3440

‎core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala‎

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ import org.apache.spark.util.Utils
2929

3030
/**
3131
* Parses and encapsulates arguments from the spark-submit script.
32+
* The env argument is used for testing.
3233
*/
33-
private[spark] class SparkSubmitArguments(args: Seq[String]) {
34+
private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, String] = sys.env) {
3435
var master: String = null
3536
var deployMode: String = null
3637
var executorMemory: String = null
@@ -86,20 +87,12 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
8687
private def mergeSparkProperties(): Unit = {
8788
// Use common defaults file, if not specified by user
8889
if (propertiesFile == null) {
89-
sys.env.get("SPARK_CONF_DIR").foreach { sparkConfDir =>
90-
val sep = File.separator
91-
val defaultPath = s"${sparkConfDir}${sep}spark-defaults.conf"
92-
val file = new File(defaultPath)
93-
if (file.exists()) {
94-
propertiesFile = file.getAbsolutePath
95-
}
96-
}
97-
}
90+
val sep = File.separator
91+
val sparkHomeConfig = env.get("SPARK_HOME").map(sparkHome => s"${sparkHome}${sep}conf")
92+
val confDir = env.get("SPARK_CONF_DIR").orElse(sparkHomeConfig)
9893

99-
if (propertiesFile == null) {
100-
sys.env.get("SPARK_HOME").foreach { sparkHome =>
101-
val sep = File.separator
102-
val defaultPath = s"${sparkHome}${sep}conf${sep}spark-defaults.conf"
94+
confDir.foreach { sparkConfDir =>
95+
val defaultPath = s"${sparkConfDir}${sep}spark-defaults.conf"
10396
val file = new File(defaultPath)
10497
if (file.exists()) {
10598
propertiesFile = file.getAbsolutePath
@@ -124,8 +117,8 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
124117
jars = Option(jars).getOrElse(properties.get("spark.jars").orNull)
125118

126119
// This supports env vars in older versions of Spark
127-
master = Option(master).getOrElse(System.getenv("MASTER"))
128-
deployMode = Option(deployMode).getOrElse(System.getenv("DEPLOY_MODE"))
120+
master = Option(master).getOrElse(env.get("MASTER").orNull)
121+
deployMode = Option(deployMode).getOrElse(env.get("DEPLOY_MODE").orNull)
129122

130123
// Try to set main class from JAR if no --class argument is given
131124
if (mainClass == null && !isPython && primaryResource != null) {
@@ -178,7 +171,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
178171
}
179172

180173
if (master.startsWith("yarn")) {
181-
val hasHadoopEnv = sys.env.contains("HADOOP_CONF_DIR") || sys.env.contains("YARN_CONF_DIR")
174+
val hasHadoopEnv = env.contains("HADOOP_CONF_DIR") || env.contains("YARN_CONF_DIR")
182175
if (!hasHadoopEnv && !Utils.isTesting) {
183176
throw new Exception(s"When running with master '$master' " +
184177
"either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment.")

‎core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.deploy
1919

20-
import java.io.{File, OutputStream, PrintStream}
20+
import java.io._
2121

2222
import scala.collection.mutable.ArrayBuffer
2323

@@ -26,6 +26,7 @@ import org.apache.spark.deploy.SparkSubmit._
2626
import org.apache.spark.util.Utils
2727
import org.scalatest.FunSuite
2828
import org.scalatest.Matchers
29+
import com.google.common.io.Files
2930

3031
class SparkSubmitSuite extends FunSuite with Matchers {
3132
def beforeAll() {
@@ -306,6 +307,21 @@ class SparkSubmitSuite extends FunSuite with Matchers {
306307
runSparkSubmit(args)
307308
}
308309

310+
test("SPARK_CONF_DIR overrides spark-defaults.conf") {
311+
forConfDir(Map("spark.executor.memory" -> "2.3g")) { path =>
312+
val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
313+
val args = Seq(
314+
"--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
315+
"--name", "testApp",
316+
"--master", "local",
317+
unusedJar.toString)
318+
val appArgs = new SparkSubmitArguments(args, Map("SPARK_CONF_DIR" -> path))
319+
assert(appArgs.propertiesFile != null)
320+
assert(appArgs.propertiesFile.startsWith(path))
321+
appArgs.executorMemory should be ("2.3g")
322+
}
323+
}
324+
309325
// NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
310326
def runSparkSubmit(args: Seq[String]): String = {
311327
val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
@@ -314,6 +330,22 @@ class SparkSubmitSuite extends FunSuite with Matchers {
314330
new File(sparkHome),
315331
Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome))
316332
}
333+
334+
def forConfDir(defaults: Map[String, String]) (f: String => Unit) = {
335+
val tmpDir = Files.createTempDir()
336+
337+
val defaultsConf = new File(tmpDir.getAbsolutePath, "spark-defaults.conf")
338+
val writer = new OutputStreamWriter(new FileOutputStream(defaultsConf))
339+
for ((key, value) <- defaults) writer.write(s"$key $value\n")
340+
341+
writer.close()
342+
343+
try {
344+
f(tmpDir.getAbsolutePath)
345+
} finally {
346+
Utils.deleteRecursively(tmpDir)
347+
}
348+
}
317349
}
318350

319351
object JarCreationTest {

‎docs/configuration.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1088,3 +1088,10 @@ compute `SPARK_LOCAL_IP` by looking up the IP of a specific network interface.
10881088
Spark uses [log4j](http://logging.apache.org/log4j/) for logging. You can configure it by adding a
10891089
`log4j.properties` file in the `conf` directory. One way to start is to copy the existing
10901090
`log4j.properties.template` located there.
1091+
1092+
# Overriding configuration directory
1093+
1094+
To specify a different configuration directory other than the default "SPARK_HOME/conf",
1095+
you can set SPARK_CONF_DIR. Spark will use the the configuration files (spark-defaults.conf, spark-env.sh, log4j.properties, etc)
1096+
from this directory.
1097+

0 commit comments

Comments
 (0)