Address review comments

sryza · sryza · commit 607539fa59e5 · 2014-03-12T18:53:24.000-07:00
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
@@ -62,7 +62,7 @@ The command to launch the Spark application on the cluster is as follows:
       --args <APP_MAIN_ARGUMENTS> \
       --num-executors <NUMBER_OF_EXECUTOR_PROCESSES> \
       --am-class <ApplicationMaster_CLASS>
-      --am-memory <MEMORY_FOR_ApplicationMaster> \
+      --driver-memory <MEMORY_FOR_ApplicationMaster> \
       --executor-memory <MEMORY_PER_EXECUTOR> \
       --executor-cores <CORES_PER_EXECUTOR> \
       --name <application_name> \
@@ -86,7 +86,7 @@ For example:
           --class org.apache.spark.examples.SparkPi \
           --args yarn-cluster \
           --num-executors 3 \
-          --am-memory 4g \
+          --driver-memory 4g \
           --executor-memory 2g \
           --executor-cores 1
 
@@ -105,7 +105,7 @@ In order to tune executor cores/number/memory etc., you need to export environme
 * `SPARK_EXECUTOR_INSTANCES`, Number of executors to start (Default: 2)
 * `SPARK_EXECUTOR_CORES`, Number of cores per executor (Default: 1).
 * `SPARK_EXECUTOR_MEMORY`, Memory per executor (e.g. 1000M, 2G) (Default: 1G)
-* `SPARK_AM_MEMORY`, Memory for application master (e.g. 1000M, 2G) (Default: 512 Mb)
+* `SPARK_DRIVER_MEMORY`, Memory for driver (e.g. 1000M, 2G) (Default: 512 Mb)
 * `SPARK_YARN_APP_NAME`, The name of your application (Default: Spark)
 * `SPARK_YARN_QUEUE`, The YARN queue to use for allocation requests (Default: 'default')
 * `SPARK_YARN_DIST_FILES`, Comma separated list of files to be distributed with the job.
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala
@@ -36,7 +36,8 @@ class ApplicationMasterArguments(val args: Array[String]) {
     var args = inputArgs
 
     while (! args.isEmpty) {
-
+      // --num-workers, --worker-memory, and --worker-cores are deprecated since 1.0,
+      // the properties with executor in their names are preferred.
       args match {
         case ("--jar") :: value :: tail =>
           userJar = value
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
@@ -74,9 +74,9 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) {
           amClass = value
           args = tail
 
-        case ("--master-memory" | "--am-memory") :: MemoryParam(value) :: tail =>
+        case ("--master-memory" | "--driver-memory") :: MemoryParam(value) :: tail =>
           if (args(0) == "--master-memory") {
-            println("--master-memory is deprecated. Use --am-memory instead.")
+            println("--master-memory is deprecated. Use --driver-memory instead.")
           }
           amMemory = value
           args = tail
@@ -151,7 +151,7 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) {
       "  --num-executors NUM        Number of executors to start (Default: 2)\n" +
       "  --executor-cores NUM       Number of cores for the executors (Default: 1).\n" +
       "  --am-class CLASS_NAME      Class Name for application master (Default: spark.deploy.yarn.ApplicationMaster)\n" +
-      "  --am-memory MEM            Memory for application master (e.g. 1000M, 2G) (Default: 512 Mb)\n" +
+      "  --driver-memory MEM        Memory for driver (e.g. 1000M, 2G) (Default: 512 Mb)\n" +
       "  --executor-memory MEM      Memory per executor (e.g. 1000M, 2G) (Default: 1G)\n" +
       "  --name NAME                The name of your application (Default: Spark)\n" +
       "  --queue QUEUE              The hadoop queue to use for allocation requests (Default: 'default')\n" +
diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -58,8 +58,8 @@ private[spark] class YarnClientSchedulerBackend(
 
     // process any optional arguments, use the defaults already defined in ClientArguments 
     // if things aren't specified
-    Map("SPARK_MASTER_MEMORY" -> "--master-memory",
-      "SPARK_AM_MEMORY" -> "--am-memory",
+    Map("SPARK_MASTER_MEMORY" -> "--driver-memory",
+      "SPARK_DRIVER_MEMORY" -> "--driver-memory",
       "SPARK_WORKER_INSTANCES" -> "--num-executors",
       "SPARK_WORKER_MEMORY" -> "--executor-memory",
       "SPARK_WORKER_CORES" -> "--executor-cores",
