diff options
| author | WangTaoTheTonic <barneystinson@aliyun.com> | 2014-10-28 12:31:42 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2014-10-28 12:31:42 -0700 |
| commit | 1ea3e3dc9dd942402731751089bab2fb6ae29c7b (patch) | |
| tree | 6fa693814b4a7a7baa1d92040637b9ea5e12aa91 /yarn | |
| parent | 44d8b45a38c8d934628373a3b21084432516ee00 (diff) | |
| download | spark-1ea3e3dc9dd942402731751089bab2fb6ae29c7b.tar.gz spark-1ea3e3dc9dd942402731751089bab2fb6ae29c7b.tar.bz2 spark-1ea3e3dc9dd942402731751089bab2fb6ae29c7b.zip | |
[SPARK-4096][YARN]let ApplicationMaster accept executor memory argument in same format as JVM memory strings
Here `ApplicationMaster` accept executor memory argument only in number format, we should let it accept JVM style memory strings as well.
Author: WangTaoTheTonic <barneystinson@aliyun.com>
Closes #2955 from WangTaoTheTonic/modifyDesc and squashes the following commits:
ab98c70 [WangTaoTheTonic] append parameter passed in
3779767 [WangTaoTheTonic] Update executor memory description in the help message
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala | 4 | ||||
| -rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 2 |
2 files changed, 3 insertions, 3 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala index 3e6b96fb63..5c54e34003 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala @@ -17,7 +17,7 @@ package org.apache.spark.deploy.yarn -import org.apache.spark.util.IntParam +import org.apache.spark.util.{MemoryParam, IntParam} import collection.mutable.ArrayBuffer class ApplicationMasterArguments(val args: Array[String]) { @@ -55,7 +55,7 @@ class ApplicationMasterArguments(val args: Array[String]) { numExecutors = value args = tail - case ("--worker-memory" | "--executor-memory") :: IntParam(value) :: tail => + case ("--worker-memory" | "--executor-memory") :: MemoryParam(value) :: tail => executorMemory = value args = tail diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 0417cdd00a..8ea0e7cf40 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -379,7 +379,7 @@ private[spark] trait ClientBase extends Logging { val amArgs = Seq(amClass) ++ userClass ++ userJar ++ userArgs ++ Seq( - "--executor-memory", args.executorMemory.toString, + "--executor-memory", args.executorMemory.toString + "m", "--executor-cores", args.executorCores.toString, "--num-executors ", args.numExecutors.toString) |