diff options
author | Andrew Or <andrew@databricks.com> | 2014-12-23 14:28:36 -0800 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2014-12-23 14:28:36 -0800 |
commit | 27c5399f4dd542e36ea579956b8cb0613de25c6d (patch) | |
tree | 6985726b9f31b14f85bf8ddd5a6f0998738f2324 /yarn/src/main | |
parent | 395b771feed3fc90e5053defbe86dbd673fde582 (diff) | |
download | spark-27c5399f4dd542e36ea579956b8cb0613de25c6d.tar.gz spark-27c5399f4dd542e36ea579956b8cb0613de25c6d.tar.bz2 spark-27c5399f4dd542e36ea579956b8cb0613de25c6d.zip |
[SPARK-4730][YARN] Warn against deprecated YARN settings
See https://issues.apache.org/jira/browse/SPARK-4730.
Author: Andrew Or <andrew@databricks.com>
Closes #3590 from andrewor14/yarn-settings and squashes the following commits:
36e0753 [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-settings
dcd1316 [Andrew Or] Warn against deprecated YARN settings
Diffstat (limited to 'yarn/src/main')
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 17 |
1 files changed, 16 insertions, 1 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index 2923e6729c..09597bd0e6 100644 --- a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -65,7 +65,8 @@ private[spark] class YarnClientSchedulerBackend( */ private def getExtraClientArguments: Seq[String] = { val extraArgs = new ArrayBuffer[String] - val optionTuples = // List of (target Client argument, environment variable, Spark property) + // List of (target Client argument, environment variable, Spark property) + val optionTuples = List( ("--driver-memory", "SPARK_MASTER_MEMORY", "spark.master.memory"), ("--driver-memory", "SPARK_DRIVER_MEMORY", "spark.driver.memory"), @@ -78,11 +79,25 @@ private[spark] class YarnClientSchedulerBackend( ("--queue", "SPARK_YARN_QUEUE", "spark.yarn.queue"), ("--name", "SPARK_YARN_APP_NAME", "spark.app.name") ) + // Warn against the following deprecated environment variables: env var -> suggestion + val deprecatedEnvVars = Map( + "SPARK_MASTER_MEMORY" -> "SPARK_DRIVER_MEMORY or --driver-memory through spark-submit", + "SPARK_WORKER_INSTANCES" -> "SPARK_WORKER_INSTANCES or --num-executors through spark-submit", + "SPARK_WORKER_MEMORY" -> "SPARK_EXECUTOR_MEMORY or --executor-memory through spark-submit", + "SPARK_WORKER_CORES" -> "SPARK_EXECUTOR_CORES or --executor-cores through spark-submit") + // Do the same for deprecated properties: property -> suggestion + val deprecatedProps = Map("spark.master.memory" -> "--driver-memory through spark-submit") optionTuples.foreach { case (optionName, envVar, sparkProp) => if (System.getenv(envVar) != null) { extraArgs += (optionName, System.getenv(envVar)) + if (deprecatedEnvVars.contains(envVar)) { + logWarning(s"NOTE: $envVar is deprecated. Use ${deprecatedEnvVars(envVar)} instead.") + } } else if (sc.getConf.contains(sparkProp)) { extraArgs += (optionName, sc.getConf.get(sparkProp)) + if (deprecatedProps.contains(sparkProp)) { + logWarning(s"NOTE: $sparkProp is deprecated. Use ${deprecatedProps(sparkProp)} instead.") + } } } extraArgs |