diff options
| -rw-r--r-- | docs/running-on-yarn.md | 8 | ||||
| -rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 19 |
2 files changed, 27 insertions, 0 deletions
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 62b317129b..b5fb077441 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -139,6 +139,14 @@ Most of the configs are the same for Spark on YARN as for other deployment modes The maximum number of threads to use in the application master for launching executor containers. </td> </tr> +<tr> + <td><code>spark.yarn.am.extraJavaOptions</code></td> + <td>(none)</td> + <td> + A string of extra JVM options to pass to the Yarn ApplicationMaster in client mode. + In cluster mode, use spark.driver.extraJavaOptions instead. + </td> +</tr> </table> # Launching Spark on YARN diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index f95d723791..5f0c67f05c 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -352,12 +352,31 @@ private[spark] trait ClientBase extends Logging { if (isLaunchingDriver) { sparkConf.getOption("spark.driver.extraJavaOptions") .orElse(sys.env.get("SPARK_JAVA_OPTS")) + .map(Utils.splitCommandString).getOrElse(Seq.empty) .foreach(opts => javaOpts += opts) val libraryPaths = Seq(sys.props.get("spark.driver.extraLibraryPath"), sys.props.get("spark.driver.libraryPath")).flatten if (libraryPaths.nonEmpty) { prefixEnv = Some(Utils.libraryPathEnvPrefix(libraryPaths)) } + if (sparkConf.getOption("spark.yarn.am.extraJavaOptions").isDefined) { + logWarning("spark.yarn.am.extraJavaOptions will not take effect in cluster mode") + } + } else { + // Validate and include yarn am specific java options in yarn-client mode. + val amOptsKey = "spark.yarn.am.extraJavaOptions" + val amOpts = sparkConf.getOption(amOptsKey) + amOpts.foreach { opts => + if (opts.contains("-Dspark")) { + val msg = s"$amOptsKey is not allowed to set Spark options (was '$opts'). " + throw new SparkException(msg) + } + if (opts.contains("-Xmx") || opts.contains("-Xms")) { + val msg = s"$amOptsKey is not allowed to alter memory settings (was '$opts')." + throw new SparkException(msg) + } + javaOpts ++= Utils.splitCommandString(opts) + } } // For log4j configuration to reference |