diff options
| author | Andrew Or <andrewor14@gmail.com> | 2014-05-10 20:58:02 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2014-05-10 20:58:11 -0700 |
| commit | 012f90427896a7ceba2466b711849dbe2fef6360 (patch) | |
| tree | 919528004bdc0d6462359e634427059bdb1eecca /yarn | |
| parent | c7253daeb567f9340528e3578300a1166167cff0 (diff) | |
| download | spark-012f90427896a7ceba2466b711849dbe2fef6360.tar.gz spark-012f90427896a7ceba2466b711849dbe2fef6360.tar.bz2 spark-012f90427896a7ceba2466b711849dbe2fef6360.zip | |
[SPARK-1774] Respect SparkSubmit --jars on YARN (client)
SparkSubmit ignores `--jars` for YARN client. This is a bug.
This PR also automatically adds the application jar to `spark.jar`. Previously, when running as yarn-client, you must specify the jar additionally through `--files` (because `--jars` didn't work). Now you don't have to explicitly specify it through either.
Tested on a YARN cluster.
Author: Andrew Or <andrewor14@gmail.com>
Closes #710 from andrewor14/yarn-jars and squashes the following commits:
35d1928 [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-jars
c27bf6c [Andrew Or] For yarn-cluster and python, do not add primaryResource to spark.jar
c92c5bf [Andrew Or] Minor cleanups
269f9f3 [Andrew Or] Fix format
013d840 [Andrew Or] Fix tests
1407474 [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-jars
3bb75e8 [Andrew Or] Allow SparkSubmit --jars to take effect in yarn-client mode
(cherry picked from commit 83e0424d87022e7a967088365931a08aa06ffd9f)
Signed-off-by: Patrick Wendell <pwendell@gmail.com>
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala | 4 |
1 files changed, 1 insertions, 3 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala index 3e4c739e34..b2c413b6d2 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala @@ -20,7 +20,7 @@ package org.apache.spark.deploy.yarn import scala.collection.mutable.{ArrayBuffer, HashMap} import org.apache.spark.SparkConf -import org.apache.spark.scheduler.{InputFormatInfo, SplitInfo} +import org.apache.spark.scheduler.InputFormatInfo import org.apache.spark.util.IntParam import org.apache.spark.util.MemoryParam @@ -40,9 +40,7 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { var amMemory: Int = 512 // MB var amClass: String = "org.apache.spark.deploy.yarn.ApplicationMaster" var appName: String = "Spark" - // TODO var inputFormatInfo: List[InputFormatInfo] = null - // TODO(harvey) var priority = 0 parseArgs(args.toList) |