diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-10-10 19:34:33 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-10-10 19:34:33 -0700 |
commit | 8f11c36fe17c2718c895b771b59a9138521e0079 (patch) | |
tree | ae2bdd4eec278538fd5c6e971b3a1e42cfa62b60 /core | |
parent | c71499b7795564e1d16495c59273ecc027070fc5 (diff) | |
parent | 0fff4ee8523ff4137eedfc314b51135427137c63 (diff) | |
download | spark-8f11c36fe17c2718c895b771b59a9138521e0079.tar.gz spark-8f11c36fe17c2718c895b771b59a9138521e0079.tar.bz2 spark-8f11c36fe17c2718c895b771b59a9138521e0079.zip |
Merge remote-tracking branch 'tgravescs/sparkYarnDistCache'
Closes #11
Conflicts:
docs/running-on-yarn.md
yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkContext.scala | 17 |
1 files changed, 14 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index f8e980fa5e..39253c67ff 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -672,10 +672,21 @@ class SparkContext( key = uri.getScheme match { case null | "file" => if (env.hadoop.isYarnMode()) { - logWarning("local jar specified as parameter to addJar under Yarn mode") - return + // In order for this to work on yarn the user must specify the --addjars option to + // the client to upload the file into the distributed cache to make it show up in the + // current working directory. + val fileName = new Path(uri.getPath).getName() + try { + env.httpFileServer.addJar(new File(fileName)) + } catch { + case e: Exception => { + logError("Error adding jar (" + e + "), was the --addJars option used?") + throw e + } + } + } else { + env.httpFileServer.addJar(new File(uri.getPath)) } - env.httpFileServer.addJar(new File(uri.getPath)) case _ => path } |