diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2014-04-17 10:29:38 -0500 |
---|---|---|
committer | Thomas Graves <tgraves@apache.org> | 2014-04-17 10:29:38 -0500 |
commit | 69047506bf97e6e37e4079c87cb0327d3760ac41 (patch) | |
tree | f1ffe2efe61f0eb5c379aab43f35c751164baf2b /core | |
parent | bb76eae1b50e4bf18360220110f7d0a4bee672ec (diff) | |
download | spark-69047506bf97e6e37e4079c87cb0327d3760ac41.tar.gz spark-69047506bf97e6e37e4079c87cb0327d3760ac41.tar.bz2 spark-69047506bf97e6e37e4079c87cb0327d3760ac41.zip |
[SPARK-1395] Allow "local:" URIs to work on Yarn.
This only works for the three paths defined in the environment
(SPARK_JAR, SPARK_YARN_APP_JAR and SPARK_LOG4J_CONF).
Tested by running SparkPi with local: and file: URIs against Yarn cluster (no "upload" shows up in logs in the local case).
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #303 from vanzin/yarn-local and squashes the following commits:
82219c1 [Marcelo Vanzin] [SPARK-1395] Allow "local:" URIs to work on Yarn.
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index e05fbfe321..e5d593cade 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy import java.io.{PrintStream, File} -import java.net.URL +import java.net.{URI, URL} import org.apache.spark.executor.ExecutorURLClassLoader @@ -216,7 +216,7 @@ object SparkSubmit { } private def addJarToClasspath(localJar: String, loader: ExecutorURLClassLoader) { - val localJarFile = new File(localJar) + val localJarFile = new File(new URI(localJar).getPath()) if (!localJarFile.exists()) { printWarning(s"Jar $localJar does not exist, skipping.") } |