diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2015-10-12 10:21:57 -0700 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2015-10-12 10:21:57 -0700 |
commit | 149472a01d12828c64b0a852982d48c123984182 (patch) | |
tree | d0a96f6d918c610b1e278a578f5194c98d9e572b /yarn/src/main/scala/org | |
parent | 64b1d00e1a7c1dc52c08a5e97baf6e7117f1a94f (diff) | |
download | spark-149472a01d12828c64b0a852982d48c123984182.tar.gz spark-149472a01d12828c64b0a852982d48c123984182.tar.bz2 spark-149472a01d12828c64b0a852982d48c123984182.zip |
[SPARK-11023] [YARN] Avoid creating URIs from local paths directly.
The issue is that local paths on Windows, when provided with drive
letters or backslashes, are not valid URIs.
Instead of trying to figure out whether paths are URIs or not, use
Utils.resolveURI() which does that for us.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #9049 from vanzin/SPARK-11023 and squashes the following commits:
77021f2 [Marcelo Vanzin] [SPARK-11023] [yarn] Avoid creating URIs from local paths directly.
Diffstat (limited to 'yarn/src/main/scala/org')
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index cec81b9406..1fbd18aa46 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -358,7 +358,8 @@ private[spark] class Client( destName: Option[String] = None, targetDir: Option[String] = None, appMasterOnly: Boolean = false): (Boolean, String) = { - val localURI = new URI(path.trim()) + val trimmedPath = path.trim() + val localURI = Utils.resolveURI(trimmedPath) if (localURI.getScheme != LOCAL_SCHEME) { if (addDistributedUri(localURI)) { val localPath = getQualifiedLocalPath(localURI, hadoopConf) @@ -374,7 +375,7 @@ private[spark] class Client( (false, null) } } else { - (true, path.trim()) + (true, trimmedPath) } } @@ -595,10 +596,10 @@ private[spark] class Client( LOCALIZED_PYTHON_DIR) } (pySparkArchives ++ pyArchives).foreach { path => - val uri = new URI(path) + val uri = Utils.resolveURI(path) if (uri.getScheme != LOCAL_SCHEME) { pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD), - new Path(path).getName()) + new Path(uri).getName()) } else { pythonPath += uri.getPath() } @@ -1229,7 +1230,7 @@ object Client extends Logging { private def getMainJarUri(mainJar: Option[String]): Option[URI] = { mainJar.flatMap { path => - val uri = new URI(path) + val uri = Utils.resolveURI(path) if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None }.orElse(Some(new URI(APP_JAR))) } |