aboutsummaryrefslogtreecommitdiff
path: root/yarn
diff options
context:
space:
mode:
authorMarcelo Vanzin <vanzin@cloudera.com>2015-10-12 10:21:57 -0700
committerMarcelo Vanzin <vanzin@cloudera.com>2015-10-12 10:21:57 -0700
commit149472a01d12828c64b0a852982d48c123984182 (patch)
treed0a96f6d918c610b1e278a578f5194c98d9e572b /yarn
parent64b1d00e1a7c1dc52c08a5e97baf6e7117f1a94f (diff)
downloadspark-149472a01d12828c64b0a852982d48c123984182.tar.gz
spark-149472a01d12828c64b0a852982d48c123984182.tar.bz2
spark-149472a01d12828c64b0a852982d48c123984182.zip
[SPARK-11023] [YARN] Avoid creating URIs from local paths directly.
The issue is that local paths on Windows, when provided with drive letters or backslashes, are not valid URIs. Instead of trying to figure out whether paths are URIs or not, use Utils.resolveURI() which does that for us. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #9049 from vanzin/SPARK-11023 and squashes the following commits: 77021f2 [Marcelo Vanzin] [SPARK-11023] [yarn] Avoid creating URIs from local paths directly.
Diffstat (limited to 'yarn')
-rw-r--r--yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala11
1 files changed, 6 insertions, 5 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index cec81b9406..1fbd18aa46 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -358,7 +358,8 @@ private[spark] class Client(
destName: Option[String] = None,
targetDir: Option[String] = None,
appMasterOnly: Boolean = false): (Boolean, String) = {
- val localURI = new URI(path.trim())
+ val trimmedPath = path.trim()
+ val localURI = Utils.resolveURI(trimmedPath)
if (localURI.getScheme != LOCAL_SCHEME) {
if (addDistributedUri(localURI)) {
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
@@ -374,7 +375,7 @@ private[spark] class Client(
(false, null)
}
} else {
- (true, path.trim())
+ (true, trimmedPath)
}
}
@@ -595,10 +596,10 @@ private[spark] class Client(
LOCALIZED_PYTHON_DIR)
}
(pySparkArchives ++ pyArchives).foreach { path =>
- val uri = new URI(path)
+ val uri = Utils.resolveURI(path)
if (uri.getScheme != LOCAL_SCHEME) {
pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD),
- new Path(path).getName())
+ new Path(uri).getName())
} else {
pythonPath += uri.getPath()
}
@@ -1229,7 +1230,7 @@ object Client extends Logging {
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
mainJar.flatMap { path =>
- val uri = new URI(path)
+ val uri = Utils.resolveURI(path)
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
}.orElse(Some(new URI(APP_JAR)))
}