diff options
author | Andrew Or <andrew@databricks.com> | 2014-12-03 13:56:23 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2014-12-03 13:56:35 -0800 |
commit | 38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce (patch) | |
tree | 6deeb499c893ce37e31ce7de06893190167228af /yarn/common/src/main/scala | |
parent | 4a71e08534b92710fd8d1eb17b077c6c7b78e55d (diff) | |
download | spark-38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce.tar.gz spark-38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce.tar.bz2 spark-38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce.zip |
[HOT FIX] [YARN] Check whether `/lib` exists before listing its files
This is caused by a975dc32799bb8a14f9e1c76defaaa7cfbaf8b53
Author: Andrew Or <andrew@databricks.com>
Closes #3589 from andrewor14/yarn-hot-fix and squashes the following commits:
a4fad5f [Andrew Or] Check whether lib directory exists before listing its files
(cherry picked from commit 90ec643e9af4c8bbb9000edca08c07afb17939c7)
Signed-off-by: Andrew Or <andrew@databricks.com>
Diffstat (limited to 'yarn/common/src/main/scala')
-rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 27 |
1 files changed, 15 insertions, 12 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 8e4360ea44..290d9943a5 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -243,18 +243,21 @@ private[spark] trait ClientBase extends Logging { val libsURI = new URI(libsDir) val jarLinks = ListBuffer.empty[String] if (libsURI.getScheme != LOCAL_SCHEME) { - val localURI = getQualifiedLocalPath(libsURI).toUri() - val jars = FileSystem.get(localURI, hadoopConf).listFiles(new Path(localURI.getPath), false) - while (jars.hasNext) { - val jar = jars.next() - val name = jar.getPath.getName - if (name.startsWith("datanucleus-")) { - // copy to remote and add to classpath - val src = jar.getPath - val destPath = copyFileToRemote(dst, src, replication) - distCacheMgr.addResource(fs, hadoopConf, destPath, - localResources, LocalResourceType.FILE, name, statCache) - jarLinks += name + val localPath = getQualifiedLocalPath(libsURI) + val localFs = FileSystem.get(localPath.toUri, hadoopConf) + if (localFs.exists(localPath)) { + val jars = localFs.listFiles(localPath, /* recursive */ false) + while (jars.hasNext) { + val jar = jars.next() + val name = jar.getPath.getName + if (name.startsWith("datanucleus-")) { + // copy to remote and add to classpath + val src = jar.getPath + val destPath = copyFileToRemote(dst, src, replication) + distCacheMgr.addResource(localFs, hadoopConf, destPath, + localResources, LocalResourceType.FILE, name, statCache) + jarLinks += name + } } } } else { |