diff options
| author | Tao Li <litao@sogou-inc.com> | 2015-07-08 19:02:24 +0100 |
|---|---|---|
| committer | Sean Owen <sowen@cloudera.com> | 2015-07-08 19:02:24 +0100 |
| commit | 26d9b6b8cae9ac6593f78ab98dd45a25d03cf71c (patch) | |
| tree | 0f1570b9bb13c416a32fe4da765e1012a1e781a3 /yarn | |
| parent | f61c989b404808f79a58b6503cf3835cf602528a (diff) | |
| download | spark-26d9b6b8cae9ac6593f78ab98dd45a25d03cf71c.tar.gz spark-26d9b6b8cae9ac6593f78ab98dd45a25d03cf71c.tar.bz2 spark-26d9b6b8cae9ac6593f78ab98dd45a25d03cf71c.zip | |
[SPARK-8657] [YARN] Fail to upload resource to viewfs
Fail to upload resource to viewfs in spark-1.4
JIRA Link: https://issues.apache.org/jira/browse/SPARK-8657
Author: Tao Li <litao@sogou-inc.com>
Closes #7125 from litao-buptsse/SPARK-8657-for-master and squashes the following commits:
65b13f4 [Tao Li] [SPARK-8657] [YARN] Fail to upload resource to viewfs
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 67a5c95400..4d52ae774e 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -321,8 +321,9 @@ private[spark] class Client( val linkname = targetDir.map(_ + "/").getOrElse("") + destName.orElse(Option(localURI.getFragment())).getOrElse(localPath.getName()) val destPath = copyFileToRemote(dst, localPath, replication) + val destFs = FileSystem.get(destPath.toUri(), hadoopConf) distCacheMgr.addResource( - fs, hadoopConf, destPath, localResources, resType, linkname, statCache, + destFs, hadoopConf, destPath, localResources, resType, linkname, statCache, appMasterOnly = appMasterOnly) (false, linkname) } else { |