diff options
author | Kishor Patil <kpatil@yahoo-inc.com> | 2016-11-08 12:13:09 -0600 |
---|---|---|
committer | Tom Graves <tgraves@yahoo-inc.com> | 2016-11-08 12:13:09 -0600 |
commit | 245e5a2f80e3195b7f8a38b480b29bfc23af66bf (patch) | |
tree | ae9a6ac9dd7c6c801a81320576d288e91d45cf46 /yarn/src | |
parent | 9c419698fe110a805570031cac3387a51957d9d1 (diff) | |
download | spark-245e5a2f80e3195b7f8a38b480b29bfc23af66bf.tar.gz spark-245e5a2f80e3195b7f8a38b480b29bfc23af66bf.tar.bz2 spark-245e5a2f80e3195b7f8a38b480b29bfc23af66bf.zip |
[SPARK-18357] Fix yarn files/archive broken issue andd unit tests
## What changes were proposed in this pull request?
The #15627 broke functionality with yarn --files --archives does not accept any files.
This patch ensures that --files and --archives accept unique files.
## How was this patch tested?
A. I added unit tests.
B. Also, manually tested --files with --archives to throw exception if duplicate files are specified and continue if unique files are specified.
Author: Kishor Patil <kpatil@yahoo-inc.com>
Closes #15810 from kishorvpatil/SPARK18357.
Diffstat (limited to 'yarn/src')
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 | ||||
-rw-r--r-- | yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 17 |
2 files changed, 18 insertions, 1 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 172fb46c98..e77fa386dc 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -604,7 +604,7 @@ private[spark] class Client( cachedSecondaryJarLinks += localizedPath } } else { - if (localizedPath != null) { + if (localizedPath == null) { throw new IllegalArgumentException(s"Attempt to add ($file) multiple times" + " to the distributed cache.") } diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index 06516c1baf..7deaf0af94 100644 --- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -322,6 +322,23 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll intercept[IllegalArgumentException] { clientArchives.prepareLocalResources(new Path(tempDirForArchives.getAbsolutePath()), Nil) } + + // Case 4: FILES_TO_DISTRIBUTE can have unique file. + val sparkConfFilesUniq = new SparkConfWithEnv(Map("SPARK_HOME" -> libs.getAbsolutePath)) + .set(FILES_TO_DISTRIBUTE, Seq(testJar.getPath)) + + val clientFilesUniq = createClient(sparkConfFilesUniq) + val tempDirForFilesUniq = Utils.createTempDir() + clientFilesUniq.prepareLocalResources(new Path(tempDirForFilesUniq.getAbsolutePath()), Nil) + + // Case 5: ARCHIVES_TO_DISTRIBUTE can have unique file. + val sparkConfArchivesUniq = new SparkConfWithEnv(Map("SPARK_HOME" -> libs.getAbsolutePath)) + .set(ARCHIVES_TO_DISTRIBUTE, Seq(testJar.getPath)) + + val clientArchivesUniq = createClient(sparkConfArchivesUniq) + val tempDirArchivesUniq = Utils.createTempDir() + clientArchivesUniq.prepareLocalResources(new Path(tempDirArchivesUniq.getAbsolutePath()), Nil) + } test("distribute local spark jars") { |