From 7e418e99cff4cf512ab2a9fa74221c4655048c8d Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Tue, 20 Sep 2016 14:17:49 -0700 Subject: [SPARK-17611][YARN][TEST] Make shuffle service test really test auth. Currently, the code is just swallowing exceptions, and not really checking whether the auth information was being recorded properly. Fix both problems, and also avoid tests inadvertently affecting other tests by modifying the shared config variable (by making it not shared). Author: Marcelo Vanzin Closes #15161 from vanzin/SPARK-17611. --- .../org/apache/spark/network/yarn/YarnShuffleService.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'common/network-yarn/src') diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java index 43c8df721d..ea726e3c82 100644 --- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java +++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java @@ -105,7 +105,8 @@ public class YarnShuffleService extends AuxiliaryService { // An entity that manages the shuffle secret per application // This is used only if authentication is enabled - private ShuffleSecretManager secretManager; + @VisibleForTesting + ShuffleSecretManager secretManager; // The actual server that serves shuffle files private TransportServer shuffleServer = null; @@ -197,7 +198,7 @@ public class YarnShuffleService extends AuxiliaryService { private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); - + // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short)0700)); @@ -306,7 +307,7 @@ public class YarnShuffleService extends AuxiliaryService { } if (db != null) { db.close(); - } + } } catch (Exception e) { logger.error("Exception when stopping service", e); } @@ -329,7 +330,7 @@ public class YarnShuffleService extends AuxiliaryService { /** * Get the path specific to this auxiliary service to use for recovery. - */ + */ protected Path getRecoveryPath(String fileName) { return _recoveryPath; } @@ -345,7 +346,7 @@ public class YarnShuffleService extends AuxiliaryService { if (recoveryFile.exists()) { return recoveryFile; } - } + } // db doesn't exist in recovery path go check local dirs for it String[] localDirs = _conf.getTrimmedStrings("yarn.nodemanager.local-dirs"); for (String dir : localDirs) { -- cgit v1.2.3