From 6bfe42a3be4fbf8bc6f93a4709038fda8ad0610b Mon Sep 17 00:00:00 2001 From: Lianhui Wang Date: Mon, 25 Apr 2016 12:33:32 -0700 Subject: [SPARK-14731][shuffle]Revert SPARK-12130 to make 2.0 shuffle service compatible with 1.x ## What changes were proposed in this pull request? SPARK-12130 make 2.0 shuffle service incompatible with 1.x. So from discussion: [http://apache-spark-developers-list.1001551.n3.nabble.com/YARN-Shuffle-service-and-its-compatibility-td17222.html](url) we should maintain compatibility between Spark 1.x and Spark 2.x's shuffle service. I put string comparison into executor's register at first avoid string comparison in getBlockData every time. ## How was this patch tested? N/A Author: Lianhui Wang Closes #12568 from lianhuiwang/SPARK-14731. --- core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala | 3 --- .../main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala | 2 -- core/src/main/scala/org/apache/spark/storage/BlockManager.scala | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) (limited to 'core') diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala index 364fad664e..4ea8a7120a 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala @@ -29,9 +29,6 @@ import org.apache.spark.{ShuffleDependency, TaskContext} */ private[spark] trait ShuffleManager { - /** Return short name for the ShuffleManager */ - val shortName: String - /** * Register a shuffle with the manager and obtain a handle for it to pass to tasks. */ diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala index 9bfd966e33..5e977a16fe 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala @@ -79,8 +79,6 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager */ private[this] val numMapsForShuffle = new ConcurrentHashMap[Int, Int]() - override val shortName: String = "sort" - override val shuffleBlockResolver = new IndexShuffleBlockResolver(conf) /** diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 22bc76b143..1c4921666f 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -182,7 +182,7 @@ private[spark] class BlockManager( val shuffleConfig = new ExecutorShuffleInfo( diskBlockManager.localDirs.map(_.toString), diskBlockManager.subDirsPerLocalDir, - shuffleManager.shortName) + shuffleManager.getClass.getName) val MAX_ATTEMPTS = 3 val SLEEP_TIME_SECS = 5 -- cgit v1.2.3