diff options
| author | Lianhui Wang <lianhuiwang09@gmail.com> | 2016-04-25 12:33:32 -0700 |
|---|---|---|
| committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-04-25 12:33:32 -0700 |
| commit | 6bfe42a3be4fbf8bc6f93a4709038fda8ad0610b (patch) | |
| tree | d98096dd1f3cba95289ed51a2f65accc38e6d7e4 /core/src/main/scala | |
| parent | 425f6916462ca5d0595c61101d52686006ed6b8b (diff) | |
| download | spark-6bfe42a3be4fbf8bc6f93a4709038fda8ad0610b.tar.gz spark-6bfe42a3be4fbf8bc6f93a4709038fda8ad0610b.tar.bz2 spark-6bfe42a3be4fbf8bc6f93a4709038fda8ad0610b.zip | |
[SPARK-14731][shuffle]Revert SPARK-12130 to make 2.0 shuffle service compatible with 1.x
## What changes were proposed in this pull request?
SPARK-12130 make 2.0 shuffle service incompatible with 1.x. So from discussion: [http://apache-spark-developers-list.1001551.n3.nabble.com/YARN-Shuffle-service-and-its-compatibility-td17222.html](url) we should maintain compatibility between Spark 1.x and Spark 2.x's shuffle service.
I put string comparison into executor's register at first avoid string comparison in getBlockData every time.
## How was this patch tested?
N/A
Author: Lianhui Wang <lianhuiwang09@gmail.com>
Closes #12568 from lianhuiwang/SPARK-14731.
Diffstat (limited to 'core/src/main/scala')
3 files changed, 1 insertions, 6 deletions
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala index 364fad664e..4ea8a7120a 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala @@ -29,9 +29,6 @@ import org.apache.spark.{ShuffleDependency, TaskContext} */ private[spark] trait ShuffleManager { - /** Return short name for the ShuffleManager */ - val shortName: String - /** * Register a shuffle with the manager and obtain a handle for it to pass to tasks. */ diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala index 9bfd966e33..5e977a16fe 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala @@ -79,8 +79,6 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager */ private[this] val numMapsForShuffle = new ConcurrentHashMap[Int, Int]() - override val shortName: String = "sort" - override val shuffleBlockResolver = new IndexShuffleBlockResolver(conf) /** diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 22bc76b143..1c4921666f 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -182,7 +182,7 @@ private[spark] class BlockManager( val shuffleConfig = new ExecutorShuffleInfo( diskBlockManager.localDirs.map(_.toString), diskBlockManager.subDirsPerLocalDir, - shuffleManager.shortName) + shuffleManager.getClass.getName) val MAX_ATTEMPTS = 3 val SLEEP_TIME_SECS = 5 |