diff options
| author | Lianhui Wang <lianhuiwang09@gmail.com> | 2015-06-19 10:47:07 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2015-06-19 10:47:15 -0700 |
| commit | 6f2e41108437b23e4b8cdcfc500f8fb2babf92c6 (patch) | |
| tree | f47cf8589d7d86574a82199fb71fb9345a3d18a8 | |
| parent | 1f2dafb77f9af52602885cd5767032a20b486b98 (diff) | |
| download | spark-6f2e41108437b23e4b8cdcfc500f8fb2babf92c6.tar.gz spark-6f2e41108437b23e4b8cdcfc500f8fb2babf92c6.tar.bz2 spark-6f2e41108437b23e4b8cdcfc500f8fb2babf92c6.zip | |
[SPARK-8430] ExternalShuffleBlockResolver of shuffle service should support UnsafeShuffleManager
andrewor14 can you take a look?thanks
Author: Lianhui Wang <lianhuiwang09@gmail.com>
Closes #6873 from lianhuiwang/SPARK-8430 and squashes the following commits:
51c47ca [Lianhui Wang] update andrewor's comments
2b27b19 [Lianhui Wang] support UnsafeShuffleManager
(cherry picked from commit 9baf093014a48c5ec49f747773f4500dafdfa4ec)
Signed-off-by: Andrew Or <andrew@databricks.com>
| -rw-r--r-- | network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java index dd08e24cad..022ed88a16 100644 --- a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java +++ b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java @@ -108,7 +108,8 @@ public class ExternalShuffleBlockResolver { if ("org.apache.spark.shuffle.hash.HashShuffleManager".equals(executor.shuffleManager)) { return getHashBasedShuffleBlockData(executor, blockId); - } else if ("org.apache.spark.shuffle.sort.SortShuffleManager".equals(executor.shuffleManager)) { + } else if ("org.apache.spark.shuffle.sort.SortShuffleManager".equals(executor.shuffleManager) + || "org.apache.spark.shuffle.unsafe.UnsafeShuffleManager".equals(executor.shuffleManager)) { return getSortBasedShuffleBlockData(executor, shuffleId, mapId, reduceId); } else { throw new UnsupportedOperationException( |