diff options
author | Reynold Xin <rxin@databricks.com> | 2015-09-18 13:48:41 -0700 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2015-09-18 13:48:41 -0700 |
commit | 348d7c9a93dd00d3d1859342a8eb0aea2e77f597 (patch) | |
tree | 95d9a65d2e95aedd3eb5f31ac41e779e79aa8d6d /network/shuffle | |
parent | 3a22b1004f527d54d399dd0225cd7f2f8ffad9c5 (diff) | |
download | spark-348d7c9a93dd00d3d1859342a8eb0aea2e77f597.tar.gz spark-348d7c9a93dd00d3d1859342a8eb0aea2e77f597.tar.bz2 spark-348d7c9a93dd00d3d1859342a8eb0aea2e77f597.zip |
[SPARK-9808] Remove hash shuffle file consolidation.
Author: Reynold Xin <rxin@databricks.com>
Closes #8812 from rxin/SPARK-9808-1.
Diffstat (limited to 'network/shuffle')
-rw-r--r-- | network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java | 4 |
1 files changed, 0 insertions, 4 deletions
diff --git a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java index 79beec4429..c5f93bb47f 100644 --- a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java +++ b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java @@ -50,9 +50,6 @@ import org.apache.spark.network.util.TransportConf; * of Executors. Each Executor must register its own configuration about where it stores its files * (local dirs) and how (shuffle manager). The logic for retrieval of individual files is replicated * from Spark's FileShuffleBlockResolver and IndexShuffleBlockResolver. - * - * Executors with shuffle file consolidation are not currently supported, as the index is stored in - * the Executor's memory, unlike the IndexShuffleBlockResolver. */ public class ExternalShuffleBlockResolver { private static final Logger logger = LoggerFactory.getLogger(ExternalShuffleBlockResolver.class); @@ -254,7 +251,6 @@ public class ExternalShuffleBlockResolver { * Hash-based shuffle data is simply stored as one file per block. * This logic is from FileShuffleBlockResolver. */ - // TODO: Support consolidated hash shuffle files private ManagedBuffer getHashBasedShuffleBlockData(ExecutorShuffleInfo executor, String blockId) { File shuffleFile = getFile(executor.localDirs, executor.subDirsPerLocalDir, blockId); return new FileSegmentManagedBuffer(conf, shuffleFile, 0, shuffleFile.length()); |