diff options
| -rw-r--r-- | core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala | 2 | ||||
| -rw-r--r-- | docs/configuration.md | 2 |
2 files changed, 2 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala index e2b24298a5..6e0ff143b7 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala @@ -64,7 +64,7 @@ class ShuffleBlockManager(blockManager: BlockManager) { // Turning off shuffle file consolidation causes all shuffle Blocks to get their own file. // TODO: Remove this once the shuffle file consolidation feature is stable. val consolidateShuffleFiles = - conf.getBoolean("spark.shuffle.consolidateFiles", false) + conf.getBoolean("spark.shuffle.consolidateFiles", true) private val bufferSize = conf.getInt("spark.shuffle.file.buffer.kb", 100) * 1024 diff --git a/docs/configuration.md b/docs/configuration.md index 6717757781..b1a0e19167 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -371,7 +371,7 @@ Apart from these, the following properties are also available, and may be useful <tr> <td>spark.shuffle.consolidateFiles</td> - <td>false</td> + <td>true</td> <td> If set to "true", consolidates intermediate files created during a shuffle. Creating fewer files can improve filesystem performance for shuffles with large numbers of reduce tasks. It is recommended to set this to "true" when using ext4 or xfs filesystems. On ext3, this option might degrade performance on machines with many (>8) cores due to filesystem limitations. </td> |