diff options
| author | Andrew Or <andrewor14@gmail.com> | 2014-01-12 22:43:01 -0800 |
|---|---|---|
| committer | Andrew Or <andrewor14@gmail.com> | 2014-01-12 22:43:01 -0800 |
| commit | 69c9aebed0dfd90e0a1c4d48cd24ea7ddd7624fa (patch) | |
| tree | d551200f1fc083efa70354bc0ab0d2e7201a77c6 | |
| parent | 8d40e7222f2a0a421349621105dc4c69bd7f1bb8 (diff) | |
| download | spark-69c9aebed0dfd90e0a1c4d48cd24ea7ddd7624fa.tar.gz spark-69c9aebed0dfd90e0a1c4d48cd24ea7ddd7624fa.tar.bz2 spark-69c9aebed0dfd90e0a1c4d48cd24ea7ddd7624fa.zip | |
Enable external sorting by default
| -rw-r--r-- | core/src/main/scala/org/apache/spark/Aggregator.scala | 2 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala | 2 |
2 files changed, 2 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/Aggregator.scala b/core/src/main/scala/org/apache/spark/Aggregator.scala index c46b7bd043..5fd90d0d88 100644 --- a/core/src/main/scala/org/apache/spark/Aggregator.scala +++ b/core/src/main/scala/org/apache/spark/Aggregator.scala @@ -32,7 +32,7 @@ case class Aggregator[K, V, C] ( mergeCombiners: (C, C) => C) { private val sparkConf = SparkEnv.get.conf - private val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", false) + private val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", true) def combineValuesByKey(iter: Iterator[_ <: Product2[K, V]], context: TaskContext) : Iterator[(K, C)] = { if (!externalSorting) { diff --git a/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala index 34e834132e..656c3efa95 100644 --- a/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala @@ -107,7 +107,7 @@ class CoGroupedRDD[K](@transient var rdds: Seq[RDD[_ <: Product2[K, _]]], part: override def compute(s: Partition, context: TaskContext): Iterator[(K, CoGroupCombiner)] = { - val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", false) + val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", true) val split = s.asInstanceOf[CoGroupPartition] val numRdds = split.deps.size |