diff options
| author | Harvey <h.feng@berkeley.edu> | 2012-08-03 12:22:07 -0700 |
|---|---|---|
| committer | Harvey <h.feng@berkeley.edu> | 2012-08-03 12:22:07 -0700 |
| commit | 5ec13327d4041df59c3c9d842658cbecbdbf2567 (patch) | |
| tree | b8208bfff03bb4743e05fe0ef6ed657420e6762c | |
| parent | 4a9c58913d0a9bd51811dedca934a2a0adbe8d37 (diff) | |
| download | spark-5ec13327d4041df59c3c9d842658cbecbdbf2567.tar.gz spark-5ec13327d4041df59c3c9d842658cbecbdbf2567.tar.bz2 spark-5ec13327d4041df59c3c9d842658cbecbdbf2567.zip | |
Fix for partitioning when sorting in descending order
| -rw-r--r-- | core/src/main/scala/spark/Partitioner.scala | 3 |
1 files changed, 1 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/Partitioner.scala b/core/src/main/scala/spark/Partitioner.scala index 2235a0ec3d..4ef871bbf9 100644 --- a/core/src/main/scala/spark/Partitioner.scala +++ b/core/src/main/scala/spark/Partitioner.scala @@ -39,8 +39,7 @@ class RangePartitioner[K <% Ordered[K]: ClassManifest, V]( val rddSize = rdd.count() val maxSampleSize = partitions * 10.0 val frac = math.min(maxSampleSize / math.max(rddSize, 1), 1.0) - val rddSample = rdd.sample(true, frac, 1).map(_._1).collect() - .sortWith((x, y) => if (ascending) x < y else x > y) + val rddSample = rdd.sample(true, frac, 1).map(_._1).collect().sortWith(_ < _) if (rddSample.length == 0) { Array() } else { |