diff options
author | Josh Rosen <joshrosen@apache.org> | 2013-12-14 12:59:37 -0800 |
---|---|---|
committer | Josh Rosen <joshrosen@apache.org> | 2013-12-14 12:59:37 -0800 |
commit | 2fd781d347107be34e6a4251b8bd0b35b351785a (patch) | |
tree | b7911943bfa5e57ae53c652a0f51c3beee892401 /core/src/main/scala/org | |
parent | 97ac06018206b593600594605be241d0cd706e08 (diff) | |
parent | 0b494f7db47cf1de35aaed046f21bbb3592c3d97 (diff) | |
download | spark-2fd781d347107be34e6a4251b8bd0b35b351785a.tar.gz spark-2fd781d347107be34e6a4251b8bd0b35b351785a.tar.bz2 spark-2fd781d347107be34e6a4251b8bd0b35b351785a.zip |
Merge pull request #249 from ngbinh/partitionInJavaSortByKey
Expose numPartitions parameter in JavaPairRDD.sortByKey()
This change makes Java and Scala API on sortByKey() the same.
Diffstat (limited to 'core/src/main/scala/org')
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala | 14 |
1 files changed, 14 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala index 1167c12022..363667fa86 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala @@ -589,6 +589,20 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])(implicit val kClassTag: ClassTag[K } /** + * Sort the RDD by key, so that each partition contains a sorted range of the elements. Calling + * `collect` or `save` on the resulting RDD will return or output an ordered list of records + * (in the `save` case, they will be written to multiple `part-X` files in the filesystem, in + * order of the keys). + */ + def sortByKey(comp: Comparator[K], ascending: Boolean, numPartitions: Int): JavaPairRDD[K, V] = { + class KeyOrdering(val a: K) extends Ordered[K] { + override def compare(b: K) = comp.compare(a, b) + } + implicit def toOrdered(x: K): Ordered[K] = new KeyOrdering(x) + fromRDD(new OrderedRDDFunctions[K, V, (K, V)](rdd).sortByKey(ascending, numPartitions)) + } + + /** * Return an RDD with the keys of each tuple. */ def keys(): JavaRDD[K] = JavaRDD.fromRDD[K](rdd.map(_._1)) |