diff options
author | Reynold Xin <rxin@apache.org> | 2013-09-26 13:48:20 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2013-09-26 13:48:20 -0700 |
commit | c514cd15879d510406712b1cebdaf0c0db9151c9 (patch) | |
tree | e028f2ab13c475389f94416cb3d93ad568b3d5ba | |
parent | 560ee5c9bba3f9fde380c831d0c6701343b2fecf (diff) | |
parent | 0cef683553414ba880d90527cc5f37e119efc782 (diff) | |
download | spark-c514cd15879d510406712b1cebdaf0c0db9151c9.tar.gz spark-c514cd15879d510406712b1cebdaf0c0db9151c9.tar.bz2 spark-c514cd15879d510406712b1cebdaf0c0db9151c9.zip |
Merge pull request #930 from holdenk/master
Add mapPartitionsWithIndex
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 10 | ||||
-rw-r--r-- | project/SparkBuild.scala | 2 |
2 files changed, 11 insertions, 1 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala index 7e6e691f11..7a3568c5ef 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala @@ -68,6 +68,16 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable { new JavaRDD(rdd.map(f)(f.returnType()))(f.returnType()) /** + * Return a new RDD by applying a function to each partition of this RDD, while tracking the index + * of the original partition. + */ + def mapPartitionsWithIndex[R: ClassManifest]( + f: JFunction2[Int, java.util.Iterator[T], java.util.Iterator[R]], + preservesPartitioning: Boolean = false): JavaRDD[R] = + new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))), + preservesPartitioning)) + + /** * Return a new RDD by applying a function to all elements of this RDD. */ def map[R](f: DoubleFunction[T]): JavaDoubleRDD = diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index a2e29591fc..99cdadb9e7 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -81,7 +81,7 @@ object SparkBuild extends Build { organization := "org.apache.spark", version := "0.9.0-incubating-SNAPSHOT", scalaVersion := "2.9.3", - scalacOptions := Seq("-unchecked", "-optimize", "-deprecation", + scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-optimize", "-deprecation", "-target:" + SCALAC_JVM_VERSION), javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION), unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath }, |