diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-03-29 21:29:39 -0400 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-03-29 21:29:39 -0400 |
commit | 4d52cc67388876bcbddd70a2875b59695a5bd437 (patch) | |
tree | d9a87ea0c48443c85069749d4184c5dcdde9cd21 | |
parent | ca64a7ae03f2ba4a965b6f2b55afbd6d9f2a397a (diff) | |
parent | d46f662c5b758cf99721a439e935638923cb168d (diff) | |
download | spark-4d52cc67388876bcbddd70a2875b59695a5bd437.tar.gz spark-4d52cc67388876bcbddd70a2875b59695a5bd437.tar.bz2 spark-4d52cc67388876bcbddd70a2875b59695a5bd437.zip |
Merge branch 'master' into mesos-0.9
-rw-r--r-- | core/src/main/scala/spark/PairRDDFunctions.scala | 40 | ||||
-rw-r--r-- | project/SparkBuild.scala | 4 |
2 files changed, 21 insertions, 23 deletions
diff --git a/core/src/main/scala/spark/PairRDDFunctions.scala b/core/src/main/scala/spark/PairRDDFunctions.scala index 295fe81ce6..4982a1aa15 100644 --- a/core/src/main/scala/spark/PairRDDFunctions.scala +++ b/core/src/main/scala/spark/PairRDDFunctions.scala @@ -359,28 +359,28 @@ class PairRDDFunctions[K: ClassManifest, V: ClassManifest]( def getValueClass() = implicitly[ClassManifest[V]].erasure } - class OrderedRDDFunctions[K <% Ordered[K]: ClassManifest, V: ClassManifest]( - self: RDD[(K, V)]) - extends Logging - with Serializable { +class OrderedRDDFunctions[K <% Ordered[K]: ClassManifest, V: ClassManifest]( + self: RDD[(K, V)]) + extends Logging + with Serializable { - def sortByKey(ascending: Boolean = true): RDD[(K,V)] = { - val rangePartitionedRDD = self.partitionBy(new RangePartitioner(self.splits.size, self, ascending)) - new SortedRDD(rangePartitionedRDD, ascending) - } + def sortByKey(ascending: Boolean = true): RDD[(K,V)] = { + val rangePartitionedRDD = self.partitionBy(new RangePartitioner(self.splits.size, self, ascending)) + new SortedRDD(rangePartitionedRDD, ascending) } - - class SortedRDD[K <% Ordered[K], V](prev: RDD[(K, V)], ascending: Boolean) - extends RDD[(K, V)](prev.context) { - - override def splits = prev.splits - override val partitioner = prev.partitioner - override val dependencies = List(new OneToOneDependency(prev)) - override def compute(split: Split) = { - prev.iterator(split).toArray - .sortWith((x, y) => if (ascending) x._1 < y._1 else x._1 > y._1).iterator - } - } +} + +class SortedRDD[K <% Ordered[K], V](prev: RDD[(K, V)], ascending: Boolean) + extends RDD[(K, V)](prev.context) { + + override def splits = prev.splits + override val partitioner = prev.partitioner + override val dependencies = List(new OneToOneDependency(prev)) + override def compute(split: Split) = { + prev.iterator(split).toArray + .sortWith((x, y) => if (ascending) x._1 < y._1 else x._1 > y._1).iterator + } +} class MappedValuesRDD[K, V, U](prev: RDD[(K, V)], f: V => U) extends RDD[(K, U)](prev.context) { override def splits = prev.splits diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f2c1e95c92..ba7969be32 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -57,8 +57,6 @@ object SparkBuild extends Build { "asm" % "asm-all" % "3.3.1", "com.google.protobuf" % "protobuf-java" % "2.3.0", "de.javakaffee" % "kryo-serializers" % "0.9", - "se.scalablesolutions.akka" % "akka-actor" % "1.2", - "se.scalablesolutions.akka" % "akka-remote" % "1.2", "org.jboss.netty" % "netty" % "3.2.6.Final", "it.unimi.dsi" % "fastutil" % "6.4.2" ) @@ -67,7 +65,7 @@ object SparkBuild extends Build { def replSettings = sharedSettings ++ Seq( name := "spark-repl", libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _) - ) + ) ++ assemblySettings ++ Seq(test in assembly := {}) def examplesSettings = sharedSettings ++ Seq( name := "spark-examples", |