diff options
| author | Aaron Davidson <aaron@databricks.com> | 2014-02-03 11:25:39 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-02-03 11:25:39 -0800 |
| commit | 1625d8c44693420de026138f3abecce2d12f895c (patch) | |
| tree | 9489d8592a805dfb218c04784eff79e91bbf2148 | |
| parent | 0386f42e383dc01b8df33c4a70b024e7902b5fdd (diff) | |
| download | spark-1625d8c44693420de026138f3abecce2d12f895c.tar.gz spark-1625d8c44693420de026138f3abecce2d12f895c.tar.bz2 spark-1625d8c44693420de026138f3abecce2d12f895c.zip | |
Merge pull request #530 from aarondav/cleanup. Closes #530.
Remove explicit conversion to PairRDDFunctions in cogroup()
As SparkContext._ is already imported, using the implicit conversion appears to make the code much cleaner. Perhaps there was some sinister reason for doing the conversion explicitly, however.
Author: Aaron Davidson <aaron@databricks.com>
== Merge branch commits ==
commit aa4a63f1bfd5b5178fe67364dd7ce4d84c357996
Author: Aaron Davidson <aaron@databricks.com>
Date: Sun Feb 2 23:48:04 2014 -0800
Remove explicit conversion to PairRDDFunctions in cogroup()
As SparkContext._ is already imported, using the implicit conversion
appears to make the code much cleaner. Perhaps there was some sinister
reason for doing the converion explicitly, however.
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 10 |
1 files changed, 2 insertions, 8 deletions
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 4148581f52..370061492d 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -458,8 +458,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) throw new SparkException("Default partitioner cannot partition array keys.") } val cg = new CoGroupedRDD[K](Seq(self, other), partitioner) - val prfs = new PairRDDFunctions[K, Seq[Seq[_]]](cg)(classTag[K], ClassTags.seqSeqClassTag) - prfs.mapValues { case Seq(vs, ws) => + cg.mapValues { case Seq(vs, ws) => (vs.asInstanceOf[Seq[V]], ws.asInstanceOf[Seq[W]]) } } @@ -474,8 +473,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) throw new SparkException("Default partitioner cannot partition array keys.") } val cg = new CoGroupedRDD[K](Seq(self, other1, other2), partitioner) - val prfs = new PairRDDFunctions[K, Seq[Seq[_]]](cg)(classTag[K], ClassTags.seqSeqClassTag) - prfs.mapValues { case Seq(vs, w1s, w2s) => + cg.mapValues { case Seq(vs, w1s, w2s) => (vs.asInstanceOf[Seq[V]], w1s.asInstanceOf[Seq[W1]], w2s.asInstanceOf[Seq[W2]]) } } @@ -749,7 +747,3 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) private[spark] def getValueClass() = implicitly[ClassTag[V]].runtimeClass } - -private[spark] object ClassTags { - val seqSeqClassTag = classTag[Seq[Seq[_]]] -} |