aboutsummaryrefslogtreecommitdiff
path: root/core/src/test/scala
diff options
context:
space:
mode:
authorGavin Li <lyo.gavin@gmail.com>2013-06-16 22:32:55 +0000
committerGavin Li <lyo.gavin@gmail.com>2013-06-16 22:32:55 +0000
commitfb6d733fa88aa124deecf155af40cc095ecca5b3 (patch)
tree33fa219e31a107bbad8e4df59d1438ce8896728a /core/src/test/scala
parente179ff8a32fc08cc308dc99bac2527d350d0d970 (diff)
downloadspark-fb6d733fa88aa124deecf155af40cc095ecca5b3.tar.gz
spark-fb6d733fa88aa124deecf155af40cc095ecca5b3.tar.bz2
spark-fb6d733fa88aa124deecf155af40cc095ecca5b3.zip
update according to comments
Diffstat (limited to 'core/src/test/scala')
-rw-r--r--core/src/test/scala/spark/PipedRDDSuite.scala13
1 files changed, 9 insertions, 4 deletions
diff --git a/core/src/test/scala/spark/PipedRDDSuite.scala b/core/src/test/scala/spark/PipedRDDSuite.scala
index d2852867de..ed075f93ec 100644
--- a/core/src/test/scala/spark/PipedRDDSuite.scala
+++ b/core/src/test/scala/spark/PipedRDDSuite.scala
@@ -22,9 +22,12 @@ class PipedRDDSuite extends FunSuite with LocalSparkContext {
test("advanced pipe") {
sc = new SparkContext("local", "test")
val nums = sc.makeRDD(Array(1, 2, 3, 4), 2)
+ val bl = sc.broadcast(List("0"))
- val piped = nums.pipe(Seq("cat"), Map[String, String](),
- (i:Int, f: String=> Unit) => f(i + "_"), sc.broadcast(List("0")))
+ val piped = nums.pipe(Seq("cat"),
+ Map[String, String](),
+ (f: String => Unit) => {bl.value.map(f(_));f("\u0001")},
+ (i:Int, f: String=> Unit) => f(i + "_"))
val c = piped.collect()
@@ -40,8 +43,10 @@ class PipedRDDSuite extends FunSuite with LocalSparkContext {
val nums1 = sc.makeRDD(Array("a\t1", "b\t2", "a\t3", "b\t4"), 2)
val d = nums1.groupBy(str=>str.split("\t")(0)).
- pipe(Seq("cat"), Map[String, String](), (i:Tuple2[String, Seq[String]], f: String=> Unit) =>
- {for (e <- i._2){ f(e + "_")}}, sc.broadcast(List("0"))).collect()
+ pipe(Seq("cat"),
+ Map[String, String](),
+ (f: String => Unit) => {bl.value.map(f(_));f("\u0001")},
+ (i:Tuple2[String, Seq[String]], f: String=> Unit) => {for (e <- i._2){ f(e + "_")}}).collect()
assert(d.size === 8)
assert(d(0) === "0")
assert(d(1) === "\u0001")