aboutsummaryrefslogtreecommitdiff
path: root/examples/src/main/scala
diff options
context:
space:
mode:
authorJosh Rosen <rosenville@gmail.com>2012-07-24 09:43:02 -0700
committerJosh Rosen <rosenville@gmail.com>2012-07-24 09:47:00 -0700
commit6a78e88237cd2d2947cf5461e9da8359a16a0228 (patch)
tree079c8f49adbacb80babe2abc50dc293a23af5593 /examples/src/main/scala
parent042dcbde33c878d165b777964f5abb646257ab94 (diff)
downloadspark-6a78e88237cd2d2947cf5461e9da8359a16a0228.tar.gz
spark-6a78e88237cd2d2947cf5461e9da8359a16a0228.tar.bz2
spark-6a78e88237cd2d2947cf5461e9da8359a16a0228.zip
Minor cleanup and optimizations in Java API.
- Add override keywords. - Cache RDDs and counts in TC example. - Clean up JavaRDDLike's abstract methods.
Diffstat (limited to 'examples/src/main/scala')
-rw-r--r--examples/src/main/scala/spark/examples/SparkTC.scala10
1 files changed, 6 insertions, 4 deletions
diff --git a/examples/src/main/scala/spark/examples/SparkTC.scala b/examples/src/main/scala/spark/examples/SparkTC.scala
index fa945b5082..a095476a23 100644
--- a/examples/src/main/scala/spark/examples/SparkTC.scala
+++ b/examples/src/main/scala/spark/examples/SparkTC.scala
@@ -28,7 +28,7 @@ object SparkTC {
}
val spark = new SparkContext(args(0), "SparkTC")
val slices = if (args.length > 1) args(1).toInt else 2
- var tc = spark.parallelize(generateGraph, slices)
+ var tc = spark.parallelize(generateGraph, slices).cache()
// Linear transitive closure: each round grows paths by one edge,
// by joining the graph's edges with the already-discovered paths.
@@ -40,12 +40,14 @@ object SparkTC {
// This join is iterated until a fixed point is reached.
var oldCount = 0L
+ var nextCount = tc.count()
do {
- oldCount = tc.count()
+ oldCount = nextCount
// Perform the join, obtaining an RDD of (y, (z, x)) pairs,
// then project the result to obtain the new (x, z) paths.
- tc = tc.union(tc.join(edges).map(x => (x._2._2, x._2._1))).distinct()
- } while (tc.count() != oldCount)
+ tc = tc.union(tc.join(edges).map(x => (x._2._2, x._2._1))).distinct().cache();
+ nextCount = tc.count()
+ } while (nextCount != oldCount)
println("TC has " + tc.count() + " edges.")
System.exit(0)