diff options
author | Josh Rosen <rosenville@gmail.com> | 2012-07-24 09:43:02 -0700 |
---|---|---|
committer | Josh Rosen <rosenville@gmail.com> | 2012-07-24 09:47:00 -0700 |
commit | 6a78e88237cd2d2947cf5461e9da8359a16a0228 (patch) | |
tree | 079c8f49adbacb80babe2abc50dc293a23af5593 /examples/src/main/scala | |
parent | 042dcbde33c878d165b777964f5abb646257ab94 (diff) | |
download | spark-6a78e88237cd2d2947cf5461e9da8359a16a0228.tar.gz spark-6a78e88237cd2d2947cf5461e9da8359a16a0228.tar.bz2 spark-6a78e88237cd2d2947cf5461e9da8359a16a0228.zip |
Minor cleanup and optimizations in Java API.
- Add override keywords.
- Cache RDDs and counts in TC example.
- Clean up JavaRDDLike's abstract methods.
Diffstat (limited to 'examples/src/main/scala')
-rw-r--r-- | examples/src/main/scala/spark/examples/SparkTC.scala | 10 |
1 files changed, 6 insertions, 4 deletions
diff --git a/examples/src/main/scala/spark/examples/SparkTC.scala b/examples/src/main/scala/spark/examples/SparkTC.scala index fa945b5082..a095476a23 100644 --- a/examples/src/main/scala/spark/examples/SparkTC.scala +++ b/examples/src/main/scala/spark/examples/SparkTC.scala @@ -28,7 +28,7 @@ object SparkTC { } val spark = new SparkContext(args(0), "SparkTC") val slices = if (args.length > 1) args(1).toInt else 2 - var tc = spark.parallelize(generateGraph, slices) + var tc = spark.parallelize(generateGraph, slices).cache() // Linear transitive closure: each round grows paths by one edge, // by joining the graph's edges with the already-discovered paths. @@ -40,12 +40,14 @@ object SparkTC { // This join is iterated until a fixed point is reached. var oldCount = 0L + var nextCount = tc.count() do { - oldCount = tc.count() + oldCount = nextCount // Perform the join, obtaining an RDD of (y, (z, x)) pairs, // then project the result to obtain the new (x, z) paths. - tc = tc.union(tc.join(edges).map(x => (x._2._2, x._2._1))).distinct() - } while (tc.count() != oldCount) + tc = tc.union(tc.join(edges).map(x => (x._2._2, x._2._1))).distinct().cache(); + nextCount = tc.count() + } while (nextCount != oldCount) println("TC has " + tc.count() + " edges.") System.exit(0) |