aboutsummaryrefslogtreecommitdiff
path: root/graphx/src/test/scala/org/apache
diff options
context:
space:
mode:
authorBrennon York <brennon.york@capitalone.com>2015-03-26 19:08:09 -0700
committerAnkur Dave <ankurdave@gmail.com>2015-03-26 19:08:09 -0700
commit39fb57968352549f2276ac4fcd2b92988ed6fe42 (patch)
tree811c1ecc0e813d299cdfdac6f1d6521883beb4b8 /graphx/src/test/scala/org/apache
parentaad00322765d6041e817a6bd3fcff2187d212057 (diff)
downloadspark-39fb57968352549f2276ac4fcd2b92988ed6fe42.tar.gz
spark-39fb57968352549f2276ac4fcd2b92988ed6fe42.tar.bz2
spark-39fb57968352549f2276ac4fcd2b92988ed6fe42.zip
[SPARK-6510][GraphX]: Add Graph#minus method to act as Set#difference
Adds a `Graph#minus` method which will return only unique `VertexId`'s from the calling `VertexRDD`. To demonstrate a basic example with pseudocode: ``` Set((0L,0),(1L,1)).minus(Set((1L,1),(2L,2))) > Set((0L,0)) ``` Author: Brennon York <brennon.york@capitalone.com> Closes #5175 from brennonyork/SPARK-6510 and squashes the following commits: 248d5c8 [Brennon York] added minus(VertexRDD[VD]) method to avoid createUsingIndex and updated the mask operations to simplify with andNot call 3fb7cce [Brennon York] updated graphx doc to reflect the addition of minus method 6575d92 [Brennon York] updated mima exclude aaa030b [Brennon York] completed graph#minus functionality 7227c0f [Brennon York] beginning work on minus functionality
Diffstat (limited to 'graphx/src/test/scala/org/apache')
-rw-r--r--graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala33
1 files changed, 31 insertions, 2 deletions
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
index 4f7a442ab5..c9443d11c7 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
@@ -47,6 +47,35 @@ class VertexRDDSuite extends FunSuite with LocalSparkContext {
}
}
+ test("minus") {
+ withSpark { sc =>
+ val vertexA = VertexRDD(sc.parallelize(0 until 75, 2).map(i => (i.toLong, 0))).cache()
+ val vertexB = VertexRDD(sc.parallelize(25 until 100, 2).map(i => (i.toLong, 1))).cache()
+ val vertexC = vertexA.minus(vertexB)
+ assert(vertexC.map(_._1).collect.toSet === (0 until 25).toSet)
+ }
+ }
+
+ test("minus with RDD[(VertexId, VD)]") {
+ withSpark { sc =>
+ val vertexA = VertexRDD(sc.parallelize(0 until 75, 2).map(i => (i.toLong, 0))).cache()
+ val vertexB: RDD[(VertexId, Int)] =
+ sc.parallelize(25 until 100, 2).map(i => (i.toLong, 1)).cache()
+ val vertexC = vertexA.minus(vertexB)
+ assert(vertexC.map(_._1).collect.toSet === (0 until 25).toSet)
+ }
+ }
+
+ test("minus with non-equal number of partitions") {
+ withSpark { sc =>
+ val vertexA = VertexRDD(sc.parallelize(0 until 75, 5).map(i => (i.toLong, 0)))
+ val vertexB = VertexRDD(sc.parallelize(50 until 100, 2).map(i => (i.toLong, 1)))
+ assert(vertexA.partitions.size != vertexB.partitions.size)
+ val vertexC = vertexA.minus(vertexB)
+ assert(vertexC.map(_._1).collect.toSet === (0 until 50).toSet)
+ }
+ }
+
test("diff") {
withSpark { sc =>
val n = 100
@@ -71,7 +100,7 @@ class VertexRDDSuite extends FunSuite with LocalSparkContext {
}
}
- test("diff vertices with the non-equal number of partitions") {
+ test("diff vertices with non-equal number of partitions") {
withSpark { sc =>
val vertexA = VertexRDD(sc.parallelize(0 until 24, 3).map(i => (i.toLong, 0)))
val vertexB = VertexRDD(sc.parallelize(8 until 16, 2).map(i => (i.toLong, 1)))
@@ -96,7 +125,7 @@ class VertexRDDSuite extends FunSuite with LocalSparkContext {
}
}
- test("leftJoin vertices with the non-equal number of partitions") {
+ test("leftJoin vertices with non-equal number of partitions") {
withSpark { sc =>
val vertexA = VertexRDD(sc.parallelize(0 until 100, 2).map(i => (i.toLong, 1)))
val vertexB = VertexRDD(