From 7201f033ce520259b6d07ea5ead92272cac92363 Mon Sep 17 00:00:00 2001 From: Guillaume Poulin Date: Tue, 5 Apr 2016 02:54:38 +0100 Subject: [SPARK-12425][STREAMING] DStream union optimisation Use PartitionerAwareUnionRDD when possbile for optimizing shuffling and preserving the partitioner. Author: Guillaume Poulin Closes #10382 from gpoulin/dstream_union_optimisation. --- core/src/main/scala/org/apache/spark/rdd/RDD.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'core/src/main/scala/org/apache') diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 4a0a2199ef..032939b49a 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -568,11 +568,7 @@ abstract class RDD[T: ClassTag]( * times (use `.distinct()` to eliminate them). */ def union(other: RDD[T]): RDD[T] = withScope { - if (partitioner.isDefined && other.partitioner == partitioner) { - new PartitionerAwareUnionRDD(sc, Array(this, other)) - } else { - new UnionRDD(sc, Array(this, other)) - } + sc.union(this, other) } /** -- cgit v1.2.3