From 671bc08ed502815bfa2254c30d64149402acb0c7 Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Wed, 15 Feb 2017 10:45:37 -0800 Subject: [SPARK-19399][SPARKR] Add R coalesce API for DataFrame and Column ## What changes were proposed in this pull request? Add coalesce on DataFrame for down partitioning without shuffle and coalesce on Column ## How was this patch tested? manual, unit tests Author: Felix Cheung Closes #16739 from felixcheung/rcoalesce. --- core/src/main/scala/org/apache/spark/rdd/RDD.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'core/src/main/scala') diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 0359508c00..e524675332 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -423,7 +423,8 @@ abstract class RDD[T: ClassTag]( * * This results in a narrow dependency, e.g. if you go from 1000 partitions * to 100 partitions, there will not be a shuffle, instead each of the 100 - * new partitions will claim 10 of the current partitions. + * new partitions will claim 10 of the current partitions. If a larger number + * of partitions is requested, it will stay at the current number of partitions. * * However, if you're doing a drastic coalesce, e.g. to numPartitions = 1, * this may result in your computation taking place on fewer nodes than -- cgit v1.2.3