aboutsummaryrefslogtreecommitdiff
path: root/core/src
diff options
context:
space:
mode:
authorDongjoon Hyun <dongjoon@apache.org>2016-05-24 18:55:23 -0700
committerReynold Xin <rxin@databricks.com>2016-05-24 18:55:23 -0700
commitf08bf587b1913c6cc8ecb34c45331cf4750961c9 (patch)
treeaaa71096ce0da7d546ae373aed76eab84ddc2f7f /core/src
parente631b819fe348729aab062207a452b8f1d1511bd (diff)
downloadspark-f08bf587b1913c6cc8ecb34c45331cf4750961c9.tar.gz
spark-f08bf587b1913c6cc8ecb34c45331cf4750961c9.tar.bz2
spark-f08bf587b1913c6cc8ecb34c45331cf4750961c9.zip
[SPARK-15512][CORE] repartition(0) should raise IllegalArgumentException
## What changes were proposed in this pull request? Previously, SPARK-8893 added the constraints on positive number of partitions for repartition/coalesce operations in general. This PR adds one missing part for that and adds explicit two testcases. **Before** ```scala scala> sc.parallelize(1 to 5).coalesce(0) java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. ... scala> sc.parallelize(1 to 5).repartition(0).collect() res1: Array[Int] = Array() // empty scala> spark.sql("select 1").coalesce(0) res2: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [1: int] scala> spark.sql("select 1").coalesce(0).collect() java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. scala> spark.sql("select 1").repartition(0) res3: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [1: int] scala> spark.sql("select 1").repartition(0).collect() res4: Array[org.apache.spark.sql.Row] = Array() // empty ``` **After** ```scala scala> sc.parallelize(1 to 5).coalesce(0) java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. ... scala> sc.parallelize(1 to 5).repartition(0) java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. ... scala> spark.sql("select 1").coalesce(0) java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. ... scala> spark.sql("select 1").repartition(0) java.lang.IllegalArgumentException: requirement failed: Number of partitions (0) must be positive. ... ``` ## How was this patch tested? Pass the Jenkins tests with new testcases. Author: Dongjoon Hyun <dongjoon@apache.org> Closes #13282 from dongjoon-hyun/SPARK-15512.
Diffstat (limited to 'core/src')
-rw-r--r--core/src/main/scala/org/apache/spark/rdd/RDD.scala1
-rw-r--r--core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala8
2 files changed, 9 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index e6db9b3eec..e251421c48 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -437,6 +437,7 @@ abstract class RDD[T: ClassTag](
partitionCoalescer: Option[PartitionCoalescer] = Option.empty)
(implicit ord: Ordering[T] = null)
: RDD[T] = withScope {
+ require(numPartitions > 0, s"Number of partitions ($numPartitions) must be positive.")
if (shuffle) {
/** Distributes elements evenly across output partitions, starting from a random partition. */
val distributePartition = (index: Int, items: Iterator[T]) => {
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
index a4992fe8ac..ad56715656 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
@@ -276,6 +276,10 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext {
test("repartitioned RDDs") {
val data = sc.parallelize(1 to 1000, 10)
+ intercept[IllegalArgumentException] {
+ data.repartition(0)
+ }
+
// Coalesce partitions
val repartitioned1 = data.repartition(2)
assert(repartitioned1.partitions.size == 2)
@@ -329,6 +333,10 @@ class RDDSuite extends SparkFunSuite with SharedSparkContext {
test("coalesced RDDs") {
val data = sc.parallelize(1 to 10, 10)
+ intercept[IllegalArgumentException] {
+ data.coalesce(0)
+ }
+
val coalesced1 = data.coalesce(2)
assert(coalesced1.collect().toList === (1 to 10).toList)
assert(coalesced1.glom().collect().map(_.toList).toList ===