From f041e55eefe1d8a995fed321c66bccbd8b8e5255 Mon Sep 17 00:00:00 2001 From: Tejas Patil Date: Wed, 15 Feb 2017 22:45:58 -0800 Subject: [SPARK-19618][SQL] Inconsistency wrt max. buckets allowed from Dataframe API vs SQL ## What changes were proposed in this pull request? Jira: https://issues.apache.org/jira/browse/SPARK-19618 Moved the check for validating number of buckets from `DataFrameWriter` to `BucketSpec` creation ## How was this patch tested? - Added more unit tests Author: Tejas Patil Closes #16948 from tejasapatil/SPARK-19618_max_buckets. --- .../main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'sql/catalyst/src') diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 353e5954fd..2b3b575b4c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -135,8 +135,9 @@ case class BucketSpec( numBuckets: Int, bucketColumnNames: Seq[String], sortColumnNames: Seq[String]) { - if (numBuckets <= 0) { - throw new AnalysisException(s"Expected positive number of buckets, but got `$numBuckets`.") + if (numBuckets <= 0 || numBuckets >= 100000) { + throw new AnalysisException( + s"Number of buckets should be greater than 0 but less than 100000. Got `$numBuckets`") } override def toString: String = { -- cgit v1.2.3