aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authorReynold Xin <rxin@databricks.com>2016-05-27 11:13:09 -0700
committerAndrew Or <andrew@databricks.com>2016-05-27 11:13:09 -0700
commita52e6813392ba4bdb1b818694b7ced8f6caa6a2b (patch)
tree7a3003092c0d745b8bd5f1100690c510509111e2 /sql
parent635fb30f83a66cc56f5fecfed5bff77873bf49a6 (diff)
downloadspark-a52e6813392ba4bdb1b818694b7ced8f6caa6a2b.tar.gz
spark-a52e6813392ba4bdb1b818694b7ced8f6caa6a2b.tar.bz2
spark-a52e6813392ba4bdb1b818694b7ced8f6caa6a2b.zip
[SPARK-15597][SQL] Add SparkSession.emptyDataset
## What changes were proposed in this pull request? This patch adds a new function emptyDataset to SparkSession, for creating an empty dataset. ## How was this patch tested? Added a test case. Author: Reynold Xin <rxin@databricks.com> Closes #13344 from rxin/SPARK-15597.
Diffstat (limited to 'sql')
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala12
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala6
2 files changed, 18 insertions, 0 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index aa60048405..c9276cf140 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -224,6 +224,18 @@ class SparkSession private(
/**
* :: Experimental ::
+ * Creates a new [[Dataset]] of type T containing zero elements.
+ *
+ * @return 2.0.0
+ */
+ @Experimental
+ def emptyDataset[T: Encoder]: Dataset[T] = {
+ val encoder = implicitly[Encoder[T]]
+ new Dataset(self, LocalRelation(encoder.schema.toAttributes), encoder)
+ }
+
+ /**
+ * :: Experimental ::
* Creates a [[DataFrame]] from an RDD of Product (e.g. case classes, tuples).
*
* @group dataframes
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 2a6591653e..e395007999 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -46,6 +46,12 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
1, 1, 1)
}
+ test("emptyDataset") {
+ val ds = spark.emptyDataset[Int]
+ assert(ds.count() == 0L)
+ assert(ds.collect() sameElements Array.empty[Int])
+ }
+
test("range") {
assert(spark.range(10).map(_ + 1).reduce(_ + _) == 55)
assert(spark.range(10).map{ case i: java.lang.Long => i + 1 }.reduce(_ + _) == 55)