diff options
author | gatorsmile <gatorsmile@gmail.com> | 2015-12-08 10:15:58 -0800 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-12-08 10:15:58 -0800 |
commit | c0b13d5565c45ae2acbe8cfb17319c92b6a634e4 (patch) | |
tree | 21457ea9d9174df6b39ae41f8b8152697e320c65 | |
parent | 381f17b540d92507cc07adf18bce8bc7e5ca5407 (diff) | |
download | spark-c0b13d5565c45ae2acbe8cfb17319c92b6a634e4.tar.gz spark-c0b13d5565c45ae2acbe8cfb17319c92b6a634e4.tar.bz2 spark-c0b13d5565c45ae2acbe8cfb17319c92b6a634e4.zip |
[SPARK-12195][SQL] Adding BigDecimal, Date and Timestamp into Encoder
This PR is to add three more data types into Encoder, including `BigDecimal`, `Date` and `Timestamp`.
marmbrus cloud-fan rxin Could you take a quick look at these three types? Not sure if it can be merged to 1.6. Thank you very much!
Author: gatorsmile <gatorsmile@gmail.com>
Closes #10188 from gatorsmile/dataTypesinEncoder.
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala | 18 | ||||
-rw-r--r-- | sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java | 17 |
2 files changed, 35 insertions, 0 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala index c40061ae0a..3ca5ade7f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala @@ -98,6 +98,24 @@ object Encoders { def STRING: Encoder[java.lang.String] = ExpressionEncoder() /** + * An encoder for nullable decimal type. + * @since 1.6.0 + */ + def DECIMAL: Encoder[java.math.BigDecimal] = ExpressionEncoder() + + /** + * An encoder for nullable date type. + * @since 1.6.0 + */ + def DATE: Encoder[java.sql.Date] = ExpressionEncoder() + + /** + * An encoder for nullable timestamp type. + * @since 1.6.0 + */ + def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder() + + /** * Creates an encoder for Java Bean of type T. * * T must be publicly accessible. diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java index ae47f4fe0e..383a2d0bad 100644 --- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java +++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java @@ -18,6 +18,9 @@ package test.org.apache.spark.sql; import java.io.Serializable; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; import java.util.*; import scala.Tuple2; @@ -386,6 +389,20 @@ public class JavaDatasetSuite implements Serializable { } @Test + public void testPrimitiveEncoder() { + Encoder<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> encoder = + Encoders.tuple(Encoders.DOUBLE(), Encoders.DECIMAL(), Encoders.DATE(), Encoders.TIMESTAMP(), + Encoders.FLOAT()); + List<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> data = + Arrays.asList(new Tuple5<Double, BigDecimal, Date, Timestamp, Float>( + 1.7976931348623157E308, new BigDecimal("0.922337203685477589"), + Date.valueOf("1970-01-01"), new Timestamp(System.currentTimeMillis()), Float.MAX_VALUE)); + Dataset<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> ds = + context.createDataset(data, encoder); + Assert.assertEquals(data, ds.collectAsList()); + } + + @Test public void testTypedAggregation() { Encoder<Tuple2<String, Integer>> encoder = Encoders.tuple(Encoders.STRING(), Encoders.INT()); List<Tuple2<String, Integer>> data = |