diff options
author | Yin Huai <huai@cse.ohio-state.edu> | 2014-06-13 23:28:57 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-06-13 23:28:57 -0700 |
commit | 891968509105d8d8cf5a608ad9473aeeed747089 (patch) | |
tree | 7c67046b2460adb67ebaf4d63a26d42862ea0c20 /sql | |
parent | edb1f0e3164b99b483b6878efe0b6ea5ae9f97ed (diff) | |
download | spark-891968509105d8d8cf5a608ad9473aeeed747089.tar.gz spark-891968509105d8d8cf5a608ad9473aeeed747089.tar.bz2 spark-891968509105d8d8cf5a608ad9473aeeed747089.zip |
[Spark-2137][SQL] Timestamp UDFs broken
https://issues.apache.org/jira/browse/SPARK-2137
Author: Yin Huai <huai@cse.ohio-state.edu>
Closes #1081 from yhuai/SPARK-2137 and squashes the following commits:
c04f910 [Yin Huai] Merge remote-tracking branch 'upstream/master' into SPARK-2137
205f17b [Yin Huai] Make Hive UDF wrapper support Timestamp.
Diffstat (limited to 'sql')
19 files changed, 17 insertions, 2 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 96e0ec5136..cc95b7af0a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -250,7 +250,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { protected val primitiveTypes = Seq(StringType, IntegerType, LongType, DoubleType, FloatType, BooleanType, ByteType, - ShortType, DecimalType) + ShortType, DecimalType, TimestampType) protected def toHiveString(a: (Any, DataType)): String = a match { case (struct: Row, StructType(fields)) => diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala index 5729020423..771d2bccf4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala @@ -187,7 +187,8 @@ private[hive] case class HiveSimpleUdf(name: String, children: Seq[Expression]) val primitiveClasses = Seq( Integer.TYPE, classOf[java.lang.Integer], classOf[java.lang.String], java.lang.Double.TYPE, classOf[java.lang.Double], java.lang.Long.TYPE, classOf[java.lang.Long], - classOf[HiveDecimal], java.lang.Byte.TYPE, classOf[java.lang.Byte] + classOf[HiveDecimal], java.lang.Byte.TYPE, classOf[java.lang.Byte], + classOf[java.sql.Timestamp] ) val matchingConstructor = argClass.getConstructors.find { c => c.getParameterTypes.size == 1 && primitiveClasses.contains(c.getParameterTypes.head) diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 b/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 new file mode 100644 index 0000000000..b3c4eec4c2 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-10-dbc23736a61d9482d13cacada02a7a09 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 b/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 new file mode 100644 index 0000000000..f69f13ed1f --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-11-442cf850a0cc1f1dcfdeaeffbffb2c35 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 2011-05-06 07:08:09.1234567 2011-05-06 02:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 b/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 new file mode 100644 index 0000000000..f14f17e692 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-12-51959036fd4ac4f1e24f4e06eb9b0b6 @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a b/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a new file mode 100644 index 0000000000..7881bff731 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-13-6ab3f356deaf807e8accc37e1f4849a @@ -0,0 +1 @@ +2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 2011-05-06 07:08:09.1234567 2011-05-06 12:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de b/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de new file mode 100644 index 0000000000..2c5e9e9656 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-14-c745a1016461403526d44928a269c1de @@ -0,0 +1 @@ +1304690889 2011 5 6 6 18 7 8 9 2011-05-06 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe b/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe new file mode 100644 index 0000000000..19497254f8 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-15-7ab76c4458c7f78038c8b1df0fdeafbe @@ -0,0 +1 @@ +2011-05-11 2011-04-26 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 b/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 new file mode 100644 index 0000000000..816f56e43e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-16-b36e87e17ca24d82072220bff559c718 @@ -0,0 +1 @@ +0 3333 -3333 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 b/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 new file mode 100644 index 0000000000..a4182d1e39 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-17-dad44d2d4a421286e9da080271bd2639 @@ -0,0 +1 @@ +2011-05-06 02:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a b/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a new file mode 100644 index 0000000000..02ccd3a2e9 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-18-cb033ecad964a2623bc633ac1d3f752a @@ -0,0 +1 @@ +2011-05-06 12:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af b/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-19-79914c5347620c6e62a8e0b9a95984af diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d b/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-20-59fc1842a23369235d42ed040d45fb3d diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc b/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-4-80ce02ec84ee8abcb046367ca37279cc diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d b/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d new file mode 100644 index 0000000000..2c5e9e9656 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-5-1124399033bcadf3874fb48f593392d @@ -0,0 +1 @@ +1304690889 2011 5 6 6 18 7 8 9 2011-05-06 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 b/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 new file mode 100644 index 0000000000..19497254f8 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-6-5810193ce35d38c23f4fc4b4979d60a4 @@ -0,0 +1 @@ +2011-05-11 2011-04-26 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 b/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 new file mode 100644 index 0000000000..816f56e43e --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-7-250e640a6a818f989f3f3280b00f64f9 @@ -0,0 +1 @@ +0 3333 -3333 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 b/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 new file mode 100644 index 0000000000..a4182d1e39 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-8-975df43df015d86422965af456f87a94 @@ -0,0 +1 @@ +2011-05-06 02:08:09.1234567 diff --git a/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 b/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 new file mode 100644 index 0000000000..02ccd3a2e9 --- /dev/null +++ b/sql/hive/src/test/resources/golden/timestamp_udf-9-287614364eaa3fb82aad08c6b62cc938 @@ -0,0 +1 @@ +2011-05-06 12:08:09.1234567 |