diff options
author | Emiliano Leporati <emiliano.leporati@gmail.com> | 2015-08-05 00:42:08 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-08-05 00:42:16 -0700 |
commit | 57596fb8c13f2c9d7f326cad84744dad16e0fc95 (patch) | |
tree | 16bc152ec39852459c17d9b5af38591822600d49 | |
parent | ea23e54ff65fa871b4da61859254ba4980250752 (diff) | |
download | spark-57596fb8c13f2c9d7f326cad84744dad16e0fc95.tar.gz spark-57596fb8c13f2c9d7f326cad84744dad16e0fc95.tar.bz2 spark-57596fb8c13f2c9d7f326cad84744dad16e0fc95.zip |
[SPARK-9581][SQL] Add unit test for JSON UDT
This brings #7416 up-to-date by drubbo.
Author: Emiliano Leporati <emiliano.leporati@gmail.com>
Author: Reynold Xin <rxin@databricks.com>
Closes #7917 from rxin/udt-json-test and squashes the following commits:
93e3954 [Reynold Xin] Fix test.
7035308 [Reynold Xin] Merge pull request #7416 from drubbo/master
b5bcd94 [Emiliano Leporati] removed unneded case in MyDenseVector::equals
508a399 [Emiliano Leporati] Merge remote branch 'upstream/master'
7569e42 [Emiliano Leporati] using checkAnswer
62daccd [Emiliano Leporati] added coverage for UDTs in JSON RDDs
(cherry picked from commit 1d1a76c8c56148beec70dd14e4fdeb86de65018a)
Signed-off-by: Reynold Xin <rxin@databricks.com>
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala | 2 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala | 20 |
2 files changed, 21 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala index f1a66c84fc..b8fd3b9cc1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala @@ -125,7 +125,7 @@ private[sql] object JacksonParser { convertMap(factory, parser, kt) case (_, udt: UserDefinedType[_]) => - udt.deserialize(convertField(factory, parser, udt.sqlType)) + convertField(factory, parser, udt.sqlType) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala index f29935224e..9181222f69 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala @@ -138,4 +138,24 @@ class UserDefinedTypeSuite extends QueryTest { val actual = openHashSetUDT.deserialize(openHashSetUDT.serialize(set)) assert(actual.iterator.toSet === set.iterator.toSet) } + + test("UDTs with JSON") { + val data = Seq( + "{\"id\":1,\"vec\":[1.1,2.2,3.3,4.4]}", + "{\"id\":2,\"vec\":[2.25,4.5,8.75]}" + ) + val schema = StructType(Seq( + StructField("id", IntegerType, false), + StructField("vec", new MyDenseVectorUDT, false) + )) + + val stringRDD = ctx.sparkContext.parallelize(data) + val jsonRDD = ctx.read.schema(schema).json(stringRDD) + checkAnswer( + jsonRDD, + Row(1, new MyDenseVector(Array(1.1, 2.2, 3.3, 4.4))) :: + Row(2, new MyDenseVector(Array(2.25, 4.5, 8.75))) :: + Nil + ) + } } |