diff options
| author | Yin Huai <huai@cse.ohio-state.edu> | 2014-08-08 11:10:11 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2014-08-08 11:10:11 -0700 |
| commit | 0489cee6b24ca34f1adab03a75d157e04a9e06b7 (patch) | |
| tree | 55c9dc3bded94e8db60929e296b534db56329057 /sql/core/src/main | |
| parent | 9016af3f2729101027e33593e094332f05f48d92 (diff) | |
| download | spark-0489cee6b24ca34f1adab03a75d157e04a9e06b7.tar.gz spark-0489cee6b24ca34f1adab03a75d157e04a9e06b7.tar.bz2 spark-0489cee6b24ca34f1adab03a75d157e04a9e06b7.zip | |
[SPARK-2908] [SQL] JsonRDD.nullTypeToStringType does not convert all NullType to StringType
JIRA: https://issues.apache.org/jira/browse/SPARK-2908
Author: Yin Huai <huai@cse.ohio-state.edu>
Closes #1840 from yhuai/SPARK-2908 and squashes the following commits:
86e833e [Yin Huai] Update test.
cb11759 [Yin Huai] nullTypeToStringType should check columns with the type of array of structs.
Diffstat (limited to 'sql/core/src/main')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala index a3d2a1c7a5..1c0b03c684 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala @@ -109,7 +109,9 @@ private[sql] object JsonRDD extends Logging { val newType = dataType match { case NullType => StringType case ArrayType(NullType, containsNull) => ArrayType(StringType, containsNull) - case struct: StructType => nullTypeToStringType(struct) + case ArrayType(struct: StructType, containsNull) => + ArrayType(nullTypeToStringType(struct), containsNull) + case struct: StructType =>nullTypeToStringType(struct) case other: DataType => other } StructField(fieldName, newType, nullable) |