diff options
author | Burak Yavuz <brkyvz@gmail.com> | 2017-02-10 12:55:06 +0100 |
---|---|---|
committer | Herman van Hovell <hvanhovell@databricks.com> | 2017-02-10 12:55:06 +0100 |
commit | d5593f7f5794bd0343e783ac4957864fed9d1b38 (patch) | |
tree | 8e58d63a0bd9999b5fff871032563215b50b9a2f /sql | |
parent | fd6c3a0b10ce43a56df845ba66d160b77f02e576 (diff) | |
download | spark-d5593f7f5794bd0343e783ac4957864fed9d1b38.tar.gz spark-d5593f7f5794bd0343e783ac4957864fed9d1b38.tar.bz2 spark-d5593f7f5794bd0343e783ac4957864fed9d1b38.zip |
[SPARK-19543] from_json fails when the input row is empty
## What changes were proposed in this pull request?
Using from_json on a column with an empty string results in: java.util.NoSuchElementException: head of empty list.
This is because `parser.parse(input)` may return `Nil` when `input.trim.isEmpty`
## How was this patch tested?
Regression test in `JsonExpressionsSuite`
Author: Burak Yavuz <brkyvz@gmail.com>
Closes #16881 from brkyvz/json-fix.
Diffstat (limited to 'sql')
2 files changed, 9 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index d55f85d562..c410e7919a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -496,7 +496,7 @@ case class JsonToStruct(schema: StructType, options: Map[String, String], child: override def dataType: DataType = schema override def nullSafeEval(json: Any): Any = { - try parser.parse(json.toString).head catch { + try parser.parse(json.toString).headOption.orNull catch { case _: SparkSQLJsonProcessingException => null } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 618b8b29e8..8e20bd1d97 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -376,6 +376,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { ) } + test("SPARK-19543: from_json empty input column") { + val schema = StructType(StructField("a", IntegerType) :: Nil) + checkEvaluation( + JsonToStruct(schema, Map.empty, Literal.create(" ", StringType)), + null + ) + } + test("to_json") { val schema = StructType(StructField("a", IntegerType) :: Nil) val struct = Literal.create(create_row(1), schema) |