diff options
Diffstat (limited to 'sql')
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala | 11 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala | 11 |
2 files changed, 6 insertions, 16 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala index 044ada2607..9f4ce8358b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala @@ -30,7 +30,8 @@ private[sql] class CSVOptions(@transient private val parameters: Map[String, Str val paramValue = parameters.get(paramName) paramValue match { case None => default - case Some(value) if value == null || value.length == 0 => '\u0000' + case Some(null) => default + case Some(value) if value.length == 0 => '\u0000' case Some(value) if value.length == 1 => value.charAt(0) case _ => throw new RuntimeException(s"$paramName cannot be more than one character") } @@ -51,12 +52,12 @@ private[sql] class CSVOptions(@transient private val parameters: Map[String, Str } private def getBool(paramName: String, default: Boolean = false): Boolean = { - val paramValue = parameters.getOrElse(paramName, default.toString) - if (paramValue == null) { + val param = parameters.getOrElse(paramName, default.toString) + if (param == null) { default - } else if (paramValue.toLowerCase == "true") { + } else if (param.toLowerCase == "true") { true - } else if (paramValue.toLowerCase == "false") { + } else if (param.toLowerCase == "false") { false } else { throw new Exception(s"$paramName flag can be true or false") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala index b26fcea759..bc95446387 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala @@ -655,15 +655,4 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils { assert(msg.contains("CSV data source does not support array<string> data type")) } } - - test("SPARK-15585 set null at quote") { - val cars = spark.read - .format("csv") - .option("header", "true") - .option("quote", null) - .load(testFile(carsUnbalancedQuotesFile)) - - verifyCars(cars, withHeader = true, checkValues = false) - } - } |