diff options
| author | hyukjinkwon <gurwls223@gmail.com> | 2016-02-02 10:41:06 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2016-02-02 10:41:06 -0800 |
| commit | b93830126cc59a26e2cfb5d7b3c17f9cfbf85988 (patch) | |
| tree | 6b34fbe65c91baea0113df76f31b0a4275a20286 | |
| parent | 29d92181d0c49988c387d34e4a71b1afe02c29e2 (diff) | |
| download | spark-b93830126cc59a26e2cfb5d7b3c17f9cfbf85988.tar.gz spark-b93830126cc59a26e2cfb5d7b3c17f9cfbf85988.tar.bz2 spark-b93830126cc59a26e2cfb5d7b3c17f9cfbf85988.zip | |
[SPARK-13114][SQL] Add a test for tokens more than the fields in schema
https://issues.apache.org/jira/browse/SPARK-13114
This PR adds a test for tokens more than the fields in schema.
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #11020 from HyukjinKwon/SPARK-13114.
| -rw-r--r-- | sql/core/src/test/resources/cars-malformed.csv | 6 | ||||
| -rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala | 12 |
2 files changed, 18 insertions, 0 deletions
diff --git a/sql/core/src/test/resources/cars-malformed.csv b/sql/core/src/test/resources/cars-malformed.csv new file mode 100644 index 0000000000..cfa378c01f --- /dev/null +++ b/sql/core/src/test/resources/cars-malformed.csv @@ -0,0 +1,6 @@ +~ All the rows here are malformed having tokens more than the schema (header). +year,make,model,comment,blank +"2012","Tesla","S","No comment",,null,null + +1997,Ford,E350,"Go get one now they are going fast",,null,null +2015,Chevy,,,, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala index a79566b1f3..fa4f137b70 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types._ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils { private val carsFile = "cars.csv" + private val carsMalformedFile = "cars-malformed.csv" private val carsFile8859 = "cars_iso-8859-1.csv" private val carsTsvFile = "cars.tsv" private val carsAltFile = "cars-alternative.csv" @@ -191,6 +192,17 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils { assert(exception.getMessage.contains("Malformed line in FAILFAST mode: 2015,Chevy,Volt")) } + test("test for tokens more than the fields in the schema") { + val cars = sqlContext + .read + .format("csv") + .option("header", "false") + .option("comment", "~") + .load(testFile(carsMalformedFile)) + + verifyCars(cars, withHeader = false, checkTypes = false) + } + test("test with null quote character") { val cars = sqlContext.read .format("csv") |