diff options
| author | Hossein <hossein@databricks.com> | 2016-01-15 11:46:46 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2016-01-15 11:46:46 -0800 |
| commit | 5f83c6991c95616ecbc2878f8860c69b2826f56c (patch) | |
| tree | 86dc70e45f1b27b67efec9724632a108d69f2ef0 /sql/core/src/test/resources | |
| parent | c5e7076da72657ea35a0aa388f8d2e6411d39280 (diff) | |
| download | spark-5f83c6991c95616ecbc2878f8860c69b2826f56c.tar.gz spark-5f83c6991c95616ecbc2878f8860c69b2826f56c.tar.bz2 spark-5f83c6991c95616ecbc2878f8860c69b2826f56c.zip | |
[SPARK-12833][SQL] Initial import of spark-csv
CSV is the most common data format in the "small data" world. It is often the first format people want to try when they see Spark on a single node. Having to rely on a 3rd party component for this leads to poor user experience for new users. This PR merges the popular spark-csv data source package (https://github.com/databricks/spark-csv) with SparkSQL.
This is a first PR to bring the functionality to spark 2.0 master. We will complete items outlines in the design document (see JIRA attachment) in follow up pull requests.
Author: Hossein <hossein@databricks.com>
Author: Reynold Xin <rxin@databricks.com>
Closes #10766 from rxin/csv.
Diffstat (limited to 'sql/core/src/test/resources')
| -rw-r--r-- | sql/core/src/test/resources/cars-alternative.csv | 5 | ||||
| -rw-r--r-- | sql/core/src/test/resources/cars-null.csv | 6 | ||||
| -rw-r--r-- | sql/core/src/test/resources/cars-unbalanced-quotes.csv | 4 | ||||
| -rw-r--r-- | sql/core/src/test/resources/cars.csv | 6 | ||||
| -rw-r--r-- | sql/core/src/test/resources/cars.tsv | 4 | ||||
| -rw-r--r-- | sql/core/src/test/resources/cars_iso-8859-1.csv | 6 | ||||
| -rw-r--r-- | sql/core/src/test/resources/comments.csv | 6 | ||||
| -rw-r--r-- | sql/core/src/test/resources/disable_comments.csv | 2 | ||||
| -rw-r--r-- | sql/core/src/test/resources/empty.csv | 0 |
9 files changed, 39 insertions, 0 deletions
diff --git a/sql/core/src/test/resources/cars-alternative.csv b/sql/core/src/test/resources/cars-alternative.csv new file mode 100644 index 0000000000..646f7c456c --- /dev/null +++ b/sql/core/src/test/resources/cars-alternative.csv @@ -0,0 +1,5 @@ +year|make|model|comment|blank +'2012'|'Tesla'|'S'| 'No comment'| + +1997|Ford|E350|'Go get one now they are going fast'| +2015|Chevy|Volt diff --git a/sql/core/src/test/resources/cars-null.csv b/sql/core/src/test/resources/cars-null.csv new file mode 100644 index 0000000000..130c0b40bb --- /dev/null +++ b/sql/core/src/test/resources/cars-null.csv @@ -0,0 +1,6 @@ +year,make,model,comment,blank +"2012","Tesla","S",null, + +1997,Ford,E350,"Go get one now they are going fast", +null,Chevy,Volt + diff --git a/sql/core/src/test/resources/cars-unbalanced-quotes.csv b/sql/core/src/test/resources/cars-unbalanced-quotes.csv new file mode 100644 index 0000000000..5ea39fcbfa --- /dev/null +++ b/sql/core/src/test/resources/cars-unbalanced-quotes.csv @@ -0,0 +1,4 @@ +year,make,model,comment,blank +"2012,Tesla,S,No comment +1997,Ford,E350,Go get one now they are going fast" +"2015,"Chevy",Volt, diff --git a/sql/core/src/test/resources/cars.csv b/sql/core/src/test/resources/cars.csv new file mode 100644 index 0000000000..2b9d74ca60 --- /dev/null +++ b/sql/core/src/test/resources/cars.csv @@ -0,0 +1,6 @@ +year,make,model,comment,blank +"2012","Tesla","S","No comment", + +1997,Ford,E350,"Go get one now they are going fast", +2015,Chevy,Volt + diff --git a/sql/core/src/test/resources/cars.tsv b/sql/core/src/test/resources/cars.tsv new file mode 100644 index 0000000000..a7bfa9a91f --- /dev/null +++ b/sql/core/src/test/resources/cars.tsv @@ -0,0 +1,4 @@ +year make model price comment blank +2012 Tesla S "80,000.65" +1997 Ford E350 35,000 "Go get one now they are going fast" +2015 Chevy Volt 5,000.10 diff --git a/sql/core/src/test/resources/cars_iso-8859-1.csv b/sql/core/src/test/resources/cars_iso-8859-1.csv new file mode 100644 index 0000000000..c51b6c5901 --- /dev/null +++ b/sql/core/src/test/resources/cars_iso-8859-1.csv @@ -0,0 +1,6 @@ +yearþmakeþmodelþcommentþblank +"2012"þ"Tesla"þ"S"þ"No comment"þ + +1997þFordþE350þ"Go get one now they are þoing fast"þ +2015þChevyþVolt + diff --git a/sql/core/src/test/resources/comments.csv b/sql/core/src/test/resources/comments.csv new file mode 100644 index 0000000000..6275be7285 --- /dev/null +++ b/sql/core/src/test/resources/comments.csv @@ -0,0 +1,6 @@ +~ Version 1.0 +~ Using a non-standard comment char to test CSV parser defaults are overridden +1,2,3,4,5.01,2015-08-20 15:57:00 +6,7,8,9,0,2015-08-21 16:58:01 +~0,9,8,7,6,2015-08-22 17:59:02 +1,2,3,4,5,2015-08-23 18:00:42 diff --git a/sql/core/src/test/resources/disable_comments.csv b/sql/core/src/test/resources/disable_comments.csv new file mode 100644 index 0000000000..304d406e4d --- /dev/null +++ b/sql/core/src/test/resources/disable_comments.csv @@ -0,0 +1,2 @@ +#1,2,3 +4,5,6 diff --git a/sql/core/src/test/resources/empty.csv b/sql/core/src/test/resources/empty.csv new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/sql/core/src/test/resources/empty.csv |