diff options
author | gatorsmile <gatorsmile@gmail.com> | 2017-02-08 09:33:18 +0800 |
---|---|---|
committer | Wenchen Fan <wenchen@databricks.com> | 2017-02-08 09:33:18 +0800 |
commit | e33aaa2ac53a6e17e160e4e63821450b3609033b (patch) | |
tree | 5874a5632df72a1941e1b7c39ea46a3719208d66 /sql/core/src/test | |
parent | 8df444403489aec0d68f7d930afdc4f7d50e0b41 (diff) | |
download | spark-e33aaa2ac53a6e17e160e4e63821450b3609033b.tar.gz spark-e33aaa2ac53a6e17e160e4e63821450b3609033b.tar.bz2 spark-e33aaa2ac53a6e17e160e4e63821450b3609033b.zip |
[SPARK-19397][SQL] Make option names of LIBSVM and TEXT case insensitive
### What changes were proposed in this pull request?
Prior to Spark 2.1, the option names are case sensitive for all the formats. Since Spark 2.1, the option key names become case insensitive except the format `Text` and `LibSVM `. This PR is to fix these issues.
Also, add a check to know whether the input option vector type is legal for `LibSVM`.
### How was this patch tested?
Added test cases
Author: gatorsmile <gatorsmile@gmail.com>
Closes #16737 from gatorsmile/libSVMTextOptions.
Diffstat (limited to 'sql/core/src/test')
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala | 22 |
1 files changed, 20 insertions, 2 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala index d11c2acb81..cb7393cdd2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala @@ -115,8 +115,7 @@ class TextSuite extends QueryTest with SharedSQLContext { ) withTempDir { dir => val testDf = spark.read.text(testFile) - val tempDir = Utils.createTempDir() - val tempDirPath = tempDir.getAbsolutePath + val tempDirPath = dir.getAbsolutePath testDf.write.option("compression", "none") .options(extraOptions).mode(SaveMode.Overwrite).text(tempDirPath) val compressedFiles = new File(tempDirPath).listFiles() @@ -125,6 +124,25 @@ class TextSuite extends QueryTest with SharedSQLContext { } } + test("case insensitive option") { + val extraOptions = Map[String, String]( + "mApReDuCe.output.fileoutputformat.compress" -> "true", + "mApReDuCe.output.fileoutputformat.compress.type" -> CompressionType.BLOCK.toString, + "mApReDuCe.map.output.compress" -> "true", + "mApReDuCe.output.fileoutputformat.compress.codec" -> classOf[GzipCodec].getName, + "mApReDuCe.map.output.compress.codec" -> classOf[GzipCodec].getName + ) + withTempDir { dir => + val testDf = spark.read.text(testFile) + val tempDirPath = dir.getAbsolutePath + testDf.write.option("CoMpReSsIoN", "none") + .options(extraOptions).mode(SaveMode.Overwrite).text(tempDirPath) + val compressedFiles = new File(tempDirPath).listFiles() + assert(compressedFiles.exists(!_.getName.endsWith(".txt.gz"))) + verifyFrame(spark.read.options(extraOptions).text(tempDirPath)) + } + } + test("SPARK-14343: select partitioning column") { withTempPath { dir => val path = dir.getCanonicalPath |