aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/test
diff options
context:
space:
mode:
authorgatorsmile <gatorsmile@gmail.com>2017-02-08 09:33:18 +0800
committerWenchen Fan <wenchen@databricks.com>2017-02-08 09:33:18 +0800
commite33aaa2ac53a6e17e160e4e63821450b3609033b (patch)
tree5874a5632df72a1941e1b7c39ea46a3719208d66 /sql/core/src/test
parent8df444403489aec0d68f7d930afdc4f7d50e0b41 (diff)
downloadspark-e33aaa2ac53a6e17e160e4e63821450b3609033b.tar.gz
spark-e33aaa2ac53a6e17e160e4e63821450b3609033b.tar.bz2
spark-e33aaa2ac53a6e17e160e4e63821450b3609033b.zip
[SPARK-19397][SQL] Make option names of LIBSVM and TEXT case insensitive
### What changes were proposed in this pull request? Prior to Spark 2.1, the option names are case sensitive for all the formats. Since Spark 2.1, the option key names become case insensitive except the format `Text` and `LibSVM `. This PR is to fix these issues. Also, add a check to know whether the input option vector type is legal for `LibSVM`. ### How was this patch tested? Added test cases Author: gatorsmile <gatorsmile@gmail.com> Closes #16737 from gatorsmile/libSVMTextOptions.
Diffstat (limited to 'sql/core/src/test')
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala22
1 files changed, 20 insertions, 2 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
index d11c2acb81..cb7393cdd2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -115,8 +115,7 @@ class TextSuite extends QueryTest with SharedSQLContext {
)
withTempDir { dir =>
val testDf = spark.read.text(testFile)
- val tempDir = Utils.createTempDir()
- val tempDirPath = tempDir.getAbsolutePath
+ val tempDirPath = dir.getAbsolutePath
testDf.write.option("compression", "none")
.options(extraOptions).mode(SaveMode.Overwrite).text(tempDirPath)
val compressedFiles = new File(tempDirPath).listFiles()
@@ -125,6 +124,25 @@ class TextSuite extends QueryTest with SharedSQLContext {
}
}
+ test("case insensitive option") {
+ val extraOptions = Map[String, String](
+ "mApReDuCe.output.fileoutputformat.compress" -> "true",
+ "mApReDuCe.output.fileoutputformat.compress.type" -> CompressionType.BLOCK.toString,
+ "mApReDuCe.map.output.compress" -> "true",
+ "mApReDuCe.output.fileoutputformat.compress.codec" -> classOf[GzipCodec].getName,
+ "mApReDuCe.map.output.compress.codec" -> classOf[GzipCodec].getName
+ )
+ withTempDir { dir =>
+ val testDf = spark.read.text(testFile)
+ val tempDirPath = dir.getAbsolutePath
+ testDf.write.option("CoMpReSsIoN", "none")
+ .options(extraOptions).mode(SaveMode.Overwrite).text(tempDirPath)
+ val compressedFiles = new File(tempDirPath).listFiles()
+ assert(compressedFiles.exists(!_.getName.endsWith(".txt.gz")))
+ verifyFrame(spark.read.options(extraOptions).text(tempDirPath))
+ }
+ }
+
test("SPARK-14343: select partitioning column") {
withTempPath { dir =>
val path = dir.getCanonicalPath