aboutsummaryrefslogtreecommitdiff
path: root/sql/hive/src/test
diff options
context:
space:
mode:
authorLiang-Chi Hsieh <viirya@appier.com>2015-09-08 23:07:34 +0800
committerCheng Lian <lian@databricks.com>2015-09-08 23:07:34 +0800
commit990c9f79c28db501018a0a3af446ff879962475d (patch)
treeef18018fb463f7ed0348bdaabb3f4509d8e06cd5 /sql/hive/src/test
parent6ceed852ab716d8acc46ce90cba9cfcff6d3616f (diff)
downloadspark-990c9f79c28db501018a0a3af446ff879962475d.tar.gz
spark-990c9f79c28db501018a0a3af446ff879962475d.tar.bz2
spark-990c9f79c28db501018a0a3af446ff879962475d.zip
[SPARK-9170] [SQL] Use OrcStructInspector to be case preserving when writing ORC files
JIRA: https://issues.apache.org/jira/browse/SPARK-9170 `StandardStructObjectInspector` will implicitly lowercase column names. But I think Orc format doesn't have such requirement. In fact, there is a `OrcStructInspector` specified for Orc format. We should use it when serialize rows to Orc file. It can be case preserving when writing ORC files. Author: Liang-Chi Hsieh <viirya@appier.com> Closes #7520 from viirya/use_orcstruct.
Diffstat (limited to 'sql/hive/src/test')
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala14
1 files changed, 14 insertions, 0 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
index 744d462938..8bc33fcf5d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala
@@ -287,6 +287,20 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest {
}
}
+ test("SPARK-9170: Don't implicitly lowercase of user-provided columns") {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+
+ sqlContext.range(0, 10).select('id as "Acol").write.format("orc").save(path)
+ sqlContext.read.format("orc").load(path).schema("Acol")
+ intercept[IllegalArgumentException] {
+ sqlContext.read.format("orc").load(path).schema("acol")
+ }
+ checkAnswer(sqlContext.read.format("orc").load(path).select("acol").sort("acol"),
+ (0 until 10).map(Row(_)))
+ }
+ }
+
test("SPARK-8501: Avoids discovery schema from empty ORC files") {
withTempPath { dir =>
val path = dir.getCanonicalPath