diff options
author | xin Wu <xinwu@us.ibm.com> | 2016-05-09 17:18:48 -0700 |
---|---|---|
committer | Yin Huai <yhuai@databricks.com> | 2016-05-09 17:18:48 -0700 |
commit | 980bba0dcfcecbfce701d765fb45bf174cea54ad (patch) | |
tree | d276e18ec3047ac011cb179c3c425f9774c41148 /sql/hive/src/test | |
parent | 3323d0f931ddd11f41abca11425b5e43a6538667 (diff) | |
download | spark-980bba0dcfcecbfce701d765fb45bf174cea54ad.tar.gz spark-980bba0dcfcecbfce701d765fb45bf174cea54ad.tar.bz2 spark-980bba0dcfcecbfce701d765fb45bf174cea54ad.zip |
[SPARK-15025][SQL] fix duplicate of PATH key in datasource table options
## What changes were proposed in this pull request?
The issue is that when the user provides the path option with uppercase "PATH" key, `options` contains `PATH` key and will get into the non-external case in the following code in `createDataSourceTables.scala`, where a new key "path" is created with a default path.
```
val optionsWithPath =
if (!options.contains("path")) {
isExternal = false
options + ("path" -> sessionState.catalog.defaultTablePath(tableIdent))
} else {
options
}
```
So before creating hive table, serdeInfo.parameters will contain both "PATH" and "path" keys and different directories. and Hive table's dataLocation contains the value of "path".
The fix in this PR is to convert `options` in the code above to `CaseInsensitiveMap` before checking for containing "path" key.
## How was this patch tested?
A testcase is added
Author: xin Wu <xinwu@us.ibm.com>
Closes #12804 from xwu0226/SPARK-15025.
Diffstat (limited to 'sql/hive/src/test')
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 29 |
1 files changed, 26 insertions, 3 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 4bdcb96feb..78c8f0043d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -944,7 +944,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv } test("CTAS: persisted partitioned data source table") { - withTempDir { dir => + withTempPath { dir => withTable("t") { val path = dir.getCanonicalPath @@ -968,7 +968,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv } test("CTAS: persisted bucketed data source table") { - withTempDir { dir => + withTempPath { dir => withTable("t") { val path = dir.getCanonicalPath @@ -988,7 +988,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv checkAnswer(table("t"), Row(1, 2)) } + } + withTempPath { dir => withTable("t") { val path = dir.getCanonicalPath @@ -1012,7 +1014,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv } test("CTAS: persisted partitioned bucketed data source table") { - withTempDir { dir => + withTempPath { dir => withTable("t") { val path = dir.getCanonicalPath @@ -1035,4 +1037,25 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv } } } + + test("SPARK-15025: create datasource table with path with select") { + withTempPath { dir => + withTable("t") { + val path = dir.getCanonicalPath + + sql( + s"""CREATE TABLE t USING PARQUET + |OPTIONS (PATH '$path') + |AS SELECT 1 AS a, 2 AS b, 3 AS c + """.stripMargin + ) + sql("insert into t values (2, 3, 4)") + checkAnswer(table("t"), Seq(Row(1, 2, 3), Row(2, 3, 4))) + val catalogTable = sharedState.externalCatalog.getTable("default", "t") + // there should not be a lowercase key 'path' now + assert(catalogTable.storage.serdeProperties.get("path").isEmpty) + assert(catalogTable.storage.serdeProperties.get("PATH").isDefined) + } + } + } } |