diff options
author | gatorsmile <gatorsmile@gmail.com> | 2016-11-30 15:17:29 +0800 |
---|---|---|
committer | Wenchen Fan <wenchen@databricks.com> | 2016-11-30 15:17:29 +0800 |
commit | a1d9138ab286dc58d7f61c27419de7ecbf5b828b (patch) | |
tree | b7932a748aea6fcec0f8511ad43f4ed185c0137e /sql/hive/src/test | |
parent | bc09a2b8c3b03a207a6e20627f2c5ec23c1efe8c (diff) | |
download | spark-a1d9138ab286dc58d7f61c27419de7ecbf5b828b.tar.gz spark-a1d9138ab286dc58d7f61c27419de7ecbf5b828b.tar.bz2 spark-a1d9138ab286dc58d7f61c27419de7ecbf5b828b.zip |
[SPARK-17680][SQL][TEST] Added a Testcase for Verifying Unicode Character Support for Column Names and Comments
### What changes were proposed in this pull request?
Spark SQL supports Unicode characters for column names when specified within backticks(`). When the Hive support is enabled, the version of the Hive metastore must be higher than 0.12, See the JIRA: https://issues.apache.org/jira/browse/HIVE-6013 Hive metastore supports Unicode characters for column names since 0.13.
In Spark SQL, table comments, and view comments always allow Unicode characters without backticks.
BTW, a separate PR has been submitted for database and table name validation because we do not support Unicode characters in these two cases.
### How was this patch tested?
N/A
Author: gatorsmile <gatorsmile@gmail.com>
Closes #15255 from gatorsmile/unicodeSupport.
Diffstat (limited to 'sql/hive/src/test')
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 45 |
1 files changed, 45 insertions, 0 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 951e070414..f313db641b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -147,6 +147,51 @@ class HiveDDLSuite } } + test("create Hive-serde table and view with unicode columns and comment") { + val catalog = spark.sessionState.catalog + val tabName = "tab1" + val viewName = "view1" + // scalastyle:off + // non ascii characters are not allowed in the source code, so we disable the scalastyle. + val colName1 = "和" + val colName2 = "尼" + val comment = "庙" + // scalastyle:on + withTable(tabName) { + sql(s""" + |CREATE TABLE $tabName(`$colName1` int COMMENT '$comment') + |COMMENT '$comment' + |PARTITIONED BY (`$colName2` int) + """.stripMargin) + sql(s"INSERT OVERWRITE TABLE $tabName partition (`$colName2`=2) SELECT 1") + withView(viewName) { + sql( + s""" + |CREATE VIEW $viewName(`$colName1` COMMENT '$comment', `$colName2`) + |COMMENT '$comment' + |AS SELECT `$colName1`, `$colName2` FROM $tabName + """.stripMargin) + val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default"))) + val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) + assert(tableMetadata.comment == Option(comment)) + assert(viewMetadata.comment == Option(comment)) + + assert(tableMetadata.schema.fields.length == 2 && viewMetadata.schema.fields.length == 2) + val column1InTable = tableMetadata.schema.fields.head + val column1InView = viewMetadata.schema.fields.head + assert(column1InTable.name == colName1 && column1InView.name == colName1) + assert(column1InTable.getComment() == Option(comment)) + assert(column1InView.getComment() == Option(comment)) + + assert(tableMetadata.schema.fields(1).name == colName2 && + viewMetadata.schema.fields(1).name == colName2) + + checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $tabName"), Row(1, 2) :: Nil) + checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $viewName"), Row(1, 2) :: Nil) + } + } + } + test("create table: partition column names exist in table definition") { val e = intercept[AnalysisException] { sql("CREATE TABLE tbl(a int) PARTITIONED BY (a string)") |