aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorxin Wu <xinwu@us.ibm.com>2016-05-11 22:17:59 +0800
committerCheng Lian <lian@databricks.com>2016-05-11 22:17:59 +0800
commit427c20dd6d84cb9de1aac322183bc6e7b72ca25d (patch)
tree476278b64b487ac1d6ca375e88a58b82ae42ac7b
parentd88afabdfa83be47f36d833105aadd6b818ceeee (diff)
downloadspark-427c20dd6d84cb9de1aac322183bc6e7b72ca25d.tar.gz
spark-427c20dd6d84cb9de1aac322183bc6e7b72ca25d.tar.bz2
spark-427c20dd6d84cb9de1aac322183bc6e7b72ca25d.zip
[SPARK-14933][SQL] Failed to create view out of a parquet or orc table
## What changes were proposed in this pull request? #### Symptom If a table is created as parquet or ORC table with hive syntaxt DDL, such as ```SQL create table t1 (c1 int, c2 string) stored as parquet ``` The following command will fail ```SQL create view v1 as select * from t1 ``` #### Root Cause Currently, `HiveMetaStoreCatalog` converts Paruqet/Orc tables to `LogicalRelation` without giving any `tableIdentifier`. `SQLBuilder` expects the `LogicalRelation` to have an associated `tableIdentifier`. However, the `LogicalRelation` created earlier does not have such a `tableIdentifier`. Thus, `SQLBuilder.toSQL` can not recognize this logical plan and issue an exception. This PR is to assign a `TableIdentifier` to the `LogicalRelation` when resolving parquet or orc tables in `HiveMetaStoreCatalog`. ## How was this patch tested? testcases created and dev/run-tests is run. Author: xin Wu <xinwu@us.ibm.com> Closes #12716 from xwu0226/SPARK_14933.
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala10
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala24
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala30
3 files changed, 62 insertions, 2 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 7a799b6c87..607f0a10ec 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -293,7 +293,10 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
fileFormat = defaultSource,
options = options)
- val created = LogicalRelation(relation)
+ val created = LogicalRelation(
+ relation,
+ metastoreTableIdentifier =
+ Some(TableIdentifier(tableIdentifier.name, Some(tableIdentifier.database))))
cachedDataSourceTables.put(tableIdentifier, created)
created
}
@@ -317,7 +320,10 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
userSpecifiedSchema = Some(metastoreRelation.schema),
bucketSpec = bucketSpec,
options = options,
- className = fileType).resolveRelation())
+ className = fileType).resolveRelation(),
+ metastoreTableIdentifier =
+ Some(TableIdentifier(tableIdentifier.name, Some(tableIdentifier.database))))
+
cachedDataSourceTables.put(tableIdentifier, created)
created
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
index 9abefa5f28..4315197e12 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
@@ -741,4 +741,28 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
test("filter after subquery") {
checkHiveQl("SELECT a FROM (SELECT key + 1 AS a FROM parquet_t1) t WHERE a > 5")
}
+
+ test("SPARK-14933 - select parquet table") {
+ withTable("parquet_t") {
+ sql(
+ """
+ |create table parquet_t (c1 int, c2 string)
+ |stored as parquet select 1, 'abc'
+ """.stripMargin)
+
+ checkHiveQl("select * from parquet_t")
+ }
+ }
+
+ test("SPARK-14933 - select orc table") {
+ withTable("orc_t") {
+ sql(
+ """
+ |create table orc_t (c1 int, c2 string)
+ |stored as orc select 1, 'abc'
+ """.stripMargin)
+
+ checkHiveQl("select * from orc_t")
+ }
+ }
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala
index f37037e3c7..5c72ec57f5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLViewSuite.scala
@@ -304,4 +304,34 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
}
+
+ test("SPARK-14933 - create view from hive parquet tabale") {
+ withTable("t_part") {
+ withView("v_part") {
+ sqlContext.sql(
+ """create table t_part (c1 int, c2 int)
+ |stored as parquet as select 1 as a, 2 as b
+ """.stripMargin)
+ sqlContext.sql("create view v_part as select * from t_part")
+ checkAnswer(
+ sql("select * from t_part"),
+ sql("select * from v_part"))
+ }
+ }
+ }
+
+ test("SPARK-14933 - create view from hive orc tabale") {
+ withTable("t_orc") {
+ withView("v_orc") {
+ sqlContext.sql(
+ """create table t_orc (c1 int, c2 int)
+ |stored as orc as select 1 as a, 2 as b
+ """.stripMargin)
+ sqlContext.sql("create view v_orc as select * from t_orc")
+ checkAnswer(
+ sql("select * from t_orc"),
+ sql("select * from v_orc"))
+ }
+ }
+ }
}