diff options
author | windpiger <songjun@outlook.com> | 2017-01-14 10:53:33 -0800 |
---|---|---|
committer | gatorsmile <gatorsmile@gmail.com> | 2017-01-14 10:53:33 -0800 |
commit | 8942353905c354c4ce31b0d1a44d33feb3dcf737 (patch) | |
tree | 23281a909c8c44da8dc13eede62dcd18a97ba6f7 /sql/hive/src/test/scala/org | |
parent | b6a7aa4f770634e6db7244e88f8b6273fb9b6d1e (diff) | |
download | spark-8942353905c354c4ce31b0d1a44d33feb3dcf737.tar.gz spark-8942353905c354c4ce31b0d1a44d33feb3dcf737.tar.bz2 spark-8942353905c354c4ce31b0d1a44d33feb3dcf737.zip |
[SPARK-19151][SQL] DataFrameWriter.saveAsTable support hive overwrite
## What changes were proposed in this pull request?
After [SPARK-19107](https://issues.apache.org/jira/browse/SPARK-19107), we now can treat hive as a data source and create hive tables with DataFrameWriter and Catalog. However, the support is not completed, there are still some cases we do not support.
This PR implement:
DataFrameWriter.saveAsTable work with hive format with overwrite mode
## How was this patch tested?
unit test added
Author: windpiger <songjun@outlook.com>
Closes #16549 from windpiger/saveAsTableWithHiveOverwrite.
Diffstat (limited to 'sql/hive/src/test/scala/org')
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 0af331e67b..e3f1667249 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -1314,7 +1314,24 @@ class HiveDDLSuite .write.format("hive").option("fileFormat", "avro").saveAsTable("t") checkAnswer(spark.table("t"), Row(1, "a")) - val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) + Seq("c" -> 1).toDF("i", "j").write.format("hive") + .mode(SaveMode.Overwrite).option("fileFormat", "parquet").saveAsTable("t") + checkAnswer(spark.table("t"), Row("c", 1)) + + var table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) + assert(DDLUtils.isHiveTable(table)) + assert(table.storage.inputFormat == + Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat")) + assert(table.storage.outputFormat == + Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat")) + assert(table.storage.serde == + Some("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")) + + Seq(9 -> "x").toDF("i", "j") + .write.format("hive").mode(SaveMode.Overwrite).option("fileFormat", "avro").saveAsTable("t") + checkAnswer(spark.table("t"), Row(9, "x")) + + table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(DDLUtils.isHiveTable(table)) assert(table.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat")) @@ -1324,7 +1341,7 @@ class HiveDDLSuite Some("org.apache.hadoop.hive.serde2.avro.AvroSerDe")) sql("INSERT INTO t SELECT 2, 'b'") - checkAnswer(spark.table("t"), Row(1, "a") :: Row(2, "b") :: Nil) + checkAnswer(spark.table("t"), Row(9, "x") :: Row(2, "b") :: Nil) val e = intercept[AnalysisException] { Seq(1 -> "a").toDF("i", "j").write.format("hive").partitionBy("i").saveAsTable("t2") @@ -1340,8 +1357,7 @@ class HiveDDLSuite val e3 = intercept[AnalysisException] { spark.table("t").write.format("hive").mode("overwrite").saveAsTable("t") } - assert(e3.message.contains( - "CTAS for hive serde tables does not support append or overwrite semantics")) + assert(e3.message.contains("Cannot overwrite table default.t that is also being read from")) } } |