diff options
author | Wenchen Fan <wenchen@databricks.com> | 2017-02-07 00:36:57 +0800 |
---|---|---|
committer | Wenchen Fan <wenchen@databricks.com> | 2017-02-07 00:36:57 +0800 |
commit | aff53021cf828cd7c139d8ec230d45593078b73a (patch) | |
tree | d5add8b21cf710eb30329b040bbcd3ee721fd00c /sql/hive/src/test/scala/org | |
parent | 0f16ff5b0ec8cd828774ba5ddb276d7b06dbe273 (diff) | |
download | spark-aff53021cf828cd7c139d8ec230d45593078b73a.tar.gz spark-aff53021cf828cd7c139d8ec230d45593078b73a.tar.bz2 spark-aff53021cf828cd7c139d8ec230d45593078b73a.zip |
[SPARK-19080][SQL] simplify data source analysis
## What changes were proposed in this pull request?
The current way of resolving `InsertIntoTable` and `CreateTable` is convoluted: sometimes we replace them with concrete implementation commands during analysis, sometimes during planning phase.
And the error checking logic is also a mess: we may put it in extended analyzer rules, or extended checking rules, or `CheckAnalysis`.
This PR simplifies the data source analysis:
1. `InsertIntoTable` and `CreateTable` are always unresolved and need to be replaced by concrete implementation commands during analysis.
2. The error checking logic is mainly in 2 rules: `PreprocessTableCreation` and `PreprocessTableInsertion`.
## How was this patch tested?
existing test.
Author: Wenchen Fan <wenchen@databricks.com>
Closes #16269 from cloud-fan/ddl.
Diffstat (limited to 'sql/hive/src/test/scala/org')
3 files changed, 9 insertions, 12 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala index 6f43b83607..0bd08877a3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala @@ -52,7 +52,7 @@ class HiveDDLCommandSuite extends PlanTest with SQLTestUtils with TestHiveSingle private def analyzeCreateTable(sql: String): CatalogTable = { TestHive.sessionState.analyzer.execute(parser.parsePlan(sql)).collect { - case CreateTable(tableDesc, mode, _) => tableDesc + case CreateTableCommand(tableDesc, _) => tableDesc }.head } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala index e3ddaf7254..71ce5a7c4a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala @@ -376,7 +376,8 @@ class InsertIntoHiveTableSuite extends QueryTest with TestHiveSingleton with Bef val e = intercept[AnalysisException] { sql(s"INSERT INTO TABLE $tableName PARTITION(b=1, c=2) SELECT 1, 2, 3") } - assert(e.message.contains("the number of columns are different")) + assert(e.message.contains( + "target table has 4 column(s) but the inserted data has 5 column(s)")) } testPartitionedTable("SPARK-16037: INSERT statement should match columns by position") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala index e9239ea56f..1a1b2571b6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala @@ -307,13 +307,11 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { """.stripMargin) val df = sql("INSERT INTO TABLE test_insert_parquet SELECT a FROM jt") - df.queryExecution.sparkPlan match { - case ExecutedCommandExec(cmd: InsertIntoHadoopFsRelationCommand) => + df.queryExecution.analyzed match { + case cmd: InsertIntoHadoopFsRelationCommand => assert(cmd.catalogTable.map(_.identifier.table) === Some("test_insert_parquet")) case o => fail("test_insert_parquet should be converted to a " + - s"${classOf[HadoopFsRelation ].getCanonicalName} and " + - s"${classOf[InsertIntoDataSourceCommand].getCanonicalName} should have been SparkPlan. " + - s"However, found a ${o.toString} ") + s"${classOf[HadoopFsRelation ].getCanonicalName}. However, found a ${o.toString}") } checkAnswer( @@ -338,13 +336,11 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { """.stripMargin) val df = sql("INSERT INTO TABLE test_insert_parquet SELECT a FROM jt_array") - df.queryExecution.sparkPlan match { - case ExecutedCommandExec(cmd: InsertIntoHadoopFsRelationCommand) => + df.queryExecution.analyzed match { + case cmd: InsertIntoHadoopFsRelationCommand => assert(cmd.catalogTable.map(_.identifier.table) === Some("test_insert_parquet")) case o => fail("test_insert_parquet should be converted to a " + - s"${classOf[HadoopFsRelation ].getCanonicalName} and " + - s"${classOf[InsertIntoDataSourceCommand].getCanonicalName} should have been SparkPlan." + - s"However, found a ${o.toString} ") + s"${classOf[HadoopFsRelation ].getCanonicalName}. However, found a ${o.toString}") } checkAnswer( |