diff options
author | Sean Zhong <seanzhong@databricks.com> | 2016-05-16 10:41:20 +0800 |
---|---|---|
committer | Wenchen Fan <wenchen@databricks.com> | 2016-05-16 10:41:20 +0800 |
commit | 4a5ee1954a6fb77231abb492355fe70313f0b35b (patch) | |
tree | 51f9b19e8c3fe0d4c1cf75df4d06abcc7896c32c | |
parent | c7efc56c7b6fc99c005b35c335716ff676856c6c (diff) | |
download | spark-4a5ee1954a6fb77231abb492355fe70313f0b35b.tar.gz spark-4a5ee1954a6fb77231abb492355fe70313f0b35b.tar.bz2 spark-4a5ee1954a6fb77231abb492355fe70313f0b35b.zip |
[SPARK-15253][SQL] Support old table schema config key "spark.sql.sources.schema" for DESCRIBE TABLE
## What changes were proposed in this pull request?
"DESCRIBE table" is broken when table schema is stored at key "spark.sql.sources.schema".
Originally, we used spark.sql.sources.schema to store the schema of a data source table.
After SPARK-6024, we removed this flag. Although we are not using spark.sql.sources.schema any more, we need to still support it.
## How was this patch tested?
Unit test.
When using spark2.0 to load a table generated by spark 1.2.
Before change:
`DESCRIBE table` => Schema of this table is inferred at runtime,,
After change:
`DESCRIBE table` => correct output.
Author: Sean Zhong <seanzhong@databricks.com>
Closes #13073 from clockfly/spark-15253.
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala | 29 | ||||
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 3 |
2 files changed, 21 insertions, 11 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 1c1716f050..49d7fe956f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -497,20 +497,27 @@ private[sql] object DDLUtils { // will be inferred at runtime when the table is referenced. def getSchemaFromTableProperties(metadata: CatalogTable): Option[StructType] = { require(isDatasourceTable(metadata)) + val props = metadata.properties + if (props.isDefinedAt("spark.sql.sources.schema")) { + // Originally, we used spark.sql.sources.schema to store the schema of a data source table. + // After SPARK-6024, we removed this flag. + // Although we are not using spark.sql.sources.schema any more, we need to still support. + props.get("spark.sql.sources.schema").map(DataType.fromJson(_).asInstanceOf[StructType]) + } else { + metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts => + val parts = (0 until numParts.toInt).map { index => + val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull + if (part == null) { + throw new AnalysisException( + "Could not read schema from the metastore because it is corrupted " + + s"(missing part $index of the schema, $numParts parts are expected).") + } - metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts => - val parts = (0 until numParts.toInt).map { index => - val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull - if (part == null) { - throw new AnalysisException( - "Could not read schema from the metastore because it is corrupted " + - s"(missing part $index of the schema, $numParts parts are expected).") + part } - - part + // Stick all parts back to a single schema string. + DataType.fromJson(parts.mkString).asInstanceOf[StructType] } - // Stick all parts back to a single schema string. - DataType.fromJson(parts.mkString).asInstanceOf[StructType] } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 676fbd0a39..b507018e58 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -746,6 +746,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv sessionState.invalidateTable(tableName) val actualSchema = table(tableName).schema assert(schema === actualSchema) + + // Checks the DESCRIBE output. + checkAnswer(sql("DESCRIBE spark6655"), Row("int", "int", "") :: Nil) } } |