aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSean Zhong <seanzhong@databricks.com>2016-05-16 10:41:20 +0800
committerWenchen Fan <wenchen@databricks.com>2016-05-16 10:41:20 +0800
commit4a5ee1954a6fb77231abb492355fe70313f0b35b (patch)
tree51f9b19e8c3fe0d4c1cf75df4d06abcc7896c32c
parentc7efc56c7b6fc99c005b35c335716ff676856c6c (diff)
downloadspark-4a5ee1954a6fb77231abb492355fe70313f0b35b.tar.gz
spark-4a5ee1954a6fb77231abb492355fe70313f0b35b.tar.bz2
spark-4a5ee1954a6fb77231abb492355fe70313f0b35b.zip
[SPARK-15253][SQL] Support old table schema config key "spark.sql.sources.schema" for DESCRIBE TABLE
## What changes were proposed in this pull request? "DESCRIBE table" is broken when table schema is stored at key "spark.sql.sources.schema". Originally, we used spark.sql.sources.schema to store the schema of a data source table. After SPARK-6024, we removed this flag. Although we are not using spark.sql.sources.schema any more, we need to still support it. ## How was this patch tested? Unit test. When using spark2.0 to load a table generated by spark 1.2. Before change: `DESCRIBE table` => Schema of this table is inferred at runtime,, After change: `DESCRIBE table` => correct output. Author: Sean Zhong <seanzhong@databricks.com> Closes #13073 from clockfly/spark-15253.
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala29
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala3
2 files changed, 21 insertions, 11 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 1c1716f050..49d7fe956f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -497,20 +497,27 @@ private[sql] object DDLUtils {
// will be inferred at runtime when the table is referenced.
def getSchemaFromTableProperties(metadata: CatalogTable): Option[StructType] = {
require(isDatasourceTable(metadata))
+ val props = metadata.properties
+ if (props.isDefinedAt("spark.sql.sources.schema")) {
+ // Originally, we used spark.sql.sources.schema to store the schema of a data source table.
+ // After SPARK-6024, we removed this flag.
+ // Although we are not using spark.sql.sources.schema any more, we need to still support.
+ props.get("spark.sql.sources.schema").map(DataType.fromJson(_).asInstanceOf[StructType])
+ } else {
+ metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts =>
+ val parts = (0 until numParts.toInt).map { index =>
+ val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull
+ if (part == null) {
+ throw new AnalysisException(
+ "Could not read schema from the metastore because it is corrupted " +
+ s"(missing part $index of the schema, $numParts parts are expected).")
+ }
- metadata.properties.get("spark.sql.sources.schema.numParts").map { numParts =>
- val parts = (0 until numParts.toInt).map { index =>
- val part = metadata.properties.get(s"spark.sql.sources.schema.part.$index").orNull
- if (part == null) {
- throw new AnalysisException(
- "Could not read schema from the metastore because it is corrupted " +
- s"(missing part $index of the schema, $numParts parts are expected).")
+ part
}
-
- part
+ // Stick all parts back to a single schema string.
+ DataType.fromJson(parts.mkString).asInstanceOf[StructType]
}
- // Stick all parts back to a single schema string.
- DataType.fromJson(parts.mkString).asInstanceOf[StructType]
}
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 676fbd0a39..b507018e58 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -746,6 +746,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
sessionState.invalidateTable(tableName)
val actualSchema = table(tableName).schema
assert(schema === actualSchema)
+
+ // Checks the DESCRIBE output.
+ checkAnswer(sql("DESCRIBE spark6655"), Row("int", "int", "") :: Nil)
}
}