diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-10-18 11:03:10 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-10-18 11:03:10 -0700 |
commit | e59df62e62ec4c5f8bd02a13f05fa3ec6f0fc694 (patch) | |
tree | b0488694540ea898dc5e7268a058e16956305983 /sql/hive | |
parent | a9e79a41ee19258e5eb8da74bef4b8af9a2ccb95 (diff) | |
download | spark-e59df62e62ec4c5f8bd02a13f05fa3ec6f0fc694.tar.gz spark-e59df62e62ec4c5f8bd02a13f05fa3ec6f0fc694.tar.bz2 spark-e59df62e62ec4c5f8bd02a13f05fa3ec6f0fc694.zip |
[SPARK-17899][SQL][FOLLOW-UP] debug mode should work for corrupted table
## What changes were proposed in this pull request?
Debug mode should work for corrupted table, so that we can really debug
## How was this patch tested?
new test in `MetastoreDataSourcesSuite`
Author: Wenchen Fan <wenchen@databricks.com>
Closes #15528 from cloud-fan/debug.
Diffstat (limited to 'sql/hive')
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 9 | ||||
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 18 |
2 files changed, 17 insertions, 10 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index ff59b54f53..2003ff42d4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -448,7 +448,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat * properties, and filter out these special entries from table properties. */ private def restoreTableMetadata(table: CatalogTable): CatalogTable = { - val catalogTable = if (table.tableType == VIEW) { + val catalogTable = if (table.tableType == VIEW || conf.get(DEBUG_MODE)) { table } else { getProviderFromTableProperties(table).map { provider => @@ -467,18 +467,13 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } else { table.storage } - val tableProps = if (conf.get(DEBUG_MODE)) { - table.properties - } else { - getOriginalTableProperties(table) - } table.copy( storage = storage, schema = getSchemaFromTableProperties(table), provider = Some(provider), partitionColumnNames = getPartitionColumnsFromTableProperties(table), bucketSpec = getBucketSpecFromTableProperties(table), - properties = tableProps) + properties = getOriginalTableProperties(table)) } getOrElse { table.copy(provider = Some("hive")) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 7cc6179d44..eaa67d370d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -1321,20 +1321,32 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv sharedState.externalCatalog.getTable("default", "t") }.getMessage assert(e.contains(s"Could not read schema from the hive metastore because it is corrupted")) + + withDebugMode { + val tableMeta = sharedState.externalCatalog.getTable("default", "t") + assert(tableMeta.identifier == TableIdentifier("t", Some("default"))) + assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json") + } } finally { hiveClient.dropTable("default", "t", ignoreIfNotExists = true, purge = true) } } test("should keep data source entries in table properties when debug mode is on") { - val previousValue = sparkSession.sparkContext.conf.get(DEBUG_MODE) - try { - sparkSession.sparkContext.conf.set(DEBUG_MODE, true) + withDebugMode { val newSession = sparkSession.newSession() newSession.sql("CREATE TABLE abc(i int) USING json") val tableMeta = newSession.sessionState.catalog.getTableMetadata(TableIdentifier("abc")) assert(tableMeta.properties(DATASOURCE_SCHEMA_NUMPARTS).toInt == 1) assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json") + } + } + + private def withDebugMode(f: => Unit): Unit = { + val previousValue = sparkSession.sparkContext.conf.get(DEBUG_MODE) + try { + sparkSession.sparkContext.conf.set(DEBUG_MODE, true) + f } finally { sparkSession.sparkContext.conf.set(DEBUG_MODE, previousValue) } |