diff options
| author | Wenchen Fan <wenchen@databricks.com> | 2016-11-22 15:25:22 -0500 |
|---|---|---|
| committer | Andrew Or <andrewor14@gmail.com> | 2016-11-22 15:25:22 -0500 |
| commit | 702cd403fc8e5ce8281fe8828197ead46bdb8832 (patch) | |
| tree | 3dbb963defc71d365b9b19856ad40ed3cee4eae3 | |
| parent | 45ea46b7b397f023b4da878eb11e21b08d931115 (diff) | |
| download | spark-702cd403fc8e5ce8281fe8828197ead46bdb8832.tar.gz spark-702cd403fc8e5ce8281fe8828197ead46bdb8832.tar.bz2 spark-702cd403fc8e5ce8281fe8828197ead46bdb8832.zip | |
[SPARK-18507][SQL] HiveExternalCatalog.listPartitions should only call getTable once
## What changes were proposed in this pull request?
HiveExternalCatalog.listPartitions should only call `getTable` once, instead of calling it for every partitions.
## How was this patch tested?
N/A
Author: Wenchen Fan <wenchen@databricks.com>
Closes #15978 from cloud-fan/perf.
| -rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 5dbb4024bb..ff0923f048 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -907,8 +907,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient { + val actualPartColNames = getTable(db, table).partitionColumnNames client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part => - part.copy(spec = restorePartitionSpec(part.spec, getTable(db, table).partitionColumnNames)) + part.copy(spec = restorePartitionSpec(part.spec, actualPartColNames)) } } |