diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-12-21 19:39:00 +0800 |
---|---|---|
committer | Wenchen Fan <wenchen@databricks.com> | 2016-12-21 19:39:00 +0800 |
commit | b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff (patch) | |
tree | 0fb44521c0512d0b3d8a87db23114a91c14e3d3e | |
parent | ba4468bb24f2e13b98e7b66b44203c20393b8ab8 (diff) | |
download | spark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.tar.gz spark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.tar.bz2 spark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.zip |
[SPARK-18947][SQL] SQLContext.tableNames should not call Catalog.listTables
## What changes were proposed in this pull request?
It's a huge waste to call `Catalog.listTables` in `SQLContext.tableNames`, which only need the table names, while `Catalog.listTables` will get the table metadata for each table name.
## How was this patch tested?
N/A
Author: Wenchen Fan <wenchen@databricks.com>
Closes #16352 from cloud-fan/minor.
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 4 | ||||
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala | 9 |
2 files changed, 7 insertions, 6 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 6554359806..1a7fd689a0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -747,7 +747,7 @@ class SQLContext private[sql](val sparkSession: SparkSession) * @since 1.3.0 */ def tableNames(): Array[String] = { - sparkSession.catalog.listTables().collect().map(_.name) + tableNames(sparkSession.catalog.currentDatabase) } /** @@ -757,7 +757,7 @@ class SQLContext private[sql](val sparkSession: SparkSession) * @since 1.3.0 */ def tableNames(databaseName: String): Array[String] = { - sparkSession.catalog.listTables(databaseName).collect().map(_.name) + sessionState.catalog.listTables(databaseName).map(_.table).toArray } //////////////////////////////////////////////////////////////////////////// diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala index 80bbad47f8..e56c33e4b5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala @@ -276,11 +276,12 @@ private[sql] object SQLUtils extends Logging { } def getTableNames(sparkSession: SparkSession, databaseName: String): Array[String] = { - databaseName match { - case n: String if n != null && n.trim.nonEmpty => - sparkSession.catalog.listTables(n).collect().map(_.name) + val db = databaseName match { + case _ if databaseName != null && databaseName.trim.nonEmpty => + databaseName case _ => - sparkSession.catalog.listTables().collect().map(_.name) + sparkSession.catalog.currentDatabase } + sparkSession.sessionState.catalog.listTables(db).map(_.table).toArray } } |