aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWenchen Fan <wenchen@databricks.com>2016-12-21 19:39:00 +0800
committerWenchen Fan <wenchen@databricks.com>2016-12-21 19:39:00 +0800
commitb7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff (patch)
tree0fb44521c0512d0b3d8a87db23114a91c14e3d3e
parentba4468bb24f2e13b98e7b66b44203c20393b8ab8 (diff)
downloadspark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.tar.gz
spark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.tar.bz2
spark-b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff.zip
[SPARK-18947][SQL] SQLContext.tableNames should not call Catalog.listTables
## What changes were proposed in this pull request? It's a huge waste to call `Catalog.listTables` in `SQLContext.tableNames`, which only need the table names, while `Catalog.listTables` will get the table metadata for each table name. ## How was this patch tested? N/A Author: Wenchen Fan <wenchen@databricks.com> Closes #16352 from cloud-fan/minor.
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala4
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala9
2 files changed, 7 insertions, 6 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 6554359806..1a7fd689a0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -747,7 +747,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
def tableNames(): Array[String] = {
- sparkSession.catalog.listTables().collect().map(_.name)
+ tableNames(sparkSession.catalog.currentDatabase)
}
/**
@@ -757,7 +757,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
def tableNames(databaseName: String): Array[String] = {
- sparkSession.catalog.listTables(databaseName).collect().map(_.name)
+ sessionState.catalog.listTables(databaseName).map(_.table).toArray
}
////////////////////////////////////////////////////////////////////////////
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
index 80bbad47f8..e56c33e4b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
@@ -276,11 +276,12 @@ private[sql] object SQLUtils extends Logging {
}
def getTableNames(sparkSession: SparkSession, databaseName: String): Array[String] = {
- databaseName match {
- case n: String if n != null && n.trim.nonEmpty =>
- sparkSession.catalog.listTables(n).collect().map(_.name)
+ val db = databaseName match {
+ case _ if databaseName != null && databaseName.trim.nonEmpty =>
+ databaseName
case _ =>
- sparkSession.catalog.listTables().collect().map(_.name)
+ sparkSession.catalog.currentDatabase
}
+ sparkSession.sessionState.catalog.listTables(db).map(_.table).toArray
}
}