aboutsummaryrefslogtreecommitdiff
path: root/sql/hive
diff options
context:
space:
mode:
authorReynold Xin <rxin@databricks.com>2016-03-14 23:09:10 -0700
committerReynold Xin <rxin@databricks.com>2016-03-14 23:09:10 -0700
commite64958001cb95d53c441131f8c7a92556f49fd7d (patch)
treeb2b1471353c076d8f9c0cdc50700072910867030 /sql/hive
parentf72743d971a38d3d08984ef4b66e0955945d2f58 (diff)
downloadspark-e64958001cb95d53c441131f8c7a92556f49fd7d.tar.gz
spark-e64958001cb95d53c441131f8c7a92556f49fd7d.tar.bz2
spark-e64958001cb95d53c441131f8c7a92556f49fd7d.zip
[SPARK-13884][SQL] Remove DescribeCommand's dependency on LogicalPlan
## What changes were proposed in this pull request? This patch removes DescribeCommand's dependency on LogicalPlan. After this patch, DescribeCommand simply accepts a TableIdentifier. It minimizes the dependency, and blocks my next patch (removes SQLContext dependency from SparkPlanner). ## How was this patch tested? Should be covered by existing unit tests and Hive compatibility tests that run describe table. Author: Reynold Xin <rxin@databricks.com> Closes #11710 from rxin/SPARK-13884.
Diffstat (limited to 'sql/hive')
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala14
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala57
2 files changed, 37 insertions, 34 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
index a19dc21638..f44937ec6f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
@@ -102,18 +102,8 @@ private[hive] trait HiveStrategies {
case class HiveCommandStrategy(context: HiveContext) extends Strategy {
def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
case describe: DescribeCommand =>
- val resolvedTable = context.executePlan(describe.table).analyzed
- resolvedTable match {
- case t: MetastoreRelation =>
- ExecutedCommand(
- DescribeHiveTableCommand(t, describe.output, describe.isExtended)) :: Nil
-
- case o: LogicalPlan =>
- val resultPlan = context.executePlan(o).executedPlan
- ExecutedCommand(RunnableDescribeCommand(
- resultPlan, describe.output, describe.isExtended)) :: Nil
- }
-
+ ExecutedCommand(
+ DescribeHiveTableCommand(describe.table, describe.output, describe.isExtended)) :: Nil
case _ => Nil
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
index 57293fce97..8481324086 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
@@ -22,8 +22,10 @@ import scala.collection.JavaConverters._
import org.apache.hadoop.hive.metastore.api.FieldSchema
import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.execution.command.RunnableCommand
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.command.{DescribeCommand, RunnableCommand}
import org.apache.spark.sql.hive.MetastoreRelation
/**
@@ -31,33 +33,44 @@ import org.apache.spark.sql.hive.MetastoreRelation
*/
private[hive]
case class DescribeHiveTableCommand(
- table: MetastoreRelation,
+ tableId: TableIdentifier,
override val output: Seq[Attribute],
isExtended: Boolean) extends RunnableCommand {
override def run(sqlContext: SQLContext): Seq[Row] = {
- // Trying to mimic the format of Hive's output. But not exactly the same.
- var results: Seq[(String, String, String)] = Nil
-
- val columns: Seq[FieldSchema] = table.hiveQlTable.getCols.asScala
- val partitionColumns: Seq[FieldSchema] = table.hiveQlTable.getPartCols.asScala
- results ++= columns.map(field => (field.getName, field.getType, field.getComment))
- if (partitionColumns.nonEmpty) {
- val partColumnInfo =
- partitionColumns.map(field => (field.getName, field.getType, field.getComment))
- results ++=
- partColumnInfo ++
- Seq(("# Partition Information", "", "")) ++
- Seq((s"# ${output(0).name}", output(1).name, output(2).name)) ++
- partColumnInfo
- }
+ // There are two modes here:
+ // For metastore tables, create an output similar to Hive's.
+ // For other tables, delegate to DescribeCommand.
- if (isExtended) {
- results ++= Seq(("Detailed Table Information", table.hiveQlTable.getTTable.toString, ""))
- }
+ // In the future, we will consolidate the two and simply report what the catalog reports.
+ sqlContext.sessionState.catalog.lookupRelation(tableId) match {
+ case table: MetastoreRelation =>
+ // Trying to mimic the format of Hive's output. But not exactly the same.
+ var results: Seq[(String, String, String)] = Nil
+
+ val columns: Seq[FieldSchema] = table.hiveQlTable.getCols.asScala
+ val partitionColumns: Seq[FieldSchema] = table.hiveQlTable.getPartCols.asScala
+ results ++= columns.map(field => (field.getName, field.getType, field.getComment))
+ if (partitionColumns.nonEmpty) {
+ val partColumnInfo =
+ partitionColumns.map(field => (field.getName, field.getType, field.getComment))
+ results ++=
+ partColumnInfo ++
+ Seq(("# Partition Information", "", "")) ++
+ Seq((s"# ${output(0).name}", output(1).name, output(2).name)) ++
+ partColumnInfo
+ }
+
+ if (isExtended) {
+ results ++= Seq(("Detailed Table Information", table.hiveQlTable.getTTable.toString, ""))
+ }
+
+ results.map { case (name, dataType, comment) =>
+ Row(name, dataType, comment)
+ }
- results.map { case (name, dataType, comment) =>
- Row(name, dataType, comment)
+ case o: LogicalPlan =>
+ DescribeCommand(tableId, output, isExtended).run(sqlContext)
}
}
}