diff options
author | Matei Zaharia <matei@databricks.com> | 2016-04-03 21:08:54 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-04-03 21:08:54 -0700 |
commit | 76f3c735aa300d7ea6b17e64cc22d7e8fc3a8322 (patch) | |
tree | 2f0d787bd6dd0a7a0627731dc6bbf505bfa5845e | |
parent | 3f749f7ed443899d667c9e2b2a11bc595d6fc7f6 (diff) | |
download | spark-76f3c735aa300d7ea6b17e64cc22d7e8fc3a8322.tar.gz spark-76f3c735aa300d7ea6b17e64cc22d7e8fc3a8322.tar.bz2 spark-76f3c735aa300d7ea6b17e64cc22d7e8fc3a8322.zip |
[SPARK-14356] Update spark.sql.execution.debug to work on Datasets
## What changes were proposed in this pull request?
Update DebugQuery to work on Datasets of any type, not just DataFrames.
## How was this patch tested?
Added unit tests, checked in spark-shell.
Author: Matei Zaharia <matei@databricks.com>
Closes #12140 from mateiz/debug-dataset.
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala | 4 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala | 6 |
2 files changed, 8 insertions, 2 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala index 7b0c8ebdfa..17eae88b49 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala @@ -77,9 +77,9 @@ package object debug { } /** - * Augments [[DataFrame]]s with debug methods. + * Augments [[Dataset]]s with debug methods. */ - implicit class DebugQuery(query: DataFrame) extends Logging { + implicit class DebugQuery(query: Dataset[_]) extends Logging { def debug(): Unit = { val plan = query.queryExecution.executedPlan val visited = new collection.mutable.HashSet[TreeNodeRef]() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala index c0fce4b96a..8aa0114d98 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.debug import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSQLContext +import org.apache.spark.sql.test.SQLTestData.TestData class DebuggingSuite extends SparkFunSuite with SharedSQLContext { @@ -26,6 +27,11 @@ class DebuggingSuite extends SparkFunSuite with SharedSQLContext { testData.debug() } + test("Dataset.debug()") { + import testImplicits._ + testData.as[TestData].debug() + } + test("debugCodegen") { val res = codegenString(sqlContext.range(10).groupBy("id").count().queryExecution.executedPlan) assert(res.contains("Subtree 1 / 2")) |