aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src
diff options
context:
space:
mode:
authorgatorsmile <gatorsmile@gmail.com>2016-03-24 11:13:36 +0800
committerWenchen Fan <wenchen@databricks.com>2016-03-24 11:13:36 +0800
commitf42eaf42bdca8bc6f390f1f31ee60faa1662489b (patch)
treed5f84d22eb06d95fd4b20a79a6ccaaa34c911d89 /sql/core/src
parentde4e48b62b998d45d4a749234741a45534719497 (diff)
downloadspark-f42eaf42bdca8bc6f390f1f31ee60faa1662489b.tar.gz
spark-f42eaf42bdca8bc6f390f1f31ee60faa1662489b.tar.bz2
spark-f42eaf42bdca8bc6f390f1f31ee60faa1662489b.zip
[SPARK-14085][SQL] Star Expansion for Hash
#### What changes were proposed in this pull request? This PR is to support star expansion in hash. For example, ```SQL val structDf = testData2.select("a", "b").as("record") structDf.select(hash($"*") ``` In addition, it refactors the codes for the rule `ResolveStar` and fixes a regression for star expansion in group by when using SQL API. For example, ```SQL SELECT * FROM testData2 group by a, b ``` cc cloud-fan Now, the code for star resolution is much cleaner. The coverage is better. Could you check if this refactoring is good? Thanks! #### How was this patch tested? Added a few test cases to cover it. Author: gatorsmile <gatorsmile@gmail.com> Closes #11904 from gatorsmile/starResolution.
Diffstat (limited to 'sql/core/src')
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala19
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala8
2 files changed, 27 insertions, 0 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index e6b7dc9199..ec4e7b2042 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -181,6 +181,25 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(structDf.select(array($"record.*").as("a")).first().getAs[Seq[Int]](0) === Seq(1, 1))
}
+ test("Star Expansion - hash") {
+ val structDf = testData2.select("a", "b").as("record")
+ checkAnswer(
+ structDf.groupBy($"a", $"b").agg(min(hash($"a", $"*"))),
+ structDf.groupBy($"a", $"b").agg(min(hash($"a", $"a", $"b"))))
+
+ checkAnswer(
+ structDf.groupBy($"a", $"b").agg(hash($"a", $"*")),
+ structDf.groupBy($"a", $"b").agg(hash($"a", $"a", $"b")))
+
+ checkAnswer(
+ structDf.select(hash($"*")),
+ structDf.select(hash($"record.*")))
+
+ checkAnswer(
+ structDf.select(hash($"a", $"*")),
+ structDf.select(hash($"a", $"record.*")))
+ }
+
test("Star Expansion - explode should fail with a meaningful message if it takes a star") {
val df = Seq(("1", "1,2"), ("2", "4"), ("3", "7,8,9")).toDF("prefix", "csv")
val e = intercept[AnalysisException] {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index bd13474e73..4f36b1b42a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1941,6 +1941,14 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
}
}
+ test("Star Expansion - group by") {
+ withSQLConf("spark.sql.retainGroupColumns" -> "false") {
+ checkAnswer(
+ testData2.groupBy($"a", $"b").agg($"*"),
+ sql("SELECT * FROM testData2 group by a, b"))
+ }
+ }
+
test("Common subexpression elimination") {
// TODO: support subexpression elimination in whole stage codegen
withSQLConf("spark.sql.codegen.wholeStage" -> "false") {