diff options
| author | Takeshi YAMAMURO <linguin.m.s@gmail.com> | 2017-01-12 09:46:53 -0800 |
|---|---|---|
| committer | gatorsmile <gatorsmile@gmail.com> | 2017-01-12 09:46:53 -0800 |
| commit | 5585ed93b09bc05cdd7a731650eca50d43d7159b (patch) | |
| tree | d9c005ab1485887d4d064726daca4dd3f8f693ac /sql/core | |
| parent | 2bc4d4e286e65f8b4e9ee21bccd913b62e6061f2 (diff) | |
| download | spark-5585ed93b09bc05cdd7a731650eca50d43d7159b.tar.gz spark-5585ed93b09bc05cdd7a731650eca50d43d7159b.tar.bz2 spark-5585ed93b09bc05cdd7a731650eca50d43d7159b.zip | |
[SPARK-17237][SQL] Remove backticks in a pivot result schema
## What changes were proposed in this pull request?
Pivoting adds backticks (e.g. 3_count(\`c\`)) in column names and, in some cases,
thes causes analysis exceptions like;
```
scala> val df = Seq((2, 3, 4), (3, 4, 5)).toDF("a", "x", "y")
scala> df.groupBy("a").pivot("x").agg(count("y"), avg("y")).na.fill(0)
org.apache.spark.sql.AnalysisException: syntax error in attribute name: `3_count(`y`)`;
at org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute$.e$1(unresolved.scala:134)
at org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute$.parseAttributeName(unresolved.scala:144)
...
```
So, this pr proposes to remove these backticks from column names.
## How was this patch tested?
Added a test in `DataFrameAggregateSuite`.
Author: Takeshi YAMAMURO <linguin.m.s@gmail.com>
Closes #14812 from maropu/SPARK-17237.
Diffstat (limited to 'sql/core')
| -rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala | 8 | ||||
| -rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala | 2 |
2 files changed, 9 insertions, 1 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 7853b22fec..e7079120bb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -530,4 +530,12 @@ class DataFrameAggregateSuite extends QueryTest with SharedSQLContext { limit2Df.groupBy("id").count().select($"id"), limit2Df.select($"id")) } + + test("SPARK-17237 remove backticks in a pivot result schema") { + val df = Seq((2, 3, 4), (3, 4, 5)).toDF("a", "x", "y") + checkAnswer( + df.groupBy("a").pivot("x").agg(count("y"), avg("y")).na.fill(0), + Seq(Row(3, 0, 0.0, 1, 5.0), Row(2, 1, 4.0, 0, 0.0)) + ) + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala index a8d854ccbc..51ffe34172 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala @@ -200,7 +200,7 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext{ test("pivot preserves aliases if given") { assertResult( - Array("year", "dotNET_foo", "dotNET_avg(`earnings`)", "Java_foo", "Java_avg(`earnings`)") + Array("year", "dotNET_foo", "dotNET_avg(earnings)", "Java_foo", "Java_avg(earnings)") )( courseSales.groupBy($"year") .pivot("course", Seq("dotNET", "Java")) |