diff options
| author | Nathan Howell <nhowell@godaddy.com> | 2014-10-09 15:03:01 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2014-10-09 15:03:01 -0700 |
| commit | bc3b6cb06153d6b05f311dd78459768b6cf6a404 (patch) | |
| tree | e09f9a9b7a4bc7dd2001b8eb73c34ec5ad58d4cf /sql | |
| parent | 0c0e09f567deb775ee378f5385a16884f68b332d (diff) | |
| download | spark-bc3b6cb06153d6b05f311dd78459768b6cf6a404.tar.gz spark-bc3b6cb06153d6b05f311dd78459768b6cf6a404.tar.bz2 spark-bc3b6cb06153d6b05f311dd78459768b6cf6a404.zip | |
[SPARK-3858][SQL] Pass the generator alias into logical plan node
The alias parameter is being ignored, which makes it more difficult to specify a qualifier for Generator expressions.
Author: Nathan Howell <nhowell@godaddy.com>
Closes #2721 from NathanHowell/SPARK-3858 and squashes the following commits:
8aa0f43 [Nathan Howell] [SPARK-3858][SQL] Pass the generator alias into logical plan node
Diffstat (limited to 'sql')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 2 | ||||
| -rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala | 8 |
2 files changed, 9 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala index 594bf8ffc2..948122d42f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala @@ -360,7 +360,7 @@ class SchemaRDD( join: Boolean = false, outer: Boolean = false, alias: Option[String] = None) = - new SchemaRDD(sqlContext, Generate(generator, join, outer, None, logicalPlan)) + new SchemaRDD(sqlContext, Generate(generator, join, outer, alias, logicalPlan)) /** * Returns this RDD as a SchemaRDD. Intended primarily to force the invocation of the implicit diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala index d001abb7e1..45e58afe9d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala @@ -147,6 +147,14 @@ class DslQuerySuite extends QueryTest { (1, 1, 1, 2) :: Nil) } + test("SPARK-3858 generator qualifiers are discarded") { + checkAnswer( + arrayData.as('ad) + .generate(Explode("data" :: Nil, 'data), alias = Some("ex")) + .select("ex.data".attr), + Seq(1, 2, 3, 2, 3, 4).map(Seq(_))) + } + test("average") { checkAnswer( testData2.groupBy()(avg('a)), |