diff options
| author | Bogdan Raducanu <bogdan@databricks.com> | 2017-02-22 15:42:40 +0100 |
|---|---|---|
| committer | Herman van Hovell <hvanhovell@databricks.com> | 2017-02-22 15:42:40 +0100 |
| commit | 10c566cc3b5f93ddd823b1c987d9de8286cdea60 (patch) | |
| tree | c3c0578d27be89d8e589fe7e4885eb58ac0afc07 /sql/catalyst | |
| parent | bf7bb497784dd97de75516f300e3add478ab472d (diff) | |
| download | spark-10c566cc3b5f93ddd823b1c987d9de8286cdea60.tar.gz spark-10c566cc3b5f93ddd823b1c987d9de8286cdea60.tar.bz2 spark-10c566cc3b5f93ddd823b1c987d9de8286cdea60.zip | |
[SPARK-13721][SQL] Make GeneratorOuter unresolved.
## What changes were proposed in this pull request?
This is a small change to make GeneratorOuter always unresolved. It is mostly no-op change but makes it more clear since GeneratorOuter shouldn't survive analysis phase.
This requires also handling in ResolveAliases rule.
## How was this patch tested?
Existing generator tests.
Author: Bogdan Raducanu <bogdan@databricks.com>
Author: Reynold Xin <rxin@databricks.com>
Closes #17026 from bogdanrdc/PR16958.
Diffstat (limited to 'sql/catalyst')
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 | ||||
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala | 8 |
2 files changed, 8 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 39a276284c..c477cb48d0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -222,6 +222,7 @@ class Analyzer( expr.transformUp { case u @ UnresolvedAlias(child, optGenAliasFunc) => child match { case ne: NamedExpression => ne + case go @ GeneratorOuter(g: Generator) if g.resolved => MultiAlias(go, Nil) case e if !e.resolved => u case g: Generator => MultiAlias(g, Nil) case c @ Cast(ne: NamedExpression, _, _) => Alias(c, ne.name)() @@ -1665,7 +1666,6 @@ class Analyzer( var resolvedGenerator: Generate = null val newProjectList = projectList.flatMap { - case AliasedGenerator(generator, names, outer) if generator.childrenResolved => // It's a sanity check, this should not happen as the previous case will throw // exception earlier. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 1b98c30d37..e84796f2ed 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -173,7 +173,6 @@ case class Stack(children: Seq[Expression]) extends Generator { } } - /** * Only support code generation when stack produces 50 rows or less. */ @@ -204,6 +203,10 @@ case class Stack(children: Seq[Expression]) extends Generator { } } +/** + * Wrapper around another generator to specify outer behavior. This is used to implement functions + * such as explode_outer. This expression gets replaced during analysis. + */ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator { final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] = throw new UnsupportedOperationException(s"Cannot evaluate expression: $this") @@ -212,7 +215,10 @@ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generat throw new UnsupportedOperationException(s"Cannot evaluate expression: $this") override def elementSchema: StructType = child.elementSchema + + override lazy val resolved: Boolean = false } + /** * A base class for [[Explode]] and [[PosExplode]]. */ |