diff options
author | gatorsmile <gatorsmile@gmail.com> | 2016-03-07 12:09:27 -0800 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2016-03-07 12:09:27 -0800 |
commit | b6071a7001aff7a8319e13b31c59e3cc86aad523 (patch) | |
tree | 8193db6fba36302664c7102a77ac219711c821b0 | |
parent | a3ec50a4bc867aec7c0796457c4442c14d1bcc2c (diff) | |
download | spark-b6071a7001aff7a8319e13b31c59e3cc86aad523.tar.gz spark-b6071a7001aff7a8319e13b31c59e3cc86aad523.tar.bz2 spark-b6071a7001aff7a8319e13b31c59e3cc86aad523.zip |
[SPARK-13722][SQL] No Push Down for Non-deterministics Predicates through Generate
#### What changes were proposed in this pull request?
Non-deterministic predicates should not be pushed through Generate.
#### How was this patch tested?
Added a test case in `FilterPushdownSuite.scala`
Author: gatorsmile <gatorsmile@gmail.com>
Closes #11562 from gatorsmile/pushPredicateDownWindow.
2 files changed, 19 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 69ceea6329..deea7238f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -901,7 +901,7 @@ object PushPredicateThroughGenerate extends Rule[LogicalPlan] with PredicateHelp // Predicates that reference attributes produced by the `Generate` operator cannot // be pushed below the operator. val (pushDown, stayUp) = splitConjunctivePredicates(condition).partition { cond => - cond.references subsetOf g.child.outputSet + cond.references.subsetOf(g.child.outputSet) && cond.deterministic } if (pushDown.nonEmpty) { val pushDownPredicate = pushDown.reduce(And) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala index 1292aa0003..97a0cde381 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala @@ -496,6 +496,24 @@ class FilterPushdownSuite extends PlanTest { comparePlans(optimized, correctAnswer) } + test("generate: non-deterministic predicate referenced no generated column") { + val originalQuery = { + testRelationWithArrayType + .generate(Explode('c_arr), true, false, Some("arr")) + .where(('b >= 5) && ('a + Rand(10).as("rnd") > 6)) + } + val optimized = Optimize.execute(originalQuery.analyze) + val correctAnswer = { + testRelationWithArrayType + .where('b >= 5) + .generate(Explode('c_arr), true, false, Some("arr")) + .where('a + Rand(10).as("rnd") > 6) + .analyze + } + + comparePlans(optimized, correctAnswer) + } + test("generate: part of conjuncts referenced generated column") { val generator = Explode('c_arr) val originalQuery = { |