aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authorgatorsmile <gatorsmile@gmail.com>2016-03-07 12:09:27 -0800
committerMichael Armbrust <michael@databricks.com>2016-03-07 12:09:27 -0800
commitb6071a7001aff7a8319e13b31c59e3cc86aad523 (patch)
tree8193db6fba36302664c7102a77ac219711c821b0 /sql
parenta3ec50a4bc867aec7c0796457c4442c14d1bcc2c (diff)
downloadspark-b6071a7001aff7a8319e13b31c59e3cc86aad523.tar.gz
spark-b6071a7001aff7a8319e13b31c59e3cc86aad523.tar.bz2
spark-b6071a7001aff7a8319e13b31c59e3cc86aad523.zip
[SPARK-13722][SQL] No Push Down for Non-deterministics Predicates through Generate
#### What changes were proposed in this pull request? Non-deterministic predicates should not be pushed through Generate. #### How was this patch tested? Added a test case in `FilterPushdownSuite.scala` Author: gatorsmile <gatorsmile@gmail.com> Closes #11562 from gatorsmile/pushPredicateDownWindow.
Diffstat (limited to 'sql')
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala2
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala18
2 files changed, 19 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 69ceea6329..deea7238f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -901,7 +901,7 @@ object PushPredicateThroughGenerate extends Rule[LogicalPlan] with PredicateHelp
// Predicates that reference attributes produced by the `Generate` operator cannot
// be pushed below the operator.
val (pushDown, stayUp) = splitConjunctivePredicates(condition).partition { cond =>
- cond.references subsetOf g.child.outputSet
+ cond.references.subsetOf(g.child.outputSet) && cond.deterministic
}
if (pushDown.nonEmpty) {
val pushDownPredicate = pushDown.reduce(And)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 1292aa0003..97a0cde381 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -496,6 +496,24 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("generate: non-deterministic predicate referenced no generated column") {
+ val originalQuery = {
+ testRelationWithArrayType
+ .generate(Explode('c_arr), true, false, Some("arr"))
+ .where(('b >= 5) && ('a + Rand(10).as("rnd") > 6))
+ }
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = {
+ testRelationWithArrayType
+ .where('b >= 5)
+ .generate(Explode('c_arr), true, false, Some("arr"))
+ .where('a + Rand(10).as("rnd") > 6)
+ .analyze
+ }
+
+ comparePlans(optimized, correctAnswer)
+ }
+
test("generate: part of conjuncts referenced generated column") {
val generator = Explode('c_arr)
val originalQuery = {