diff options
author | Liang-Chi Hsieh <simonh@tw.ibm.com> | 2016-08-18 12:45:56 +0200 |
---|---|---|
committer | Herman van Hovell <hvanhovell@databricks.com> | 2016-08-18 12:45:56 +0200 |
commit | e82dbe600e0d36d76cd5607a77c3243a26777b77 (patch) | |
tree | b6b33883c56e28b4d18e194ae2b0024d053f8f2d /sql/catalyst/src/main | |
parent | 1748f824101870b845dbbd118763c6885744f98a (diff) | |
download | spark-e82dbe600e0d36d76cd5607a77c3243a26777b77.tar.gz spark-e82dbe600e0d36d76cd5607a77c3243a26777b77.tar.bz2 spark-e82dbe600e0d36d76cd5607a77c3243a26777b77.zip |
[SPARK-17107][SQL] Remove redundant pushdown rule for Union
## What changes were proposed in this pull request?
The `Optimizer` rules `PushThroughSetOperations` and `PushDownPredicate` have a redundant rule to push down `Filter` through `Union`. We should remove it.
## How was this patch tested?
Jenkins tests.
Author: Liang-Chi Hsieh <simonh@tw.ibm.com>
Closes #14687 from viirya/remove-extra-pushdown.
Diffstat (limited to 'sql/catalyst/src/main')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 21 |
1 files changed, 5 insertions, 16 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index b53c0b5bec..f7aa6da0a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -75,7 +75,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf) RemoveRepetitionFromGroupExpressions) :: Batch("Operator Optimizations", fixedPoint, // Operator push down - PushThroughSetOperations, + PushProjectionThroughUnion, ReorderJoin, EliminateOuterJoin, PushPredicateThroughJoin, @@ -302,14 +302,14 @@ object LimitPushDown extends Rule[LogicalPlan] { } /** - * Pushes certain operations to both sides of a Union operator. + * Pushes Project operator to both sides of a Union operator. * Operations that are safe to pushdown are listed as follows. * Union: * Right now, Union means UNION ALL, which does not de-duplicate rows. So, it is - * safe to pushdown Filters and Projections through it. Once we add UNION DISTINCT, - * we will not be able to pushdown Projections. + * safe to pushdown Filters and Projections through it. Filter pushdown is handled by another + * rule PushDownPredicate. Once we add UNION DISTINCT, we will not be able to pushdown Projections. */ -object PushThroughSetOperations extends Rule[LogicalPlan] with PredicateHelper { +object PushProjectionThroughUnion extends Rule[LogicalPlan] with PredicateHelper { /** * Maps Attributes from the left side to the corresponding Attribute on the right side. @@ -364,17 +364,6 @@ object PushThroughSetOperations extends Rule[LogicalPlan] with PredicateHelper { } else { p } - - // Push down filter into union - case Filter(condition, Union(children)) => - assert(children.nonEmpty) - val (deterministic, nondeterministic) = partitionByDeterministic(condition) - val newFirstChild = Filter(deterministic, children.head) - val newOtherChildren = children.tail.map { child => - val rewrites = buildRewrites(children.head, child) - Filter(pushToRight(deterministic, rewrites), child) - } - Filter(nondeterministic, Union(newFirstChild +: newOtherChildren)) } } |