diff options
author | Reynold Xin <rxin@databricks.com> | 2016-06-17 15:51:20 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-06-17 15:51:20 -0700 |
commit | 1a65e62a7ff961ba140b9189e5335736334edbd6 (patch) | |
tree | 192338eecb52a5fb64ca5d9cc7f42d15f401c559 /sql | |
parent | 62d8fe2089659e8212753a622708517e0f4a77bc (diff) | |
download | spark-1a65e62a7ff961ba140b9189e5335736334edbd6.tar.gz spark-1a65e62a7ff961ba140b9189e5335736334edbd6.tar.bz2 spark-1a65e62a7ff961ba140b9189e5335736334edbd6.zip |
[SPARK-16014][SQL] Rename optimizer rules to be more consistent
## What changes were proposed in this pull request?
This small patch renames a few optimizer rules to make the naming more consistent, e.g. class name start with a verb. The main important "fix" is probably SamplePushDown -> PushProjectThroughSample. SamplePushDown is actually the wrong name, since the rule is not about pushing Sample down.
## How was this patch tested?
Updated test cases.
Author: Reynold Xin <rxin@databricks.com>
Closes #13732 from rxin/SPARK-16014.
Diffstat (limited to 'sql')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala (renamed from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala) | 2 | ||||
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 31 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala | 2 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala | 2 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala | 2 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala | 2 |
6 files changed, 19 insertions, 22 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala index 063eff4f98..8afd28dbba 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala @@ -99,7 +99,7 @@ import org.apache.spark.sql.types.IntegerType * we could improve this in the current rule by applying more advanced expression canonicalization * techniques. */ -object DistinctAggregationRewriter extends Rule[LogicalPlan] { +object RewriteDistinctAggregates extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformUp { case a: Aggregate => rewrite(a) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 7b9b21f416..6e78ad0e77 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -52,7 +52,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf) ReplaceExpressions, ComputeCurrentTime, GetCurrentDatabase(sessionCatalog), - DistinctAggregationRewriter) :: + RewriteDistinctAggregates) :: ////////////////////////////////////////////////////////////////////////////////////////// // Optimizer rules start here ////////////////////////////////////////////////////////////////////////////////////////// @@ -74,10 +74,10 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf) RemoveRepetitionFromGroupExpressions) :: Batch("Operator Optimizations", fixedPoint, // Operator push down - SetOperationPushDown, - SamplePushDown, + PushThroughSetOperations, + PushProjectThroughSample, ReorderJoin, - OuterJoinElimination, + EliminateOuterJoin, PushPredicateThroughJoin, PushDownPredicate, LimitPushDown, @@ -99,7 +99,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf) BooleanSimplification, SimplifyConditionals, RemoveDispensableExpressions, - BinaryComparisonSimplification, + SimplifyBinaryComparison, PruneFilters, EliminateSorts, SimplifyCasts, @@ -148,10 +148,9 @@ class SimpleTestOptimizer extends Optimizer( new SimpleCatalystConf(caseSensitiveAnalysis = true)) /** - * Pushes operations down into a Sample. + * Pushes projects down beneath Sample to enable column pruning with sampling. */ -object SamplePushDown extends Rule[LogicalPlan] { - +object PushProjectThroughSample extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { // Push down projection into sample case Project(projectList, Sample(lb, up, replace, seed, child)) => @@ -171,7 +170,7 @@ object RemoveAliasOnlyProject extends Rule[LogicalPlan] { projectList: Seq[NamedExpression], childOutput: Seq[Attribute]): Boolean = { if (!projectList.forall(_.isInstanceOf[Alias]) || projectList.length != childOutput.length) { - return false + false } else { projectList.map(_.asInstanceOf[Alias]).zip(childOutput).forall { case (a, o) => a.child match { @@ -183,11 +182,9 @@ object RemoveAliasOnlyProject extends Rule[LogicalPlan] { } def apply(plan: LogicalPlan): LogicalPlan = { - val aliasOnlyProject = plan.find { p => - p match { - case Project(pList, child) if isAliasOnly(pList, child.output) => true - case _ => false - } + val aliasOnlyProject = plan.find { + case Project(pList, child) if isAliasOnly(pList, child.output) => true + case _ => false } aliasOnlyProject.map { case p: Project => @@ -292,7 +289,7 @@ object LimitPushDown extends Rule[LogicalPlan] { * safe to pushdown Filters and Projections through it. Once we add UNION DISTINCT, * we will not be able to pushdown Projections. */ -object SetOperationPushDown extends Rule[LogicalPlan] with PredicateHelper { +object PushThroughSetOperations extends Rule[LogicalPlan] with PredicateHelper { /** * Maps Attributes from the left side to the corresponding Attribute on the right side. @@ -918,7 +915,7 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper { * 2) Replace '=', '<=', and '>=' with 'true' literal if both operands are non-nullable. * 3) Replace '<' and '>' with 'false' literal if both operands are non-nullable. */ -object BinaryComparisonSimplification extends Rule[LogicalPlan] with PredicateHelper { +object SimplifyBinaryComparison extends Rule[LogicalPlan] with PredicateHelper { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsUp { // True with equality @@ -1256,7 +1253,7 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper { * * This rule should be executed before pushing down the Filter */ -object OuterJoinElimination extends Rule[LogicalPlan] with PredicateHelper { +object EliminateOuterJoin extends Rule[LogicalPlan] with PredicateHelper { /** * Returns whether the expression returns null or false when all inputs are nulls. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala index 7cd038570b..a313681eeb 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala @@ -36,7 +36,7 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper NullPropagation, ConstantFolding, BooleanSimplification, - BinaryComparisonSimplification, + SimplifyBinaryComparison, PruneFilters) :: Nil } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala index fcc14a803b..b8f28e83e7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala @@ -34,7 +34,7 @@ class FilterPushdownSuite extends PlanTest { Batch("Subqueries", Once, EliminateSubqueryAliases) :: Batch("Filter Pushdown", FixedPoint(10), - SamplePushDown, + PushProjectThroughSample, CombineFilters, PushDownPredicate, BooleanSimplification, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala index 5e6e54dc74..41754adef4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala @@ -30,7 +30,7 @@ class OuterJoinEliminationSuite extends PlanTest { Batch("Subqueries", Once, EliminateSubqueryAliases) :: Batch("Outer Join Elimination", Once, - OuterJoinElimination, + EliminateOuterJoin, PushPredicateThroughJoin) :: Nil } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala index 83ca9d5ec9..dab45a6b16 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala @@ -31,7 +31,7 @@ class SetOperationSuite extends PlanTest { EliminateSubqueryAliases) :: Batch("Union Pushdown", Once, CombineUnions, - SetOperationPushDown, + PushThroughSetOperations, PruneFilters) :: Nil } |