aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authorReynold Xin <rxin@databricks.com>2016-06-17 15:51:20 -0700
committerReynold Xin <rxin@databricks.com>2016-06-17 15:51:20 -0700
commit1a65e62a7ff961ba140b9189e5335736334edbd6 (patch)
tree192338eecb52a5fb64ca5d9cc7f42d15f401c559 /sql
parent62d8fe2089659e8212753a622708517e0f4a77bc (diff)
downloadspark-1a65e62a7ff961ba140b9189e5335736334edbd6.tar.gz
spark-1a65e62a7ff961ba140b9189e5335736334edbd6.tar.bz2
spark-1a65e62a7ff961ba140b9189e5335736334edbd6.zip
[SPARK-16014][SQL] Rename optimizer rules to be more consistent
## What changes were proposed in this pull request? This small patch renames a few optimizer rules to make the naming more consistent, e.g. class name start with a verb. The main important "fix" is probably SamplePushDown -> PushProjectThroughSample. SamplePushDown is actually the wrong name, since the rule is not about pushing Sample down. ## How was this patch tested? Updated test cases. Author: Reynold Xin <rxin@databricks.com> Closes #13732 from rxin/SPARK-16014.
Diffstat (limited to 'sql')
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala (renamed from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala)2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala31
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala2
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala2
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala2
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala2
6 files changed, 19 insertions, 22 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala
index 063eff4f98..8afd28dbba 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DistinctAggregationRewriter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteDistinctAggregates.scala
@@ -99,7 +99,7 @@ import org.apache.spark.sql.types.IntegerType
* we could improve this in the current rule by applying more advanced expression canonicalization
* techniques.
*/
-object DistinctAggregationRewriter extends Rule[LogicalPlan] {
+object RewriteDistinctAggregates extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
case a: Aggregate => rewrite(a)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 7b9b21f416..6e78ad0e77 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -52,7 +52,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf)
ReplaceExpressions,
ComputeCurrentTime,
GetCurrentDatabase(sessionCatalog),
- DistinctAggregationRewriter) ::
+ RewriteDistinctAggregates) ::
//////////////////////////////////////////////////////////////////////////////////////////
// Optimizer rules start here
//////////////////////////////////////////////////////////////////////////////////////////
@@ -74,10 +74,10 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf)
RemoveRepetitionFromGroupExpressions) ::
Batch("Operator Optimizations", fixedPoint,
// Operator push down
- SetOperationPushDown,
- SamplePushDown,
+ PushThroughSetOperations,
+ PushProjectThroughSample,
ReorderJoin,
- OuterJoinElimination,
+ EliminateOuterJoin,
PushPredicateThroughJoin,
PushDownPredicate,
LimitPushDown,
@@ -99,7 +99,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog, conf: CatalystConf)
BooleanSimplification,
SimplifyConditionals,
RemoveDispensableExpressions,
- BinaryComparisonSimplification,
+ SimplifyBinaryComparison,
PruneFilters,
EliminateSorts,
SimplifyCasts,
@@ -148,10 +148,9 @@ class SimpleTestOptimizer extends Optimizer(
new SimpleCatalystConf(caseSensitiveAnalysis = true))
/**
- * Pushes operations down into a Sample.
+ * Pushes projects down beneath Sample to enable column pruning with sampling.
*/
-object SamplePushDown extends Rule[LogicalPlan] {
-
+object PushProjectThroughSample extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
// Push down projection into sample
case Project(projectList, Sample(lb, up, replace, seed, child)) =>
@@ -171,7 +170,7 @@ object RemoveAliasOnlyProject extends Rule[LogicalPlan] {
projectList: Seq[NamedExpression],
childOutput: Seq[Attribute]): Boolean = {
if (!projectList.forall(_.isInstanceOf[Alias]) || projectList.length != childOutput.length) {
- return false
+ false
} else {
projectList.map(_.asInstanceOf[Alias]).zip(childOutput).forall { case (a, o) =>
a.child match {
@@ -183,11 +182,9 @@ object RemoveAliasOnlyProject extends Rule[LogicalPlan] {
}
def apply(plan: LogicalPlan): LogicalPlan = {
- val aliasOnlyProject = plan.find { p =>
- p match {
- case Project(pList, child) if isAliasOnly(pList, child.output) => true
- case _ => false
- }
+ val aliasOnlyProject = plan.find {
+ case Project(pList, child) if isAliasOnly(pList, child.output) => true
+ case _ => false
}
aliasOnlyProject.map { case p: Project =>
@@ -292,7 +289,7 @@ object LimitPushDown extends Rule[LogicalPlan] {
* safe to pushdown Filters and Projections through it. Once we add UNION DISTINCT,
* we will not be able to pushdown Projections.
*/
-object SetOperationPushDown extends Rule[LogicalPlan] with PredicateHelper {
+object PushThroughSetOperations extends Rule[LogicalPlan] with PredicateHelper {
/**
* Maps Attributes from the left side to the corresponding Attribute on the right side.
@@ -918,7 +915,7 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
* 2) Replace '=', '<=', and '>=' with 'true' literal if both operands are non-nullable.
* 3) Replace '<' and '>' with 'false' literal if both operands are non-nullable.
*/
-object BinaryComparisonSimplification extends Rule[LogicalPlan] with PredicateHelper {
+object SimplifyBinaryComparison extends Rule[LogicalPlan] with PredicateHelper {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case q: LogicalPlan => q transformExpressionsUp {
// True with equality
@@ -1256,7 +1253,7 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
*
* This rule should be executed before pushing down the Filter
*/
-object OuterJoinElimination extends Rule[LogicalPlan] with PredicateHelper {
+object EliminateOuterJoin extends Rule[LogicalPlan] with PredicateHelper {
/**
* Returns whether the expression returns null or false when all inputs are nulls.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
index 7cd038570b..a313681eeb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
@@ -36,7 +36,7 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper
NullPropagation,
ConstantFolding,
BooleanSimplification,
- BinaryComparisonSimplification,
+ SimplifyBinaryComparison,
PruneFilters) :: Nil
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index fcc14a803b..b8f28e83e7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -34,7 +34,7 @@ class FilterPushdownSuite extends PlanTest {
Batch("Subqueries", Once,
EliminateSubqueryAliases) ::
Batch("Filter Pushdown", FixedPoint(10),
- SamplePushDown,
+ PushProjectThroughSample,
CombineFilters,
PushDownPredicate,
BooleanSimplification,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala
index 5e6e54dc74..41754adef4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OuterJoinEliminationSuite.scala
@@ -30,7 +30,7 @@ class OuterJoinEliminationSuite extends PlanTest {
Batch("Subqueries", Once,
EliminateSubqueryAliases) ::
Batch("Outer Join Elimination", Once,
- OuterJoinElimination,
+ EliminateOuterJoin,
PushPredicateThroughJoin) :: Nil
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
index 83ca9d5ec9..dab45a6b16 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala
@@ -31,7 +31,7 @@ class SetOperationSuite extends PlanTest {
EliminateSubqueryAliases) ::
Batch("Union Pushdown", Once,
CombineUnions,
- SetOperationPushDown,
+ PushThroughSetOperations,
PruneFilters) :: Nil
}