diff options
Diffstat (limited to 'sql')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 4 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala | 17 | ||||
-rw-r--r-- | sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql (renamed from sql/core/src/test/resources/sql-tests/inputs/anti-join.sql) | 7 | ||||
-rw-r--r-- | sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out (renamed from sql/core/src/test/resources/sql-tests/results/anti-join.sql.out) | 13 |
4 files changed, 37 insertions, 4 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 805cad5cb9..37f0c8ed19 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -932,7 +932,7 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper { split(joinCondition.map(splitConjunctivePredicates).getOrElse(Nil), left, right) joinType match { - case _: InnerLike | LeftSemi | ExistenceJoin(_) => + case _: InnerLike | LeftSemi => // push down the single side only join filter for both sides sub queries val newLeft = leftJoinConditions. reduceLeftOption(And).map(Filter(_, left)).getOrElse(left) @@ -949,7 +949,7 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper { val newJoinCond = (rightJoinConditions ++ commonJoinCondition).reduceLeftOption(And) Join(newLeft, newRight, RightOuter, newJoinCond) - case LeftOuter | LeftAnti => + case LeftOuter | LeftAnti | ExistenceJoin(_) => // push down the right side only join filter for right sub query val newLeft = left val newRight = rightJoinConditions. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala index 3e67282d68..6feea4060f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala @@ -546,6 +546,23 @@ class FilterPushdownSuite extends PlanTest { comparePlans(optimized, analysis.EliminateSubqueryAliases(correctAnswer)) } + test("joins: only push down join conditions to the right of an existence join") { + val x = testRelation.subquery('x) + val y = testRelation.subquery('y) + val fillerVal = 'val.boolean + val originalQuery = + x.join(y, + ExistenceJoin(fillerVal), + Some("x.a".attr > 1 && "y.b".attr > 2)).analyze + val optimized = Optimize.execute(originalQuery) + val correctAnswer = + x.join( + y.where("y.b".attr > 2), + ExistenceJoin(fillerVal), + Some("x.a".attr > 1)) + .analyze + comparePlans(optimized, analysis.EliminateSubqueryAliases(correctAnswer)) + } val testRelationWithArrayType = LocalRelation('a.int, 'b.int, 'c_arr.array(IntegerType)) diff --git a/sql/core/src/test/resources/sql-tests/inputs/anti-join.sql b/sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql index 0346f57d60..eff258a066 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/anti-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql @@ -1,7 +1,12 @@ --- SPARK-18597: Do not push down predicates to left hand side in an anti-join CREATE OR REPLACE TEMPORARY VIEW tbl_a AS VALUES (1, 1), (2, 1), (3, 6) AS T(c1, c2); CREATE OR REPLACE TEMPORARY VIEW tbl_b AS VALUES 1 AS T(c1); +-- SPARK-18597: Do not push down predicates to left hand side in an anti-join SELECT * FROM tbl_a LEFT ANTI JOIN tbl_b ON ((tbl_a.c1 = tbl_a.c2) IS NULL OR tbl_a.c1 = tbl_a.c2); + +-- SPARK-18614: Do not push down predicates on left table below ExistenceJoin +SELECT l.c1, l.c2 +FROM tbl_a l +WHERE EXISTS (SELECT 1 FROM tbl_b r WHERE l.c1 = l.c2) OR l.c2 < 2; diff --git a/sql/core/src/test/resources/sql-tests/results/anti-join.sql.out b/sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out index 6f38c4d08b..1b8ddbe4c7 100644 --- a/sql/core/src/test/resources/sql-tests/results/anti-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 3 +-- Number of queries: 4 -- !query 0 @@ -27,3 +27,14 @@ struct<c1:int,c2:int> -- !query 2 output 2 1 3 6 + + +-- !query 3 +SELECT l.c1, l.c2 +FROM tbl_a l +WHERE EXISTS (SELECT 1 FROM tbl_b r WHERE l.c1 = l.c2) OR l.c2 < 2 +-- !query 3 schema +struct<c1:int,c2:int> +-- !query 3 output +1 1 +2 1 |