aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
Diffstat (limited to 'sql')
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala4
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala17
-rw-r--r--sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql (renamed from sql/core/src/test/resources/sql-tests/inputs/anti-join.sql)7
-rw-r--r--sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out (renamed from sql/core/src/test/resources/sql-tests/results/anti-join.sql.out)13
4 files changed, 37 insertions, 4 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 805cad5cb9..37f0c8ed19 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -932,7 +932,7 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
split(joinCondition.map(splitConjunctivePredicates).getOrElse(Nil), left, right)
joinType match {
- case _: InnerLike | LeftSemi | ExistenceJoin(_) =>
+ case _: InnerLike | LeftSemi =>
// push down the single side only join filter for both sides sub queries
val newLeft = leftJoinConditions.
reduceLeftOption(And).map(Filter(_, left)).getOrElse(left)
@@ -949,7 +949,7 @@ object PushPredicateThroughJoin extends Rule[LogicalPlan] with PredicateHelper {
val newJoinCond = (rightJoinConditions ++ commonJoinCondition).reduceLeftOption(And)
Join(newLeft, newRight, RightOuter, newJoinCond)
- case LeftOuter | LeftAnti =>
+ case LeftOuter | LeftAnti | ExistenceJoin(_) =>
// push down the right side only join filter for right sub query
val newLeft = left
val newRight = rightJoinConditions.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 3e67282d68..6feea4060f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -546,6 +546,23 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, analysis.EliminateSubqueryAliases(correctAnswer))
}
+ test("joins: only push down join conditions to the right of an existence join") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val fillerVal = 'val.boolean
+ val originalQuery =
+ x.join(y,
+ ExistenceJoin(fillerVal),
+ Some("x.a".attr > 1 && "y.b".attr > 2)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ val correctAnswer =
+ x.join(
+ y.where("y.b".attr > 2),
+ ExistenceJoin(fillerVal),
+ Some("x.a".attr > 1))
+ .analyze
+ comparePlans(optimized, analysis.EliminateSubqueryAliases(correctAnswer))
+ }
val testRelationWithArrayType = LocalRelation('a.int, 'b.int, 'c_arr.array(IntegerType))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/anti-join.sql b/sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql
index 0346f57d60..eff258a066 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/anti-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/pred-pushdown.sql
@@ -1,7 +1,12 @@
--- SPARK-18597: Do not push down predicates to left hand side in an anti-join
CREATE OR REPLACE TEMPORARY VIEW tbl_a AS VALUES (1, 1), (2, 1), (3, 6) AS T(c1, c2);
CREATE OR REPLACE TEMPORARY VIEW tbl_b AS VALUES 1 AS T(c1);
+-- SPARK-18597: Do not push down predicates to left hand side in an anti-join
SELECT *
FROM tbl_a
LEFT ANTI JOIN tbl_b ON ((tbl_a.c1 = tbl_a.c2) IS NULL OR tbl_a.c1 = tbl_a.c2);
+
+-- SPARK-18614: Do not push down predicates on left table below ExistenceJoin
+SELECT l.c1, l.c2
+FROM tbl_a l
+WHERE EXISTS (SELECT 1 FROM tbl_b r WHERE l.c1 = l.c2) OR l.c2 < 2;
diff --git a/sql/core/src/test/resources/sql-tests/results/anti-join.sql.out b/sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out
index 6f38c4d08b..1b8ddbe4c7 100644
--- a/sql/core/src/test/resources/sql-tests/results/anti-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pred-pushdown.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 3
+-- Number of queries: 4
-- !query 0
@@ -27,3 +27,14 @@ struct<c1:int,c2:int>
-- !query 2 output
2 1
3 6
+
+
+-- !query 3
+SELECT l.c1, l.c2
+FROM tbl_a l
+WHERE EXISTS (SELECT 1 FROM tbl_b r WHERE l.c1 = l.c2) OR l.c2 < 2
+-- !query 3 schema
+struct<c1:int,c2:int>
+-- !query 3 output
+1 1
+2 1