diff options
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 5 | ||||
-rw-r--r-- | sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala | 51 |
2 files changed, 55 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index d37f43888f..338c5193cb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -417,6 +417,11 @@ object NullPropagation extends Rule[LogicalPlan] { case left :: Literal(null, _) :: Nil => Literal.create(null, e.dataType) case _ => e } + + // If the value expression is NULL then transform the In expression to + // Literal(null) + case In(Literal(null, _), list) => Literal.create(null, BooleanType) + } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala index 6f7b5b9572..48cab01ac1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala @@ -35,7 +35,8 @@ class OptimizeInSuite extends PlanTest { val batches = Batch("AnalysisNodes", Once, EliminateSubQueries) :: - Batch("ConstantFolding", Once, + Batch("ConstantFolding", FixedPoint(10), + NullPropagation, ConstantFolding, BooleanSimplification, OptimizeIn) :: Nil @@ -82,4 +83,52 @@ class OptimizeInSuite extends PlanTest { comparePlans(optimized, correctAnswer) } + + test("OptimizedIn test: NULL IN (expr1, ..., exprN) gets transformed to Filter(null)") { + val originalQuery = + testRelation + .where(In(Literal.create(null, NullType), Seq(Literal(1), Literal(2)))) + .analyze + + val optimized = Optimize.execute(originalQuery.analyze) + val correctAnswer = + testRelation + .where(Literal.create(null, BooleanType)) + .analyze + + comparePlans(optimized, correctAnswer) + } + + test("OptimizedIn test: Inset optimization disabled as " + + "list expression contains attribute)") { + val originalQuery = + testRelation + .where(In(Literal.create(null, StringType), Seq(Literal(1), UnresolvedAttribute("b")))) + .analyze + + val optimized = Optimize.execute(originalQuery.analyze) + val correctAnswer = + testRelation + .where(Literal.create(null, BooleanType)) + .analyze + + comparePlans(optimized, correctAnswer) + } + + test("OptimizedIn test: Inset optimization disabled as " + + "list expression contains attribute - select)") { + val originalQuery = + testRelation + .select(In(Literal.create(null, StringType), + Seq(Literal(1), UnresolvedAttribute("b"))).as("a")).analyze + + val optimized = Optimize.execute(originalQuery.analyze) + val correctAnswer = + testRelation + .select(Literal.create(null, BooleanType).as("a")) + .analyze + + comparePlans(optimized, correctAnswer) + } + } |