diff options
author | Dilip Biswal <dbiswal@us.ibm.com> | 2015-10-08 10:41:45 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-10-08 10:41:45 -0700 |
commit | dcbd58a929be0058b1cfa59b14898c4c428a7680 (patch) | |
tree | 8191a8bfb7550e3919ab1fdd390ec93f06b0a0f2 | |
parent | 5c9fdf74e328c067388ba1109eb48bf9d128bcf4 (diff) | |
download | spark-dcbd58a929be0058b1cfa59b14898c4c428a7680.tar.gz spark-dcbd58a929be0058b1cfa59b14898c4c428a7680.tar.bz2 spark-dcbd58a929be0058b1cfa59b14898c4c428a7680.zip |
[SPARK-8654] [SQL] Fix Analysis exception when using NULL IN (...)
In the analysis phase , while processing the rules for IN predicate, we
compare the in-list types to the lhs expression type and generate
cast operation if necessary. In the case of NULL [NOT] IN expr1 , we end up
generating cast between in list types to NULL like cast (1 as NULL) which
is not a valid cast.
The fix is to not generate such a cast if the lhs type is a NullType instead
we translate the expression to Literal(Null).
Author: Dilip Biswal <dbiswal@us.ibm.com>
Closes #8983 from dilipbiswal/spark_8654.
2 files changed, 29 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 87a3845b2d..7192c931d2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -304,7 +304,10 @@ object HiveTypeCoercion { } /** - * Convert all expressions in in() list to the left operator type + * Convert the value and in list expressions to the common operator type + * by looking at all the argument types and finding the closest one that + * all the arguments can be cast to. When no common operator type is found + * an Analysis Exception is raised. */ object InConversion extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions { @@ -312,7 +315,10 @@ object HiveTypeCoercion { case e if !e.childrenResolved => e case i @ In(a, b) if b.exists(_.dataType != a.dataType) => - i.makeCopy(Array(a, b.map(Cast(_, a.dataType)))) + findWiderCommonType(i.children.map(_.dataType)) match { + case Some(finalDataType) => i.withNewChildren(i.children.map(Cast(_, finalDataType))) + case None => i + } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 820b336aac..77a4765e77 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -135,4 +135,25 @@ class AnalysisSuite extends AnalysisTest { plan = testRelation.select(CreateStructUnsafe(Seq(a, (a + 1).as("a+1"))).as("col")) checkAnalysis(plan, plan) } + + test("SPARK-8654: invalid CAST in NULL IN(...) expression") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(2))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisSuccess(plan) + } + + test("SPARK-8654: different types in inlist but can be converted to a commmon type") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(1.2345))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisSuccess(plan) + } + + test("SPARK-8654: check type compatibility error") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(true), Literal(1))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisError(plan, Seq("data type mismatch: Arguments must be same type")) + } } |