aboutsummaryrefslogtreecommitdiff
path: root/sql/catalyst
diff options
context:
space:
mode:
authorDongjoon Hyun <dongjoon@apache.org>2016-07-19 10:28:17 -0700
committerReynold Xin <rxin@databricks.com>2016-07-19 10:28:17 -0700
commit162d04a30e38bb83d35865679145f8ea80b84c26 (patch)
treecc8d6f2871f792bcd2bc1cd03ef4f0699cda3e89 /sql/catalyst
parent0bd76e872b60cb80295fc12654e370cf22390056 (diff)
downloadspark-162d04a30e38bb83d35865679145f8ea80b84c26.tar.gz
spark-162d04a30e38bb83d35865679145f8ea80b84c26.tar.bz2
spark-162d04a30e38bb83d35865679145f8ea80b84c26.zip
[SPARK-16602][SQL] `Nvl` function should support numeric-string cases
## What changes were proposed in this pull request? `Nvl` function should support numeric-straing cases like Hive/Spark1.6. Currently, `Nvl` finds the tightest common types among numeric types. This PR extends that to consider `String` type, too. ```scala - TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype => + TypeCoercion.findTightestCommonTypeToString(left.dataType, right.dataType).map { dtype => ``` **Before** ```scala scala> sql("select nvl('0', 1)").collect() org.apache.spark.sql.AnalysisException: cannot resolve `nvl("0", 1)` due to data type mismatch: input to function coalesce should all be the same type, but it's [string, int]; line 1 pos 7 ``` **After** ```scala scala> sql("select nvl('0', 1)").collect() res0: Array[org.apache.spark.sql.Row] = Array([0]) ``` ## How was this patch tested? Pass the Jenkins tests. Author: Dongjoon Hyun <dongjoon@apache.org> Closes #14251 from dongjoon-hyun/SPARK-16602.
Diffstat (limited to 'sql/catalyst')
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala2
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala15
3 files changed, 17 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index baec6d14a2..9a040f8644 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -100,7 +100,7 @@ object TypeCoercion {
}
/** Similar to [[findTightestCommonType]], but can promote all the way to StringType. */
- private def findTightestCommonTypeToString(left: DataType, right: DataType): Option[DataType] = {
+ def findTightestCommonTypeToString(left: DataType, right: DataType): Option[DataType] = {
findTightestCommonTypeOfTwo(left, right).orElse((left, right) match {
case (StringType, t2: AtomicType) if t2 != BinaryType && t2 != BooleanType => Some(StringType)
case (t1: AtomicType, StringType) if t1 != BinaryType && t1 != BooleanType => Some(StringType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index 523fb05397..1c18265e0f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -134,7 +134,7 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable {
override def replaceForTypeCoercion(): Expression = {
if (left.dataType != right.dataType) {
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType, right.dataType).map { dtype =>
copy(left = Cast(left, dtype), right = Cast(right, dtype))
}.getOrElse(this)
} else {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
index ace6c15dc8..712fe35f47 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullFunctionsSuite.scala
@@ -77,6 +77,21 @@ class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
}
+ test("SPARK-16602 Nvl should support numeric-string cases") {
+ val intLit = Literal.create(1, IntegerType)
+ val doubleLit = Literal.create(2.2, DoubleType)
+ val stringLit = Literal.create("c", StringType)
+ val nullLit = Literal.create(null, NullType)
+
+ assert(Nvl(intLit, doubleLit).replaceForTypeCoercion().dataType == DoubleType)
+ assert(Nvl(intLit, stringLit).replaceForTypeCoercion().dataType == StringType)
+ assert(Nvl(stringLit, doubleLit).replaceForTypeCoercion().dataType == StringType)
+
+ assert(Nvl(nullLit, intLit).replaceForTypeCoercion().dataType == IntegerType)
+ assert(Nvl(doubleLit, nullLit).replaceForTypeCoercion().dataType == DoubleType)
+ assert(Nvl(nullLit, stringLit).replaceForTypeCoercion().dataType == StringType)
+ }
+
test("AtLeastNNonNulls") {
val mix = Seq(Literal("x"),
Literal.create(null, StringType),