aboutsummaryrefslogtreecommitdiff
path: root/sql/catalyst
diff options
context:
space:
mode:
authorTakuya UESHIN <ueshin@happy-camper.st>2014-11-17 16:28:07 -0800
committerMichael Armbrust <michael@databricks.com>2014-11-17 16:28:39 -0800
commit1a650e7d863b72025625c3140b038ab12ec86eca (patch)
tree41b52d1de9749cdf295f97a0f25fa1902e00cfa6 /sql/catalyst
parent1ca39b723fa1d9c3d3525f1e32e0a19770563d4e (diff)
downloadspark-1a650e7d863b72025625c3140b038ab12ec86eca.tar.gz
spark-1a650e7d863b72025625c3140b038ab12ec86eca.tar.bz2
spark-1a650e7d863b72025625c3140b038ab12ec86eca.zip
[SPARK-4425][SQL] Handle NaN or Infinity cast to Timestamp correctly.
`Cast` from `NaN` or `Infinity` of `Double` or `Float` to `TimestampType` throws `NumberFormatException`. Author: Takuya UESHIN <ueshin@happy-camper.st> Closes #3283 from ueshin/issues/SPARK-4425 and squashes the following commits: 14def0c [Takuya UESHIN] Fix Cast to be able to handle NaN or Infinity to TimestampType. (cherry picked from commit 566c791931645bfaaaf57ee5a15b9ffad534f81e) Signed-off-by: Michael Armbrust <michael@databricks.com>
Diffstat (limited to 'sql/catalyst')
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala14
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala5
2 files changed, 17 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index b401096ce1..b47865f87a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -32,6 +32,8 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
override def nullable = (child.dataType, dataType) match {
case (StringType, _: NumericType) => true
case (StringType, TimestampType) => true
+ case (DoubleType, TimestampType) => true
+ case (FloatType, TimestampType) => true
case (StringType, DateType) => true
case (_: NumericType, DateType) => true
case (BooleanType, DateType) => true
@@ -117,10 +119,18 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
buildCast[Decimal](_, d => decimalToTimestamp(d))
// TimestampWritable.doubleToTimestamp
case DoubleType =>
- buildCast[Double](_, d => decimalToTimestamp(Decimal(d)))
+ buildCast[Double](_, d => try {
+ decimalToTimestamp(Decimal(d))
+ } catch {
+ case _: NumberFormatException => null
+ })
// TimestampWritable.floatToTimestamp
case FloatType =>
- buildCast[Float](_, f => decimalToTimestamp(Decimal(f)))
+ buildCast[Float](_, f => try {
+ decimalToTimestamp(Decimal(f))
+ } catch {
+ case _: NumberFormatException => null
+ })
}
private[this] def decimalToTimestamp(d: Decimal) = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
index 3a6a0203af..3f5b9f698f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
@@ -450,6 +450,11 @@ class ExpressionEvaluationSuite extends FunSuite {
// A test for higher precision than millis
checkEvaluation(Cast(Cast(0.00000001, TimestampType), DoubleType), 0.00000001)
+
+ checkEvaluation(Cast(Literal(Double.NaN), TimestampType), null)
+ checkEvaluation(Cast(Literal(1.0 / 0.0), TimestampType), null)
+ checkEvaluation(Cast(Literal(Float.NaN), TimestampType), null)
+ checkEvaluation(Cast(Literal(1.0f / 0.0f), TimestampType), null)
}
test("null checking") {