diff options
author | Reynold Xin <rxin@databricks.com> | 2015-07-03 00:25:02 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-07-03 00:25:02 -0700 |
commit | 2848f4da47d5c395de93ab9960bd905edfbd3439 (patch) | |
tree | f0c60ad63b20a672c7fa3e52380c31c92941314d /sql | |
parent | 9b23e92c727881ff9038b4fe9643c49b96914159 (diff) | |
download | spark-2848f4da47d5c395de93ab9960bd905edfbd3439.tar.gz spark-2848f4da47d5c395de93ab9960bd905edfbd3439.tar.bz2 spark-2848f4da47d5c395de93ab9960bd905edfbd3439.zip |
[SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
"NaN" from string to double is already handled by Cast expression itself.
Author: Reynold Xin <rxin@databricks.com>
Closes #7206 from rxin/convertnans and squashes the following commits:
3d99c33 [Reynold Xin] [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
Diffstat (limited to 'sql')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala | 33 |
1 files changed, 0 insertions, 33 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 6006e7bf00..38eb8322c8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -35,7 +35,6 @@ object HiveTypeCoercion { val typeCoercionRules = PropagateTypes :: - ConvertNaNs :: InConversion :: WidenTypes :: PromoteStrings :: @@ -149,38 +148,6 @@ object HiveTypeCoercion { } /** - * Converts string "NaN"s that are in binary operators with a NaN-able types (Float / Double) to - * the appropriate numeric equivalent. - */ - // TODO: remove this rule and make Cast handle Nan. - object ConvertNaNs extends Rule[LogicalPlan] { - private val StringNaN = Literal("NaN") - - def apply(plan: LogicalPlan): LogicalPlan = plan transform { - case q: LogicalPlan => q transformExpressions { - // Skip nodes who's children have not been resolved yet. - case e if !e.childrenResolved => e - - /* Double Conversions */ - case b @ BinaryOperator(StringNaN, right @ DoubleType()) => - b.makeCopy(Array(Literal(Double.NaN), right)) - case b @ BinaryOperator(left @ DoubleType(), StringNaN) => - b.makeCopy(Array(left, Literal(Double.NaN))) - - /* Float Conversions */ - case b @ BinaryOperator(StringNaN, right @ FloatType()) => - b.makeCopy(Array(Literal(Float.NaN), right)) - case b @ BinaryOperator(left @ FloatType(), StringNaN) => - b.makeCopy(Array(left, Literal(Float.NaN))) - - /* Use float NaN by default to avoid unnecessary type widening */ - case b @ BinaryOperator(left @ StringNaN, StringNaN) => - b.makeCopy(Array(left, Literal(Float.NaN))) - } - } - } - - /** * Widens numeric types and converts strings to numbers when appropriate. * * Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White |