diff options
author | Herman van Hovell <hvanhovell@questtec.nl> | 2016-01-20 15:13:01 -0800 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-01-20 15:13:01 -0800 |
commit | 10173279305a0e8a62bfbfe7a9d5d1fd558dd8e1 (patch) | |
tree | 94fce3f9499b51ef69ad6bf49a132f54cb8afe6b /sql/core | |
parent | f3934a8d656f1668bec065751b2a11411229b6f5 (diff) | |
download | spark-10173279305a0e8a62bfbfe7a9d5d1fd558dd8e1.tar.gz spark-10173279305a0e8a62bfbfe7a9d5d1fd558dd8e1.tar.bz2 spark-10173279305a0e8a62bfbfe7a9d5d1fd558dd8e1.zip |
[SPARK-12848][SQL] Change parsed decimal literal datatype from Double to Decimal
The current parser turns a decimal literal, for example ```12.1```, into a Double. The problem with this approach is that we convert an exact literal into a non-exact ```Double```. The PR changes this behavior, a Decimal literal is now converted into an extact ```BigDecimal```.
The behavior for scientific decimals, for example ```12.1e01```, is unchanged. This will be converted into a Double.
This PR replaces the ```BigDecimal``` literal by a ```Double``` literal, because the ```BigDecimal``` is the default now. You can use the double literal by appending a 'D' to the value, for instance: ```3.141527D```
cc davies rxin
Author: Herman van Hovell <hvanhovell@questtec.nl>
Closes #10796 from hvanhovell/SPARK-12848.
Diffstat (limited to 'sql/core')
3 files changed, 38 insertions, 28 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala index aec450e0a6..013a90875e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala @@ -212,7 +212,7 @@ class MathExpressionsSuite extends QueryTest with SharedSQLContext { Seq(Row(5, 10, 0), Row(55, 60, 100), Row(555, 560, 600)) ) - val pi = "3.1415BD" + val pi = "3.1415" checkAnswer( sql(s"SELECT round($pi, -3), round($pi, -2), round($pi, -1), " + s"round($pi, 0), round($pi, 1), round($pi, 2), round($pi, 3)"), @@ -367,6 +367,16 @@ class MathExpressionsSuite extends QueryTest with SharedSQLContext { checkAnswer( input.toDF("key", "value").selectExpr("abs(key) a").sort("a"), input.map(pair => Row(pair._2))) + + checkAnswer( + sql("select abs(0), abs(-1), abs(123), abs(-9223372036854775807), abs(9223372036854775807)"), + Row(0, 1, 123, 9223372036854775807L, 9223372036854775807L) + ) + + checkAnswer( + sql("select abs(0.0), abs(-3.14159265), abs(3.14159265)"), + Row(BigDecimal("0.0"), BigDecimal("3.14159265"), BigDecimal("3.14159265")) + ) } test("log2") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index b159346bed..47308966e9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -1174,19 +1174,19 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { test("Floating point number format") { checkAnswer( - sql("SELECT 0.3"), Row(0.3) + sql("SELECT 0.3"), Row(BigDecimal(0.3)) ) checkAnswer( - sql("SELECT -0.8"), Row(-0.8) + sql("SELECT -0.8"), Row(BigDecimal(-0.8)) ) checkAnswer( - sql("SELECT .5"), Row(0.5) + sql("SELECT .5"), Row(BigDecimal(0.5)) ) checkAnswer( - sql("SELECT -.18"), Row(-0.18) + sql("SELECT -.18"), Row(BigDecimal(-0.18)) ) } @@ -1200,11 +1200,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { ) checkAnswer( - sql("SELECT 9223372036854775808BD"), Row(new java.math.BigDecimal("9223372036854775808")) + sql("SELECT 9223372036854775808"), Row(new java.math.BigDecimal("9223372036854775808")) ) checkAnswer( - sql("SELECT -9223372036854775809BD"), Row(new java.math.BigDecimal("-9223372036854775809")) + sql("SELECT -9223372036854775809"), Row(new java.math.BigDecimal("-9223372036854775809")) ) } @@ -1219,11 +1219,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { ) checkAnswer( - sql("SELECT -5.2BD"), Row(BigDecimal(-5.2)) + sql("SELECT -5.2"), Row(BigDecimal(-5.2)) ) checkAnswer( - sql("SELECT +6.8"), Row(6.8d) + sql("SELECT +6.8e0"), Row(6.8d) ) checkAnswer( @@ -1598,20 +1598,20 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("decimal precision with multiply/division") { - checkAnswer(sql("select 10.3BD * 3.0BD"), Row(BigDecimal("30.90"))) - checkAnswer(sql("select 10.3000BD * 3.0BD"), Row(BigDecimal("30.90000"))) - checkAnswer(sql("select 10.30000BD * 30.0BD"), Row(BigDecimal("309.000000"))) - checkAnswer(sql("select 10.300000000000000000BD * 3.000000000000000000BD"), + checkAnswer(sql("select 10.3 * 3.0"), Row(BigDecimal("30.90"))) + checkAnswer(sql("select 10.3000 * 3.0"), Row(BigDecimal("30.90000"))) + checkAnswer(sql("select 10.30000 * 30.0"), Row(BigDecimal("309.000000"))) + checkAnswer(sql("select 10.300000000000000000 * 3.000000000000000000"), Row(BigDecimal("30.900000000000000000000000000000000000", new MathContext(38)))) - checkAnswer(sql("select 10.300000000000000000BD * 3.0000000000000000000BD"), + checkAnswer(sql("select 10.300000000000000000 * 3.0000000000000000000"), Row(null)) - checkAnswer(sql("select 10.3BD / 3.0BD"), Row(BigDecimal("3.433333"))) - checkAnswer(sql("select 10.3000BD / 3.0BD"), Row(BigDecimal("3.4333333"))) - checkAnswer(sql("select 10.30000BD / 30.0BD"), Row(BigDecimal("0.343333333"))) - checkAnswer(sql("select 10.300000000000000000BD / 3.00000000000000000BD"), + checkAnswer(sql("select 10.3 / 3.0"), Row(BigDecimal("3.433333"))) + checkAnswer(sql("select 10.3000 / 3.0"), Row(BigDecimal("3.4333333"))) + checkAnswer(sql("select 10.30000 / 30.0"), Row(BigDecimal("0.343333333"))) + checkAnswer(sql("select 10.300000000000000000 / 3.00000000000000000"), Row(BigDecimal("3.433333333333333333333333333", new MathContext(38)))) - checkAnswer(sql("select 10.3000000000000000000BD / 3.00000000000000000BD"), + checkAnswer(sql("select 10.3000000000000000000 / 3.00000000000000000"), Row(BigDecimal("3.4333333333333333333333333333", new MathContext(38)))) } @@ -1637,13 +1637,13 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("precision smaller than scale") { - checkAnswer(sql("select 10.00BD"), Row(BigDecimal("10.00"))) - checkAnswer(sql("select 1.00BD"), Row(BigDecimal("1.00"))) - checkAnswer(sql("select 0.10BD"), Row(BigDecimal("0.10"))) - checkAnswer(sql("select 0.01BD"), Row(BigDecimal("0.01"))) - checkAnswer(sql("select 0.001BD"), Row(BigDecimal("0.001"))) - checkAnswer(sql("select -0.01BD"), Row(BigDecimal("-0.01"))) - checkAnswer(sql("select -0.001BD"), Row(BigDecimal("-0.001"))) + checkAnswer(sql("select 10.00"), Row(BigDecimal("10.00"))) + checkAnswer(sql("select 1.00"), Row(BigDecimal("1.00"))) + checkAnswer(sql("select 0.10"), Row(BigDecimal("0.10"))) + checkAnswer(sql("select 0.01"), Row(BigDecimal("0.01"))) + checkAnswer(sql("select 0.001"), Row(BigDecimal("0.001"))) + checkAnswer(sql("select -0.01"), Row(BigDecimal("-0.01"))) + checkAnswer(sql("select -0.001"), Row(BigDecimal("-0.001"))) } test("external sorting updates peak execution memory") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 8de8ba355e..a3c6a1d7b2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -442,13 +442,13 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { // Number and String conflict: resolve the type as number in this query. checkAnswer( - sql("select num_str + 1.2BD from jsonTable where num_str > 14"), + sql("select num_str + 1.2 from jsonTable where num_str > 14"), Row(BigDecimal("92233720368547758071.2")) ) // Number and String conflict: resolve the type as number in this query. checkAnswer( - sql("select num_str + 1.2BD from jsonTable where num_str >= 92233720368547758060BD"), + sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"), Row(new java.math.BigDecimal("92233720368547758071.2")) ) |