diff options
| author | 0x0FFF <programmerag@gmail.com> | 2015-09-01 14:58:49 -0700 |
|---|---|---|
| committer | Davies Liu <davies.liu@gmail.com> | 2015-09-01 14:58:49 -0700 |
| commit | 00d9af5e190475affffb8b50467fcddfc40f50dc (patch) | |
| tree | 39ade3aeb2ebd7b55bbc66a7491a1685b79d6457 /python | |
| parent | bf550a4b551b6dd18fea3eb3f70497f9a6ad8e6c (diff) | |
| download | spark-00d9af5e190475affffb8b50467fcddfc40f50dc.tar.gz spark-00d9af5e190475affffb8b50467fcddfc40f50dc.tar.bz2 spark-00d9af5e190475affffb8b50467fcddfc40f50dc.zip | |
[SPARK-10392] [SQL] Pyspark - Wrong DateType support on JDBC connection
This PR addresses issue [SPARK-10392](https://issues.apache.org/jira/browse/SPARK-10392)
The problem is that for "start of epoch" date (01 Jan 1970) PySpark class DateType returns 0 instead of the `datetime.date` due to implementation of its return statement
Issue reproduction on master:
```
>>> from pyspark.sql.types import *
>>> a = DateType()
>>> a.fromInternal(0)
0
>>> a.fromInternal(1)
datetime.date(1970, 1, 2)
```
Author: 0x0FFF <programmerag@gmail.com>
Closes #8556 from 0x0FFF/SPARK-10392.
Diffstat (limited to 'python')
| -rw-r--r-- | python/pyspark/sql/tests.py | 5 | ||||
| -rw-r--r-- | python/pyspark/sql/types.py | 6 |
2 files changed, 9 insertions, 2 deletions
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index 59a891bd7c..fc778631d9 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -168,6 +168,11 @@ class DataTypeTests(unittest.TestCase): t3 = DecimalType(8) self.assertNotEqual(t2, t3) + # regression test for SPARK-10392 + def test_datetype_equal_zero(self): + dt = DateType() + self.assertEqual(dt.fromInternal(0), datetime.date(1970, 1, 1)) + class SQLTests(ReusedPySparkTestCase): diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index f84d08d709..8bd58d69ee 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -168,10 +168,12 @@ class DateType(AtomicType): return True def toInternal(self, d): - return d and d.toordinal() - self.EPOCH_ORDINAL + if d is not None: + return d.toordinal() - self.EPOCH_ORDINAL def fromInternal(self, v): - return v and datetime.date.fromordinal(v + self.EPOCH_ORDINAL) + if v is not None: + return datetime.date.fromordinal(v + self.EPOCH_ORDINAL) class TimestampType(AtomicType): |