diff options
| author | Davies Liu <davies.liu@gmail.com> | 2014-08-04 15:54:52 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@apache.org> | 2014-08-04 15:55:12 -0700 |
| commit | 2225d18a751b7a4470a93f3d9edebe0d33df75c8 (patch) | |
| tree | 745eec88071ffd166a208de21ce900a8c9e45c1f /python | |
| parent | aa7a48ee905b95e57f64051ea887d4775b427603 (diff) | |
| download | spark-2225d18a751b7a4470a93f3d9edebe0d33df75c8.tar.gz spark-2225d18a751b7a4470a93f3d9edebe0d33df75c8.tar.bz2 spark-2225d18a751b7a4470a93f3d9edebe0d33df75c8.zip | |
[SPARK-1687] [PySpark] fix unit tests related to pickable namedtuple
serializer is imported multiple times during doctests, so it's better to make _hijack_namedtuple() safe to be called multiple times.
Author: Davies Liu <davies.liu@gmail.com>
Closes #1771 from davies/fix and squashes the following commits:
1a9e336 [Davies Liu] fix unit tests
(cherry picked from commit 9fd82dbbcb8b10debbe95f1acab53ae8b340f38e)
Signed-off-by: Josh Rosen <joshrosen@apache.org>
Diffstat (limited to 'python')
| -rw-r--r-- | python/pyspark/serializers.py | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py index 1b52c144df..a10f85b55a 100644 --- a/python/pyspark/serializers.py +++ b/python/pyspark/serializers.py @@ -297,8 +297,11 @@ def _hack_namedtuple(cls): def _hijack_namedtuple(): """ Hack namedtuple() to make it picklable """ - global _old_namedtuple # or it will put in closure + # hijack only one time + if hasattr(collections.namedtuple, "__hijack"): + return + global _old_namedtuple # or it will put in closure def _copy_func(f): return types.FunctionType(f.func_code, f.func_globals, f.func_name, f.func_defaults, f.func_closure) @@ -313,6 +316,7 @@ def _hijack_namedtuple(): collections.namedtuple.func_globals["_old_namedtuple"] = _old_namedtuple collections.namedtuple.func_globals["_hack_namedtuple"] = _hack_namedtuple collections.namedtuple.func_code = namedtuple.func_code + collections.namedtuple.__hijack = 1 # hack the cls already generated by namedtuple # those created in other module can be pickled as normal, |