diff options
| author | Josh Rosen <joshrosen@apache.org> | 2013-11-28 23:44:56 -0800 |
|---|---|---|
| committer | Josh Rosen <joshrosen@apache.org> | 2013-11-28 23:44:56 -0800 |
| commit | 3787f514d9a8e45d2c257b4696e30bc1a1935748 (patch) | |
| tree | 572553edf58b4d97b54afe1a536f30288bc1db4f /python/pyspark/rdd.py | |
| parent | 743a31a7ca4421cbbd5b615b773997a06a7ab4ee (diff) | |
| download | spark-3787f514d9a8e45d2c257b4696e30bc1a1935748.tar.gz spark-3787f514d9a8e45d2c257b4696e30bc1a1935748.tar.bz2 spark-3787f514d9a8e45d2c257b4696e30bc1a1935748.zip | |
Fix UnicodeEncodeError in PySpark saveAsTextFile().
Fixes SPARK-970.
Diffstat (limited to 'python/pyspark/rdd.py')
| -rw-r--r-- | python/pyspark/rdd.py | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index 957f3f89c0..d8da02072c 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -605,7 +605,10 @@ class RDD(object): '0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n' """ def func(split, iterator): - return (str(x).encode("utf-8") for x in iterator) + for x in iterator: + if not isinstance(x, basestring): + x = unicode(x) + yield x.encode("utf-8") keyed = PipelinedRDD(self, func) keyed._bypass_serializer = True keyed._jrdd.map(self.ctx._jvm.BytesToString()).saveAsTextFile(path) |