diff options
| author | Josh Rosen <joshrosen@apache.org> | 2013-11-05 17:52:39 -0800 |
|---|---|---|
| committer | Josh Rosen <joshrosen@apache.org> | 2013-11-10 16:45:38 -0800 |
| commit | cbb7f04aef2220ece93dea9f3fa98b5db5f270d6 (patch) | |
| tree | 5feaed6b6064b81272fcb74b48ee2579e32de4e6 /python/pyspark/tests.py | |
| parent | 7d68a81a8ed5f49fefb3bd0fa0b9d3835cc7d86e (diff) | |
| download | spark-cbb7f04aef2220ece93dea9f3fa98b5db5f270d6.tar.gz spark-cbb7f04aef2220ece93dea9f3fa98b5db5f270d6.tar.bz2 spark-cbb7f04aef2220ece93dea9f3fa98b5db5f270d6.zip | |
Add custom serializer support to PySpark.
For now, this only adds MarshalSerializer, but it lays the groundwork
for other supporting custom serializers. Many of these mechanisms
can also be used to support deserialization of different data formats
sent by Java, such as data encoded by MsgPack.
This also fixes a bug in SparkContext.union().
Diffstat (limited to 'python/pyspark/tests.py')
| -rw-r--r-- | python/pyspark/tests.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 29d6a128f6..621e1cb58c 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -86,7 +86,8 @@ class TestCheckpoint(PySparkTestCase): time.sleep(1) # 1 second self.assertTrue(flatMappedRDD.getCheckpointFile() is not None) - recovered = self.sc._checkpointFile(flatMappedRDD.getCheckpointFile()) + recovered = self.sc._checkpointFile(flatMappedRDD.getCheckpointFile(), + flatMappedRDD._jrdd_deserializer) self.assertEquals([1, 2, 3, 4], recovered.collect()) |