aboutsummaryrefslogtreecommitdiff
path: root/python/pyspark/tests.py
diff options
context:
space:
mode:
Diffstat (limited to 'python/pyspark/tests.py')
-rw-r--r--python/pyspark/tests.py18
1 files changed, 4 insertions, 14 deletions
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 29bcd38908..32645778c2 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -48,7 +48,7 @@ from pyspark.conf import SparkConf
from pyspark.context import SparkContext
from pyspark.files import SparkFiles
from pyspark.serializers import read_int, BatchedSerializer, MarshalSerializer, PickleSerializer, \
- CloudPickleSerializer, SizeLimitedStream, CompressedSerializer, LargeObjectSerializer
+ CloudPickleSerializer, CompressedSerializer
from pyspark.shuffle import Aggregator, InMemoryMerger, ExternalMerger, ExternalSorter
from pyspark.sql import SQLContext, IntegerType, Row, ArrayType, StructType, StructField, \
UserDefinedType, DoubleType
@@ -237,26 +237,16 @@ class SerializationTestCase(unittest.TestCase):
self.assertTrue("exit" in foo.func_code.co_names)
ser.dumps(foo)
- def _test_serializer(self, ser):
+ def test_compressed_serializer(self):
+ ser = CompressedSerializer(PickleSerializer())
from StringIO import StringIO
io = StringIO()
ser.dump_stream(["abc", u"123", range(5)], io)
io.seek(0)
self.assertEqual(["abc", u"123", range(5)], list(ser.load_stream(io)))
- size = io.tell()
ser.dump_stream(range(1000), io)
io.seek(0)
- first = SizeLimitedStream(io, size)
- self.assertEqual(["abc", u"123", range(5)], list(ser.load_stream(first)))
- self.assertEqual(range(1000), list(ser.load_stream(io)))
-
- def test_compressed_serializer(self):
- ser = CompressedSerializer(PickleSerializer())
- self._test_serializer(ser)
-
- def test_large_object_serializer(self):
- ser = LargeObjectSerializer()
- self._test_serializer(ser)
+ self.assertEqual(["abc", u"123", range(5)] + range(1000), list(ser.load_stream(io)))
class PySparkTestCase(unittest.TestCase):