diff options
| author | Sandy Ryza <sandy@cloudera.com> | 2015-06-27 14:33:31 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@databricks.com> | 2015-06-27 14:33:31 -0700 |
| commit | 4153776fd840ae075e6bb608f054091b6d3ec0c4 (patch) | |
| tree | e25178c8d88647b99d35d5579332cc977ff6529d /core | |
| parent | d48e78934a346f023bd5cf44a34320f4d5a88e12 (diff) | |
| download | spark-4153776fd840ae075e6bb608f054091b6d3ec0c4.tar.gz spark-4153776fd840ae075e6bb608f054091b6d3ec0c4.tar.bz2 spark-4153776fd840ae075e6bb608f054091b6d3ec0c4.zip | |
[SPARK-8623] Hadoop RDDs fail to properly serialize configuration
Author: Sandy Ryza <sandy@cloudera.com>
Closes #7050 from sryza/sandy-spark-8623 and squashes the following commits:
58a8079 [Sandy Ryza] SPARK-8623. Hadoop RDDs fail to properly serialize configuration
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index cd8a82347a..ed35cffe96 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -36,7 +36,7 @@ import org.apache.spark.network.nio.{GetBlock, GotBlock, PutBlock} import org.apache.spark.network.util.ByteUnit import org.apache.spark.scheduler.{CompressedMapStatus, HighlyCompressedMapStatus} import org.apache.spark.storage._ -import org.apache.spark.util.BoundedPriorityQueue +import org.apache.spark.util.{BoundedPriorityQueue, SerializableConfiguration, SerializableJobConf} import org.apache.spark.util.collection.CompactBuffer /** @@ -94,8 +94,10 @@ class KryoSerializer(conf: SparkConf) // For results returned by asJavaIterable. See JavaIterableWrapperSerializer. kryo.register(JavaIterableWrapperSerializer.wrapperClass, new JavaIterableWrapperSerializer) - // Allow sending SerializableWritable + // Allow sending classes with custom Java serializers kryo.register(classOf[SerializableWritable[_]], new KryoJavaSerializer()) + kryo.register(classOf[SerializableConfiguration], new KryoJavaSerializer()) + kryo.register(classOf[SerializableJobConf], new KryoJavaSerializer()) kryo.register(classOf[HttpBroadcast[_]], new KryoJavaSerializer()) kryo.register(classOf[PythonBroadcast], new KryoJavaSerializer()) |