diff options
| author | rotems <roter> | 2015-12-04 16:58:31 -0800 |
|---|---|---|
| committer | Marcelo Vanzin <vanzin@cloudera.com> | 2015-12-04 16:58:34 -0800 |
| commit | f30373f5ee60f9892c28771e34b208e4f1f675a6 (patch) | |
| tree | 383e6745fa9a9e078f9ba736a83c135b6ced1adf | |
| parent | bbfc16ec9d690c2dfa20896bd6d33f9783b9c109 (diff) | |
| download | spark-f30373f5ee60f9892c28771e34b208e4f1f675a6.tar.gz spark-f30373f5ee60f9892c28771e34b208e4f1f675a6.tar.bz2 spark-f30373f5ee60f9892c28771e34b208e4f1f675a6.zip | |
[SPARK-12080][CORE] Kryo - Support multiple user registrators
Author: rotems <roter>
Closes #10078 from Botnaim/KryoMultipleCustomRegistrators.
| -rw-r--r-- | core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala | 6 | ||||
| -rw-r--r-- | docs/configuration.md | 4 |
2 files changed, 6 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index d5ba690ed0..7b77f78ce6 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -70,7 +70,9 @@ class KryoSerializer(conf: SparkConf) private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true) private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false) - private val userRegistrator = conf.getOption("spark.kryo.registrator") + private val userRegistrators = conf.get("spark.kryo.registrator", "") + .split(',') + .filter(!_.isEmpty) private val classesToRegister = conf.get("spark.kryo.classesToRegister", "") .split(',') .filter(!_.isEmpty) @@ -119,7 +121,7 @@ class KryoSerializer(conf: SparkConf) classesToRegister .foreach { className => kryo.register(Class.forName(className, true, classLoader)) } // Allow the user to register their own classes by setting spark.kryo.registrator. - userRegistrator + userRegistrators .map(Class.forName(_, true, classLoader).newInstance().asInstanceOf[KryoRegistrator]) .foreach { reg => reg.registerClasses(kryo) } // scalastyle:on classforname diff --git a/docs/configuration.md b/docs/configuration.md index c39b489085..fd61ddc244 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -647,10 +647,10 @@ Apart from these, the following properties are also available, and may be useful <td><code>spark.kryo.registrator</code></td> <td>(none)</td> <td> - If you use Kryo serialization, set this class to register your custom classes with Kryo. This + If you use Kryo serialization, give a comma-separated list of classes that register your custom classes with Kryo. This property is useful if you need to register your classes in a custom way, e.g. to specify a custom field serializer. Otherwise <code>spark.kryo.classesToRegister</code> is simpler. It should be - set to a class that extends + set to classes that extend <a href="api/scala/index.html#org.apache.spark.serializer.KryoRegistrator"> <code>KryoRegistrator</code></a>. See the <a href="tuning.html#data-serialization">tuning guide</a> for more details. |