diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2016-08-25 16:11:42 -0700 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-08-25 16:11:42 -0700 |
commit | 9b5a1d1d53bc4412de3cbc86dc819b0c213229a8 (patch) | |
tree | 44734abfc7b061da7a6fc1bce3ab3fa6ec528beb /core/src/test | |
parent | 3e4c7db4d11c474457e7886a5501108ebab0cf6d (diff) | |
download | spark-9b5a1d1d53bc4412de3cbc86dc819b0c213229a8.tar.gz spark-9b5a1d1d53bc4412de3cbc86dc819b0c213229a8.tar.bz2 spark-9b5a1d1d53bc4412de3cbc86dc819b0c213229a8.zip |
[SPARK-17240][CORE] Make SparkConf serializable again.
Make the config reader transient, and initialize it lazily so that
serialization works with both java and kryo (and hopefully any other
custom serializer).
Added unit test to make sure SparkConf remains serializable and the
reader works with both built-in serializers.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #14813 from vanzin/SPARK-17240.
Diffstat (limited to 'core/src/test')
-rw-r--r-- | core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 22 |
1 files changed, 21 insertions, 1 deletions
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala index 1f0f655a15..83906cff12 100644 --- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala @@ -26,8 +26,9 @@ import scala.util.{Random, Try} import com.esotericsoftware.kryo.Kryo +import org.apache.spark.internal.config._ import org.apache.spark.network.util.ByteUnit -import org.apache.spark.serializer.{KryoRegistrator, KryoSerializer} +import org.apache.spark.serializer.{JavaSerializer, KryoRegistrator, KryoSerializer} import org.apache.spark.util.{ResetSystemProperties, RpcUtils} class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSystemProperties { @@ -283,6 +284,25 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst assert(conf.contains("spark.io.compression.lz4.blockSize")) assert(conf.contains("spark.io.unknown") === false) } + + val serializers = Map( + "java" -> new JavaSerializer(new SparkConf()), + "kryo" -> new KryoSerializer(new SparkConf())) + + serializers.foreach { case (name, ser) => + test(s"SPARK-17240: SparkConf should be serializable ($name)") { + val conf = new SparkConf() + conf.set(DRIVER_CLASS_PATH, "${" + DRIVER_JAVA_OPTIONS.key + "}") + conf.set(DRIVER_JAVA_OPTIONS, "test") + + val serializer = ser.newInstance() + val bytes = serializer.serialize(conf) + val deser = serializer.deserialize[SparkConf](bytes) + + assert(conf.get(DRIVER_CLASS_PATH) === deser.get(DRIVER_CLASS_PATH)) + } + } + } class Class1 {} |