diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-08 11:58:19 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-08 11:58:19 -0800 |
commit | 8ee3ec29ee919217768caf6a897c9e9d7e8ff920 (patch) | |
tree | da3bedbf19d99dc4ddff1b27bc8851dae2bebe18 /core | |
parent | d39f5dd15e45099b51d64eb9b9ede26dafa0b003 (diff) | |
parent | 7408230bfab0d374eb2eefd3f181e4d997c66061 (diff) | |
download | spark-8ee3ec29ee919217768caf6a897c9e9d7e8ff920.tar.gz spark-8ee3ec29ee919217768caf6a897c9e9d7e8ff920.tar.bz2 spark-8ee3ec29ee919217768caf6a897c9e9d7e8ff920.zip |
Merge remote branch 'origin/custom-serialization' into new-rdds
Diffstat (limited to 'core')
-rw-r--r-- | core/lib/kryo-1.04-mod/kryo-1.04-mod.jar | bin | 86177 -> 86081 bytes | |||
-rw-r--r-- | core/lib/kryo-1.04-mod/objenesis-1.2.jar | bin | 0 -> 36034 bytes | |||
-rw-r--r-- | core/src/main/scala/spark/KryoSerialization.scala | 20 |
3 files changed, 18 insertions, 2 deletions
diff --git a/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar b/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar Binary files differindex 7c4a8d3af8..815c1c8d94 100644 --- a/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar +++ b/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar diff --git a/core/lib/kryo-1.04-mod/objenesis-1.2.jar b/core/lib/kryo-1.04-mod/objenesis-1.2.jar Binary files differnew file mode 100644 index 0000000000..45cb641683 --- /dev/null +++ b/core/lib/kryo-1.04-mod/objenesis-1.2.jar diff --git a/core/src/main/scala/spark/KryoSerialization.scala b/core/src/main/scala/spark/KryoSerialization.scala index cd169fa7ab..462dee217e 100644 --- a/core/src/main/scala/spark/KryoSerialization.scala +++ b/core/src/main/scala/spark/KryoSerialization.scala @@ -8,6 +8,7 @@ import scala.collection.immutable import scala.collection.mutable import com.esotericsoftware.kryo._ +import com.esotericsoftware.kryo.{Serializer => KSerializer} object ZigZag { def writeInt(n: Int, out: OutputStream) { @@ -115,6 +116,8 @@ class KryoSerialization extends SerializationStrategy with Logging { def createKryo(): Kryo = { val kryo = new Kryo() + + // Register some commonly used classes val toRegister: Seq[AnyRef] = Seq( // Arrays Array(1), Array(1.0), Array(1.0f), Array(1L), Array(""), Array(("", "")), @@ -123,16 +126,29 @@ class KryoSerialization extends SerializationStrategy with Logging { ("", ""), (1, 1), (1.0, 1.0), (1L, 1L), (1, 1.0), (1.0, 1), (1L, 1.0), (1.0, 1L), (1, 1L), (1L, 1), // Scala collections - Nil, List(1), immutable.Map(1 -> 1), immutable.HashMap(1 -> 1), + List(1), immutable.Map(1 -> 1), immutable.HashMap(1 -> 1), mutable.Map(1 -> 1), mutable.HashMap(1 -> 1), mutable.ArrayBuffer(1), // Options and Either - Some(1), None, Left(1), Right(1), + Some(1), Left(1), Right(1), // Higher-dimensional tuples (1, 1, 1), (1, 1, 1, 1), (1, 1, 1, 1, 1) ) for (obj <- toRegister) { kryo.register(obj.getClass) } + + // Register some commonly used Scala singleton objects. Because these + // are singletons, we must return the exact same local object when we + // deserialize rather than returning a clone as FieldSerializer would. + kryo.register(None.getClass, new KSerializer { + override def writeObjectData(buf: ByteBuffer, obj: AnyRef) {} + override def readObjectData[T](buf: ByteBuffer, cls: Class[T]): T = None.asInstanceOf[T] + }) + kryo.register(Nil.getClass, new KSerializer { + override def writeObjectData(buf: ByteBuffer, obj: AnyRef) {} + override def readObjectData[T](buf: ByteBuffer, cls: Class[T]): T = Nil.asInstanceOf[T] + }) + val regCls = System.getProperty("spark.kryo.registrator") if (regCls != null) { logInfo("Running user registrator: " + regCls) |