diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-07 19:16:24 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-07 19:16:24 -0800 |
commit | 38f6bce33d63f793d814385047b9bac34e5947f7 (patch) | |
tree | 2e2c892890565c1a729252884bb89c30ca18258d | |
parent | 6316c7979d0151ac226a4b740ad32ec45672435e (diff) | |
download | spark-38f6bce33d63f793d814385047b9bac34e5947f7.tar.gz spark-38f6bce33d63f793d814385047b9bac34e5947f7.tar.bz2 spark-38f6bce33d63f793d814385047b9bac34e5947f7.zip |
Added SerializingCache
-rw-r--r-- | core/src/main/scala/spark/KryoSerialization.scala | 2 | ||||
-rw-r--r-- | core/src/main/scala/spark/SerializingCache.scala | 26 |
2 files changed, 27 insertions, 1 deletions
diff --git a/core/src/main/scala/spark/KryoSerialization.scala b/core/src/main/scala/spark/KryoSerialization.scala index f17051f5a8..f6e818adb0 100644 --- a/core/src/main/scala/spark/KryoSerialization.scala +++ b/core/src/main/scala/spark/KryoSerialization.scala @@ -117,7 +117,7 @@ class KryoSerialization extends SerializationStrategy with Logging { val kryo = new Kryo() val toRegister: Seq[AnyRef] = Seq( // Arrays - Array(1), Array(1.0), Array(1.0f), Array(1L), Array(""), + Array(1), Array(1.0), Array(1.0f), Array(1L), Array(""), Array(("", "")), // Specialized Tuple2s ("", ""), (1, 1), (1.0, 1.0), (1L, 1L), (1, 1.0), (1.0, 1), (1L, 1.0), (1.0, 1L), (1, 1L), (1L, 1), diff --git a/core/src/main/scala/spark/SerializingCache.scala b/core/src/main/scala/spark/SerializingCache.scala new file mode 100644 index 0000000000..cbc64736e6 --- /dev/null +++ b/core/src/main/scala/spark/SerializingCache.scala @@ -0,0 +1,26 @@ +package spark + +import java.io._ + +/** + * Wrapper around a BoundedMemoryCache that stores serialized objects as + * byte arrays in order to reduce storage cost and GC overhead + */ +class SerializingCache extends Cache with Logging { + val bmc = new BoundedMemoryCache + + override def put(key: Any, value: Any) { + val ser = Serializer.newInstance() + bmc.put(key, ser.serialize(value)) + } + + override def get(key: Any): Any = { + val bytes = bmc.get(key) + if (bytes != null) { + val ser = Serializer.newInstance() + return ser.deserialize(bytes.asInstanceOf[Array[Byte]]) + } else { + return null + } + } +} |