1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
package spark
import scala.collection.mutable
import scala.collection.immutable
import org.scalatest.FunSuite
import com.esotericsoftware.kryo._
import SparkContext._
class KryoSerializerSuite extends FunSuite {
test("basic types") {
val ser = (new KryoSerializer).newInstance()
def check[T](t: T) {
assert(ser.deserialize[T](ser.serialize(t)) === t)
}
check(1)
check(1L)
check(1.0f)
check(1.0)
check(1.toByte)
check(1.toShort)
check("")
check("hello")
check(Integer.MAX_VALUE)
check(Integer.MIN_VALUE)
check(java.lang.Long.MAX_VALUE)
check(java.lang.Long.MIN_VALUE)
check[String](null)
check(Array(1, 2, 3))
check(Array(1L, 2L, 3L))
check(Array(1.0, 2.0, 3.0))
check(Array(1.0f, 2.9f, 3.9f))
check(Array("aaa", "bbb", "ccc"))
check(Array("aaa", "bbb", null))
check(Array(true, false, true))
check(Array('a', 'b', 'c'))
check(Array[Int]())
}
test("pairs") {
val ser = (new KryoSerializer).newInstance()
def check[T](t: T) {
assert(ser.deserialize[T](ser.serialize(t)) === t)
}
check((1, 1))
check((1, 1L))
check((1L, 1))
check((1L, 1L))
check((1.0, 1))
check((1, 1.0))
check((1.0, 1.0))
check((1.0, 1L))
check((1L, 1.0))
check((1.0, 1L))
check(("x", 1))
check(("x", 1.0))
check(("x", 1L))
check((1, "x"))
check((1.0, "x"))
check((1L, "x"))
check(("x", "x"))
}
test("Scala data structures") {
val ser = (new KryoSerializer).newInstance()
def check[T](t: T) {
assert(ser.deserialize[T](ser.serialize(t)) === t)
}
check(List[Int]())
check(List[Int](1, 2, 3))
check(List[String]())
check(List[String]("x", "y", "z"))
check(None)
check(Some(1))
check(Some("hi"))
check(mutable.ArrayBuffer(1, 2, 3))
check(mutable.ArrayBuffer("1", "2", "3"))
check(mutable.Map())
check(mutable.Map(1 -> "one", 2 -> "two"))
check(mutable.Map("one" -> 1, "two" -> 2))
check(mutable.HashMap(1 -> "one", 2 -> "two"))
check(mutable.HashMap("one" -> 1, "two" -> 2))
check(List(Some(mutable.HashMap(1->1, 2->2)), None, Some(mutable.HashMap(3->4))))
check(List(mutable.HashMap("one" -> 1, "two" -> 2),mutable.HashMap(1->"one",2->"two",3->"three")))
}
test("custom registrator") {
import spark.test._
System.setProperty("spark.kryo.registrator", classOf[MyRegistrator].getName)
val ser = (new KryoSerializer).newInstance()
def check[T](t: T) {
assert(ser.deserialize[T](ser.serialize(t)) === t)
}
check(CaseClass(17, "hello"))
val c1 = new ClassWithNoArgConstructor
c1.x = 32
check(c1)
val c2 = new ClassWithoutNoArgConstructor(47)
check(c2)
val hashMap = new java.util.HashMap[String, String]
hashMap.put("foo", "bar")
check(hashMap)
System.clearProperty("spark.kryo.registrator")
}
}
package test {
case class CaseClass(i: Int, s: String) {}
class ClassWithNoArgConstructor {
var x: Int = 0
override def equals(other: Any) = other match {
case c: ClassWithNoArgConstructor => x == c.x
case _ => false
}
}
class ClassWithoutNoArgConstructor(val x: Int) {
override def equals(other: Any) = other match {
case c: ClassWithoutNoArgConstructor => x == c.x
case _ => false
}
}
class MyRegistrator extends KryoRegistrator {
override def registerClasses(k: Kryo) {
k.register(classOf[CaseClass])
k.register(classOf[ClassWithNoArgConstructor])
k.register(classOf[ClassWithoutNoArgConstructor])
k.register(classOf[java.util.HashMap[_, _]])
}
}
}
|
