aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
authorzsxwing <zsxwing@gmail.com>2014-10-28 17:59:10 -0700
committerAaron Davidson <aaron@databricks.com>2014-10-28 17:59:10 -0700
commit1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35 (patch)
tree8bc09d016bb146a9917f5ce25e9f3a4feccad1c8 /core
parent84e5da87e32256ba4f3dee6f8bf532ce88322028 (diff)
downloadspark-1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35.tar.gz
spark-1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35.tar.bz2
spark-1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35.zip
[SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
`zeroValue` will be serialized by `spark.closure.serializer` but `spark.closure.serializer` only supports the default Java serializer. So it must not be `ClassWithoutNoArgConstructor`, which can not be serialized by the Java serializer. This PR changed `zeroValue` to null and updated the test to make it work correctly. Author: zsxwing <zsxwing@gmail.com> Closes #2856 from zsxwing/SPARK-4008 and squashes the following commits: 51da655 [zsxwing] [SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
Diffstat (limited to 'core')
-rw-r--r--core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala13
1 files changed, 9 insertions, 4 deletions
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 64ac6d2d92..a70f67af2e 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -201,12 +201,17 @@ class KryoSerializerSuite extends FunSuite with SharedSparkContext {
assert(control.sum === result)
}
- // TODO: this still doesn't work
- ignore("kryo with fold") {
+ test("kryo with fold") {
val control = 1 :: 2 :: Nil
+ // zeroValue must not be a ClassWithoutNoArgConstructor instance because it will be
+ // serialized by spark.closure.serializer but spark.closure.serializer only supports
+ // the default Java serializer.
val result = sc.parallelize(control, 2).map(new ClassWithoutNoArgConstructor(_))
- .fold(new ClassWithoutNoArgConstructor(10))((t1, t2) => new ClassWithoutNoArgConstructor(t1.x + t2.x)).x
- assert(10 + control.sum === result)
+ .fold(null)((t1, t2) => {
+ val t1x = if (t1 == null) 0 else t1.x
+ new ClassWithoutNoArgConstructor(t1x + t2.x)
+ }).x
+ assert(control.sum === result)
}
test("kryo with nonexistent custom registrator should fail") {