aboutsummaryrefslogtreecommitdiff
path: root/core/src/test/java
diff options
context:
space:
mode:
authorSean Owen <sowen@cloudera.com>2014-12-08 16:13:03 -0800
committerJosh Rosen <joshrosen@databricks.com>2014-12-08 16:13:03 -0800
commite829bfa1ab9b68f44c489d26efb042f793fd9362 (patch)
tree22f28aea8b5e18d2a95ab0d9636d4173f99460c4 /core/src/test/java
parent65f929d5b3a50a73cd6397bd4b72c3e7d94c99d7 (diff)
downloadspark-e829bfa1ab9b68f44c489d26efb042f793fd9362.tar.gz
spark-e829bfa1ab9b68f44c489d26efb042f793fd9362.tar.bz2
spark-e829bfa1ab9b68f44c489d26efb042f793fd9362.zip
SPARK-3926 [CORE] Reopened: result of JavaRDD collectAsMap() is not serializable
My original 'fix' didn't fix at all. Now, there's a unit test to check whether it works. Of the two options to really fix it -- copy the `Map` to a `java.util.HashMap`, or copy and modify Scala's implementation in `Wrappers.MapWrapper`, I went with the latter. Author: Sean Owen <sowen@cloudera.com> Closes #3587 from srowen/SPARK-3926 and squashes the following commits: 8586bb9 [Sean Owen] Remove unneeded no-arg constructor, and add additional note about copied code in LICENSE 7bb0e66 [Sean Owen] Make SerializableMapWrapper actually serialize, and add unit test
Diffstat (limited to 'core/src/test/java')
-rw-r--r--core/src/test/java/org/apache/spark/JavaAPISuite.java13
1 files changed, 13 insertions, 0 deletions
diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
index 3ad4f2f193..e5bdad6bda 100644
--- a/core/src/test/java/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -1357,6 +1357,19 @@ public class JavaAPISuite implements Serializable {
pairRDD.collectAsMap(); // Used to crash with ClassCastException
}
+ @SuppressWarnings("unchecked")
+ @Test
+ public void collectAsMapAndSerialize() throws Exception {
+ JavaPairRDD<String,Integer> rdd =
+ sc.parallelizePairs(Arrays.asList(new Tuple2<String,Integer>("foo", 1)));
+ Map<String,Integer> map = rdd.collectAsMap();
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ new ObjectOutputStream(bytes).writeObject(map);
+ Map<String,Integer> deserializedMap = (Map<String,Integer>)
+ new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray())).readObject();
+ Assert.assertEquals(1, deserializedMap.get("foo").intValue());
+ }
+
@Test
@SuppressWarnings("unchecked")
public void sampleByKey() {