From 4cc06f4eb1fbf1ba1fc6165783e22f93dc3b14ac Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 17 Feb 2017 11:28:16 -0800 Subject: [SPARK-18986][CORE] ExternalAppendOnlyMap shouldn't fail when forced to spill before calling its iterator ## What changes were proposed in this pull request? `ExternalAppendOnlyMap.forceSpill` now uses an assert to check if an iterator is not null in the map. However, the assertion is only true after the map is asked for iterator. Before it, if another memory consumer asks more memory than currently available, `ExternalAppendOnlyMap.forceSpill` is also be called too. In this case, we will see failure like this: [info] java.lang.AssertionError: assertion failed [info] at scala.Predef$.assert(Predef.scala:156) [info] at org.apache.spark.util.collection.ExternalAppendOnlyMap.forceSpill(ExternalAppendOnlyMap.scala:196) [info] at org.apache.spark.util.collection.Spillable.spill(Spillable.scala:111) [info] at org.apache.spark.util.collection.ExternalAppendOnlyMapSuite$$anonfun$13.apply$mcV$sp(ExternalAppendOnlyMapSuite.scala:294) This fixing is motivated by http://apache-spark-developers-list.1001551.n3.nabble.com/java-lang-AssertionError-assertion-failed-tc20277.html. ## How was this patch tested? Jenkins tests. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Liang-Chi Hsieh Closes #16387 from viirya/fix-externalappendonlymap. --- .../spark/util/collection/ExternalAppendOnlyMapSuite.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'core/src/test/scala/org') diff --git a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala index c8b6a3346a..35312f2d71 100644 --- a/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala @@ -283,6 +283,17 @@ class ExternalAppendOnlyMapSuite extends SparkFunSuite with LocalSparkContext { sc.stop() } + test("ExternalAppendOnlyMap shouldn't fail when forced to spill before calling its iterator") { + val size = 1000 + val conf = createSparkConf(loadDefaults = true) + conf.set("spark.shuffle.spill.numElementsForceSpillThreshold", (size / 2).toString) + sc = new SparkContext("local-cluster[1,1,1024]", "test", conf) + val map = createExternalMap[String] + val consumer = createExternalMap[String] + map.insertAll((1 to size).iterator.map(_.toString).map(i => (i, i))) + assert(map.spill(10000, consumer) == 0L) + } + test("spilling with hash collisions") { val size = 1000 val conf = createSparkConf(loadDefaults = true) -- cgit v1.2.3