diff options
author | Andrew Or <andrew@databricks.com> | 2016-02-17 16:17:20 -0800 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2016-02-17 16:17:20 -0800 |
commit | 9451fed52cb8a00c706b582a0b51d8cd832f9350 (patch) | |
tree | 8af563fc2be9a2396257a36072c6cd29e434d9ce | |
parent | 97ee85daf68345cf5c3c11ae5bf288cc697bdf9e (diff) | |
download | spark-9451fed52cb8a00c706b582a0b51d8cd832f9350.tar.gz spark-9451fed52cb8a00c706b582a0b51d8cd832f9350.tar.bz2 spark-9451fed52cb8a00c706b582a0b51d8cd832f9350.zip |
[SPARK-13344][TEST] Fix harmless accumulator not found exceptions
See [JIRA](https://issues.apache.org/jira/browse/SPARK-13344) for more detail. This was caused by #10835.
Author: Andrew Or <andrew@databricks.com>
Closes #11222 from andrewor14/fix-test-accum-exceptions.
3 files changed, 30 insertions, 4 deletions
diff --git a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala index 4d49fe5159..8acd0439b6 100644 --- a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala +++ b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala @@ -34,6 +34,14 @@ import org.apache.spark.serializer.JavaSerializer class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContext { import AccumulatorParam._ + override def afterEach(): Unit = { + try { + Accumulators.clear() + } finally { + super.afterEach() + } + } + implicit def setAccum[A]: AccumulableParam[mutable.Set[A], A] = new AccumulableParam[mutable.Set[A], A] { def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]) : mutable.Set[A] = { diff --git a/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala b/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala index c426bb7a4e..474550608b 100644 --- a/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala +++ b/core/src/test/scala/org/apache/spark/InternalAccumulatorSuite.scala @@ -28,6 +28,14 @@ class InternalAccumulatorSuite extends SparkFunSuite with LocalSparkContext { import InternalAccumulator._ import AccumulatorParam._ + override def afterEach(): Unit = { + try { + Accumulators.clear() + } finally { + super.afterEach() + } + } + test("get param") { assert(getParam(EXECUTOR_DESERIALIZE_TIME) === LongAccumulatorParam) assert(getParam(EXECUTOR_RUN_TIME) === LongAccumulatorParam) diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala index d3359c7406..99366a32c4 100644 --- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala @@ -18,14 +18,26 @@ package org.apache.spark // scalastyle:off -import org.scalatest.{FunSuite, Outcome} +import org.scalatest.{BeforeAndAfterAll, FunSuite, Outcome} /** * Base abstract class for all unit tests in Spark for handling common functionality. */ -private[spark] abstract class SparkFunSuite extends FunSuite with Logging { +private[spark] abstract class SparkFunSuite + extends FunSuite + with BeforeAndAfterAll + with Logging { // scalastyle:on + protected override def afterAll(): Unit = { + try { + // Avoid leaking map entries in tests that use accumulators without SparkContext + Accumulators.clear() + } finally { + super.afterAll() + } + } + /** * Log the suite name and the test name before and after each test. * @@ -42,8 +54,6 @@ private[spark] abstract class SparkFunSuite extends FunSuite with Logging { test() } finally { logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") - // Avoid leaking map entries in tests that use accumulators without SparkContext - Accumulators.clear() } } |