diff options
author | Bogdan Raducanu <bogdan@databricks.com> | 2017-04-20 18:49:39 +0200 |
---|---|---|
committer | Herman van Hovell <hvanhovell@databricks.com> | 2017-04-20 18:49:39 +0200 |
commit | c5a31d160f47ba51bb9f8a4f3141851034640fc7 (patch) | |
tree | ad4eb98ddd4b85baa4f94eb95f3cd898c9dff33a /sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala | |
parent | b91873db0930c6fe885c27936e1243d5fabd03ed (diff) | |
download | spark-c5a31d160f47ba51bb9f8a4f3141851034640fc7.tar.gz spark-c5a31d160f47ba51bb9f8a4f3141851034640fc7.tar.bz2 spark-c5a31d160f47ba51bb9f8a4f3141851034640fc7.zip |
[SPARK-20407][TESTS] ParquetQuerySuite 'Enabling/disabling ignoreCorruptFiles' flaky test
## What changes were proposed in this pull request?
SharedSQLContext.afterEach now calls DebugFilesystem.assertNoOpenStreams inside eventually.
SQLTestUtils withTempDir calls waitForTasksToFinish before deleting the directory.
## How was this patch tested?
Added new test in ParquetQuerySuite based on the flaky test
Author: Bogdan Raducanu <bogdan@databricks.com>
Closes #17701 from bogdanrdc/SPARK-20407.
Diffstat (limited to 'sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala')
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala index e122b39f6f..3d76e05f61 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala @@ -17,17 +17,18 @@ package org.apache.spark.sql.test +import scala.concurrent.duration._ + import org.scalatest.BeforeAndAfterEach +import org.scalatest.concurrent.Eventually import org.apache.spark.{DebugFilesystem, SparkConf} import org.apache.spark.sql.{SparkSession, SQLContext} -import org.apache.spark.sql.internal.SQLConf - /** * Helper trait for SQL test suites where all tests share a single [[TestSparkSession]]. */ -trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach { +trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach with Eventually { protected val sparkConf = new SparkConf() @@ -84,6 +85,10 @@ trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach { protected override def afterEach(): Unit = { super.afterEach() - DebugFilesystem.assertNoOpenStreams() + // files can be closed from other threads, so wait a bit + // normally this doesn't take more than 1s + eventually(timeout(10.seconds)) { + DebugFilesystem.assertNoOpenStreams() + } } } |