aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorhyukjinkwon <gurwls223@gmail.com>2016-12-03 10:12:28 +0000
committerSean Owen <sowen@cloudera.com>2016-12-03 10:12:28 +0000
commitd1312fb7edffd6e10c86f69ddfff05f8915856ac (patch)
tree6fcae59a04f023c1c8a06937bfc91473cc9274fb
parent553aac56bd5284e84391c05e2ef54d8bd7ad3a12 (diff)
downloadspark-d1312fb7edffd6e10c86f69ddfff05f8915856ac.tar.gz
spark-d1312fb7edffd6e10c86f69ddfff05f8915856ac.tar.bz2
spark-d1312fb7edffd6e10c86f69ddfff05f8915856ac.zip
[SPARK-18685][TESTS] Fix URI and release resources after opening in tests at ExecutorClassLoaderSuite
## What changes were proposed in this pull request? This PR fixes two problems as below: - Close `BufferedSource` after `Source.fromInputStream(...)` to release resource and make the tests pass on Windows in `ExecutorClassLoaderSuite` ``` [info] Exception encountered when attempting to run a suite with class name: org.apache.spark.repl.ExecutorClassLoaderSuite *** ABORTED *** (7 seconds, 333 milliseconds) [info] java.io.IOException: Failed to delete: C:\projects\spark\target\tmp\spark-77b2f37b-6405-47c4-af1c-4a6a206511f2 [info] at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:1010) [info] at org.apache.spark.repl.ExecutorClassLoaderSuite.afterAll(ExecutorClassLoaderSuite.scala:76) [info] at org.scalatest.BeforeAndAfterAll$class.afterAll(BeforeAndAfterAll.scala:213) ... ``` - Fix URI correctly so that related tests can be passed on Windows. ``` [info] - child first *** FAILED *** (78 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - parent first *** FAILED *** (15 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - child first can fall back *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - child first can fail *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - resource from parent *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ... [info] - resources from parent *** FAILED *** (0 milliseconds) [info] java.net.URISyntaxException: Illegal character in authority at index 7: file://C:\projects\spark\target\tmp\spark-00b66070-0548-463c-b6f3-8965d173da9b [info] at java.net.URI$Parser.fail(URI.java:2848) [info] at java.net.URI$Parser.parseAuthority(URI.java:3186) ``` ## How was this patch tested? Manually tested via AppVeyor. **Before** https://ci.appveyor.com/project/spark-test/spark/build/102-rpel-ExecutorClassLoaderSuite **After** https://ci.appveyor.com/project/spark-test/spark/build/108-rpel-ExecutorClassLoaderSuite Author: hyukjinkwon <gurwls223@gmail.com> Closes #16116 from HyukjinKwon/close-after-open.
-rw-r--r--repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala25
1 files changed, 17 insertions, 8 deletions
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index 3d622d42f4..6d274bddb7 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -24,7 +24,6 @@ import java.nio.charset.StandardCharsets
import java.nio.file.{Paths, StandardOpenOption}
import java.util
-import scala.concurrent.duration._
import scala.io.Source
import scala.language.implicitConversions
@@ -34,8 +33,6 @@ import org.mockito.Mockito._
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import org.scalatest.BeforeAndAfterAll
-import org.scalatest.concurrent.Interruptor
-import org.scalatest.concurrent.Timeouts._
import org.scalatest.mock.MockitoSugar
import org.apache.spark._
@@ -61,7 +58,7 @@ class ExecutorClassLoaderSuite
super.beforeAll()
tempDir1 = Utils.createTempDir()
tempDir2 = Utils.createTempDir()
- url1 = "file://" + tempDir1
+ url1 = tempDir1.toURI.toURL.toString
urls2 = List(tempDir2.toURI.toURL).toArray
childClassNames.foreach(TestUtils.createCompiledClass(_, tempDir1, "1"))
parentResourceNames.foreach { x =>
@@ -118,8 +115,14 @@ class ExecutorClassLoaderSuite
val resourceName: String = parentResourceNames.head
val is = classLoader.getResourceAsStream(resourceName)
assert(is != null, s"Resource $resourceName not found")
- val content = Source.fromInputStream(is, "UTF-8").getLines().next()
- assert(content.contains("resource"), "File doesn't contain 'resource'")
+
+ val bufferedSource = Source.fromInputStream(is, "UTF-8")
+ Utils.tryWithSafeFinally {
+ val content = bufferedSource.getLines().next()
+ assert(content.contains("resource"), "File doesn't contain 'resource'")
+ } {
+ bufferedSource.close()
+ }
}
test("resources from parent") {
@@ -128,8 +131,14 @@ class ExecutorClassLoaderSuite
val resourceName: String = parentResourceNames.head
val resources: util.Enumeration[URL] = classLoader.getResources(resourceName)
assert(resources.hasMoreElements, s"Resource $resourceName not found")
- val fileReader = Source.fromInputStream(resources.nextElement().openStream()).bufferedReader()
- assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
+
+ val bufferedSource = Source.fromInputStream(resources.nextElement().openStream())
+ Utils.tryWithSafeFinally {
+ val fileReader = bufferedSource.bufferedReader()
+ assert(fileReader.readLine().contains("resource"), "File doesn't contain 'resource'")
+ } {
+ bufferedSource.close()
+ }
}
test("fetch classes using Spark's RpcEnv") {