diff options
author | Aaron Davidson <aaron@databricks.com> | 2014-11-02 16:26:24 -0800 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-11-02 16:26:24 -0800 |
commit | 2ebd1df3f17993f3cb472ec44c8832213976d99a (patch) | |
tree | 27369ea3bbc025e1c43f282fea96129db7d879d9 /network/shuffle | |
parent | 9081b9f9f79b78f0b20a5fc3bc4e7c1d3e717130 (diff) | |
download | spark-2ebd1df3f17993f3cb472ec44c8832213976d99a.tar.gz spark-2ebd1df3f17993f3cb472ec44c8832213976d99a.tar.bz2 spark-2ebd1df3f17993f3cb472ec44c8832213976d99a.zip |
[SPARK-4183] Close transport-related resources between SparkContexts
A leak of event loops may be causing test failures.
Author: Aaron Davidson <aaron@databricks.com>
Closes #3053 from aarondav/leak and squashes the following commits:
e676d18 [Aaron Davidson] Typo!
8f96475 [Aaron Davidson] Keep original ssc semantics
7e49f10 [Aaron Davidson] A leak of event loops may be causing test failures.
Diffstat (limited to 'network/shuffle')
-rw-r--r-- | network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleClient.java | 7 | ||||
-rw-r--r-- | network/shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleClient.java | 4 |
2 files changed, 10 insertions, 1 deletions
diff --git a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleClient.java b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleClient.java index cc2f6261ca..6bbabc44b9 100644 --- a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleClient.java +++ b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleClient.java @@ -17,6 +17,8 @@ package org.apache.spark.network.shuffle; +import java.io.Closeable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,4 +87,9 @@ public class ExternalShuffleClient implements ShuffleClient { JavaUtils.serialize(new RegisterExecutor(appId, execId, executorInfo)); client.sendRpcSync(registerExecutorMessage, 5000 /* timeoutMs */); } + + @Override + public void close() { + clientFactory.close(); + } } diff --git a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleClient.java b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleClient.java index 9fa87c2c6e..d46a562394 100644 --- a/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleClient.java +++ b/network/shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleClient.java @@ -17,8 +17,10 @@ package org.apache.spark.network.shuffle; +import java.io.Closeable; + /** Provides an interface for reading shuffle files, either from an Executor or external service. */ -public interface ShuffleClient { +public interface ShuffleClient extends Closeable { /** * Fetch a sequence of blocks from a remote node asynchronously, * |