diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2015-12-18 09:49:08 -0800 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2015-12-18 09:49:08 -0800 |
commit | 2782818287a71925523c1320291db6cb25221e9f (patch) | |
tree | 86e0ed3adb3957b7fae3afbbf96d6dfaa4b1fb8d /network/common | |
parent | ea59b0f3a6600f8046e5f3f55e89257614fb1f10 (diff) | |
download | spark-2782818287a71925523c1320291db6cb25221e9f.tar.gz spark-2782818287a71925523c1320291db6cb25221e9f.tar.bz2 spark-2782818287a71925523c1320291db6cb25221e9f.zip |
[SPARK-12350][CORE] Don't log errors when requested stream is not found.
If a client requests a non-existent stream, just send a failure message
back, without logging any error on the server side (since it's not a
server error).
On the executor side, avoid error logs by translating any errors during
transfer to a `ClassNotFoundException`, so that loading the class is
retried on a the parent class loader. This can mask IO errors during
transmission, but the most common cause is that the class is not
served by the remote end.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #10337 from vanzin/SPARK-12350.
Diffstat (limited to 'network/common')
-rw-r--r-- | network/common/src/main/java/org/apache/spark/network/server/StreamManager.java | 1 | ||||
-rw-r--r-- | network/common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java | 7 |
2 files changed, 7 insertions, 1 deletions
diff --git a/network/common/src/main/java/org/apache/spark/network/server/StreamManager.java b/network/common/src/main/java/org/apache/spark/network/server/StreamManager.java index 3f0155957a..07f161a29c 100644 --- a/network/common/src/main/java/org/apache/spark/network/server/StreamManager.java +++ b/network/common/src/main/java/org/apache/spark/network/server/StreamManager.java @@ -54,6 +54,7 @@ public abstract class StreamManager { * {@link #getChunk(long, int)} method. * * @param streamId id of a stream that has been previously registered with the StreamManager. + * @return A managed buffer for the stream, or null if the stream was not found. */ public ManagedBuffer openStream(String streamId) { throw new UnsupportedOperationException(); diff --git a/network/common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java b/network/common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java index c864d7ce16..105f538831 100644 --- a/network/common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java +++ b/network/common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java @@ -141,7 +141,12 @@ public class TransportRequestHandler extends MessageHandler<RequestMessage> { return; } - respond(new StreamResponse(req.streamId, buf.size(), buf)); + if (buf != null) { + respond(new StreamResponse(req.streamId, buf.size(), buf)); + } else { + respond(new StreamFailure(req.streamId, String.format( + "Stream '%s' was not found.", req.streamId))); + } } private void processRpcRequest(final RpcRequest req) { |