diff options
author | Andrew Or <andrew@databricks.com> | 2014-11-06 17:18:49 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2014-11-06 17:18:49 -0800 |
commit | 96136f222abd4f3abd10cb78a4ebecdb21f3bde7 (patch) | |
tree | 7cf8e6a9cdd93e1e9b5e31fa0cc548a5976d1a63 /network/shuffle | |
parent | 470881b24a503c9edcaed159c29bafa446ab0e9a (diff) | |
download | spark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.tar.gz spark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.tar.bz2 spark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.zip |
[SPARK-3797] Minor addendum to Yarn shuffle service
I did not realize there was a `network.util.JavaUtils` when I wrote this code. This PR moves the `ByteBuffer` string conversion to the appropriate place. I tested the changes on a stable yarn cluster.
Author: Andrew Or <andrew@databricks.com>
Closes #3144 from andrewor14/yarn-shuffle-util and squashes the following commits:
b6c08bf [Andrew Or] Remove unused import
94e205c [Andrew Or] Use netty Unpooled
85202a5 [Andrew Or] Use guava Charsets
057135b [Andrew Or] Reword comment
adf186d [Andrew Or] Move byte buffer String conversion logic to JavaUtils
Diffstat (limited to 'network/shuffle')
-rw-r--r-- | network/shuffle/src/main/java/org/apache/spark/network/sasl/ShuffleSecretManager.java | 24 |
1 files changed, 2 insertions, 22 deletions
diff --git a/network/shuffle/src/main/java/org/apache/spark/network/sasl/ShuffleSecretManager.java b/network/shuffle/src/main/java/org/apache/spark/network/sasl/ShuffleSecretManager.java index e66c4af0f1..351c7930a9 100644 --- a/network/shuffle/src/main/java/org/apache/spark/network/sasl/ShuffleSecretManager.java +++ b/network/shuffle/src/main/java/org/apache/spark/network/sasl/ShuffleSecretManager.java @@ -19,13 +19,13 @@ package org.apache.spark.network.sasl; import java.lang.Override; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.spark.network.sasl.SecretKeyHolder; +import org.apache.spark.network.util.JavaUtils; /** * A class that manages shuffle secret used by the external shuffle service. @@ -34,30 +34,10 @@ public class ShuffleSecretManager implements SecretKeyHolder { private final Logger logger = LoggerFactory.getLogger(ShuffleSecretManager.class); private final ConcurrentHashMap<String, String> shuffleSecretMap; - private static final Charset UTF8_CHARSET = Charset.forName("UTF-8"); - // Spark user used for authenticating SASL connections // Note that this must match the value in org.apache.spark.SecurityManager private static final String SPARK_SASL_USER = "sparkSaslUser"; - /** - * Convert the given string to a byte buffer. The resulting buffer can be converted back to - * the same string through {@link #bytesToString(ByteBuffer)}. This is used if the external - * shuffle service represents shuffle secrets as bytes buffers instead of strings. - */ - public static ByteBuffer stringToBytes(String s) { - return ByteBuffer.wrap(s.getBytes(UTF8_CHARSET)); - } - - /** - * Convert the given byte buffer to a string. The resulting string can be converted back to - * the same byte buffer through {@link #stringToBytes(String)}. This is used if the external - * shuffle service represents shuffle secrets as bytes buffers instead of strings. - */ - public static String bytesToString(ByteBuffer b) { - return new String(b.array(), UTF8_CHARSET); - } - public ShuffleSecretManager() { shuffleSecretMap = new ConcurrentHashMap<String, String>(); } @@ -80,7 +60,7 @@ public class ShuffleSecretManager implements SecretKeyHolder { * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { - registerApp(appId, bytesToString(shuffleSecret)); + registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); } /** |