aboutsummaryrefslogtreecommitdiff
path: root/network/common
diff options
context:
space:
mode:
authorAndrew Or <andrew@databricks.com>2014-11-06 17:18:49 -0800
committerAndrew Or <andrew@databricks.com>2014-11-06 17:18:49 -0800
commit96136f222abd4f3abd10cb78a4ebecdb21f3bde7 (patch)
tree7cf8e6a9cdd93e1e9b5e31fa0cc548a5976d1a63 /network/common
parent470881b24a503c9edcaed159c29bafa446ab0e9a (diff)
downloadspark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.tar.gz
spark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.tar.bz2
spark-96136f222abd4f3abd10cb78a4ebecdb21f3bde7.zip
[SPARK-3797] Minor addendum to Yarn shuffle service
I did not realize there was a `network.util.JavaUtils` when I wrote this code. This PR moves the `ByteBuffer` string conversion to the appropriate place. I tested the changes on a stable yarn cluster. Author: Andrew Or <andrew@databricks.com> Closes #3144 from andrewor14/yarn-shuffle-util and squashes the following commits: b6c08bf [Andrew Or] Remove unused import 94e205c [Andrew Or] Use netty Unpooled 85202a5 [Andrew Or] Use guava Charsets 057135b [Andrew Or] Reword comment adf186d [Andrew Or] Move byte buffer String conversion logic to JavaUtils
Diffstat (limited to 'network/common')
-rw-r--r--network/common/src/main/java/org/apache/spark/network/util/JavaUtils.java20
1 files changed, 20 insertions, 0 deletions
diff --git a/network/common/src/main/java/org/apache/spark/network/util/JavaUtils.java b/network/common/src/main/java/org/apache/spark/network/util/JavaUtils.java
index 40b71b0c87..2856d1c8c9 100644
--- a/network/common/src/main/java/org/apache/spark/network/util/JavaUtils.java
+++ b/network/common/src/main/java/org/apache/spark/network/util/JavaUtils.java
@@ -17,6 +17,8 @@
package org.apache.spark.network.util;
+import java.nio.ByteBuffer;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
@@ -25,6 +27,8 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import com.google.common.io.Closeables;
+import com.google.common.base.Charsets;
+import io.netty.buffer.Unpooled;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,4 +77,20 @@ public class JavaUtils {
int hash = obj.hashCode();
return hash != Integer.MIN_VALUE ? Math.abs(hash) : 0;
}
+
+ /**
+ * Convert the given string to a byte buffer. The resulting buffer can be
+ * converted back to the same string through {@link #bytesToString(ByteBuffer)}.
+ */
+ public static ByteBuffer stringToBytes(String s) {
+ return Unpooled.wrappedBuffer(s.getBytes(Charsets.UTF_8)).nioBuffer();
+ }
+
+ /**
+ * Convert the given byte buffer to a string. The resulting string can be
+ * converted back to the same byte buffer through {@link #stringToBytes(String)}.
+ */
+ public static String bytesToString(ByteBuffer b) {
+ return Unpooled.wrappedBuffer(b).toString(Charsets.UTF_8);
+ }
}