diff options
Diffstat (limited to 'core')
3 files changed, 14 insertions, 15 deletions
diff --git a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala index 86874e2067..df520f804b 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala @@ -17,6 +17,8 @@ package org.apache.spark.network.netty +import scala.collection.JavaConverters._ + import org.apache.spark.SparkConf import org.apache.spark.network.util.{ConfigProvider, TransportConf} @@ -58,6 +60,10 @@ object SparkTransportConf { new TransportConf(module, new ConfigProvider { override def get(name: String): String = conf.get(name) + + override def getAll(): java.lang.Iterable[java.util.Map.Entry[String, String]] = { + conf.getAll.toMap.asJava.entrySet() + } }) } diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala index 8e3436f134..cdd3b8d851 100644 --- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala +++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala @@ -21,12 +21,15 @@ import java.util.Properties import javax.crypto.KeyGenerator import javax.crypto.spec.{IvParameterSpec, SecretKeySpec} +import scala.collection.JavaConverters._ + import org.apache.commons.crypto.random._ import org.apache.commons.crypto.stream._ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ +import org.apache.spark.network.util.CryptoUtils /** * A util class for manipulating IO encryption and decryption streams. @@ -37,8 +40,6 @@ private[spark] object CryptoStreamUtils extends Logging { val IV_LENGTH_IN_BYTES = 16 // The prefix of IO encryption related configurations in Spark configuration. val SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX = "spark.io.encryption.commons.config." - // The prefix for the configurations passing to Apache Commons Crypto library. - val COMMONS_CRYPTO_CONF_PREFIX = "commons.crypto." /** * Helper method to wrap `OutputStream` with `CryptoOutputStream` for encryption. @@ -70,18 +71,9 @@ private[spark] object CryptoStreamUtils extends Logging { new SecretKeySpec(key, "AES"), new IvParameterSpec(iv)) } - /** - * Get Commons-crypto configurations from Spark configurations identified by prefix. - */ def toCryptoConf(conf: SparkConf): Properties = { - val props = new Properties() - conf.getAll.foreach { case (k, v) => - if (k.startsWith(SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX)) { - props.put(COMMONS_CRYPTO_CONF_PREFIX + k.substring( - SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX.length()), v) - } - } - props + CryptoUtils.toCryptoConf(SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX, + conf.getAll.toMap.asJava.entrySet()) } /** diff --git a/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala b/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala index a61ec74c7d..0f3a4a0361 100644 --- a/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/security/CryptoStreamUtilsSuite.scala @@ -24,6 +24,7 @@ import com.google.common.io.ByteStreams import org.apache.spark._ import org.apache.spark.internal.config._ +import org.apache.spark.network.util.CryptoUtils import org.apache.spark.security.CryptoStreamUtils._ import org.apache.spark.serializer.{JavaSerializer, SerializerManager} import org.apache.spark.storage.TempShuffleBlockId @@ -33,11 +34,11 @@ class CryptoStreamUtilsSuite extends SparkFunSuite { test("crypto configuration conversion") { val sparkKey1 = s"${SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX}a.b.c" val sparkVal1 = "val1" - val cryptoKey1 = s"${COMMONS_CRYPTO_CONF_PREFIX}a.b.c" + val cryptoKey1 = s"${CryptoUtils.COMMONS_CRYPTO_CONFIG_PREFIX}a.b.c" val sparkKey2 = SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX.stripSuffix(".") + "A.b.c" val sparkVal2 = "val2" - val cryptoKey2 = s"${COMMONS_CRYPTO_CONF_PREFIX}A.b.c" + val cryptoKey2 = s"${CryptoUtils.COMMONS_CRYPTO_CONFIG_PREFIX}A.b.c" val conf = new SparkConf() conf.set(sparkKey1, sparkVal1) conf.set(sparkKey2, sparkVal2) |