diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2016-12-12 16:27:04 -0800 |
---|---|---|
committer | Shixiong Zhu <shixiong@databricks.com> | 2016-12-12 16:27:04 -0800 |
commit | bc59951babbe4d7d5265a5dbccd50ea84ad74592 (patch) | |
tree | 83e9864e9349eca833b4afae780ad4c77d37f0f7 /core/src/main | |
parent | 8a51cfdcad5f8397558ed2e245eb03650f37ce66 (diff) | |
download | spark-bc59951babbe4d7d5265a5dbccd50ea84ad74592.tar.gz spark-bc59951babbe4d7d5265a5dbccd50ea84ad74592.tar.bz2 spark-bc59951babbe4d7d5265a5dbccd50ea84ad74592.zip |
[SPARK-18773][CORE] Make commons-crypto config translation consistent.
This change moves the logic that translates Spark configuration to
commons-crypto configuration to the network-common module. It also
extends TransportConf and ConfigProvider to provide the necessary
interfaces for the translation to work.
As part of the change, I removed SystemPropertyConfigProvider, which
was mostly used as an "empty config" in unit tests, and adjusted the
very few tests that required a specific config.
I also changed the config keys for AES encryption to live under the
"spark.network." namespace, which is more correct than their previous
names under "spark.authenticate.".
Tested via existing unit test.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #16200 from vanzin/SPARK-18773.
Diffstat (limited to 'core/src/main')
-rw-r--r-- | core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala | 6 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala | 18 |
2 files changed, 11 insertions, 13 deletions
diff --git a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala index 86874e2067..df520f804b 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala @@ -17,6 +17,8 @@ package org.apache.spark.network.netty +import scala.collection.JavaConverters._ + import org.apache.spark.SparkConf import org.apache.spark.network.util.{ConfigProvider, TransportConf} @@ -58,6 +60,10 @@ object SparkTransportConf { new TransportConf(module, new ConfigProvider { override def get(name: String): String = conf.get(name) + + override def getAll(): java.lang.Iterable[java.util.Map.Entry[String, String]] = { + conf.getAll.toMap.asJava.entrySet() + } }) } diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala index 8e3436f134..cdd3b8d851 100644 --- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala +++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala @@ -21,12 +21,15 @@ import java.util.Properties import javax.crypto.KeyGenerator import javax.crypto.spec.{IvParameterSpec, SecretKeySpec} +import scala.collection.JavaConverters._ + import org.apache.commons.crypto.random._ import org.apache.commons.crypto.stream._ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ +import org.apache.spark.network.util.CryptoUtils /** * A util class for manipulating IO encryption and decryption streams. @@ -37,8 +40,6 @@ private[spark] object CryptoStreamUtils extends Logging { val IV_LENGTH_IN_BYTES = 16 // The prefix of IO encryption related configurations in Spark configuration. val SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX = "spark.io.encryption.commons.config." - // The prefix for the configurations passing to Apache Commons Crypto library. - val COMMONS_CRYPTO_CONF_PREFIX = "commons.crypto." /** * Helper method to wrap `OutputStream` with `CryptoOutputStream` for encryption. @@ -70,18 +71,9 @@ private[spark] object CryptoStreamUtils extends Logging { new SecretKeySpec(key, "AES"), new IvParameterSpec(iv)) } - /** - * Get Commons-crypto configurations from Spark configurations identified by prefix. - */ def toCryptoConf(conf: SparkConf): Properties = { - val props = new Properties() - conf.getAll.foreach { case (k, v) => - if (k.startsWith(SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX)) { - props.put(COMMONS_CRYPTO_CONF_PREFIX + k.substring( - SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX.length()), v) - } - } - props + CryptoUtils.toCryptoConf(SPARK_IO_ENCRYPTION_COMMONS_CONFIG_PREFIX, + conf.getAll.toMap.asJava.entrySet()) } /** |