diff options
author | Davies Liu <davies@databricks.com> | 2015-01-21 16:51:42 -0800 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2015-01-21 16:51:54 -0800 |
commit | 9bad062268676aaa66dcbddd1e0ab7f2d7742425 (patch) | |
tree | ee88b40ec5bb1d617ae0f1a63d4bbfe2e5f6f58c /core | |
parent | 3be2a887bf6107b6398e472872b22175ea4ae1f7 (diff) | |
download | spark-9bad062268676aaa66dcbddd1e0ab7f2d7742425.tar.gz spark-9bad062268676aaa66dcbddd1e0ab7f2d7742425.tar.bz2 spark-9bad062268676aaa66dcbddd1e0ab7f2d7742425.zip |
[SPARK-5355] make SparkConf thread-safe
The SparkConf is not thread-safe, but is accessed by many threads. The getAll() could return parts of the configs if another thread is access it.
This PR changes SparkConf.settings to a thread-safe TrieMap.
Author: Davies Liu <davies@databricks.com>
Closes #4143 from davies/safe-conf and squashes the following commits:
f8fa1cf [Davies Liu] change to TrieMap
a1d769a [Davies Liu] make SparkConf thread-safe
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkConf.scala | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index a0ce107f43..f9d4aa4240 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -18,6 +18,7 @@ package org.apache.spark import scala.collection.JavaConverters._ +import scala.collection.concurrent.TrieMap import scala.collection.mutable.{HashMap, LinkedHashSet} import org.apache.spark.serializer.KryoSerializer @@ -46,7 +47,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging { /** Create a SparkConf that loads defaults from system properties and the classpath */ def this() = this(true) - private[spark] val settings = new HashMap[String, String]() + private[spark] val settings = new TrieMap[String, String]() if (loadDefaults) { // Load any spark.* system properties @@ -177,7 +178,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging { } /** Get all parameters as a list of pairs */ - def getAll: Array[(String, String)] = settings.clone().toArray + def getAll: Array[(String, String)] = settings.toArray /** Get a parameter as an integer, falling back to a default if not set */ def getInt(key: String, defaultValue: Int): Int = { |