diff options
author | jerryshao <saisai.shao@intel.com> | 2013-09-18 17:33:24 +0800 |
---|---|---|
committer | jerryshao <saisai.shao@intel.com> | 2013-09-18 17:33:24 +0800 |
commit | ffa5f8e11db26dd616e85b9d941de3590ca3643e (patch) | |
tree | 758334615814c178dd1b1e21ffa2913edb399c97 | |
parent | 2aff7989ab617f33052098498119886c40794774 (diff) | |
download | spark-ffa5f8e11db26dd616e85b9d941de3590ca3643e.tar.gz spark-ffa5f8e11db26dd616e85b9d941de3590ca3643e.tar.bz2 spark-ffa5f8e11db26dd616e85b9d941de3590ca3643e.zip |
Fix issue when local properties pass from parent to child thread
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkContext.scala | 6 | ||||
-rw-r--r-- | core/src/test/scala/org/apache/spark/ThreadingSuite.scala | 38 |
2 files changed, 42 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 72540c712a..3922e9a7fa 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -256,7 +256,9 @@ class SparkContext( private[spark] var checkpointDir: Option[String] = None // Thread Local variable that can be used by users to pass information down the stack - private val localProperties = new ThreadLocal[Properties] + private val localProperties = new InheritableThreadLocal[Properties] { + override protected def childValue(parent: Properties): Properties = new Properties(parent) + } def initLocalProperties() { localProperties.set(new Properties()) @@ -273,6 +275,8 @@ class SparkContext( } } + def getLocalProperty(key: String): String = Option(localProperties.get).map(_.getProperty(key)).getOrElse(null) + /** Set a human readable description of the current job. */ def setJobDescription(value: String) { setLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION, value) diff --git a/core/src/test/scala/org/apache/spark/ThreadingSuite.scala b/core/src/test/scala/org/apache/spark/ThreadingSuite.scala index 69383ddfb8..331f79dba1 100644 --- a/core/src/test/scala/org/apache/spark/ThreadingSuite.scala +++ b/core/src/test/scala/org/apache/spark/ThreadingSuite.scala @@ -40,7 +40,7 @@ object ThreadingSuiteState { } class ThreadingSuite extends FunSuite with LocalSparkContext { - + test("accessing SparkContext form a different thread") { sc = new SparkContext("local", "test") val nums = sc.parallelize(1 to 10, 2) @@ -149,4 +149,40 @@ class ThreadingSuite extends FunSuite with LocalSparkContext { fail("One or more threads didn't see runningThreads = 4") } } + + test("set local properties in different thread") { + sc = new SparkContext("local", "test") + + val threads = (1 to 5).map{ i => + new Thread() { + override def run() { + sc.setLocalProperty("test", i.toString) + assert(sc.getLocalProperty("test") === i.toString) + } + } + } + + threads.foreach(_.start()) + + assert(sc.getLocalProperty("test") === null) + } + + test("set and get local properties in parent-children thread") { + sc = new SparkContext("local", "test") + sc.setLocalProperty("test", "parent") + + val threads = (1 to 5).map{ i => + new Thread() { + override def run() { + assert(sc.getLocalProperty("test") === "parent") + sc.setLocalProperty("test", i.toString) + assert(sc.getLocalProperty("test") === i.toString) + } + } + } + + threads.foreach(_.start()) + assert(sc.getLocalProperty("test") === "parent") + assert(sc.getLocalProperty("Foo") === null) + } } |