diff options
author | Bouke van der Bijl <boukevanderbijl@gmail.com> | 2014-05-05 11:19:35 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-05-05 11:19:36 -0700 |
commit | 3292e2a71bfb5df5ba156cf7557747d164d12291 (patch) | |
tree | 97911ecad7a0231b56cbd439b1996362cb2dd008 /core | |
parent | 73b0cbcc241cca3d318ff74340e80b02f884acbd (diff) | |
download | spark-3292e2a71bfb5df5ba156cf7557747d164d12291.tar.gz spark-3292e2a71bfb5df5ba156cf7557747d164d12291.tar.bz2 spark-3292e2a71bfb5df5ba156cf7557747d164d12291.zip |
SPARK-1721: Reset the thread classLoader in the Mesos Executor
This is because Mesos calls it with a different environment or something, the result is that the Spark jar is missing and it can't load classes.
This fixes http://apache-spark-user-list.1001560.n3.nabble.com/java-lang-ClassNotFoundException-spark-on-mesos-td3510.html
I have no idea whether this is the right fix, I can only confirm that it fixes the issue for us.
The `registered` method is called from mesos (https://github.com/apache/mesos/blob/765ff9bc2ac5a12d4362f8235b572a37d646390a/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp)
I am unsure which commit caused this regression
Author: Bouke van der Bijl <boukevanderbijl@gmail.com>
Closes #620 from bouk/mesos-classloader-fix and squashes the following commits:
c13eae0 [Bouke van der Bijl] Use getContextOrSparkClassLoader in SparkEnv and CompressionCodec
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkEnv.scala | 4 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/io/CompressionCodec.scala | 3 |
2 files changed, 3 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index bea435ec34..d40ed27da5 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -156,13 +156,11 @@ object SparkEnv extends Logging { conf.set("spark.driver.port", boundPort.toString) } - val classLoader = Thread.currentThread.getContextClassLoader - // Create an instance of the class named by the given Java system property, or by // defaultClassName if the property is not set, and return it as a T def instantiateClass[T](propertyName: String, defaultClassName: String): T = { val name = conf.get(propertyName, defaultClassName) - val cls = Class.forName(name, true, classLoader) + val cls = Class.forName(name, true, Utils.getContextOrSparkClassLoader) // First try with the constructor that takes SparkConf. If we can't find one, // use a no-arg constructor instead. try { diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala index e1a5ee316b..4b0fe1ab82 100644 --- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala +++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala @@ -24,6 +24,7 @@ import org.xerial.snappy.{SnappyInputStream, SnappyOutputStream} import org.apache.spark.SparkConf import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.util.Utils /** * :: DeveloperApi :: @@ -49,7 +50,7 @@ private[spark] object CompressionCodec { } def createCodec(conf: SparkConf, codecName: String): CompressionCodec = { - val ctor = Class.forName(codecName, true, Thread.currentThread.getContextClassLoader) + val ctor = Class.forName(codecName, true, Utils.getContextOrSparkClassLoader) .getConstructor(classOf[SparkConf]) ctor.newInstance(conf).asInstanceOf[CompressionCodec] } |