diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2016-07-11 15:20:48 -0700 |
---|---|---|
committer | Yin Huai <yhuai@databricks.com> | 2016-07-11 15:20:48 -0700 |
commit | b4fbe140be158f576706f21fa69f40d6e14e4a43 (patch) | |
tree | 0238a12ddf8c15a2d20148571327fb543a9ddf35 | |
parent | 7f38b9d5f469b2550bc481cbf9adb9acc3779712 (diff) | |
download | spark-b4fbe140be158f576706f21fa69f40d6e14e4a43.tar.gz spark-b4fbe140be158f576706f21fa69f40d6e14e4a43.tar.bz2 spark-b4fbe140be158f576706f21fa69f40d6e14e4a43.zip |
[SPARK-16349][SQL] Fall back to isolated class loader when classes not found.
Some Hadoop classes needed by the Hive metastore client jars are not present
in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So
if the parent class loader fails to find a class, try to load it from the
isolated class loader, in case it's available there.
Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop
libraries and verifying that Spark can talk to the metastore.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #14020 from vanzin/SPARK-16349.
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala | 12 |
1 files changed, 9 insertions, 3 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index e1950d181d..a72a13b778 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader( logDebug(s"hive class: $name - ${getResource(classToPath(name))}") super.loadClass(name, resolve) } else { - // For shared classes, we delegate to baseClassLoader. + // For shared classes, we delegate to baseClassLoader, but fall back in case the + // class is not found. logDebug(s"shared class: $name") - baseClassLoader.loadClass(name) + try { + baseClassLoader.loadClass(name) + } catch { + case _: ClassNotFoundException => + super.loadClass(name, resolve) + } } } } @@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader( throw new ClassNotFoundException( s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" + "Please make sure that jars for your version of hive and hadoop are included in the " + - s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e) + s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e) } else { throw e } |