From b4fbe140be158f576706f21fa69f40d6e14e4a43 Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Mon, 11 Jul 2016 15:20:48 -0700 Subject: [SPARK-16349][SQL] Fall back to isolated class loader when classes not found. Some Hadoop classes needed by the Hive metastore client jars are not present in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So if the parent class loader fails to find a class, try to load it from the isolated class loader, in case it's available there. Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop libraries and verifying that Spark can talk to the metastore. Author: Marcelo Vanzin Closes #14020 from vanzin/SPARK-16349. --- .../apache/spark/sql/hive/client/IsolatedClientLoader.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index e1950d181d..a72a13b778 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader( logDebug(s"hive class: $name - ${getResource(classToPath(name))}") super.loadClass(name, resolve) } else { - // For shared classes, we delegate to baseClassLoader. + // For shared classes, we delegate to baseClassLoader, but fall back in case the + // class is not found. logDebug(s"shared class: $name") - baseClassLoader.loadClass(name) + try { + baseClassLoader.loadClass(name) + } catch { + case _: ClassNotFoundException => + super.loadClass(name, resolve) + } } } } @@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader( throw new ClassNotFoundException( s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" + "Please make sure that jars for your version of hive and hadoop are included in the " + - s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e) + s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e) } else { throw e } -- cgit v1.2.3