aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYin Huai <yhuai@databricks.com>2015-08-13 15:08:57 +0800
committerCheng Lian <lian@databricks.com>2015-08-13 15:08:57 +0800
commit84a27916a62980c8fcb0977c3a7fdb73c0bd5812 (patch)
tree665f3d140705195e58c2d87659b193cc0ee9eaa0
parent68f99571492f67596b3656e9f076deeb96616f4a (diff)
downloadspark-84a27916a62980c8fcb0977c3a7fdb73c0bd5812.tar.gz
spark-84a27916a62980c8fcb0977c3a7fdb73c0bd5812.tar.bz2
spark-84a27916a62980c8fcb0977c3a7fdb73c0bd5812.zip
[SPARK-9885] [SQL] Also pass barrierPrefixes and sharedPrefixes to IsolatedClientLoader when hiveMetastoreJars is set to maven.
https://issues.apache.org/jira/browse/SPARK-9885 cc marmbrus liancheng Author: Yin Huai <yhuai@databricks.com> Closes #8158 from yhuai/classloaderMaven.
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala6
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala11
2 files changed, 14 insertions, 3 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index f17177a771..17762649fd 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -231,7 +231,11 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) with Logging {
// TODO: Support for loading the jars from an already downloaded location.
logInfo(
s"Initializing HiveMetastoreConnection version $hiveMetastoreVersion using maven.")
- IsolatedClientLoader.forVersion(hiveMetastoreVersion, allConfig)
+ IsolatedClientLoader.forVersion(
+ version = hiveMetastoreVersion,
+ config = allConfig,
+ barrierPrefixes = hiveMetastoreBarrierPrefixes,
+ sharedPrefixes = hiveMetastoreSharedPrefixes)
} else {
// Convert to files and expand any directories.
val jars =
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index a7d5a99194..7856037508 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -42,11 +42,18 @@ private[hive] object IsolatedClientLoader {
def forVersion(
version: String,
config: Map[String, String] = Map.empty,
- ivyPath: Option[String] = None): IsolatedClientLoader = synchronized {
+ ivyPath: Option[String] = None,
+ sharedPrefixes: Seq[String] = Seq.empty,
+ barrierPrefixes: Seq[String] = Seq.empty): IsolatedClientLoader = synchronized {
val resolvedVersion = hiveVersion(version)
val files = resolvedVersions.getOrElseUpdate(resolvedVersion,
downloadVersion(resolvedVersion, ivyPath))
- new IsolatedClientLoader(hiveVersion(version), files, config)
+ new IsolatedClientLoader(
+ version = hiveVersion(version),
+ execJars = files,
+ config = config,
+ sharedPrefixes = sharedPrefixes,
+ barrierPrefixes = barrierPrefixes)
}
def hiveVersion(version: String): HiveVersion = version match {