aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrew Or <andrew@databricks.com>2016-05-09 11:24:58 -0700
committerAndrew Or <andrew@databricks.com>2016-05-09 11:24:58 -0700
commit7bf9b12019bb20470b726a7233d60ce38a9c52cc (patch)
treee924f574465971ab13735b19cb34bce5b3ca57cc
parentdfdcab00c7b6200c22883baa3ebc5818be09556f (diff)
downloadspark-7bf9b12019bb20470b726a7233d60ce38a9c52cc.tar.gz
spark-7bf9b12019bb20470b726a7233d60ce38a9c52cc.tar.bz2
spark-7bf9b12019bb20470b726a7233d60ce38a9c52cc.zip
[SPARK-15166][SQL] Move some hive-specific code from SparkSession
## What changes were proposed in this pull request? This also simplifies the code being moved. ## How was this patch tested? Existing tests. Author: Andrew Or <andrew@databricks.com> Closes #12941 from andrewor14/move-code.
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala13
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala2
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala17
3 files changed, 13 insertions, 19 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 2a893c6478..c7fa8f71bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -73,19 +73,6 @@ class SparkSession private(
| Session-related state |
* ----------------------- */
- {
- val defaultWarehousePath =
- SQLConf.WAREHOUSE_PATH
- .defaultValueString
- .replace("${system:user.dir}", System.getProperty("user.dir"))
- val warehousePath = sparkContext.conf.get(
- SQLConf.WAREHOUSE_PATH.key,
- defaultWarehousePath)
- sparkContext.conf.set(SQLConf.WAREHOUSE_PATH.key, warehousePath)
- sparkContext.conf.set("hive.metastore.warehouse.dir", warehousePath)
- logInfo(s"Setting warehouse location to $warehousePath")
- }
-
/**
* State shared across sessions, including the [[SparkContext]], cached data, listener,
* and a catalog that interacts with external systems.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
index 31f28f205f..46579ecd85 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.hive
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.Analyzer
import org.apache.spark.sql.execution.SparkPlanner
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
index 0ea5ce9196..f0d96403e8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
@@ -18,8 +18,9 @@
package org.apache.spark.sql.hive
import org.apache.spark.SparkContext
-import org.apache.spark.sql.hive.client.{HiveClient, HiveClientImpl}
-import org.apache.spark.sql.internal.SharedState
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.hive.client.HiveClient
+import org.apache.spark.sql.internal.{SharedState, SQLConf}
/**
@@ -27,9 +28,17 @@ import org.apache.spark.sql.internal.SharedState
* [[org.apache.spark.sql.SparkSession]] backed by Hive.
*/
private[hive] class HiveSharedState(override val sparkContext: SparkContext)
- extends SharedState(sparkContext) {
+ extends SharedState(sparkContext) with Logging {
- // TODO: just share the IsolatedClientLoader instead of the client instances themselves
+ // TODO: just share the IsolatedClientLoader instead of the client instance itself
+
+ {
+ // Set the Hive metastore warehouse path to the one we use
+ val tempConf = new SQLConf
+ sparkContext.conf.getAll.foreach { case (k, v) => tempConf.setConfString(k, v) }
+ sparkContext.conf.set("hive.metastore.warehouse.dir", tempConf.warehousePath)
+ logInfo(s"Setting Hive metastore warehouse path to '${tempConf.warehousePath}'")
+ }
/**
* A Hive client used to interact with the metastore.