diff options
author | Yin Huai <yhuai@databricks.com> | 2015-06-02 00:16:56 -0700 |
---|---|---|
committer | Yin Huai <yhuai@databricks.com> | 2015-06-02 00:17:09 -0700 |
commit | 4940630f56d3e95a01526bf1fdfc88517b8e661b (patch) | |
tree | 33430b9ebc66ebd17e8ea1e51233cf949aab56f8 | |
parent | 9d6475b93d1e35e2cd5d7ffd1ef28faf6ce0a425 (diff) | |
download | spark-4940630f56d3e95a01526bf1fdfc88517b8e661b.tar.gz spark-4940630f56d3e95a01526bf1fdfc88517b8e661b.tar.bz2 spark-4940630f56d3e95a01526bf1fdfc88517b8e661b.zip |
[SPARK-8020] [SQL] Spark SQL conf in spark-defaults.conf make metadataHive get constructed too early
https://issues.apache.org/jira/browse/SPARK-8020
Author: Yin Huai <yhuai@databricks.com>
Closes #6571 from yhuai/SPARK-8020-1 and squashes the following commits:
0398f5b [Yin Huai] First populate the SQLConf and then construct executionHive and metadataHive.
(cherry picked from commit 7b7f7b6c6fd903e2ecfc886d29eaa9df58adcfc3)
Signed-off-by: Yin Huai <yhuai@databricks.com>
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 25 |
1 files changed, 22 insertions, 3 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 7384b24c50..91e6385dec 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -182,9 +182,28 @@ class SQLContext(@transient val sparkContext: SparkContext) conf.dialect } - sparkContext.getConf.getAll.foreach { - case (key, value) if key.startsWith("spark.sql") => setConf(key, value) - case _ => + { + // We extract spark sql settings from SparkContext's conf and put them to + // Spark SQL's conf. + // First, we populate the SQLConf (conf). So, we can make sure that other values using + // those settings in their construction can get the correct settings. + // For example, metadataHive in HiveContext may need both spark.sql.hive.metastore.version + // and spark.sql.hive.metastore.jars to get correctly constructed. + val properties = new Properties + sparkContext.getConf.getAll.foreach { + case (key, value) if key.startsWith("spark.sql") => properties.setProperty(key, value) + case _ => + } + // We directly put those settings to conf to avoid of calling setConf, which may have + // side-effects. For example, in HiveContext, setConf may cause executionHive and metadataHive + // get constructed. If we call setConf directly, the constructed metadataHive may have + // wrong settings, or the construction may fail. + conf.setConf(properties) + // After we have populated SQLConf, we call setConf to populate other confs in the subclass + // (e.g. hiveconf in HiveContext). + properties.foreach { + case (key, value) => setConf(key, value) + } } @transient |