diff options
| author | Alex Liu <alex_liu68@yahoo.com> | 2014-10-27 20:43:29 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2014-10-27 20:43:29 -0700 |
| commit | 698a7eab7710cccdfd3a5234dc9572e4e674fff6 (patch) | |
| tree | 48800a7657a92387c58fe0dfd292057002d2b2ac /sql | |
| parent | 418ad83fe113f2f90552eb7247670279b55aed28 (diff) | |
| download | spark-698a7eab7710cccdfd3a5234dc9572e4e674fff6.tar.gz spark-698a7eab7710cccdfd3a5234dc9572e4e674fff6.tar.bz2 spark-698a7eab7710cccdfd3a5234dc9572e4e674fff6.zip | |
[SPARK-3816][SQL] Add table properties from storage handler to output jobConf
...ob conf in SparkHadoopWriter class
Author: Alex Liu <alex_liu68@yahoo.com>
Closes #2677 from alexliu68/SPARK-SQL-3816 and squashes the following commits:
79c269b [Alex Liu] [SPARK-3816][SQL] Add table properties from storage handler to job conf
Diffstat (limited to 'sql')
| -rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala index 981ab954da..bf2ce9df67 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.ql.exec.{FileSinkOperator, Utilities} import org.apache.hadoop.hive.ql.io.{HiveFileFormatUtils, HiveOutputFormat} +import org.apache.hadoop.hive.ql.plan.{PlanUtils, TableDesc} import org.apache.hadoop.io.Writable import org.apache.hadoop.mapred._ @@ -47,6 +48,13 @@ private[hive] class SparkHiveWriterContainer( with Serializable { private val now = new Date() + private val tableDesc: TableDesc = fileSinkConf.getTableInfo + // Add table properties from storage handler to jobConf, so any custom storage + // handler settings can be set to jobConf + if (tableDesc != null) { + PlanUtils.configureOutputJobPropertiesForStorageHandler(tableDesc) + Utilities.copyTableJobPropertiesToConf(tableDesc, jobConf) + } protected val conf = new SerializableWritable(jobConf) private var jobID = 0 |