diff options
author | Daoyuan Wang <daoyuan.wang@intel.com> | 2015-03-25 18:43:26 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-03-25 18:43:26 -0700 |
commit | 5ab6e9f0c06b763b06a0e1396bdb08a823146c32 (patch) | |
tree | 88c12ca4d9b026d035df2d8b2921f4183435634f | |
parent | 328daf65f8e320d3bdfd12397a8f85891f1d14c7 (diff) | |
download | spark-5ab6e9f0c06b763b06a0e1396bdb08a823146c32.tar.gz spark-5ab6e9f0c06b763b06a0e1396bdb08a823146c32.tar.bz2 spark-5ab6e9f0c06b763b06a0e1396bdb08a823146c32.zip |
[SPARK-6202] [SQL] enable variable substitution on test framework
Author: Daoyuan Wang <daoyuan.wang@intel.com>
Closes #4930 from adrian-wang/testvs and squashes the following commits:
2ce590f [Daoyuan Wang] add explicit function types
b1d68bf [Daoyuan Wang] only substitute for parseSql
9c4a950 [Daoyuan Wang] add a comment explaining
18fb481 [Daoyuan Wang] enable variable substitute on test framework
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala index dc61e9d2e3..a3497eadd6 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -23,6 +23,7 @@ import java.util.{Set => JavaSet} import org.apache.hadoop.hive.ql.exec.FunctionRegistry import org.apache.hadoop.hive.ql.io.avro.{AvroContainerInputFormat, AvroContainerOutputFormat} import org.apache.hadoop.hive.ql.metadata.Table +import org.apache.hadoop.hive.ql.parse.VariableSubstitution import org.apache.hadoop.hive.ql.processors._ import org.apache.hadoop.hive.serde2.RegexSerDe import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe @@ -153,8 +154,13 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { val describedTable = "DESCRIBE (\\w+)".r + val vs = new VariableSubstitution() + + // we should substitute variables in hql to pass the text to parseSql() as a parameter. + // Hive parser need substituted text. HiveContext.sql() does this but return a DataFrame, + // while we need a logicalPlan so we cannot reuse that. protected[hive] class HiveQLQueryExecution(hql: String) - extends this.QueryExecution(HiveQl.parseSql(hql)) { + extends this.QueryExecution(HiveQl.parseSql(vs.substitute(hiveconf, hql))) { def hiveExec(): Seq[String] = runSqlHive(hql) override def toString: String = hql + "\n" + super.toString } |