aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Armbrust <michael@databricks.com>2015-03-02 16:10:54 -0800
committerMichael Armbrust <michael@databricks.com>2015-03-02 16:10:54 -0800
commit8223ce6a81e4cc9fdf816892365fcdff4006c35e (patch)
treeffe535079f37013b96a50d6f78a125a3d96894f2
parent26c1c56dea5d4160913bb65bb743aeb63fee3240 (diff)
downloadspark-8223ce6a81e4cc9fdf816892365fcdff4006c35e.tar.gz
spark-8223ce6a81e4cc9fdf816892365fcdff4006c35e.tar.bz2
spark-8223ce6a81e4cc9fdf816892365fcdff4006c35e.zip
[SPARK-6114][SQL] Avoid metastore conversions before plan is resolved
Author: Michael Armbrust <michael@databricks.com> Closes #4855 from marmbrus/explodeBug and squashes the following commits: a712249 [Michael Armbrust] [SPARK-6114][SQL] Avoid metastore conversions before plan is resolved
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala4
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala10
2 files changed, 14 insertions, 0 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index d3ad364328..74b4e767ca 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -444,6 +444,10 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
*/
object ParquetConversions extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = {
+ if (!plan.resolved) {
+ return plan
+ }
+
// Collects all `MetastoreRelation`s which should be replaced
val toBeReplaced = plan.collect {
// Write path
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index f2bc73bf3b..22ea19bd82 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -31,6 +31,9 @@ case class Nested1(f1: Nested2)
case class Nested2(f2: Nested3)
case class Nested3(f3: Int)
+case class NestedArray2(b: Seq[Int])
+case class NestedArray1(a: NestedArray2)
+
/**
* A collection of hive query tests where we generate the answers ourselves instead of depending on
* Hive to generate them (in contrast to HiveQuerySuite). Often this is because the query is
@@ -38,6 +41,13 @@ case class Nested3(f3: Int)
*/
class SQLQuerySuite extends QueryTest {
+ test("explode nested Field") {
+ Seq(NestedArray1(NestedArray2(Seq(1,2,3)))).toDF.registerTempTable("nestedArray")
+ checkAnswer(
+ sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
+ Row(1) :: Row(2) :: Row(3) :: Nil)
+ }
+
test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") {
checkAnswer(
sql("SELECT * FROM (SELECT key + key AS a FROM src SORT BY value) t ORDER BY t.a"),