aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Armbrust <michael@databricks.com>2014-07-15 14:01:48 -0700
committerMichael Armbrust <michael@databricks.com>2014-07-15 14:02:14 -0700
commit3aa120cd4914add55b1661237416de3df775445d (patch)
treeca88f6dc3fa81aa8ee479276e906433e227d7e31
parent53a6399e54b1843528e1a0b45de5996694747957 (diff)
downloadspark-3aa120cd4914add55b1661237416de3df775445d.tar.gz
spark-3aa120cd4914add55b1661237416de3df775445d.tar.bz2
spark-3aa120cd4914add55b1661237416de3df775445d.zip
[SPARK-2483][SQL] Fix parsing of repeated, nested data access.
Author: Michael Armbrust <michael@databricks.com> Closes #1411 from marmbrus/nestedRepeated and squashes the following commits: 044fa09 [Michael Armbrust] Fix parsing of repeated, nested data access. (cherry picked from commit 0f98ef1a2c9ecf328f6c5918808fa5ca486e8afd) Signed-off-by: Michael Armbrust <michael@databricks.com>
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala5
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala10
2 files changed, 9 insertions, 6 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index b70104dd5b..56aa27a208 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -870,10 +870,7 @@ private[hive] object HiveQl {
nodeToExpr(qualifier) match {
case UnresolvedAttribute(qualifierName) =>
UnresolvedAttribute(qualifierName + "." + cleanIdentifier(attr))
- // The precidence for . seems to be wrong, so [] binds tighter an we need to go inside to
- // find the underlying attribute references.
- case GetItem(UnresolvedAttribute(qualifierName), ordinal) =>
- GetItem(UnresolvedAttribute(qualifierName + "." + cleanIdentifier(attr)), ordinal)
+ case other => GetField(other, attr)
}
/* Stars (*) */
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
index 67594b57d3..fb03db12a0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.hive.test.TestHive._
-case class Data(a: Int, B: Int, n: Nested)
+case class Data(a: Int, B: Int, n: Nested, nestedArray: Seq[Nested])
case class Nested(a: Int, B: Int)
/**
@@ -53,12 +53,18 @@ class HiveResolutionSuite extends HiveComparisonTest {
test("case insensitivity with scala reflection") {
// Test resolution with Scala Reflection
- TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2)) :: Nil)
+ TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2), Seq(Nested(1,2))) :: Nil)
.registerAsTable("caseSensitivityTest")
hql("SELECT a, b, A, B, n.a, n.b, n.A, n.B FROM caseSensitivityTest")
}
+ test("nested repeated resolution") {
+ TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2), Seq(Nested(1,2))) :: Nil)
+ .registerAsTable("nestedRepeatedTest")
+ assert(hql("SELECT nestedArray[0].a FROM nestedRepeatedTest").collect().head(0) === 1)
+ }
+
/**
* Negative examples. Currently only left here for documentation purposes.
* TODO(marmbrus): Test that catalyst fails on these queries.