diff options
| author | Michael Armbrust <michael@databricks.com> | 2014-07-21 18:18:17 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2014-07-21 18:18:35 -0700 |
| commit | 84bbfbd843a5c73ff4998efe8a2dabc2574f033a (patch) | |
| tree | 4122d18fdfef4c22c6568debd112497f27d23cb7 /sql | |
| parent | cdcd467176dd7c615868053898c621d69c6c162c (diff) | |
| download | spark-84bbfbd843a5c73ff4998efe8a2dabc2574f033a.tar.gz spark-84bbfbd843a5c73ff4998efe8a2dabc2574f033a.tar.bz2 spark-84bbfbd843a5c73ff4998efe8a2dabc2574f033a.zip | |
[SPARK-2561][SQL] Fix apply schema
We need to use the analyzed attributes otherwise we end up with a tree that will never resolve.
Author: Michael Armbrust <michael@databricks.com>
Closes #1470 from marmbrus/fixApplySchema and squashes the following commits:
f968195 [Michael Armbrust] Use analyzed attributes when applying the schema.
4969015 [Michael Armbrust] Add test case.
(cherry picked from commit 511a7314037219c23e824ea5363bf7f1df55bab3)
Signed-off-by: Michael Armbrust <michael@databricks.com>
Diffstat (limited to 'sql')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 2 | ||||
| -rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala | 6 |
2 files changed, 7 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala index d5214a34e5..d0d2db27f6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala @@ -410,7 +410,7 @@ class SchemaRDD( * @group schema */ private def applySchema(rdd: RDD[Row]): SchemaRDD = { - new SchemaRDD(sqlContext, SparkLogicalPlan(ExistingRdd(logicalPlan.output, rdd))) + new SchemaRDD(sqlContext, SparkLogicalPlan(ExistingRdd(queryExecution.analyzed.output, rdd))) } // ======================================================================= diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala index 05aac66d81..0a69cbcc83 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala @@ -34,6 +34,12 @@ class DslQuerySuite extends QueryTest { testData.collect().toSeq) } + test("repartition") { + checkAnswer( + testData.select('key).repartition(10).select('key), + testData.select('key).collect().toSeq) + } + test("agg") { checkAnswer( testData2.groupBy('a)('a, Sum('b)), |