diff options
| author | Wenchen Fan <wenchen@databricks.com> | 2016-05-04 00:10:20 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2016-05-04 00:10:20 -0700 |
| commit | 6c12e801e84565265d2939b920901d1805d5935f (patch) | |
| tree | b09ff9210ae23abe346523da253baa9e751fc6da /sql/hive/src/main | |
| parent | bc3760d405cc8c3ffcd957b188afa8b7e3b1f824 (diff) | |
| download | spark-6c12e801e84565265d2939b920901d1805d5935f.tar.gz spark-6c12e801e84565265d2939b920901d1805d5935f.tar.bz2 spark-6c12e801e84565265d2939b920901d1805d5935f.zip | |
[SPARK-15029] improve error message for Generate
## What changes were proposed in this pull request?
This PR improve the error message for `Generate` in 3 cases:
1. generator is nested in expressions, e.g. `SELECT explode(list) + 1 FROM tbl`
2. generator appears more than one time in SELECT, e.g. `SELECT explode(list), explode(list) FROM tbl`
3. generator appears in other operator which is not project, e.g. `SELECT * FROM tbl SORT BY explode(list)`
## How was this patch tested?
new tests in `AnalysisErrorSuite`
Author: Wenchen Fan <wenchen@databricks.com>
Closes #12810 from cloud-fan/bug.
Diffstat (limited to 'sql/hive/src/main')
| -rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala | 2 | ||||
| -rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala | 7 |
2 files changed, 5 insertions, 4 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala index f023edbd96..3220f143aa 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala @@ -145,7 +145,7 @@ private[sql] class HiveSessionCatalog( udaf } else if (classOf[GenericUDTF].isAssignableFrom(clazz)) { val udtf = HiveGenericUDTF(name, new HiveFunctionWrapper(clazz.getName), children) - udtf.elementTypes // Force it to check input data types. + udtf.elementSchema // Force it to check input data types. udtf } else { throw new AnalysisException(s"No handler for Hive UDF '${clazz.getCanonicalName}'") diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala index 5aab4132bc..c53675694f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala @@ -202,9 +202,10 @@ private[hive] case class HiveGenericUDTF( @transient protected lazy val collector = new UDTFCollector - override lazy val elementTypes = outputInspector.getAllStructFieldRefs.asScala.map { - field => (inspectorToDataType(field.getFieldObjectInspector), true, field.getFieldName) - } + override lazy val elementSchema = StructType(outputInspector.getAllStructFieldRefs.asScala.map { + field => StructField(field.getFieldName, inspectorToDataType(field.getFieldObjectInspector), + nullable = true) + }) @transient private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray |