aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDongjoon Hyun <dongjoon@apache.org>2016-10-03 21:28:16 -0700
committerReynold Xin <rxin@databricks.com>2016-10-03 21:28:16 -0700
commitc571cfb2d0e1e224107fc3f0c672730cae9804cb (patch)
tree8a0ca7e3262d9c3ee84b334fe02fbe39f484669c
parent2bbecdec2023143fd144e4242ff70822e0823986 (diff)
downloadspark-c571cfb2d0e1e224107fc3f0c672730cae9804cb.tar.gz
spark-c571cfb2d0e1e224107fc3f0c672730cae9804cb.tar.bz2
spark-c571cfb2d0e1e224107fc3f0c672730cae9804cb.zip
[SPARK-17112][SQL] "select null" via JDBC triggers IllegalArgumentException in Thriftserver
## What changes were proposed in this pull request? Currently, Spark Thrift Server raises `IllegalArgumentException` for queries whose column types are `NullType`, e.g., `SELECT null` or `SELECT if(true,null,null)`. This PR fixes that by returning `void` like Hive 1.2. **Before** ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ Error: java.lang.IllegalArgumentException: Unrecognized type name: null (state=,code=0) Closing: 0: jdbc:hive2://localhost:10000 $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ Error: java.lang.IllegalArgumentException: Unrecognized type name: null (state=,code=0) Closing: 0: jdbc:hive2://localhost:10000 ``` **After** ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ +-------+--+ | NULL | +-------+--+ | NULL | +-------+--+ 1 row selected (3.242 seconds) Beeline version 1.2.1.spark2 by Apache Hive Closing: 0: jdbc:hive2://localhost:10000 $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ +-------------------------+--+ | (IF(true, NULL, NULL)) | +-------------------------+--+ | NULL | +-------------------------+--+ 1 row selected (0.201 seconds) Beeline version 1.2.1.spark2 by Apache Hive Closing: 0: jdbc:hive2://localhost:10000 ``` ## How was this patch tested? * Pass the Jenkins test with a new testsuite. * Also, Manually, after starting Spark Thrift Server, run the following command. ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" ``` **Hive 1.2** ```sql hive> create table null_table as select null; hive> desc null_table; OK _c0 void ``` Author: Dongjoon Hyun <dongjoon@apache.org> Closes #15325 from dongjoon-hyun/SPARK-17112.
-rw-r--r--sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala19
-rw-r--r--sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala33
2 files changed, 46 insertions, 6 deletions
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index e555ebd623..aeabd6a158 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -56,14 +56,11 @@ private[hive] class SparkExecuteStatementOperation(
private var statementId: String = _
private lazy val resultSchema: TableSchema = {
- if (result == null || result.queryExecution.analyzed.output.size == 0) {
+ if (result == null || result.schema.isEmpty) {
new TableSchema(Arrays.asList(new FieldSchema("Result", "string", "")))
} else {
- logInfo(s"Result Schema: ${result.queryExecution.analyzed.output}")
- val schema = result.queryExecution.analyzed.output.map { attr =>
- new FieldSchema(attr.name, attr.dataType.catalogString, "")
- }
- new TableSchema(schema.asJava)
+ logInfo(s"Result Schema: ${result.schema}")
+ SparkExecuteStatementOperation.getTableSchema(result.schema)
}
}
@@ -282,3 +279,13 @@ private[hive] class SparkExecuteStatementOperation(
}
}
}
+
+object SparkExecuteStatementOperation {
+ def getTableSchema(structType: StructType): TableSchema = {
+ val schema = structType.map { field =>
+ val attrTypeString = if (field.dataType == NullType) "void" else field.dataType.catalogString
+ new FieldSchema(field.name, attrTypeString, "")
+ }
+ new TableSchema(schema.asJava)
+ }
+}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
new file mode 100644
index 0000000000..32ded0d254
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.types.{NullType, StructField, StructType}
+
+class SparkExecuteStatementOperationSuite extends SparkFunSuite {
+ test("SPARK-17112 `select null` via JDBC triggers IllegalArgumentException in ThriftServer") {
+ val field1 = StructField("NULL", NullType)
+ val field2 = StructField("(IF(true, NULL, NULL))", NullType)
+ val tableSchema = StructType(Seq(field1, field2))
+ val columns = SparkExecuteStatementOperation.getTableSchema(tableSchema).getColumnDescriptors()
+ assert(columns.size() == 2)
+ assert(columns.get(0).getType() == org.apache.hive.service.cli.Type.NULL_TYPE)
+ assert(columns.get(1).getType() == org.apache.hive.service.cli.Type.NULL_TYPE)
+ }
+}