aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authornavis.ryu <navis@apache.org>2015-11-06 17:13:46 -0800
committerMichael Armbrust <michael@databricks.com>2015-11-06 17:13:46 -0800
commit1c80d66e52c0bcc4e5adda78b3d8e5bf55e4f128 (patch)
tree93c3d19e9ba7701cdd81f0118ab1daeef3713607 /sql
parent6d0ead322e72303c6444c6ac641378a4690cde96 (diff)
downloadspark-1c80d66e52c0bcc4e5adda78b3d8e5bf55e4f128.tar.gz
spark-1c80d66e52c0bcc4e5adda78b3d8e5bf55e4f128.tar.bz2
spark-1c80d66e52c0bcc4e5adda78b3d8e5bf55e4f128.zip
[SPARK-11546] Thrift server makes too many logs about result schema
SparkExecuteStatementOperation logs result schema for each getNextRowSet() calls which is by default every 1000 rows, overwhelming whole log file. Author: navis.ryu <navis@apache.org> Closes #9514 from navis/SPARK-11546.
Diffstat (limited to 'sql')
-rw-r--r--sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala24
1 files changed, 13 insertions, 11 deletions
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 719b03e1c7..82fef92dcb 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -53,6 +53,18 @@ private[hive] class SparkExecuteStatementOperation(
private var dataTypes: Array[DataType] = _
private var statementId: String = _
+ private lazy val resultSchema: TableSchema = {
+ if (result == null || result.queryExecution.analyzed.output.size == 0) {
+ new TableSchema(Arrays.asList(new FieldSchema("Result", "string", "")))
+ } else {
+ logInfo(s"Result Schema: ${result.queryExecution.analyzed.output}")
+ val schema = result.queryExecution.analyzed.output.map { attr =>
+ new FieldSchema(attr.name, HiveMetastoreTypes.toMetastoreType(attr.dataType), "")
+ }
+ new TableSchema(schema.asJava)
+ }
+ }
+
def close(): Unit = {
// RDDs will be cleaned automatically upon garbage collection.
hiveContext.sparkContext.clearJobGroup()
@@ -120,17 +132,7 @@ private[hive] class SparkExecuteStatementOperation(
}
}
- def getResultSetSchema: TableSchema = {
- if (result == null || result.queryExecution.analyzed.output.size == 0) {
- new TableSchema(Arrays.asList(new FieldSchema("Result", "string", "")))
- } else {
- logInfo(s"Result Schema: ${result.queryExecution.analyzed.output}")
- val schema = result.queryExecution.analyzed.output.map { attr =>
- new FieldSchema(attr.name, HiveMetastoreTypes.toMetastoreType(attr.dataType), "")
- }
- new TableSchema(schema.asJava)
- }
- }
+ def getResultSetSchema: TableSchema = resultSchema
override def run(): Unit = {
setState(OperationState.PENDING)