From dab37966b0cfd290919ca5c005f59dde00615c0e Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Fri, 1 Aug 2014 23:55:30 -0700 Subject: Revert "[SPARK-1470][SPARK-1842] Use the scala-logging wrapper instead of the directly sfl4j api" This reverts commit adc8303294e26efb4ed15e5f5ba1062f7988625d. --- core/pom.xml | 4 - core/src/main/scala/org/apache/spark/Logging.scala | 39 ++++------ .../scala/org/apache/spark/util/SignalLogger.scala | 2 +- mllib/pom.xml | 4 - pom.xml | 5 -- project/MimaExcludes.scala | 91 +--------------------- sql/catalyst/pom.xml | 5 ++ .../spark/sql/catalyst/analysis/Analyzer.scala | 4 +- .../sql/catalyst/analysis/HiveTypeCoercion.scala | 8 +- .../sql/catalyst/expressions/BoundAttribute.scala | 2 +- .../expressions/codegen/GenerateOrdering.scala | 4 +- .../org/apache/spark/sql/catalyst/package.scala | 1 + .../spark/sql/catalyst/planning/QueryPlanner.scala | 2 +- .../spark/sql/catalyst/planning/patterns.scala | 6 +- .../org/apache/spark/sql/catalyst/rules/Rule.scala | 2 +- .../spark/sql/catalyst/rules/RuleExecutor.scala | 12 +-- .../apache/spark/sql/catalyst/trees/package.scala | 8 +- .../scala/org/apache/spark/sql/SQLContext.scala | 2 +- .../compression/CompressibleColumnBuilder.scala | 5 +- .../org/apache/spark/sql/execution/Exchange.scala | 2 +- .../scala/org/apache/spark/sql/json/JsonRDD.scala | 2 +- .../main/scala/org/apache/spark/sql/package.scala | 2 + .../spark/sql/columnar/ColumnTypeSuite.scala | 4 +- .../sql/hive/thriftserver/HiveThriftServer2.scala | 12 +-- .../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 2 +- .../sql/hive/thriftserver/SparkSQLDriver.scala | 6 +- .../spark/sql/hive/thriftserver/SparkSQLEnv.scala | 6 +- .../server/SparkSQLOperationManager.scala | 13 ++-- .../hive/thriftserver/HiveThriftServer2Suite.scala | 2 +- .../org/apache/spark/sql/hive/HiveContext.scala | 2 +- .../spark/sql/hive/HiveMetastoreCatalog.scala | 3 +- .../scala/org/apache/spark/sql/hive/TestHive.scala | 10 +-- .../scala/org/apache/spark/sql/hive/hiveUdfs.scala | 4 +- .../sql/hive/execution/HiveComparisonTest.scala | 22 +++--- .../sql/hive/execution/HiveQueryFileTest.scala | 2 +- 35 files changed, 97 insertions(+), 203 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 47766ae5fb..7c60cf10c3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -98,10 +98,6 @@ org.slf4j jcl-over-slf4j - - com.typesafe.scala-logging - scala-logging-slf4j_${scala.binary.version} - log4j log4j diff --git a/core/src/main/scala/org/apache/spark/Logging.scala b/core/src/main/scala/org/apache/spark/Logging.scala index 6e61c00b8d..807ef3e9c9 100644 --- a/core/src/main/scala/org/apache/spark/Logging.scala +++ b/core/src/main/scala/org/apache/spark/Logging.scala @@ -18,9 +18,8 @@ package org.apache.spark import org.apache.log4j.{LogManager, PropertyConfigurator} -import org.slf4j.LoggerFactory +import org.slf4j.{Logger, LoggerFactory} import org.slf4j.impl.StaticLoggerBinder -import com.typesafe.scalalogging.slf4j.Logger import org.apache.spark.annotation.DeveloperApi import org.apache.spark.util.Utils @@ -40,69 +39,61 @@ trait Logging { // be serialized and used on another machine @transient private var log_ : Logger = null - // Method to get the logger name for this object - protected def logName = { - var className = this.getClass.getName - // Ignore trailing $'s in the class names for Scala objects - if (className.endsWith("$")) { - className = className.substring(0, className.length - 1) - } - className - } - // Method to get or create the logger for this object protected def log: Logger = { if (log_ == null) { initializeIfNecessary() - log_ = Logger(LoggerFactory.getLogger(logName)) + var className = this.getClass.getName + // Ignore trailing $'s in the class names for Scala objects + log_ = LoggerFactory.getLogger(className.stripSuffix("$")) } log_ } // Log methods that take only a String protected def logInfo(msg: => String) { - log.info(msg) + if (log.isInfoEnabled) log.info(msg) } protected def logDebug(msg: => String) { - log.debug(msg) + if (log.isDebugEnabled) log.debug(msg) } protected def logTrace(msg: => String) { - log.trace(msg) + if (log.isTraceEnabled) log.trace(msg) } protected def logWarning(msg: => String) { - log.warn(msg) + if (log.isWarnEnabled) log.warn(msg) } protected def logError(msg: => String) { - log.error(msg) + if (log.isErrorEnabled) log.error(msg) } // Log methods that take Throwables (Exceptions/Errors) too protected def logInfo(msg: => String, throwable: Throwable) { - log.info(msg, throwable) + if (log.isInfoEnabled) log.info(msg, throwable) } protected def logDebug(msg: => String, throwable: Throwable) { - log.debug(msg, throwable) + if (log.isDebugEnabled) log.debug(msg, throwable) } protected def logTrace(msg: => String, throwable: Throwable) { - log.trace(msg, throwable) + if (log.isTraceEnabled) log.trace(msg, throwable) } protected def logWarning(msg: => String, throwable: Throwable) { - log.warn(msg, throwable) + if (log.isWarnEnabled) log.warn(msg, throwable) } protected def logError(msg: => String, throwable: Throwable) { - log.error(msg, throwable) + if (log.isErrorEnabled) log.error(msg, throwable) } protected def isTraceEnabled(): Boolean = { - log.underlying.isTraceEnabled + log.isTraceEnabled } private def initializeIfNecessary() { diff --git a/core/src/main/scala/org/apache/spark/util/SignalLogger.scala b/core/src/main/scala/org/apache/spark/util/SignalLogger.scala index e84a6b951f..f77488ef3d 100644 --- a/core/src/main/scala/org/apache/spark/util/SignalLogger.scala +++ b/core/src/main/scala/org/apache/spark/util/SignalLogger.scala @@ -18,7 +18,7 @@ package org.apache.spark.util import org.apache.commons.lang3.SystemUtils -import com.typesafe.scalalogging.slf4j.Logger +import org.slf4j.Logger import sun.misc.{Signal, SignalHandler} /** diff --git a/mllib/pom.xml b/mllib/pom.xml index 3007681a44..9a33bd1cf6 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -59,10 +59,6 @@ breeze_${scala.binary.version} 0.7 - - com.typesafe - scalalogging-slf4j_${scala.binary.version} - diff --git a/pom.xml b/pom.xml index 9d62cea689..ae97bf03c5 100644 --- a/pom.xml +++ b/pom.xml @@ -279,11 +279,6 @@ slf4j-log4j12 ${slf4j.version} - - com.typesafe.scala-logging - scala-logging-slf4j_${scala.binary.version} - 2.1.2 - org.slf4j jul-to-slf4j diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index a0cee1d765..537ca0dcf2 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -103,101 +103,14 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleMethTypeProblem]( "org.apache.spark.mllib.tree.impurity.Variance.calculate") ) ++ - Seq( // Package-private classes removed in SPARK-2341 + Seq ( // Package-private classes removed in SPARK-2341 ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.BinaryLabelParser"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.BinaryLabelParser$"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.LabelParser"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.LabelParser$"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.MulticlassLabelParser"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.MulticlassLabelParser$") - ) ++ - Seq( - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.bagel.Bagel.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.StreamingContext.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.dstream.DStream.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.mllib.recommendation.ALS.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.mllib.clustering.KMeans.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.mllib.classification.NaiveBayes.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.kafka.KafkaReceiver.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.SparkContext.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.rdd.PairRDDFunctions.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.rdd.OrderedRDDFunctions.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.rdd.SequenceFileRDDFunctions.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.rdd.DoubleRDDFunctions.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.twitter.TwitterReceiver.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.zeromq.ZeroMQReceiver.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.flume.FlumeReceiver.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.rdd.RDD.log"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.SparkConf.log"), - - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.SparkConf.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.bagel.Bagel.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.StreamingContext.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.dstream.DStream.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.mllib.recommendation.ALS.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.mllib.clustering.KMeans.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.mllib.classification.NaiveBayes.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.twitter.TwitterReceiver.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.zeromq.ZeroMQReceiver.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.SparkContext.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.rdd.RDD.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.rdd.SequenceFileRDDFunctions.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.rdd.OrderedRDDFunctions.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.kafka.KafkaReceiver.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.rdd.DoubleRDDFunctions.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.flume.FlumeReceiver.org$apache$spark$Logging$$log__="), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.kafka.KafkaReceiver.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleMethTypeProblem] - ("org.apache.spark.streaming.twitter.TwitterReceiver.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.twitter.TwitterReceiver.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.zeromq.ZeroMQReceiver.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.bagel.Bagel.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.bagel.Bagel.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.flume.FlumeReceiver.org$apache$spark$Logging$$log_"), - ProblemFilters.exclude[IncompatibleResultTypeProblem] - ("org.apache.spark.streaming.kafka.KafkaReceiver.org$apache$spark$Logging$$log_") - ) + ) case v if v.startsWith("1.0") => Seq( MimaBuild.excludeSparkPackage("api.java"), diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 58d44e7923..54fa96baa1 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -54,6 +54,11 @@ spark-core_${scala.binary.version} ${project.version} + + com.typesafe + scalalogging-slf4j_${scala.binary.version} + 1.0.1 + org.scalatest scalatest_${scala.binary.version} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 2b36582215..74c0104e5b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -109,12 +109,12 @@ class Analyzer(catalog: Catalog, registry: FunctionRegistry, caseSensitive: Bool object ResolveReferences extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformUp { case q: LogicalPlan if q.childrenResolved => - log.trace(s"Attempting to resolve ${q.simpleString}") + logger.trace(s"Attempting to resolve ${q.simpleString}") q transformExpressions { case u @ UnresolvedAttribute(name) => // Leave unchanged if resolution fails. Hopefully will be resolved next round. val result = q.resolve(name).getOrElse(u) - log.debug(s"Resolving $u to $result") + logger.debug(s"Resolving $u to $result") result } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index eafbb70dc3..47c7ad076a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -75,7 +75,7 @@ trait HiveTypeCoercion { // Leave the same if the dataTypes match. case Some(newType) if a.dataType == newType.dataType => a case Some(newType) => - log.debug(s"Promoting $a to $newType in ${q.simpleString}}") + logger.debug(s"Promoting $a to $newType in ${q.simpleString}}") newType } } @@ -154,7 +154,7 @@ trait HiveTypeCoercion { (Alias(Cast(l, StringType), l.name)(), r) case (l, r) if l.dataType != r.dataType => - log.debug(s"Resolving mismatched union input ${l.dataType}, ${r.dataType}") + logger.debug(s"Resolving mismatched union input ${l.dataType}, ${r.dataType}") findTightestCommonType(l.dataType, r.dataType).map { widestType => val newLeft = if (l.dataType == widestType) l else Alias(Cast(l, widestType), l.name)() @@ -170,7 +170,7 @@ trait HiveTypeCoercion { val newLeft = if (castedLeft.map(_.dataType) != left.output.map(_.dataType)) { - log.debug(s"Widening numeric types in union $castedLeft ${left.output}") + logger.debug(s"Widening numeric types in union $castedLeft ${left.output}") Project(castedLeft, left) } else { left @@ -178,7 +178,7 @@ trait HiveTypeCoercion { val newRight = if (castedRight.map(_.dataType) != right.output.map(_.dataType)) { - log.debug(s"Widening numeric types in union $castedRight ${right.output}") + logger.debug(s"Widening numeric types in union $castedRight ${right.output}") Project(castedRight, right) } else { right diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala index 0913f15888..f38f99569f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.expressions -import org.apache.spark.Logging +import org.apache.spark.sql.catalyst.Logging import org.apache.spark.sql.catalyst.errors.attachTree import org.apache.spark.sql.catalyst.types._ import org.apache.spark.sql.catalyst.trees diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala index e2552d432c..4211998f75 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.expressions.codegen -import org.apache.spark.Logging +import com.typesafe.scalalogging.slf4j.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.types.{StringType, NumericType} @@ -92,7 +92,7 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[Row]] wit } new $orderingName() """ - log.debug(s"Generated Ordering: $code") + logger.debug(s"Generated Ordering: $code") toolBox.eval(code).asInstanceOf[Ordering[Row]] } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala index bdd07bbeb2..ca9642954e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala @@ -25,4 +25,5 @@ package object catalyst { */ protected[catalyst] object ScalaReflectionLock + protected[catalyst] type Logging = com.typesafe.scalalogging.slf4j.Logging } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala index 5839c9f7c4..781ba489b4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.planning -import org.apache.spark.Logging +import org.apache.spark.sql.catalyst.Logging import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.TreeNode diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala index 06c5ffe92a..bc763a4e06 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.planning import scala.annotation.tailrec import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.Logging +import org.apache.spark.sql.catalyst.Logging import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ @@ -184,7 +184,7 @@ object ExtractEquiJoinKeys extends Logging with PredicateHelper { def unapply(plan: LogicalPlan): Option[ReturnType] = plan match { case join @ Join(left, right, joinType, condition) => - log.debug(s"Considering join on: $condition") + logger.debug(s"Considering join on: $condition") // Find equi-join predicates that can be evaluated before the join, and thus can be used // as join keys. val (joinPredicates, otherPredicates) = @@ -202,7 +202,7 @@ object ExtractEquiJoinKeys extends Logging with PredicateHelper { val rightKeys = joinKeys.map(_._2) if (joinKeys.nonEmpty) { - log.debug(s"leftKeys:${leftKeys} | rightKeys:${rightKeys}") + logger.debug(s"leftKeys:${leftKeys} | rightKeys:${rightKeys}") Some((joinType, leftKeys, rightKeys, otherPredicates.reduceOption(And), left, right)) } else { None diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala index 03414b2301..f8960b3fe7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/Rule.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.rules -import org.apache.spark.Logging +import org.apache.spark.sql.catalyst.Logging import org.apache.spark.sql.catalyst.trees.TreeNode abstract class Rule[TreeType <: TreeNode[_]] extends Logging { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala index 20bf8eed7d..6aa407c836 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.rules -import org.apache.spark.Logging +import org.apache.spark.sql.catalyst.Logging import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.catalyst.util.sideBySide @@ -60,7 +60,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging { case (plan, rule) => val result = rule(plan) if (!result.fastEquals(plan)) { - log.trace( + logger.trace( s""" |=== Applying Rule ${rule.ruleName} === |${sideBySide(plan.treeString, result.treeString).mkString("\n")} @@ -73,26 +73,26 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging { if (iteration > batch.strategy.maxIterations) { // Only log if this is a rule that is supposed to run more than once. if (iteration != 2) { - log.info(s"Max iterations (${iteration - 1}) reached for batch ${batch.name}") + logger.info(s"Max iterations (${iteration - 1}) reached for batch ${batch.name}") } continue = false } if (curPlan.fastEquals(lastPlan)) { - log.trace(s"Fixed point reached for batch ${batch.name} after $iteration iterations.") + logger.trace(s"Fixed point reached for batch ${batch.name} after $iteration iterations.") continue = false } lastPlan = curPlan } if (!batchStartPlan.fastEquals(curPlan)) { - log.debug( + logger.debug( s""" |=== Result of Batch ${batch.name} === |${sideBySide(plan.treeString, curPlan.treeString).mkString("\n")} """.stripMargin) } else { - log.trace(s"Batch ${batch.name} has no effect.") + logger.trace(s"Batch ${batch.name} has no effect.") } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/package.scala index d725a92c06..9a28d035a1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/package.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.catalyst -import org.apache.spark.Logging - /** * A library for easily manipulating trees of operators. Operators that extend TreeNode are * granted the following interface: @@ -33,8 +31,8 @@ import org.apache.spark.Logging *
  • debugging support - pretty printing, easy splicing of trees, etc.
    • * */ -package object trees extends Logging { +package object trees { // Since we want tree nodes to be lightweight, we create one logger for all treenode instances. - protected override def logName = "catalyst.trees" - + protected val logger = + com.typesafe.scalalogging.slf4j.Logger(org.slf4j.LoggerFactory.getLogger("catalyst.trees")) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 00dd34aabc..dad71079c2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.SparkStrategies import org.apache.spark.sql.json._ import org.apache.spark.sql.parquet.ParquetRelation -import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.SparkContext /** * :: AlphaComponent :: diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala index 828a8896ff..4c6675c3c8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala @@ -19,8 +19,7 @@ package org.apache.spark.sql.columnar.compression import java.nio.{ByteBuffer, ByteOrder} -import org.apache.spark.Logging -import org.apache.spark.sql.Row +import org.apache.spark.sql.{Logging, Row} import org.apache.spark.sql.catalyst.types.NativeType import org.apache.spark.sql.columnar.{ColumnBuilder, NativeColumnBuilder} @@ -102,7 +101,7 @@ private[sql] trait CompressibleColumnBuilder[T <: NativeType] copyColumnHeader(rawBuffer, compressedBuffer) - log.info(s"Compressor for [$columnName]: $encoder, ratio: ${encoder.compressionRatio}") + logger.info(s"Compressor for [$columnName]: $encoder, ratio: ${encoder.compressionRatio}") encoder.compress(rawBuffer, compressedBuffer, columnType) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Exchange.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Exchange.scala index 0c3d537ccb..30712f03ca 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Exchange.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Exchange.scala @@ -101,7 +101,7 @@ private[sql] case class AddExchange(sqlContext: SQLContext) extends Rule[SparkPl !operator.requiredChildDistribution.zip(operator.children).map { case (required, child) => val valid = child.outputPartitioning.satisfies(required) - log.debug( + logger.debug( s"${if (valid) "Valid" else "Invalid"} distribution," + s"required: $required current: ${child.outputPartitioning}") valid diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala index a3d2a1c7a5..70db1ebd3a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.HiveTypeCoercion import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.types._ import org.apache.spark.sql.catalyst.ScalaReflection -import org.apache.spark.Logging +import org.apache.spark.sql.Logging private[sql] object JsonRDD extends Logging { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala index f513eae9c2..0995a4eb62 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala @@ -32,6 +32,8 @@ import org.apache.spark.annotation.DeveloperApi */ package object sql { + protected[sql] type Logging = com.typesafe.scalalogging.slf4j.Logging + /** * :: DeveloperApi :: * diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala index a165531573..829342215e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala @@ -22,7 +22,7 @@ import java.sql.Timestamp import org.scalatest.FunSuite -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.catalyst.types._ import org.apache.spark.sql.columnar.ColumnarTestUtils._ import org.apache.spark.sql.execution.SparkSqlSerializer @@ -166,7 +166,7 @@ class ColumnTypeSuite extends FunSuite with Logging { buffer.rewind() seq.foreach { expected => - log.info("buffer = " + buffer + ", expected = " + expected) + logger.info("buffer = " + buffer + ", expected = " + expected) val extracted = columnType.extract(buffer) assert( expected === extracted, diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala index 5959ba3d23..ddbc2a79fb 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService import org.apache.hive.service.server.{HiveServer2, ServerOptionsProcessor} -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.hive.HiveContext import org.apache.spark.sql.hive.thriftserver.ReflectionUtils._ @@ -40,7 +40,7 @@ private[hive] object HiveThriftServer2 extends Logging { val optionsProcessor = new ServerOptionsProcessor("HiveThriftServer2") if (!optionsProcessor.process(args)) { - log.warn("Error starting HiveThriftServer2 with given arguments") + logger.warn("Error starting HiveThriftServer2 with given arguments") System.exit(-1) } @@ -49,12 +49,12 @@ private[hive] object HiveThriftServer2 extends Logging { // Set all properties specified via command line. val hiveConf: HiveConf = ss.getConf hiveConf.getAllProperties.toSeq.sortBy(_._1).foreach { case (k, v) => - log.debug(s"HiveConf var: $k=$v") + logger.debug(s"HiveConf var: $k=$v") } SessionState.start(ss) - log.info("Starting SparkContext") + logger.info("Starting SparkContext") SparkSQLEnv.init() SessionState.start(ss) @@ -70,10 +70,10 @@ private[hive] object HiveThriftServer2 extends Logging { val server = new HiveThriftServer2(SparkSQLEnv.hiveContext) server.init(hiveConf) server.start() - log.info("HiveThriftServer2 started") + logger.info("HiveThriftServer2 started") } catch { case e: Exception => - log.error("Error starting HiveThriftServer2", e) + logger.error("Error starting HiveThriftServer2", e) System.exit(-1) } } diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 4d0c506c5a..cb17d7ce58 100755 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.shims.ShimLoader import org.apache.thrift.transport.TSocket -import org.apache.spark.Logging +import org.apache.spark.sql.Logging private[hive] object SparkSQLCLIDriver { private var prompt = "spark-sql" diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala index 276723990b..a56b19a4bc 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema} import org.apache.hadoop.hive.ql.Driver import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.hive.{HiveContext, HiveMetastoreTypes} private[hive] class SparkSQLDriver(val context: HiveContext = SparkSQLEnv.hiveContext) @@ -40,7 +40,7 @@ private[hive] class SparkSQLDriver(val context: HiveContext = SparkSQLEnv.hiveCo private def getResultSetSchema(query: context.QueryExecution): Schema = { val analyzed = query.analyzed - log.debug(s"Result Schema: ${analyzed.output}") + logger.debug(s"Result Schema: ${analyzed.output}") if (analyzed.output.size == 0) { new Schema(new FieldSchema("Response code", "string", "") :: Nil, null) } else { @@ -61,7 +61,7 @@ private[hive] class SparkSQLDriver(val context: HiveContext = SparkSQLEnv.hiveCo new CommandProcessorResponse(0) } catch { case cause: Throwable => - log.error(s"Failed in [$command]", cause) + logger.error(s"Failed in [$command]", cause) new CommandProcessorResponse(-3, ExceptionUtils.getFullStackTrace(cause), null) } } diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala index dfc93b19d0..451c3bd7b9 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala @@ -20,13 +20,13 @@ package org.apache.spark.sql.hive.thriftserver import org.apache.hadoop.hive.ql.session.SessionState import org.apache.spark.scheduler.{SplitInfo, StatsReportListener} -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.hive.HiveContext import org.apache.spark.{SparkConf, SparkContext} /** A singleton object for the master program. The slaves should not access this. */ private[hive] object SparkSQLEnv extends Logging { - log.debug("Initializing SparkSQLEnv") + logger.debug("Initializing SparkSQLEnv") var hiveContext: HiveContext = _ var sparkContext: SparkContext = _ @@ -47,7 +47,7 @@ private[hive] object SparkSQLEnv extends Logging { /** Cleans up and shuts down the Spark SQL environments. */ def stop() { - log.debug("Shutting down Spark SQL Environment") + logger.debug("Shutting down Spark SQL Environment") // Stop the SparkContext if (SparkSQLEnv.sparkContext != null) { sparkContext.stop() diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala index 2c6e24e80d..a4e1f3e762 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala @@ -30,11 +30,10 @@ import org.apache.hive.service.cli._ import org.apache.hive.service.cli.operation.{ExecuteStatementOperation, Operation, OperationManager} import org.apache.hive.service.cli.session.HiveSession -import org.apache.spark.Logging import org.apache.spark.sql.catalyst.types._ import org.apache.spark.sql.hive.thriftserver.ReflectionUtils import org.apache.spark.sql.hive.{HiveContext, HiveMetastoreTypes} -import org.apache.spark.sql.{SchemaRDD, Row => SparkRow} +import org.apache.spark.sql.{Logging, SchemaRDD, Row => SparkRow} /** * Executes queries using Spark SQL, and maintains a list of handles to active queries. @@ -56,7 +55,7 @@ class SparkSQLOperationManager(hiveContext: HiveContext) extends OperationManage def close(): Unit = { // RDDs will be cleaned automatically upon garbage collection. - log.debug("CLOSING") + logger.debug("CLOSING") } def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = { @@ -113,7 +112,7 @@ class SparkSQLOperationManager(hiveContext: HiveContext) extends OperationManage } def getResultSetSchema: TableSchema = { - log.warn(s"Result Schema: ${result.queryExecution.analyzed.output}") + logger.warn(s"Result Schema: ${result.queryExecution.analyzed.output}") if (result.queryExecution.analyzed.output.size == 0) { new TableSchema(new FieldSchema("Result", "string", "") :: Nil) } else { @@ -125,11 +124,11 @@ class SparkSQLOperationManager(hiveContext: HiveContext) extends OperationManage } def run(): Unit = { - log.info(s"Running query '$statement'") + logger.info(s"Running query '$statement'") setState(OperationState.RUNNING) try { result = hiveContext.hql(statement) - log.debug(result.queryExecution.toString()) + logger.debug(result.queryExecution.toString()) val groupId = round(random * 1000000).toString hiveContext.sparkContext.setJobGroup(groupId, statement) iter = result.queryExecution.toRdd.toLocalIterator @@ -139,7 +138,7 @@ class SparkSQLOperationManager(hiveContext: HiveContext) extends OperationManage // Actually do need to catch Throwable as some failures don't inherit from Exception and // HiveServer will silently swallow them. case e: Throwable => - log.error("Error executing query:",e) + logger.error("Error executing query:",e) throw new HiveSQLException(e.toString) } setState(OperationState.FINISHED) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala index b7b7c9957a..fe3403b329 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala @@ -27,7 +27,7 @@ import java.sql.{Connection, DriverManager, Statement} import org.scalatest.{BeforeAndAfterAll, FunSuite} -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.catalyst.util.getTempFilePath /** diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 1f31d35eaa..7e3b8727be 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -207,7 +207,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { } } catch { case e: Exception => - log.error( + logger.error( s""" |====================== |HIVE FAILURE OUTPUT diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala index df3604439e..fa4e78439c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala @@ -28,8 +28,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc import org.apache.hadoop.hive.serde2.Deserializer import org.apache.spark.annotation.DeveloperApi -import org.apache.spark.Logging -import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.{SQLContext, Logging} import org.apache.spark.sql.catalyst.analysis.{EliminateAnalysisOperators, Catalog} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala index 7376fb5dc8..c50e8c4b5c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala @@ -148,7 +148,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { describedTables ++ logical.collect { case UnresolvedRelation(databaseName, name, _) => name } val referencedTestTables = referencedTables.filter(testTables.contains) - log.debug(s"Query references test tables: ${referencedTestTables.mkString(", ")}") + logger.debug(s"Query references test tables: ${referencedTestTables.mkString(", ")}") referencedTestTables.foreach(loadTestTable) // Proceed with analysis. analyzer(logical) @@ -273,7 +273,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { if (!(loadedTables contains name)) { // Marks the table as loaded first to prevent infite mutually recursive table loading. loadedTables += name - log.info(s"Loading test table $name") + logger.info(s"Loading test table $name") val createCmds = testTables.get(name).map(_.commands).getOrElse(sys.error(s"Unknown test table $name")) createCmds.foreach(_()) @@ -312,7 +312,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { loadedTables.clear() catalog.client.getAllTables("default").foreach { t => - log.debug(s"Deleting table $t") + logger.debug(s"Deleting table $t") val table = catalog.client.getTable("default", t) catalog.client.getIndexes("default", t, 255).foreach { index => @@ -325,7 +325,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { } catalog.client.getAllDatabases.filterNot(_ == "default").foreach { db => - log.debug(s"Dropping Database: $db") + logger.debug(s"Dropping Database: $db") catalog.client.dropDatabase(db, true, false, true) } @@ -347,7 +347,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { loadTestTable("srcpart") } catch { case e: Exception => - log.error(s"FATAL ERROR: Failed to reset TestDB state. $e") + logger.error(s"FATAL ERROR: Failed to reset TestDB state. $e") // At this point there is really no reason to continue, but the test framework traps exits. // So instead we just pause forever so that at least the developer can see where things // started to go wrong. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala index 4d8eaa18d7..7582b4743d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.{FunctionInfo, FunctionRegistry} import org.apache.hadoop.hive.ql.udf.{UDFType => HiveUDFType} import org.apache.hadoop.hive.ql.udf.generic._ -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.catalyst.analysis import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.types._ @@ -119,7 +119,7 @@ private[hive] case class HiveSimpleUdf(functionClassName: String, children: Seq[ sys.error(s"No matching wrapper found, options: ${argClass.getConstructors.toSeq}.")) (a: Any) => { - log.debug( + logger.debug( s"Wrapping $a of type ${if (a == null) "null" else a.getClass.getName} using $constructor.") // We must make sure that primitives get boxed java style. if (a == null) { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index 52cb1cf986..6c8fe4b196 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -21,7 +21,7 @@ import java.io._ import org.scalatest.{BeforeAndAfterAll, FunSuite, GivenWhenThen} -import org.apache.spark.Logging +import org.apache.spark.sql.Logging import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.plans.logical.{NativeCommand => LogicalNativeCommand} @@ -197,7 +197,7 @@ abstract class HiveComparisonTest // If test sharding is enable, skip tests that are not in the correct shard. shardInfo.foreach { case (shardId, numShards) if testCaseName.hashCode % numShards != shardId => return - case (shardId, _) => log.debug(s"Shard $shardId includes test '$testCaseName'") + case (shardId, _) => logger.debug(s"Shard $shardId includes test '$testCaseName'") } // Skip tests found in directories specified by user. @@ -213,13 +213,13 @@ abstract class HiveComparisonTest .map(new File(_, testCaseName)) .filter(_.exists) if (runOnlyDirectories.nonEmpty && runIndicators.isEmpty) { - log.debug( + logger.debug( s"Skipping test '$testCaseName' not found in ${runOnlyDirectories.map(_.getCanonicalPath)}") return } test(testCaseName) { - log.debug(s"=== HIVE TEST: $testCaseName ===") + logger.debug(s"=== HIVE TEST: $testCaseName ===") // Clear old output for this testcase. outputDirectories.map(new File(_, testCaseName)).filter(_.exists()).foreach(_.delete()) @@ -235,7 +235,7 @@ abstract class HiveComparisonTest .filterNot(_ contains "hive.outerjoin.supports.filters") if (allQueries != queryList) - log.warn(s"Simplifications made on unsupported operations for test $testCaseName") + logger.warn(s"Simplifications made on unsupported operations for test $testCaseName") lazy val consoleTestCase = { val quotes = "\"\"\"" @@ -257,11 +257,11 @@ abstract class HiveComparisonTest } val hiveCachedResults = hiveCacheFiles.flatMap { cachedAnswerFile => - log.debug(s"Looking for cached answer file $cachedAnswerFile.") + logger.debug(s"Looking for cached answer file $cachedAnswerFile.") if (cachedAnswerFile.exists) { Some(fileToString(cachedAnswerFile)) } else { - log.debug(s"File $cachedAnswerFile not found") + logger.debug(s"File $cachedAnswerFile not found") None } }.map { @@ -272,7 +272,7 @@ abstract class HiveComparisonTest val hiveResults: Seq[Seq[String]] = if (hiveCachedResults.size == queryList.size) { - log.info(s"Using answer cache for test: $testCaseName") + logger.info(s"Using answer cache for test: $testCaseName") hiveCachedResults } else { @@ -287,7 +287,7 @@ abstract class HiveComparisonTest if (installHooksCommand.findAllMatchIn(queryString).nonEmpty) sys.error("hive exec hooks not supported for tests.") - log.warn(s"Running query ${i+1}/${queryList.size} with hive.") + logger.warn(s"Running query ${i+1}/${queryList.size} with hive.") // Analyze the query with catalyst to ensure test tables are loaded. val answer = hiveQuery.analyzed match { case _: ExplainCommand => Nil // No need to execute EXPLAIN queries as we don't check the output. @@ -351,7 +351,7 @@ abstract class HiveComparisonTest val resultComparison = sideBySide(hivePrintOut, catalystPrintOut).mkString("\n") if (recomputeCache) { - log.warn(s"Clearing cache files for failed test $testCaseName") + logger.warn(s"Clearing cache files for failed test $testCaseName") hiveCacheFiles.foreach(_.delete()) } @@ -380,7 +380,7 @@ abstract class HiveComparisonTest TestHive.runSqlHive("SELECT key FROM src") } catch { case e: Exception => - log.error(s"FATAL ERROR: Canary query threw $e This implies that the testing environment has likely been corrupted.") + logger.error(s"FATAL ERROR: Canary query threw $e This implies that the testing environment has likely been corrupted.") // The testing setup traps exits so wait here for a long time so the developer can see when things started // to go wrong. Thread.sleep(1000000) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala index 9ca5575c1b..50ab71a900 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala @@ -53,7 +53,7 @@ abstract class HiveQueryFileTest extends HiveComparisonTest { testCases.sorted.foreach { case (testCaseName, testCaseFile) => if (blackList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) { - log.debug(s"Blacklisted test skipped $testCaseName") + logger.debug(s"Blacklisted test skipped $testCaseName") } else if (realWhiteList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) || runAll) { // Build a test case and submit it to scala test framework... val queriesString = fileToString(testCaseFile) -- cgit v1.2.3