From e07838aab7b059a37bfaa8ff23335cc6e1c9867e Mon Sep 17 00:00:00 2001 From: John St John Date: Tue, 26 Sep 2017 15:51:38 -0700 Subject: Catch and log google tracing exceptions rather than crashing the entire app --- ...ceptionLoggingSizedBufferingTraceConsumer.scala | 26 ++++++++++++++++++++++ .../driver/core/trace/GoogleStackdriverTrace.scala | 2 +- .../trace/GoogleStackdriverTraceWithConsumer.scala | 10 ++++++--- .../scala/xyz/driver/core/trace/LoggingTrace.scala | 3 ++- 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala (limited to 'src/main') diff --git a/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala new file mode 100644 index 0000000..534df8c --- /dev/null +++ b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala @@ -0,0 +1,26 @@ +package xyz.driver.core.trace + +import com.google.cloud.trace.v1.consumer.{FlushableTraceConsumer} +import com.google.devtools.cloudtrace.v1.Traces +import com.typesafe.scalalogging.Logger +import scala.util.Try + +/** + * ExceptionLoggingFlushableTraceConsumer simply wraps a flushable trace consumer and catches/logs any exceptions + * @param traceConsumer the flusable trace consumer to wrap + * @param log where to log any exceptions + */ +class ExceptionLoggingFlushableTraceConsumer(traceConsumer: FlushableTraceConsumer, log: Logger) + extends FlushableTraceConsumer { + + private val flushableTraceConsumer = traceConsumer + + private def exceptionLogger(exception: Throwable): Unit = + log.trace(s"Encountered exception logging to google $exception") + + override def receive(trace: Traces): Unit = + Try(flushableTraceConsumer.receive(trace)).fold(exceptionLogger, identity) + + override def flush(): Unit = + Try(flushableTraceConsumer.flush()).fold(exceptionLogger, identity) +} diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala index 1ff8d10..fa98ef4 100644 --- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala +++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala @@ -38,7 +38,7 @@ final class GoogleStackdriverTrace(projectId: String, } private val googleServiceTracer = - new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer) + new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer, log) override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan = googleServiceTracer.startSpan(httpRequest) diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala index 7fed3c7..81619d2 100644 --- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala +++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala @@ -8,18 +8,22 @@ import com.google.cloud.trace.v1.consumer.{SizedBufferingTraceConsumer, TraceCon import com.google.cloud.trace.v1.producer.TraceProducer import com.google.cloud.trace.v1.util.RoughTraceSizer import com.google.cloud.trace.{SpanContextHandler, SpanContextHandlerTracer, Tracer} +import com.typesafe.scalalogging.Logger import scala.compat.java8.OptionConverters._ final class GoogleStackdriverTraceWithConsumer(projectId: String, appName: String, appEnvironment: String, - traceConsumer: TraceConsumer) + traceConsumer: TraceConsumer, + log: Logger) extends GoogleServiceTracer { private val traceProducer: TraceProducer = new TraceProducer() - private val threadSafeBufferingTraceConsumer = - new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), 100) + private val threadSafeBufferingTraceConsumer = new ExceptionLoggingFlushableTraceConsumer( + new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), 100), + log + ) private val traceSink: TraceSink = new TraceSinkV1(projectId, traceProducer, threadSafeBufferingTraceConsumer) diff --git a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala index 9db85b7..e75de53 100644 --- a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala +++ b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala @@ -11,7 +11,8 @@ final class LoggingTrace(appName: String, appEnvironment: String, log: Logger) e "logging-tracer", appName, appEnvironment, - traceConsumer + traceConsumer, + log ) override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan = -- cgit v1.2.3 From c1628abaad79de60c7eafb27a5914dabd3fb75dc Mon Sep 17 00:00:00 2001 From: John St John Date: Tue, 26 Sep 2017 17:03:32 -0700 Subject: only push traces every 10 requests by default --- src/main/scala/xyz/driver/core/app.scala | 6 ++++-- src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala | 7 +++++-- .../driver/core/trace/GoogleStackdriverTraceWithConsumer.scala | 8 +++++--- src/main/scala/xyz/driver/core/trace/LoggingTrace.scala | 8 ++++++-- src/main/scala/xyz/driver/core/trace/ServiceTracer.scala | 2 ++ 5 files changed, 22 insertions(+), 9 deletions(-) (limited to 'src/main') diff --git a/src/main/scala/xyz/driver/core/app.scala b/src/main/scala/xyz/driver/core/app.scala index 763a363..a7f58e3 100644 --- a/src/main/scala/xyz/driver/core/app.scala +++ b/src/main/scala/xyz/driver/core/app.scala @@ -53,7 +53,8 @@ object app { implicit private lazy val materializer = ActorMaterializer()(actorSystem) private lazy val http = Http()(actorSystem) val appEnvironment = config.getString("application.environment") - val serviceTracer = tracer.getOrElse(new LoggingTrace(appName, config.getString("application.environment"), log)) + val serviceTracer = + tracer.getOrElse(new LoggingTrace(appName, config.getString("application.environment"), log, 10)) def run(): Unit = { activateServices(modules) scheduleServicesDeactivation(modules) @@ -63,7 +64,8 @@ object app { def stop(): Unit = { http.shutdownAllConnectionPools().onComplete { _ => - val _ = actorSystem.terminate() + val _ = actorSystem.terminate() + serviceTracer.flush() // flush out any remaining traces from the buffer val terminated = Await.result(actorSystem.whenTerminated, 30.seconds) val addressTerminated = if (terminated.addressTerminated) "is" else "is not" Console.print(s"${this.getClass.getName} App $addressTerminated stopped ") diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala index fa98ef4..04f2ec6 100644 --- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala +++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala @@ -14,7 +14,8 @@ final class GoogleStackdriverTrace(projectId: String, clientSecretsFile: String, appName: String, appEnvironment: String, - log: Logger) + log: Logger, + bufferSize: Int = 10) extends GoogleServiceTracer { // initialize our various tracking storage systems @@ -38,10 +39,12 @@ final class GoogleStackdriverTrace(projectId: String, } private val googleServiceTracer = - new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer, log) + new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer, log, bufferSize) override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan = googleServiceTracer.startSpan(httpRequest) override def endSpan(span: GoogleStackdriverTraceSpan): Unit = googleServiceTracer.endSpan(span) + + override def flush(): Unit = googleServiceTracer.flush() } diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala index 81619d2..13b80bb 100644 --- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala +++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala @@ -16,12 +16,13 @@ final class GoogleStackdriverTraceWithConsumer(projectId: String, appName: String, appEnvironment: String, traceConsumer: TraceConsumer, - log: Logger) + log: Logger, + bufferSize: Int) extends GoogleServiceTracer { private val traceProducer: TraceProducer = new TraceProducer() private val threadSafeBufferingTraceConsumer = new ExceptionLoggingFlushableTraceConsumer( - new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), 100), + new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), bufferSize), log ) @@ -73,7 +74,8 @@ final class GoogleStackdriverTraceWithConsumer(projectId: String, override def endSpan(span: TracerSpanPayload): Unit = { span.tracer.endSpan(span.context) - threadSafeBufferingTraceConsumer.flush() // flush out the thread safe buffer } + override def flush(): Unit = threadSafeBufferingTraceConsumer.flush() // flush out the thread safe buffer + } diff --git a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala index e75de53..cd920f0 100644 --- a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala +++ b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala @@ -4,7 +4,8 @@ import akka.http.scaladsl.model.HttpRequest import com.google.cloud.trace.v1.consumer.TraceConsumer import com.typesafe.scalalogging.Logger -final class LoggingTrace(appName: String, appEnvironment: String, log: Logger) extends GoogleServiceTracer { +final class LoggingTrace(appName: String, appEnvironment: String, log: Logger, bufferSize: Int) + extends GoogleServiceTracer { private val traceConsumer: TraceConsumer = new LoggingTraceConsumer(log) private val googleServiceTracer = new GoogleStackdriverTraceWithConsumer( @@ -12,11 +13,14 @@ final class LoggingTrace(appName: String, appEnvironment: String, log: Logger) e appName, appEnvironment, traceConsumer, - log + log, + bufferSize ) override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan = googleServiceTracer.startSpan(httpRequest) override def endSpan(span: GoogleStackdriverTraceSpan): Unit = googleServiceTracer.endSpan(span) + + override def flush(): Unit = googleServiceTracer.flush() } diff --git a/src/main/scala/xyz/driver/core/trace/ServiceTracer.scala b/src/main/scala/xyz/driver/core/trace/ServiceTracer.scala index 25562cd..1413b63 100644 --- a/src/main/scala/xyz/driver/core/trace/ServiceTracer.scala +++ b/src/main/scala/xyz/driver/core/trace/ServiceTracer.scala @@ -14,4 +14,6 @@ trait ServiceTracer { def startSpan(httpRequest: HttpRequest): TracerSpanPayload def endSpan(span: TracerSpanPayload): Unit + + def flush(): Unit } -- cgit v1.2.3 From 77d464e2db5a46319b3687db2353120fddaab097 Mon Sep 17 00:00:00 2001 From: John St John Date: Tue, 26 Sep 2017 17:16:10 -0700 Subject: just use a unit size per span --- .../xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala | 4 ++-- src/main/scala/xyz/driver/core/trace/UnitTraceSizer.scala | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/xyz/driver/core/trace/UnitTraceSizer.scala (limited to 'src/main') diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala index 13b80bb..cd9170a 100644 --- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala +++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala @@ -6,7 +6,6 @@ import com.google.cloud.trace.sink.TraceSink import com.google.cloud.trace.v1.TraceSinkV1 import com.google.cloud.trace.v1.consumer.{SizedBufferingTraceConsumer, TraceConsumer} import com.google.cloud.trace.v1.producer.TraceProducer -import com.google.cloud.trace.v1.util.RoughTraceSizer import com.google.cloud.trace.{SpanContextHandler, SpanContextHandlerTracer, Tracer} import com.typesafe.scalalogging.Logger @@ -21,8 +20,9 @@ final class GoogleStackdriverTraceWithConsumer(projectId: String, extends GoogleServiceTracer { private val traceProducer: TraceProducer = new TraceProducer() + // use a UnitTraceSizer so the interpretation of bufferSize is # of spans to hold in memory prior to flushing private val threadSafeBufferingTraceConsumer = new ExceptionLoggingFlushableTraceConsumer( - new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), bufferSize), + new SizedBufferingTraceConsumer(traceConsumer, new UnitTraceSizer(), bufferSize), log ) diff --git a/src/main/scala/xyz/driver/core/trace/UnitTraceSizer.scala b/src/main/scala/xyz/driver/core/trace/UnitTraceSizer.scala new file mode 100644 index 0000000..a7d6032 --- /dev/null +++ b/src/main/scala/xyz/driver/core/trace/UnitTraceSizer.scala @@ -0,0 +1,7 @@ +package xyz.driver.core.trace +import com.google.cloud.trace.v1.util.Sizer +import com.google.devtools.cloudtrace.v1.Trace + +class UnitTraceSizer extends Sizer[Trace] { + override def size(sizeable: Trace) = 1 +} -- cgit v1.2.3 From 8a66e1a0e62187d850e665af0d17431272775705 Mon Sep 17 00:00:00 2001 From: John St John Date: Wed, 27 Sep 2017 11:20:39 -0700 Subject: log error as error --- .../driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/main') diff --git a/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala index 534df8c..64ae15a 100644 --- a/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala +++ b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala @@ -16,7 +16,7 @@ class ExceptionLoggingFlushableTraceConsumer(traceConsumer: FlushableTraceConsum private val flushableTraceConsumer = traceConsumer private def exceptionLogger(exception: Throwable): Unit = - log.trace(s"Encountered exception logging to google $exception") + log.error(s"Encountered exception logging to google $exception") override def receive(trace: Traces): Unit = Try(flushableTraceConsumer.receive(trace)).fold(exceptionLogger, identity) -- cgit v1.2.3