Catch and log google tracing exceptions rather than crashing the entire app

4 files changed, 36 insertions, 5 deletions

diff --git a/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala
new file mode 100644
index 0000000..534df8c
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/trace/ExceptionLoggingSizedBufferingTraceConsumer.scala
@@ -0,0 +1,26 @@
+package xyz.driver.core.trace
+
+import com.google.cloud.trace.v1.consumer.{FlushableTraceConsumer}
+import com.google.devtools.cloudtrace.v1.Traces
+import com.typesafe.scalalogging.Logger
+import scala.util.Try
+
+/**
+  * ExceptionLoggingFlushableTraceConsumer simply wraps a flushable trace consumer and catches/logs any exceptions
+  * @param traceConsumer the flusable trace consumer to wrap
+  * @param log where to log any exceptions
+  */
+class ExceptionLoggingFlushableTraceConsumer(traceConsumer: FlushableTraceConsumer, log: Logger)
+    extends FlushableTraceConsumer {
+
+  private val flushableTraceConsumer = traceConsumer
+
+  private def exceptionLogger(exception: Throwable): Unit =
+    log.trace(s"Encountered exception logging to google $exception")
+
+  override def receive(trace: Traces): Unit =
+    Try(flushableTraceConsumer.receive(trace)).fold(exceptionLogger, identity)
+
+  override def flush(): Unit =
+    Try(flushableTraceConsumer.flush()).fold(exceptionLogger, identity)
+}
diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala
index 1ff8d10..fa98ef4 100644
--- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala
+++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTrace.scala
@@ -38,7 +38,7 @@ final class GoogleStackdriverTrace(projectId: String,
   }
 
   private val googleServiceTracer =
-    new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer)
+    new GoogleStackdriverTraceWithConsumer(projectId, appName, appEnvironment, traceConsumer, log)
 
   override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan =
     googleServiceTracer.startSpan(httpRequest)
diff --git a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala
index 7fed3c7..81619d2 100644
--- a/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala
+++ b/src/main/scala/xyz/driver/core/trace/GoogleStackdriverTraceWithConsumer.scala
@@ -8,18 +8,22 @@ import com.google.cloud.trace.v1.consumer.{SizedBufferingTraceConsumer, TraceCon
 import com.google.cloud.trace.v1.producer.TraceProducer
 import com.google.cloud.trace.v1.util.RoughTraceSizer
 import com.google.cloud.trace.{SpanContextHandler, SpanContextHandlerTracer, Tracer}
+import com.typesafe.scalalogging.Logger
 
 import scala.compat.java8.OptionConverters._
 
 final class GoogleStackdriverTraceWithConsumer(projectId: String,
                                                appName: String,
                                                appEnvironment: String,
-                                               traceConsumer: TraceConsumer)
+                                               traceConsumer: TraceConsumer,
+                                               log: Logger)
     extends GoogleServiceTracer {
 
   private val traceProducer: TraceProducer = new TraceProducer()
-  private val threadSafeBufferingTraceConsumer =
-    new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), 100)
+  private val threadSafeBufferingTraceConsumer = new ExceptionLoggingFlushableTraceConsumer(
+    new SizedBufferingTraceConsumer(traceConsumer, new RoughTraceSizer(), 100),
+    log
+  )
 
   private val traceSink: TraceSink = new TraceSinkV1(projectId, traceProducer, threadSafeBufferingTraceConsumer)
 
diff --git a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala
index 9db85b7..e75de53 100644
--- a/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala
+++ b/src/main/scala/xyz/driver/core/trace/LoggingTrace.scala
@@ -11,7 +11,8 @@ final class LoggingTrace(appName: String, appEnvironment: String, log: Logger) e
     "logging-tracer",
     appName,
     appEnvironment,
-    traceConsumer
+    traceConsumer,
+    log
   )
 
   override def startSpan(httpRequest: HttpRequest): GoogleStackdriverTraceSpan =