! all: refactor the core metric recording instruments and accomodate UserMetrics

This PR is including several changes to the kamon-core, most notably:
  - Formalize the interface for Histograms, Counters and MinMaxCounters. Making sure
    that the interfaces are as clean as possible.
  - Move away from the all Vector[Measurement] based Histogram snapshot to a new approach
    in which we use a single long to store both the index in the counts array and the
    frequency on that bucket. The leftmost 2 bytes of each long are used for storing the
    counts array index and the remaining 6 bytes are used for the actual count, and
    everything is put into a simple long array. This way only the buckets that actually
    have values will be included in the snapshot with the smallest possible memory
    footprint.
  - Introduce Gauges.
  - Reorganize the instrumentation for Akka and Scala and rewrite most of the tests
    of this components to avoid going through the subscription protocol to test.
  - Introduce trace tests and fixes on various tests.
  - Necessary changes on new relic, datadog and statsd modules to compile with the new
    codebase.

Pending:
  - Finish the upgrade of the new relic to the current model.
  - Introduce proper limit checks for histograms to ensure that we never pass the 2/6 bytes
    limits.
  - More testing, more testing, more testing.
  - Create the KamonStandalone module.

4 files changed, 0 insertions, 426 deletions

diff --git a/kamon-core/src/main/scala/akka/instrumentation/ActorMessagePassingTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/ActorMessagePassingTracing.scala
deleted file mode 100644
index 6db86828..00000000
--- a/kamon-core/src/main/scala/akka/instrumentation/ActorMessagePassingTracing.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-/* ===================================================
- * Copyright © 2013-2014 the kamon project <http://kamon.io/>
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-package akka.instrumentation
-
-import org.aspectj.lang.annotation._
-import org.aspectj.lang.ProceedingJoinPoint
-import akka.actor._
-import akka.dispatch.{ Envelope, MessageDispatcher }
-import kamon.trace._
-import kamon.metrics.{ ActorMetrics, Metrics }
-import kamon.Kamon
-import kamon.metrics.ActorMetrics.ActorMetricRecorder
-import kamon.metrics.instruments.MinMaxCounter
-import kamon.metrics.instruments.MinMaxCounter.CounterMeasurement
-
-@Aspect
-class BehaviourInvokeTracing {
-
-  @Pointcut("execution(akka.actor.ActorCell.new(..)) && this(cell) && args(system, ref, props, dispatcher, parent)")
-  def actorCellCreation(cell: ActorCell, system: ActorSystem, ref: ActorRef, props: Props, dispatcher: MessageDispatcher, parent: ActorRef): Unit = {}
-
-  @After("actorCellCreation(cell, system, ref, props, dispatcher, parent)")
-  def afterCreation(cell: ActorCell, system: ActorSystem, ref: ActorRef, props: Props, dispatcher: MessageDispatcher, parent: ActorRef): Unit = {
-
-    val metricsExtension = Kamon(Metrics)(system)
-    val metricIdentity = ActorMetrics(ref.path.elements.mkString("/"))
-    val cellWithMetrics = cell.asInstanceOf[ActorCellMetrics]
-
-    cellWithMetrics.metricIdentity = metricIdentity
-    cellWithMetrics.actorMetricsRecorder = metricsExtension.register(metricIdentity, ActorMetrics.Factory)
-
-    if (cellWithMetrics.actorMetricsRecorder.isDefined) {
-      cellWithMetrics.mailboxSizeCollectorCancellable = metricsExtension.scheduleGaugeRecorder {
-        cellWithMetrics.actorMetricsRecorder.map { am ⇒
-          import am.mailboxSize._
-          val CounterMeasurement(min, max, current) = cellWithMetrics.queueSize.collect()
-
-          record(min)
-          record(max)
-          record(current)
-        }
-      }
-    }
-  }
-
-  @Pointcut("(execution(* akka.actor.ActorCell.invoke(*)) || execution(* akka.routing.RoutedActorCell.sendMessage(*))) && this(cell) && args(envelope)")
-  def invokingActorBehaviourAtActorCell(cell: ActorCell, envelope: Envelope) = {}
-
-  @Around("invokingActorBehaviourAtActorCell(cell, envelope)")
-  def aroundBehaviourInvoke(pjp: ProceedingJoinPoint, cell: ActorCell, envelope: Envelope): Any = {
-    val timestampBeforeProcessing = System.nanoTime()
-    val contextAndTimestamp = envelope.asInstanceOf[TraceContextAware]
-    val cellWithMetrics = cell.asInstanceOf[ActorCellMetrics]
-
-    try {
-      TraceRecorder.withTraceContext(contextAndTimestamp.traceContext) {
-        pjp.proceed()
-      }
-    } finally {
-      cellWithMetrics.actorMetricsRecorder.map {
-        am ⇒
-          am.processingTime.record(System.nanoTime() - timestampBeforeProcessing)
-          am.timeInMailbox.record(timestampBeforeProcessing - contextAndTimestamp.captureNanoTime)
-          cellWithMetrics.queueSize.decrement()
-      }
-    }
-  }
-
-  @Pointcut("execution(* akka.actor.ActorCell.sendMessage(*)) && this(cell)")
-  def sendingMessageToActorCell(cell: ActorCell): Unit = {}
-
-  @After("sendingMessageToActorCell(cell)")
-  def afterSendMessageToActorCell(cell: ActorCell): Unit = {
-    val cellWithMetrics = cell.asInstanceOf[ActorCellMetrics]
-    cellWithMetrics.actorMetricsRecorder.map(am ⇒ cellWithMetrics.queueSize.increment())
-  }
-
-  @Pointcut("execution(* akka.actor.ActorCell.stop()) && this(cell)")
-  def actorStop(cell: ActorCell): Unit = {}
-
-  @After("actorStop(cell)")
-  def afterStop(cell: ActorCell): Unit = {
-    val cellWithMetrics = cell.asInstanceOf[ActorCellMetrics]
-
-    cellWithMetrics.actorMetricsRecorder.map { p ⇒
-      cellWithMetrics.mailboxSizeCollectorCancellable.cancel()
-      Kamon(Metrics)(cell.system).unregister(cellWithMetrics.metricIdentity)
-    }
-  }
-
-  @Pointcut("execution(* akka.actor.ActorCell.handleInvokeFailure(..)) && this(cell)")
-  def actorInvokeFailure(cell: ActorCell): Unit = {}
-
-  @Before("actorInvokeFailure(cell)")
-  def beforeInvokeFailure(cell: ActorCell): Unit = {
-    val cellWithMetrics = cell.asInstanceOf[ActorCellMetrics]
-
-    cellWithMetrics.actorMetricsRecorder.map {
-      am ⇒ am.errorCounter.record(1L)
-    }
-  }
-}
-
-trait ActorCellMetrics {
-  var metricIdentity: ActorMetrics = _
-  var actorMetricsRecorder: Option[ActorMetricRecorder] = _
-  var mailboxSizeCollectorCancellable: Cancellable = _
-  val queueSize = MinMaxCounter()
-}
-
-@Aspect
-class ActorCellMetricsMixin {
-
-  @DeclareMixin("akka.actor.ActorCell")
-  def mixinActorCellMetricsToActorCell: ActorCellMetrics = new ActorCellMetrics {}
-}
-
-@Aspect
-class EnvelopeTraceContextMixin {
-
-  @DeclareMixin("akka.dispatch.Envelope")
-  def mixinTraceContextAwareToEnvelope: TraceContextAware = TraceContextAware.default
-
-  @Pointcut("execution(akka.dispatch.Envelope.new(..)) && this(ctx)")
-  def envelopeCreation(ctx: TraceContextAware): Unit = {}
-
-  @After("envelopeCreation(ctx)")
-  def afterEnvelopeCreation(ctx: TraceContextAware): Unit = {
-    // Necessary to force the initialization of ContextAware at the moment of creation.
-    ctx.traceContext
-  }
-}
\ No newline at end of file
diff --git a/kamon-core/src/main/scala/akka/instrumentation/ActorSystemMessagePassingTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/ActorSystemMessagePassingTracing.scala
deleted file mode 100644
index 7d03d946..00000000
--- a/kamon-core/src/main/scala/akka/instrumentation/ActorSystemMessagePassingTracing.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-package akka.instrumentation
-
-import org.aspectj.lang.annotation._
-import akka.dispatch.sysmsg.EarliestFirstSystemMessageList
-import org.aspectj.lang.ProceedingJoinPoint
-import kamon.trace.{ TraceRecorder, TraceContextAware }
-
-@Aspect
-class SystemMessageTraceContextMixin {
-
-  @DeclareMixin("akka.dispatch.sysmsg.SystemMessage+")
-  def mixinTraceContextAwareToSystemMessage: TraceContextAware = TraceContextAware.default
-
-  @Pointcut("execution(akka.dispatch.sysmsg.SystemMessage+.new(..)) && this(ctx)")
-  def envelopeCreation(ctx: TraceContextAware): Unit = {}
-
-  @After("envelopeCreation(ctx)")
-  def afterEnvelopeCreation(ctx: TraceContextAware): Unit = {
-    // Necessary to force the initialization of ContextAware at the moment of creation.
-    ctx.traceContext
-  }
-}
-
-@Aspect
-class RepointableActorRefTraceContextMixin {
-
-  @DeclareMixin("akka.actor.RepointableActorRef")
-  def mixinTraceContextAwareToRepointableActorRef: TraceContextAware = TraceContextAware.default
-
-  @Pointcut("execution(akka.actor.RepointableActorRef.new(..)) && this(ctx)")
-  def envelopeCreation(ctx: TraceContextAware): Unit = {}
-
-  @After("envelopeCreation(ctx)")
-  def afterEnvelopeCreation(ctx: TraceContextAware): Unit = {
-    // Necessary to force the initialization of ContextAware at the moment of creation.
-    ctx.traceContext
-  }
-
-  @Pointcut("execution(* akka.actor.RepointableActorRef.point(..)) && this(repointableActorRef)")
-  def repointableActorRefCreation(repointableActorRef: TraceContextAware): Unit = {}
-
-  @Around("repointableActorRefCreation(repointableActorRef)")
-  def afterRepointableActorRefCreation(pjp: ProceedingJoinPoint, repointableActorRef: TraceContextAware): Any = {
-    TraceRecorder.withTraceContext(repointableActorRef.traceContext) {
-      pjp.proceed()
-    }
-  }
-
-}
-
-@Aspect
-class ActorSystemMessagePassingTracing {
-
-  @Pointcut("execution(* akka.actor.ActorCell.invokeAll$1(..)) && args(messages, *)")
-  def systemMessageProcessing(messages: EarliestFirstSystemMessageList): Unit = {}
-
-  @Around("systemMessageProcessing(messages)")
-  def aroundSystemMessageInvoke(pjp: ProceedingJoinPoint, messages: EarliestFirstSystemMessageList): Any = {
-    if (messages.nonEmpty) {
-      val ctx = messages.head.asInstanceOf[TraceContextAware].traceContext
-      TraceRecorder.withTraceContext(ctx)(pjp.proceed())
-
-    } else pjp.proceed()
-  }
-}
diff --git a/kamon-core/src/main/scala/akka/instrumentation/AskPatternTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/AskPatternTracing.scala
deleted file mode 100644
index 31ec92a4..00000000
--- a/kamon-core/src/main/scala/akka/instrumentation/AskPatternTracing.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * =========================================================================================
- * Copyright © 2013 the kamon project <http://kamon.io/>
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
- * except in compliance with the License. You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific language governing permissions
- * and limitations under the License.
- * =========================================================================================
- */
-
-package akka.instrumentation
-
-import org.aspectj.lang.annotation.{ AfterReturning, Pointcut, Aspect }
-import akka.event.Logging.Warning
-import scala.compat.Platform.EOL
-import akka.actor.ActorRefProvider
-import akka.pattern.{ AskTimeoutException, PromiseActorRef }
-import kamon.trace.Trace
-import kamon.Kamon
-
-@Aspect
-class AskPatternTracing {
-
-  class StackTraceCaptureException extends Throwable
-
-  @Pointcut(value = "execution(* akka.pattern.PromiseActorRef$.apply(..)) && args(provider, *, *)", argNames = "provider")
-  def promiseActorRefApply(provider: ActorRefProvider): Unit = {}
-
-  @AfterReturning(pointcut = "promiseActorRefApply(provider)", returning = "promiseActor")
-  def hookAskTimeoutWarning(provider: ActorRefProvider, promiseActor: PromiseActorRef): Unit = {
-    val system = promiseActor.provider.guardian.underlying.system
-    val traceExtension = Kamon(Trace)(system)
-
-    if (traceExtension.enableAskPatternTracing) {
-      val future = promiseActor.result.future
-      implicit val ec = system.dispatcher
-      val stack = new StackTraceCaptureException
-
-      future onFailure {
-        case timeout: AskTimeoutException ⇒
-          val stackString = stack.getStackTrace.drop(3).mkString("", EOL, EOL)
-
-          system.eventStream.publish(Warning("AskPatternTracing", classOf[AskPatternTracing],
-            "Timeout triggered for ask pattern registered at: " + stackString))
-      }
-    }
-  }
-}
diff --git a/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala
deleted file mode 100644
index 60cc4ddf..00000000
--- a/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * =========================================================================================
- * Copyright © 2013-2014 the kamon project <http://kamon.io/>
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
- * except in compliance with the License. You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific language governing permissions
- * and limitations under the License.
- * =========================================================================================
- */
-
-package akka.instrumentation
-
-import org.aspectj.lang.annotation._
-import akka.dispatch.{ Dispatchers, ExecutorServiceDelegate, Dispatcher, MessageDispatcher }
-import kamon.metrics.{ Metrics, DispatcherMetrics }
-import kamon.metrics.DispatcherMetrics.DispatcherMetricRecorder
-import kamon.Kamon
-import akka.actor.{ Cancellable, ActorSystemImpl }
-import scala.concurrent.forkjoin.ForkJoinPool
-import java.util.concurrent.ThreadPoolExecutor
-import java.lang.reflect.Method
-import akka.instrumentation.DispatcherMetricsCollector.DispatcherMetricsMeasurement
-
-@Aspect
-class DispatcherTracing {
-
-  @Pointcut("execution(akka.dispatch.Dispatchers.new(..)) && this(dispatchers) && cflow(execution(akka.actor.ActorSystemImpl.new(..)) && this(system))")
-  def onActorSystemStartup(dispatchers: Dispatchers, system: ActorSystemImpl) = {}
-
-  @Before("onActorSystemStartup(dispatchers, system)")
-  def beforeActorSystemStartup(dispatchers: Dispatchers, system: ActorSystemImpl): Unit = {
-    val currentDispatchers = dispatchers.asInstanceOf[DispatchersWithActorSystem]
-    currentDispatchers.actorSystem = system
-  }
-
-  @Pointcut("execution(* akka.dispatch.Dispatchers.lookup(..)) && this(dispatchers)")
-  def onDispatchersLookup(dispatchers: Dispatchers) = {}
-
-  @AfterReturning(pointcut = "onDispatchersLookup(dispatchers)", returning = "dispatcher")
-  def afterReturningLookup(dispatchers: Dispatchers, dispatcher: Dispatcher): Unit = {
-    val dispatchersWithActorSystem = dispatchers.asInstanceOf[DispatchersWithActorSystem]
-    val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics]
-
-    dispatcherWithMetrics.actorSystem = dispatchersWithActorSystem.actorSystem
-  }
-
-  @Pointcut("call(* akka.dispatch.ExecutorServiceFactory.createExecutorService(..))")
-  def onCreateExecutorService(): Unit = {}
-
-  @Pointcut("cflow((execution(* akka.dispatch.MessageDispatcher.registerForExecution(..)) || execution(* akka.dispatch.MessageDispatcher.executeTask(..))) && this(dispatcher))")
-  def onCflowMessageDispatcher(dispatcher: Dispatcher): Unit = {}
-
-  @Pointcut("onCreateExecutorService() && onCflowMessageDispatcher(dispatcher)")
-  def onDispatcherStartup(dispatcher: Dispatcher): Unit = {}
-
-  @After("onDispatcherStartup(dispatcher)")
-  def afterDispatcherStartup(dispatcher: MessageDispatcher): Unit = {
-
-    val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics]
-    val metricsExtension = Kamon(Metrics)(dispatcherWithMetrics.actorSystem)
-    val metricIdentity = DispatcherMetrics(dispatcher.id)
-
-    dispatcherWithMetrics.metricIdentity = metricIdentity
-    dispatcherWithMetrics.dispatcherMetricsRecorder = metricsExtension.register(metricIdentity, DispatcherMetrics.Factory)
-
-    if (dispatcherWithMetrics.dispatcherMetricsRecorder.isDefined) {
-      dispatcherWithMetrics.dispatcherCollectorCancellable = metricsExtension.scheduleGaugeRecorder {
-        dispatcherWithMetrics.dispatcherMetricsRecorder.map {
-          dm ⇒
-            val DispatcherMetricsMeasurement(maximumPoolSize, runningThreadCount, queueTaskCount, poolSize) =
-              DispatcherMetricsCollector.collect(dispatcher)
-
-            dm.maximumPoolSize.record(maximumPoolSize)
-            dm.runningThreadCount.record(runningThreadCount)
-            dm.queueTaskCount.record(queueTaskCount)
-            dm.poolSize.record(poolSize)
-        }
-      }
-    }
-  }
-
-  @Pointcut("execution(* akka.dispatch.MessageDispatcher.shutdown(..)) && this(dispatcher)")
-  def onDispatcherShutdown(dispatcher: MessageDispatcher): Unit = {}
-
-  @After("onDispatcherShutdown(dispatcher)")
-  def afterDispatcherShutdown(dispatcher: MessageDispatcher): Unit = {
-    val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics]
-
-    dispatcherWithMetrics.dispatcherMetricsRecorder.map {
-      dispatcher ⇒
-        dispatcherWithMetrics.dispatcherCollectorCancellable.cancel()
-        Kamon(Metrics)(dispatcherWithMetrics.actorSystem).unregister(dispatcherWithMetrics.metricIdentity)
-    }
-  }
-}
-
-@Aspect
-class DispatcherMetricsMixin {
-
-  @DeclareMixin("akka.dispatch.Dispatcher")
-  def mixinDispatcherMetricsToMessageDispatcher: DispatcherMessageMetrics = new DispatcherMessageMetrics {}
-
-  @DeclareMixin("akka.dispatch.Dispatchers")
-  def mixinDispatchersToDispatchersWithActorSystem: DispatchersWithActorSystem = new DispatchersWithActorSystem {}
-}
-
-trait DispatcherMessageMetrics {
-  var metricIdentity: DispatcherMetrics = _
-  var dispatcherMetricsRecorder: Option[DispatcherMetricRecorder] = _
-  var dispatcherCollectorCancellable: Cancellable = _
-  var actorSystem: ActorSystemImpl = _
-}
-
-trait DispatchersWithActorSystem {
-  var actorSystem: ActorSystemImpl = _
-}
-
-object DispatcherMetricsCollector {
-
-  case class DispatcherMetricsMeasurement(maximumPoolSize: Long, runningThreadCount: Long, queueTaskCount: Long, poolSize: Long)
-
-  private def collectForkJoinMetrics(pool: ForkJoinPool): DispatcherMetricsMeasurement = {
-    DispatcherMetricsMeasurement(pool.getParallelism, pool.getActiveThreadCount,
-      (pool.getQueuedTaskCount + pool.getQueuedSubmissionCount), pool.getPoolSize)
-  }
-
-  private def collectExecutorMetrics(pool: ThreadPoolExecutor): DispatcherMetricsMeasurement = {
-    DispatcherMetricsMeasurement(pool.getMaximumPoolSize, pool.getActiveCount, pool.getQueue.size(), pool.getPoolSize)
-  }
-
-  private val executorServiceMethod: Method = {
-    // executorService is protected
-    val method = classOf[Dispatcher].getDeclaredMethod("executorService")
-    method.setAccessible(true)
-    method
-  }
-
-  def collect(dispatcher: MessageDispatcher): DispatcherMetricsMeasurement = {
-    dispatcher match {
-      case x: Dispatcher ⇒ {
-        val executor = executorServiceMethod.invoke(x) match {
-          case delegate: ExecutorServiceDelegate ⇒ delegate.executor
-          case other                             ⇒ other
-        }
-
-        executor match {
-          case fjp: ForkJoinPool       ⇒ collectForkJoinMetrics(fjp)
-          case tpe: ThreadPoolExecutor ⇒ collectExecutorMetrics(tpe)
-          case anything                ⇒ DispatcherMetricsMeasurement(0L, 0L, 0L, 0L)
-        }
-      }
-      case _ ⇒ new DispatcherMetricsMeasurement(0L, 0L, 0L, 0L)
-    }
-  }
-}