! all: refactor the core metric recording instruments and accomodate UserMetrics

This PR is including several changes to the kamon-core, most notably:
  - Formalize the interface for Histograms, Counters and MinMaxCounters. Making sure
    that the interfaces are as clean as possible.
  - Move away from the all Vector[Measurement] based Histogram snapshot to a new approach
    in which we use a single long to store both the index in the counts array and the
    frequency on that bucket. The leftmost 2 bytes of each long are used for storing the
    counts array index and the remaining 6 bytes are used for the actual count, and
    everything is put into a simple long array. This way only the buckets that actually
    have values will be included in the snapshot with the smallest possible memory
    footprint.
  - Introduce Gauges.
  - Reorganize the instrumentation for Akka and Scala and rewrite most of the tests
    of this components to avoid going through the subscription protocol to test.
  - Introduce trace tests and fixes on various tests.
  - Necessary changes on new relic, datadog and statsd modules to compile with the new
    codebase.

Pending:
  - Finish the upgrade of the new relic to the current model.
  - Introduce proper limit checks for histograms to ensure that we never pass the 2/6 bytes
    limits.
  - More testing, more testing, more testing.
  - Create the KamonStandalone module.

3 files changed, 42 insertions, 35 deletions

diff --git a/kamon-statsd/src/main/scala/kamon/statsd/StatsD.scala b/kamon-statsd/src/main/scala/kamon/statsd/StatsD.scala
index 1b3daa97..dcd78f78 100644
--- a/kamon-statsd/src/main/scala/kamon/statsd/StatsD.scala
+++ b/kamon-statsd/src/main/scala/kamon/statsd/StatsD.scala
@@ -18,7 +18,7 @@ package kamon.statsd
 
 import akka.actor._
 import kamon.Kamon
-import kamon.metrics._
+import kamon.metric._
 import scala.concurrent.duration._
 import scala.collection.JavaConverters._
 import com.typesafe.config.Config
diff --git a/kamon-statsd/src/main/scala/kamon/statsd/StatsDMetricsSender.scala b/kamon-statsd/src/main/scala/kamon/statsd/StatsDMetricsSender.scala
index adda18cc..94bab27c 100644
--- a/kamon-statsd/src/main/scala/kamon/statsd/StatsDMetricsSender.scala
+++ b/kamon-statsd/src/main/scala/kamon/statsd/StatsDMetricsSender.scala
@@ -20,12 +20,12 @@ import akka.actor.{ ActorSystem, Props, ActorRef, Actor }
 import akka.io.{ Udp, IO }
 import java.net.InetSocketAddress
 import akka.util.ByteString
-import kamon.metrics.Subscriptions.TickMetricSnapshot
-import kamon.metrics.MetricSnapshot.Measurement
-import kamon.metrics.InstrumentTypes.{ Counter, Gauge, Histogram, InstrumentType }
+import kamon.metric.Subscriptions.TickMetricSnapshot
 import java.text.{ DecimalFormatSymbols, DecimalFormat }
 import java.util.Locale
 
+import kamon.metric.instrument.{ Counter, Histogram }
+
 class StatsDMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long) extends Actor with UdpExtensionProvider {
   import context.system
 
@@ -48,7 +48,7 @@ class StatsDMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long)
   }
 
   def writeMetricsToRemote(tick: TickMetricSnapshot, udpSender: ActorRef): Unit = {
-    val dataBuilder = new MetricDataPacketBuilder(maxPacketSizeInBytes, udpSender, remote)
+    val packetBuilder = new MetricDataPacketBuilder(maxPacketSizeInBytes, udpSender, remote)
 
     for (
       (groupIdentity, groupSnapshot) ← tick.metrics;
@@ -57,25 +57,26 @@ class StatsDMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long)
 
       val key = metricKeyGenerator.generateKey(groupIdentity, metricIdentity)
 
-      for (measurement ← metricSnapshot.measurements) {
-        val measurementData = encodeMeasurement(measurement, metricSnapshot.instrumentType)
-        dataBuilder.appendMeasurement(key, measurementData)
+      metricSnapshot match {
+        case hs: Histogram.Snapshot ⇒
+          hs.recordsIterator.foreach { record ⇒
+            packetBuilder.appendMeasurement(key, encodeStatsDTimer(record.level, record.count))
+          }
+
+        case cs: Counter.Snapshot ⇒
+          packetBuilder.appendMeasurement(key, encodeStatsDCounter(cs.count))
       }
     }
 
-    dataBuilder.flush()
+    packetBuilder.flush()
   }
 
-  def encodeMeasurement(measurement: Measurement, instrumentType: InstrumentType): String = {
-    def statsDMetricFormat(value: String, metricType: String, samplingRate: Double = 1D): String =
-      value + "|" + metricType + (if (samplingRate != 1D) "|@" + samplingRateFormat.format(samplingRate) else "")
-
-    instrumentType match {
-      case Histogram ⇒ statsDMetricFormat(measurement.value.toString, "ms", (1D / measurement.count))
-      case Gauge     ⇒ statsDMetricFormat(measurement.value.toString, "g")
-      case Counter   ⇒ statsDMetricFormat(measurement.count.toString, "c")
-    }
+  def encodeStatsDTimer(level: Long, count: Long): String = {
+    val samplingRate: Double = 1D / count
+    level.toString + "|ms" + (if (samplingRate != 1D) "|@" + samplingRateFormat.format(samplingRate) else "")
   }
+
+  def encodeStatsDCounter(count: Long): String = count.toString + "|c"
 }
 
 object StatsDMetricsSender {
diff --git a/kamon-statsd/src/test/scala/kamon/statsd/StatsDMetricSenderSpec.scala b/kamon-statsd/src/test/scala/kamon/statsd/StatsDMetricSenderSpec.scala
index 9dfd05f7..19d8a80b 100644
--- a/kamon-statsd/src/test/scala/kamon/statsd/StatsDMetricSenderSpec.scala
+++ b/kamon-statsd/src/test/scala/kamon/statsd/StatsDMetricSenderSpec.scala
@@ -18,11 +18,12 @@ package kamon.statsd
 
 import akka.testkit.{ TestKitBase, TestProbe }
 import akka.actor.{ ActorRef, Props, ActorSystem }
+import kamon.metric.instrument.Histogram.Precision
+import kamon.metric.instrument.{ Histogram, HdrHistogram }
 import org.scalatest.{ Matchers, WordSpecLike }
-import kamon.metrics._
+import kamon.metric._
 import akka.io.Udp
-import org.HdrHistogram.HdrRecorder
-import kamon.metrics.Subscriptions.TickMetricSnapshot
+import kamon.metric.Subscriptions.TickMetricSnapshot
 import java.lang.management.ManagementFactory
 import java.net.InetSocketAddress
 import com.typesafe.config.ConfigFactory
@@ -31,14 +32,16 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
   implicit lazy val system = ActorSystem("statsd-metric-sender-spec",
     ConfigFactory.parseString("kamon.statsd.max-packet-size = 256 bytes"))
 
+  val context = CollectionContext.default
+
   "the StatsDMetricSender" should {
     "flush the metrics data after processing the tick, even if the max-packet-size is not reached" in new UdpListenerFixture {
       val testMetricName = "test-metric"
       val testMetricKey = buildMetricKey(testMetricName)
-      val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
+      val testRecorder = Histogram(1000L, Precision.Normal, Scale.Unit)
       testRecorder.record(10L)
 
-      val udp = setup(Map(testMetricName -> testRecorder.collect()))
+      val udp = setup(Map(testMetricName -> testRecorder.collect(context)))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
       data.utf8String should be(s"$testMetricKey:10|ms")
@@ -47,12 +50,12 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
     "render several measurements of the same key under a single (key + multiple measurements) packet" in new UdpListenerFixture {
       val testMetricName = "test-metric"
       val testMetricKey = buildMetricKey(testMetricName)
-      val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
+      val testRecorder = Histogram(1000L, Precision.Normal, Scale.Unit)
       testRecorder.record(10L)
       testRecorder.record(11L)
       testRecorder.record(12L)
 
-      val udp = setup(Map(testMetricName -> testRecorder.collect()))
+      val udp = setup(Map(testMetricName -> testRecorder.collect(context)))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
       data.utf8String should be(s"$testMetricKey:10|ms:11|ms:12|ms")
@@ -61,11 +64,11 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
     "include the correspondent sampling rate when rendering multiple occurrences of the same value" in new UdpListenerFixture {
       val testMetricName = "test-metric"
       val testMetricKey = buildMetricKey(testMetricName)
-      val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
+      val testRecorder = Histogram(1000L, Precision.Normal, Scale.Unit)
       testRecorder.record(10L)
       testRecorder.record(10L)
 
-      val udp = setup(Map(testMetricName -> testRecorder.collect()))
+      val udp = setup(Map(testMetricName -> testRecorder.collect(context)))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
       data.utf8String should be(s"$testMetricKey:10|ms|@0.5")
@@ -74,7 +77,7 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
     "flush the packet when the max-packet-size is reached" in new UdpListenerFixture {
       val testMetricName = "test-metric"
       val testMetricKey = buildMetricKey(testMetricName)
-      val testRecorder = HdrRecorder(testMaxPacketSize, 3, Scale.Unit)
+      val testRecorder = Histogram(10000L, Precision.Normal, Scale.Unit)
 
       var bytes = testMetricKey.length
       var level = 0
@@ -84,7 +87,7 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
         bytes += s":$level|ms".length
       }
 
-      val udp = setup(Map(testMetricName -> testRecorder.collect()))
+      val udp = setup(Map(testMetricName -> testRecorder.collect(context)))
       udp.expectMsgType[Udp.Send] // let the first flush pass
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
@@ -97,8 +100,8 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
       val secondTestMetricName = "second-test-metric"
       val secondTestMetricKey = buildMetricKey(secondTestMetricName)
 
-      val firstTestRecorder = HdrRecorder(1000L, 2, Scale.Unit)
-      val secondTestRecorder = HdrRecorder(1000L, 2, Scale.Unit)
+      val firstTestRecorder = Histogram(1000L, Precision.Normal, Scale.Unit)
+      val secondTestRecorder = Histogram(1000L, Precision.Normal, Scale.Unit)
 
       firstTestRecorder.record(10L)
       firstTestRecorder.record(10L)
@@ -108,8 +111,8 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
       secondTestRecorder.record(21L)
 
       val udp = setup(Map(
-        firstTestMetricName -> firstTestRecorder.collect(),
-        secondTestMetricName -> secondTestRecorder.collect()))
+        firstTestMetricName -> firstTestRecorder.collect(context),
+        secondTestMetricName -> secondTestRecorder.collect(context)))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
       data.utf8String should be(s"$firstTestMetricKey:10|ms|@0.5:11|ms\n$secondTestMetricKey:20|ms:21|ms")
@@ -122,7 +125,7 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
 
     def buildMetricKey(metricName: String): String = s"kamon.$localhostName.test-metric-category.test-group.$metricName"
 
-    def setup(metrics: Map[String, MetricSnapshotLike]): TestProbe = {
+    def setup(metrics: Map[String, MetricSnapshot]): TestProbe = {
       val udp = TestProbe()
       val metricsSender = system.actorOf(Props(new StatsDMetricsSender(new InetSocketAddress(localhostName, 0), testMaxPacketSize) {
         override def udpExtension(implicit system: ActorSystem): ActorRef = udp.ref
@@ -149,7 +152,10 @@ class StatsDMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers
       }
 
       metricsSender ! TickMetricSnapshot(0, 0, Map(testGroupIdentity -> new MetricGroupSnapshot {
-        val metrics: Map[MetricIdentity, MetricSnapshotLike] = testMetrics.toMap
+        type GroupSnapshotType = Histogram.Snapshot
+        def merge(that: GroupSnapshotType, context: CollectionContext): GroupSnapshotType = ???
+
+        val metrics: Map[MetricIdentity, MetricSnapshot] = testMetrics.toMap
       }))
 
       udp