+ datadog: include identification tags and clean up all irrelevant code from StatsD

4 files changed, 51 insertions, 131 deletions

diff --git a/kamon-datadog/src/main/resources/reference.conf b/kamon-datadog/src/main/resources/reference.conf
index 5aaa87fc..657d3385 100644
--- a/kamon-datadog/src/main/resources/reference.conf
+++ b/kamon-datadog/src/main/resources/reference.conf
@@ -13,9 +13,6 @@ kamon {
     # kamon.metrics.tick-interval setting.
     flush-interval = 1 second
 
-    # Max packet size for UDP metrics data sent to Datadog.
-    max-packet-size = 1024 bytes
-
     # Subscription patterns used to select which metrics will be pushed to Datadog. Note that first, metrics
     # collection for your desired entities must be activated under the kamon.metrics.filters settings.
     includes {
@@ -23,11 +20,9 @@ kamon {
       trace = [ "*" ]
     }
 
-    simple-metric-key-generator {
-      # Application prefix for all metrics pushed to Datadog. The default namespacing scheme for metrics follows
-      # this pattern:
-      #    application.host.entity.entity-name.metric-name
-      application = "kamon"
-    }
+    # Application prefix for all metrics pushed to Datadog. The default namespacing scheme for metrics follows
+    # this pattern:
+    #    application.entity-name.metric-name
+    application-name = "kamon"
   }
 }
diff --git a/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala b/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala
index b492b9fd..c6981fd7 100644
--- a/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala
+++ b/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala
@@ -44,7 +44,6 @@ class DatadogExtension(system: ExtendedActorSystem) extends Kamon.Extension {
 
   val datadogHost = new InetSocketAddress(datadogConfig.getString("hostname"), datadogConfig.getInt("port"))
   val flushInterval = datadogConfig.getDuration("flush-interval", MILLISECONDS)
-  val maxPacketSizeInBytes = datadogConfig.getBytes("max-packet-size")
   val tickInterval = system.settings.config.getDuration("kamon.metrics.tick-interval", MILLISECONDS)
 
   val datadogMetricsListener = buildMetricsListener(tickInterval, flushInterval)
@@ -64,7 +63,7 @@ class DatadogExtension(system: ExtendedActorSystem) extends Kamon.Extension {
   def buildMetricsListener(tickInterval: Long, flushInterval: Long): ActorRef = {
     assert(flushInterval >= tickInterval, "Datadog flush-interval needs to be equal or greater to the tick-interval")
 
-    val metricsTranslator = system.actorOf(DatadogMetricsSender.props(datadogHost, maxPacketSizeInBytes), "datadog-metrics-sender")
+    val metricsTranslator = system.actorOf(DatadogMetricsSender.props(datadogHost), "datadog-metrics-sender")
     if (flushInterval == tickInterval) {
       // No need to buffer the metrics, let's go straight to the metrics sender.
       metricsTranslator
@@ -74,11 +73,3 @@ class DatadogExtension(system: ExtendedActorSystem) extends Kamon.Extension {
   }
 }
 
-class SimpleMetricKeyGenerator(config: Config) extends Datadog.MetricKeyGenerator {
-  val application = config.getString("kamon.datadog.simple-metric-key-generator.application")
-  val localhostName = ManagementFactory.getRuntimeMXBean.getName.split('@')(1)
-
-  def generateKey(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity): String =
-    s"${application}.${localhostName}.${groupIdentity.category.name}.${groupIdentity.name}.${metricIdentity.name}"
-}
-
diff --git a/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala b/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala
index df072552..ec603b39 100644
--- a/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala
+++ b/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala
@@ -24,11 +24,12 @@ import kamon.metrics.Subscriptions.TickMetricSnapshot
 import kamon.metrics.MetricSnapshot.Measurement
 import kamon.metrics.InstrumentTypes.{ Counter, Gauge, Histogram, InstrumentType }
 import java.text.DecimalFormat
+import kamon.metrics.{ MetricIdentity, MetricGroupIdentity }
 
-class DatadogMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long) extends Actor with UdpExtensionProvider {
+class DatadogMetricsSender(remote: InetSocketAddress) extends Actor with UdpExtensionProvider {
   import context.system
 
-  val metricKeyGenerator = new SimpleMetricKeyGenerator(context.system.settings.config)
+  val appName = context.system.settings.config.getString("kamon.datadog.application-name")
   val samplingRateFormat = new DecimalFormat()
   samplingRateFormat.setMaximumFractionDigits(128) // Absurdly high, let the other end loss precision if it needs to.
 
@@ -44,38 +45,34 @@ class DatadogMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long
   }
 
   def writeMetricsToRemote(tick: TickMetricSnapshot, udpSender: ActorRef): Unit = {
-    val dataBuilder = new MetricDataPacketBuilder(maxPacketSizeInBytes, udpSender, remote)
+    def flushToUDP(data: String): Unit = udpSender ! Udp.Send(ByteString(data), remote)
 
     for (
       (groupIdentity, groupSnapshot) ← tick.metrics;
       (metricIdentity, metricSnapshot) ← groupSnapshot.metrics
     ) {
 
-      val key = metricKeyGenerator.generateKey(groupIdentity, metricIdentity)
-
       for (measurement ← metricSnapshot.measurements) {
-        val measurementData = encodeMeasurement(measurement, metricSnapshot.instrumentType)
-        dataBuilder.appendMeasurement(key, measurementData)
+        val measurementData = formatMeasurement(groupIdentity, metricIdentity, measurement, metricSnapshot.instrumentType)
+        flushToUDP(measurementData)
       }
     }
-
-    dataBuilder.flush()
   }
 
-  def encodeMeasurement(measurement: Measurement, instrumentType: InstrumentType): String = {
+  def formatMeasurement(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity, measurement: Measurement,
+                        instrumentType: InstrumentType): String = {
 
-    def processTags(tags: Seq[String]): String = {
-      if (tags.isEmpty) "" else {
-        tags.foldLeft(new StringBuilder("|#")) {
-          (sb, s) ⇒
-            if (sb.length > 2) sb ++= ","
-            sb ++= s
-        }.toString()
-      }
-    }
+    StringBuilder.newBuilder
+      .append(buildMetricName(groupIdentity, metricIdentity))
+      .append(":")
+      .append(buildMeasurementData(measurement, instrumentType))
+      .append(buildIdentificationTag(groupIdentity, metricIdentity))
+      .result()
+  }
 
-    def dataDogDMetricFormat(value: String, metricType: String, samplingRate: Double = 1D, tags: Seq[String] = Nil): String =
-      value + "|" + metricType + (if (samplingRate != 1D) "|@" + samplingRateFormat.format(samplingRate) else "" + processTags(tags))
+  def buildMeasurementData(measurement: Measurement, instrumentType: InstrumentType): String = {
+    def dataDogDMetricFormat(value: String, metricType: String, samplingRate: Double = 1D): String =
+      value + "|" + metricType + (if (samplingRate != 1D) "|@" + samplingRateFormat.format(samplingRate) else "")
 
     instrumentType match {
       case Histogram ⇒ dataDogDMetricFormat(measurement.value.toString, "ms", (1D / measurement.count))
@@ -83,42 +80,18 @@ class DatadogMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long
       case Counter   ⇒ dataDogDMetricFormat(measurement.count.toString, "c")
     }
   }
+
+  def buildMetricName(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity): String =
+    appName + "." + groupIdentity.category.name + "." + metricIdentity.name
+
+  def buildIdentificationTag(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity): String =
+    "|#" + groupIdentity.category.name + ":" + groupIdentity.name
 }
 
 object DatadogMetricsSender {
-  def props(remote: InetSocketAddress, maxPacketSize: Long): Props = Props(new DatadogMetricsSender(remote, maxPacketSize))
+  def props(remote: InetSocketAddress): Props = Props(new DatadogMetricsSender(remote))
 }
 
 trait UdpExtensionProvider {
   def udpExtension(implicit system: ActorSystem): ActorRef = IO(Udp)
 }
-
-class MetricDataPacketBuilder(maxPacketSizeInBytes: Long, udpSender: ActorRef, remote: InetSocketAddress) {
-  val metricSeparator = "\n"
-  val measurementSeparator = ":"
-
-  var lastKey = ""
-  var buffer = new StringBuilder()
-
-  def appendMeasurement(key: String, measurementData: String): Unit = {
-    val data = key + measurementSeparator + measurementData
-
-    if (fitsOnBuffer(metricSeparator + data)) {
-      val mSeparator = if (buffer.length > 0) metricSeparator else ""
-      buffer.append(mSeparator).append(data)
-    } else {
-      flushToUDP(buffer.toString())
-      buffer.clear()
-      buffer.append(data)
-    }
-  }
-
-  def fitsOnBuffer(data: String): Boolean = (buffer.length + data.length) <= maxPacketSizeInBytes
-
-  private def flushToUDP(data: String): Unit = udpSender ! Udp.Send(ByteString(data), remote)
-
-  def flush(): Unit = {
-    flushToUDP(buffer.toString)
-    buffer.clear()
-  }
-}
diff --git a/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala b/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala
index 98b36159..c649a044 100644
--- a/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala
+++ b/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala
@@ -17,7 +17,7 @@
 package kamon.datadog
 
 import akka.testkit.{ TestKitBase, TestProbe }
-import akka.actor.{Props, ActorRef, ActorSystem}
+import akka.actor.{ Props, ActorRef, ActorSystem }
 import kamon.metrics.instruments.CounterRecorder
 import org.scalatest.{ Matchers, WordSpecLike }
 import kamon.metrics._
@@ -33,21 +33,19 @@ class DatadogMetricSenderSpec extends TestKitBase with WordSpecLike with Matcher
     ConfigFactory.parseString("kamon.datadog.max-packet-size = 256 bytes"))
 
   "the DataDogMetricSender" should {
-    "flush the metrics data after processing the tick, even if the max-packet-size is not reached" in new UdpListenerFixture {
-      val testMetricName = "test-metric"
-      val testMetricKey = buildMetricKey(testMetricName)
+    "send latency measurements" in new UdpListenerFixture {
+      val testMetricName = "processing-time"
       val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
       testRecorder.record(10L)
 
       val udp = setup(Map(testMetricName -> testRecorder.collect()))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
-      data.utf8String should be(s"$testMetricKey:10|ms")
+      data.utf8String should be(s"kamon.actor.processing-time:10|ms|#actor:user/kamon")
     }
 
-    "include the correspondent sampling rate when rendering multiple occurrences of the same value" in new UdpListenerFixture {
-      val testMetricName = "test-metric"
-      val testMetricKey = buildMetricKey(testMetricName)
+    "include the sampling rate in case of multiple measurements of the same value" in new UdpListenerFixture {
+      val testMetricName = "processing-time"
       val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
       testRecorder.record(10L)
       testRecorder.record(10L)
@@ -55,59 +53,23 @@ class DatadogMetricSenderSpec extends TestKitBase with WordSpecLike with Matcher
       val udp = setup(Map(testMetricName -> testRecorder.collect()))
       val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
-      data.utf8String should be(s"$testMetricKey:10|ms|@0.5")
+      data.utf8String should be(s"kamon.actor.processing-time:10|ms|@0.5|#actor:user/kamon")
     }
 
-    "flush the packet when the max-packet-size is reached" in new UdpListenerFixture {
-      val testMetricName = "test-metric"
-      val testMetricKey = buildMetricKey(testMetricName)
-      val testRecorder = HdrRecorder(testMaxPacketSize, 3, Scale.Unit)
-
-      var bytes = 0//testMetricKey.length
-      var level = 0
-      while (bytes <= testMaxPacketSize) {
-        level += 1
-        testRecorder.record(level)
-        bytes += s"$testMetricKey:$level|ms".length
-      }
+    "send only one packet per measurement" in new UdpListenerFixture {
+      val testMetricName = "processing-time"
+      val testRecorder = HdrRecorder(1000L, 2, Scale.Unit)
+      testRecorder.record(10L)
+      testRecorder.record(10L)
+      testRecorder.record(20L)
 
       val udp = setup(Map(testMetricName -> testRecorder.collect()))
-      udp.expectMsgType[Udp.Send] // let the first flush pass
-      val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
 
-      data.utf8String should be(s"$testMetricKey:$level|ms")
-    }
-
-    "render multiple keys in the same packet using newline as separator" in new UdpListenerFixture {
-      val firstTestMetricName = "first-test-metric"
-      val firstTestMetricKey = buildMetricKey(firstTestMetricName)
-      val secondTestMetricName = "second-test-metric"
-      val secondTestMetricKey = buildMetricKey(secondTestMetricName)
-      val thirdTestMetricName = "third-test-metric"
-      val thirdTestMetricKey = buildMetricKey(thirdTestMetricName)
-
-      val firstTestRecorder = HdrRecorder(1000L, 2, Scale.Unit)
-      val secondTestRecorder = HdrRecorder(1000L, 2, Scale.Unit)
-      val thirdTestRecorder = CounterRecorder()
-
-      firstTestRecorder.record(10L)
-      firstTestRecorder.record(10L)
-
-      secondTestRecorder.record(21L)
-
-      thirdTestRecorder.record(1L)
-      thirdTestRecorder.record(1L)
-      thirdTestRecorder.record(1L)
-      thirdTestRecorder.record(1L)
-
-      val t = thirdTestRecorder.collect()
-      val udp = setup(Map(
-        firstTestMetricName -> firstTestRecorder.collect(),
-        secondTestMetricName -> secondTestRecorder.collect(),
-        thirdTestMetricName -> t))
-      val Udp.Send(data, _, _) = udp.expectMsgType[Udp.Send]
+      val Udp.Send(data1, _, _) = udp.expectMsgType[Udp.Send]
+      data1.utf8String should be(s"kamon.actor.processing-time:10|ms|@0.5|#actor:user/kamon")
 
-      data.utf8String should be(s"$firstTestMetricKey:10|ms|@0.5\n$secondTestMetricKey:21|ms\n$thirdTestMetricKey:4|c")
+      val Udp.Send(data2, _, _) = udp.expectMsgType[Udp.Send]
+      data2.utf8String should be(s"kamon.actor.processing-time:20|ms|#actor:user/kamon")
     }
   }
 
@@ -115,11 +77,9 @@ class DatadogMetricSenderSpec extends TestKitBase with WordSpecLike with Matcher
     val localhostName = ManagementFactory.getRuntimeMXBean.getName.split('@')(1)
     val testMaxPacketSize = system.settings.config.getBytes("kamon.datadog.max-packet-size")
 
-    def buildMetricKey(metricName: String): String = s"kamon.$localhostName.test-metric-category.test-group.$metricName"
-
     def setup(metrics: Map[String, MetricSnapshotLike]): TestProbe = {
       val udp = TestProbe()
-      val metricsSender = system.actorOf(Props(new DatadogMetricsSender(new InetSocketAddress(localhostName, 0), testMaxPacketSize) {
+      val metricsSender = system.actorOf(Props(new DatadogMetricsSender(new InetSocketAddress(localhostName, 0)) {
         override def udpExtension(implicit system: ActorSystem): ActorRef = udp.ref
       }))
 
@@ -127,10 +87,11 @@ class DatadogMetricSenderSpec extends TestKitBase with WordSpecLike with Matcher
       udp.expectMsgType[Udp.SimpleSender]
       udp.reply(Udp.SimpleSenderReady)
 
+      // These names are not intented to match the real actor metrics, it's just about seeing more familiar data in tests.
       val testGroupIdentity = new MetricGroupIdentity {
-        val name: String = "test-group"
+        val name: String = "user/kamon"
         val category: MetricGroupCategory = new MetricGroupCategory {
-          val name: String = "test-metric-category"
+          val name: String = "actor"
         }
       }