! datadog: first commit of datadog integration module

4 files changed, 272 insertions, 0 deletions

diff --git a/kamon-datadog/src/main/resources/reference.conf b/kamon-datadog/src/main/resources/reference.conf
new file mode 100644
index 00000000..5aaa87fc
--- /dev/null
+++ b/kamon-datadog/src/main/resources/reference.conf
@@ -0,0 +1,33 @@
+# ==================================== #
+# Kamon-Datadog Reference Configuration #
+# ==================================== #
+
+kamon {
+  datadog {
+    # Hostname and port in which your Datadog is running. Remember that Datadog packets are sent using UDP and
+    # setting unreachable hosts and/or not open ports wont be warned by the Kamon, your data wont go anywhere.
+    hostname = "127.0.0.1"
+    port = 8125
+
+    # Interval between metrics data flushes to Datadog. It's value must be equal or greater than the
+    # kamon.metrics.tick-interval setting.
+    flush-interval = 1 second
+
+    # Max packet size for UDP metrics data sent to Datadog.
+    max-packet-size = 1024 bytes
+
+    # Subscription patterns used to select which metrics will be pushed to Datadog. Note that first, metrics
+    # collection for your desired entities must be activated under the kamon.metrics.filters settings.
+    includes {
+      actor = [ "*" ]
+      trace = [ "*" ]
+    }
+
+    simple-metric-key-generator {
+      # Application prefix for all metrics pushed to Datadog. The default namespacing scheme for metrics follows
+      # this pattern:
+      #    application.host.entity.entity-name.metric-name
+      application = "kamon"
+    }
+  }
+}
diff --git a/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala b/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala
new file mode 100644
index 00000000..b492b9fd
--- /dev/null
+++ b/kamon-datadog/src/main/scala/kamon/datadog/Datadog.scala
@@ -0,0 +1,84 @@
+/*
+ * =========================================================================================
+ * Copyright © 2013-2014 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+package kamon.datadog
+
+import akka.actor._
+import kamon.Kamon
+import kamon.metrics._
+import scala.concurrent.duration._
+import scala.collection.JavaConverters._
+import com.typesafe.config.Config
+import java.lang.management.ManagementFactory
+import akka.event.Logging
+import java.net.InetSocketAddress
+import java.util.concurrent.TimeUnit.MILLISECONDS
+
+object Datadog extends ExtensionId[DatadogExtension] with ExtensionIdProvider {
+  override def lookup(): ExtensionId[_ <: Extension] = Datadog
+  override def createExtension(system: ExtendedActorSystem): DatadogExtension = new DatadogExtension(system)
+
+  trait MetricKeyGenerator {
+    def generateKey(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity): String
+  }
+}
+
+class DatadogExtension(system: ExtendedActorSystem) extends Kamon.Extension {
+  val log = Logging(system, classOf[DatadogExtension])
+  log.info("Starting the Kamon(Datadog) extension")
+
+  private val datadogConfig = system.settings.config.getConfig("kamon.datadog")
+
+  val datadogHost = new InetSocketAddress(datadogConfig.getString("hostname"), datadogConfig.getInt("port"))
+  val flushInterval = datadogConfig.getDuration("flush-interval", MILLISECONDS)
+  val maxPacketSizeInBytes = datadogConfig.getBytes("max-packet-size")
+  val tickInterval = system.settings.config.getDuration("kamon.metrics.tick-interval", MILLISECONDS)
+
+  val datadogMetricsListener = buildMetricsListener(tickInterval, flushInterval)
+
+  // Subscribe to Actors
+  val includedActors = datadogConfig.getStringList("includes.actor").asScala
+  for (actorPathPattern ← includedActors) {
+    Kamon(Metrics)(system).subscribe(ActorMetrics, actorPathPattern, datadogMetricsListener, permanently = true)
+  }
+
+  // Subscribe to Traces
+  val includedTraces = datadogConfig.getStringList("includes.trace").asScala
+  for (tracePathPattern ← includedTraces) {
+    Kamon(Metrics)(system).subscribe(TraceMetrics, tracePathPattern, datadogMetricsListener, permanently = true)
+  }
+
+  def buildMetricsListener(tickInterval: Long, flushInterval: Long): ActorRef = {
+    assert(flushInterval >= tickInterval, "Datadog flush-interval needs to be equal or greater to the tick-interval")
+
+    val metricsTranslator = system.actorOf(DatadogMetricsSender.props(datadogHost, maxPacketSizeInBytes), "datadog-metrics-sender")
+    if (flushInterval == tickInterval) {
+      // No need to buffer the metrics, let's go straight to the metrics sender.
+      metricsTranslator
+    } else {
+      system.actorOf(TickMetricSnapshotBuffer.props(flushInterval.toInt.millis, metricsTranslator), "datadog-metrics-buffer")
+    }
+  }
+}
+
+class SimpleMetricKeyGenerator(config: Config) extends Datadog.MetricKeyGenerator {
+  val application = config.getString("kamon.datadog.simple-metric-key-generator.application")
+  val localhostName = ManagementFactory.getRuntimeMXBean.getName.split('@')(1)
+
+  def generateKey(groupIdentity: MetricGroupIdentity, metricIdentity: MetricIdentity): String =
+    s"${application}.${localhostName}.${groupIdentity.category.name}.${groupIdentity.name}.${metricIdentity.name}"
+}
+
diff --git a/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala b/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala
new file mode 100644
index 00000000..9d7b34e6
--- /dev/null
+++ b/kamon-datadog/src/main/scala/kamon/datadog/DatadogMetricsSender.scala
@@ -0,0 +1,124 @@
+/*
+ * =========================================================================================
+ * Copyright © 2013-2014 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+package kamon.datadog
+
+import akka.actor.{ ActorSystem, Props, ActorRef, Actor }
+import akka.io.{ Udp, IO }
+import java.net.InetSocketAddress
+import akka.util.ByteString
+import kamon.metrics.Subscriptions.TickMetricSnapshot
+import kamon.metrics.MetricSnapshot.Measurement
+import kamon.metrics.InstrumentTypes.{ Counter, Gauge, Histogram, InstrumentType }
+import java.text.DecimalFormat
+
+class DatadogMetricsSender(remote: InetSocketAddress, maxPacketSizeInBytes: Long) extends Actor with UdpExtensionProvider {
+  import context.system
+
+  val metricKeyGenerator = new SimpleMetricKeyGenerator(context.system.settings.config)
+  val samplingRateFormat = new DecimalFormat()
+  samplingRateFormat.setMaximumFractionDigits(128) // Absurdly high, let the other end loss precision if it needs to.
+
+  udpExtension ! Udp.SimpleSender
+
+  def receive = {
+    case Udp.SimpleSenderReady ⇒
+      context.become(ready(sender))
+  }
+
+  def ready(udpSender: ActorRef): Receive = {
+    case tick: TickMetricSnapshot ⇒ writeMetricsToRemote(tick, udpSender)
+  }
+
+  def writeMetricsToRemote(tick: TickMetricSnapshot, udpSender: ActorRef): Unit = {
+    val dataBuilder = new MetricDataPacketBuilder(maxPacketSizeInBytes, udpSender, remote)
+
+    for (
+      (groupIdentity, groupSnapshot) ← tick.metrics;
+      (metricIdentity, metricSnapshot) ← groupSnapshot.metrics
+    ) {
+
+      val key = metricKeyGenerator.generateKey(groupIdentity, metricIdentity)
+
+      for (measurement ← metricSnapshot.measurements) {
+        val measurementData = encodeMeasurement(measurement, metricSnapshot.instrumentType)
+        dataBuilder.appendMeasurement(key, measurementData)
+      }
+    }
+
+    dataBuilder.flush()
+  }
+
+  def encodeMeasurement(measurement: Measurement, instrumentType: InstrumentType): String = {
+    def statsDMetricFormat(value: String, metricType: String, samplingRate: Double = 1D): String =
+      value + "|" + metricType + (if (samplingRate != 1D) "|@" + samplingRateFormat.format(samplingRate) else "")
+
+    instrumentType match {
+      case Histogram ⇒ statsDMetricFormat(measurement.value.toString, "ms", (1D / measurement.count))
+      case Gauge     ⇒ statsDMetricFormat(measurement.value.toString, "g")
+      case Counter   ⇒ "" // TODO: Need to decide how to report counters, when we have them!
+    }
+  }
+}
+
+object DatadogMetricsSender {
+  def props(remote: InetSocketAddress, maxPacketSize: Long): Props = Props(new DatadogMetricsSender(remote, maxPacketSize))
+}
+
+trait UdpExtensionProvider {
+  def udpExtension(implicit system: ActorSystem): ActorRef = IO(Udp)
+}
+
+class MetricDataPacketBuilder(maxPacketSizeInBytes: Long, udpSender: ActorRef, remote: InetSocketAddress) {
+  val metricSeparator = "\n"
+  val measurementSeparator = ":"
+
+  var lastKey = ""
+  var buffer = new StringBuilder()
+
+  def appendMeasurement(key: String, measurementData: String): Unit = {
+    if (key == lastKey) {
+      val dataWithoutKey = measurementSeparator + measurementData
+      if (fitsOnBuffer(dataWithoutKey))
+        buffer.append(dataWithoutKey)
+      else {
+        flushToUDP(buffer.toString())
+        buffer.clear()
+        buffer.append(key).append(dataWithoutKey)
+      }
+    } else {
+      lastKey = key
+      val dataWithoutSeparator = key + measurementSeparator + measurementData
+      if (fitsOnBuffer(metricSeparator + dataWithoutSeparator)) {
+        val mSeparator = if (buffer.length > 0) metricSeparator else ""
+        buffer.append(mSeparator).append(dataWithoutSeparator)
+      } else {
+        flushToUDP(buffer.toString())
+        buffer.clear()
+        buffer.append(dataWithoutSeparator)
+      }
+    }
+  }
+
+  def fitsOnBuffer(data: String): Boolean = (buffer.length + data.length) <= maxPacketSizeInBytes
+
+  private def flushToUDP(data: String): Unit = udpSender ! Udp.Send(ByteString(data), remote)
+
+  def flush(): Unit = {
+    flushToUDP(buffer.toString)
+    buffer.clear()
+  }
+}
diff --git a/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala b/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala
new file mode 100644
index 00000000..cd41a112
--- /dev/null
+++ b/kamon-datadog/src/test/scala/kamon/datadog/DatadogMetricSenderSpec.scala
@@ -0,0 +1,31 @@
+/*
+ * =========================================================================================
+ * Copyright © 2013 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+package kamon.datadog
+
+import akka.testkit.{ TestKitBase, TestProbe }
+import akka.actor.{ ActorRef, Props, ActorSystem }
+import org.scalatest.{ Matchers, WordSpecLike }
+import kamon.metrics._
+import akka.io.Udp
+import org.HdrHistogram.HdrRecorder
+import kamon.metrics.Subscriptions.TickMetricSnapshot
+import java.lang.management.ManagementFactory
+import java.net.InetSocketAddress
+import com.typesafe.config.ConfigFactory
+
+class DatadogMetricSenderSpec extends TestKitBase with WordSpecLike with Matchers {
+}