+ core: added dispatchers metrics instrumentation

6 files changed, 260 insertions, 4 deletions

diff --git a/kamon-core/src/main/resources/META-INF/aop.xml b/kamon-core/src/main/resources/META-INF/aop.xml
index 180d905b..3f7dd42d 100644
--- a/kamon-core/src/main/resources/META-INF/aop.xml
+++ b/kamon-core/src/main/resources/META-INF/aop.xml
@@ -11,6 +11,10 @@
     <aspect name="akka.instrumentation.BehaviourInvokeTracing"/>
     <aspect name="kamon.instrumentation.ActorLoggingTracing"/>
 
+    <!-- Dispatchers -->
+    <aspect name="akka.instrumentation.DispatcherTracing"/>
+    <aspect name="akka.instrumentation.DispatcherMetricsMixin"/>
+
     <!-- Futures -->
     <aspect name="kamon.instrumentation.FutureTracing"/>
 
diff --git a/kamon-core/src/main/resources/reference.conf b/kamon-core/src/main/resources/reference.conf
index 9abfd26a..d2830892 100644
--- a/kamon-core/src/main/resources/reference.conf
+++ b/kamon-core/src/main/resources/reference.conf
@@ -73,6 +73,25 @@ kamon {
           significant-value-digits = 2
         }
       }
+
+      dispatcher {
+        maximum-pool-size {
+          highest-trackable-value = 999999999
+          significant-value-digits = 2
+        }
+        running-thread-count {
+          highest-trackable-value = 999999999
+          significant-value-digits = 2
+        }
+        queued-task-count {
+          highest-trackable-value = 999999999
+          significant-value-digits = 2
+        }
+        pool-size {
+          highest-trackable-value = 999999999
+          significant-value-digits = 2
+        }
+      }
     }
   }
 
diff --git a/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala
new file mode 100644
index 00000000..63e96fa0
--- /dev/null
+++ b/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala
@@ -0,0 +1,97 @@
+/*
+ * =========================================================================================
+ * Copyright © 2013-2014 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+package akka.instrumentation
+
+import org.aspectj.lang.annotation._
+import akka.dispatch.{ Dispatcher, MessageDispatcher }
+import kamon.metrics.{ Metrics, DispatcherMetrics }
+import kamon.metrics.DispatcherMetrics.DispatcherMetricRecorder
+import kamon.Kamon
+import akka.actor.{ Cancellable, ActorSystemImpl }
+import kamon.metrics.dispatcher.DispatcherMetricsCollector
+
+@Aspect
+class DispatcherTracing {
+
+  private[this] var actorSystem: ActorSystemImpl = _
+
+  @Pointcut("execution(akka.actor.ActorSystemImpl.new(..)) && this(system)")
+  def onActorSystemStartup(system: ActorSystemImpl) = {}
+
+  @Before("onActorSystemStartup(system)")
+  def beforeActorSystemStartup(system: ActorSystemImpl): Unit = {
+    actorSystem = system
+  }
+
+  @Pointcut("call(* akka.dispatch.ExecutorServiceFactory.createExecutorService(..))")
+  def onCreateExecutorService(): Unit = {}
+
+  @Pointcut("cflow((execution(* akka.dispatch.MessageDispatcher.registerForExecution(..)) || execution(* akka.dispatch.MessageDispatcher.executeTask(..))) && this(dispatcher))")
+  def onCflowMessageDispatcher(dispatcher: Dispatcher): Unit = {}
+
+  @Pointcut("onCreateExecutorService() && onCflowMessageDispatcher(dispatcher)")
+  def onDispatcherStartup(dispatcher: Dispatcher): Unit = {}
+
+  @After("onDispatcherStartup(dispatcher)")
+  def afterDispatcherStartup(dispatcher: MessageDispatcher): Unit = {
+
+    val metricsExtension = Kamon(Metrics)(actorSystem)
+    val metricIdentity = DispatcherMetrics(dispatcher.id)
+    val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics]
+
+    dispatcherWithMetrics.metricIdentity = metricIdentity
+    dispatcherWithMetrics.dispatcherMetricsRecorder = metricsExtension.register(metricIdentity, DispatcherMetrics.Factory)
+
+    if (dispatcherWithMetrics.dispatcherMetricsRecorder.isDefined) {
+      dispatcherWithMetrics.dispatcherCollectorCancellable = metricsExtension.scheduleGaugeRecorder {
+        dispatcherWithMetrics.dispatcherMetricsRecorder.map {
+          dm ⇒
+            val (maximumPoolSize, runningThreadCount, queueTaskCount, poolSize) = DispatcherMetricsCollector.collect(dispatcher)
+
+            dm.maximumPoolSize.record(maximumPoolSize)
+            dm.runningThreadCount.record(runningThreadCount)
+            dm.queueTaskCount.record(queueTaskCount)
+            dm.poolSize.record(poolSize)
+        }
+      }
+    }
+  }
+
+  @Pointcut("execution(* akka.dispatch.MessageDispatcher.shutdown(..)) && this(dispatcher)")
+  def onDispatcherShutdown(dispatcher: MessageDispatcher): Unit = {}
+
+  @After("onDispatcherShutdown(dispatcher)")
+  def afterDispatcherShutdown(dispatcher: MessageDispatcher): Unit = {
+    val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics]
+    dispatcherWithMetrics.dispatcherCollectorCancellable.cancel()
+    Kamon(Metrics)(actorSystem).unregister(dispatcherWithMetrics.metricIdentity)
+  }
+}
+
+@Aspect
+class DispatcherMetricsMixin {
+
+  @DeclareMixin("akka.dispatch.Dispatcher")
+  def mixinDispatcherMetricsToMessageDispatcher: DispatcherMessageMetrics = new DispatcherMessageMetrics {}
+}
+
+trait DispatcherMessageMetrics {
+  var metricIdentity: DispatcherMetrics = _
+  var dispatcherMetricsRecorder: Option[DispatcherMetricRecorder] = _
+  var dispatcherCollectorCancellable: Cancellable = _
+}
+
diff --git a/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala b/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala
new file mode 100644
index 00000000..77a1c99b
--- /dev/null
+++ b/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala
@@ -0,0 +1,71 @@
+/*
+ * =========================================================================================
+ * Copyright © 2013-2014 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+package kamon.metrics
+
+import com.typesafe.config.Config
+import org.HdrHistogram.HdrRecorder
+
+case class DispatcherMetrics(name: String) extends MetricGroupIdentity {
+  val category = DispatcherMetrics
+}
+
+object DispatcherMetrics extends MetricGroupCategory {
+  val name = "dispatcher"
+
+  case object MaximunPoolSize extends MetricIdentity { val name, tag = "maximum-pool-size" }
+  case object RunningThreadCount extends MetricIdentity { val name, tag = "running-thread-count" }
+  case object QueueTaskCount extends MetricIdentity { val name, tag = "queued-task-count" }
+  case object PoolSize extends MetricIdentity { val name, tag = "pool-size" }
+
+  case class DispatcherMetricRecorder(maximumPoolSize: MetricRecorder, runningThreadCount: MetricRecorder, queueTaskCount: MetricRecorder, poolSize: MetricRecorder)
+      extends MetricGroupRecorder {
+
+    def collect: MetricGroupSnapshot = {
+      DispatcherMetricSnapshot(maximumPoolSize.collect(), runningThreadCount.collect(), queueTaskCount.collect(), poolSize.collect())
+    }
+  }
+
+  case class DispatcherMetricSnapshot(maximumPoolSize: MetricSnapshotLike, runningThreadCount: MetricSnapshotLike, queueTaskCount: MetricSnapshotLike, poolSize: MetricSnapshotLike)
+      extends MetricGroupSnapshot {
+
+    val metrics: Map[MetricIdentity, MetricSnapshotLike] = Map(
+      (MaximunPoolSize -> maximumPoolSize),
+      (RunningThreadCount -> runningThreadCount),
+      (QueueTaskCount -> queueTaskCount),
+      (PoolSize -> poolSize))
+  }
+
+  val Factory = new MetricGroupFactory {
+    type GroupRecorder = DispatcherMetricRecorder
+
+    def create(config: Config): DispatcherMetricRecorder = {
+      val settings = config.getConfig("precision.dispatcher")
+
+      val threadCountConfig = extractPrecisionConfig(settings.getConfig("maximum-pool-size"))
+      val RunningThreadCountConfig = extractPrecisionConfig(settings.getConfig("running-thread-count"))
+      val QueueTaskCountConfig = extractPrecisionConfig(settings.getConfig("queued-task-count"))
+      val PoolSizeConfig = extractPrecisionConfig(settings.getConfig("pool-size"))
+
+      new DispatcherMetricRecorder(
+        HdrRecorder(threadCountConfig.highestTrackableValue, threadCountConfig.significantValueDigits, Scale.Unit),
+        HdrRecorder(RunningThreadCountConfig.highestTrackableValue, RunningThreadCountConfig.significantValueDigits, Scale.Unit),
+        HdrRecorder(QueueTaskCountConfig.highestTrackableValue, QueueTaskCountConfig.significantValueDigits, Scale.Unit),
+        HdrRecorder(PoolSizeConfig.highestTrackableValue, PoolSizeConfig.significantValueDigits, Scale.Unit))
+    }
+  }
+}
+
diff --git a/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala b/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala
new file mode 100644
index 00000000..e7edce51
--- /dev/null
+++ b/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala
@@ -0,0 +1,59 @@
+package kamon.metrics.dispatcher
+
+/*
+ * =========================================================================================
+ * Copyright © 2013-2014 the kamon project <http://kamon.io/>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ * =========================================================================================
+ */
+
+import scala.concurrent.forkjoin.ForkJoinPool
+import java.util.concurrent.ThreadPoolExecutor
+import akka.dispatch.{ExecutorServiceDelegate, Dispatcher, MessageDispatcher}
+import java.lang.reflect.Method
+
+object DispatcherMetricsCollector {
+
+  private[this] type DispatcherMetrics = (Long, Long, Long, Long)
+
+  private[this] def collectForkJoinMetrics(pool: ForkJoinPool):DispatcherMetrics = {
+    (pool.getParallelism, pool.getActiveThreadCount, (pool.getQueuedTaskCount + pool.getQueuedSubmissionCount), pool.getPoolSize)
+  }
+  private[this] def collectExecutorMetrics(pool: ThreadPoolExecutor):DispatcherMetrics = {
+    (pool.getMaximumPoolSize, pool.getActiveCount, pool.getQueue.size(), pool.getPoolSize)
+  }
+
+  private[this] val executorServiceMethod: Method = {
+    // executorService is protected
+    val method = classOf[Dispatcher].getDeclaredMethod("executorService")
+    method.setAccessible(true)
+    method
+  }
+
+  def collect(dispatcher: MessageDispatcher): (Long, Long, Long, Long) = {
+    dispatcher match {
+      case x: Dispatcher ⇒ {
+        val executor = executorServiceMethod.invoke(x) match {
+          case delegate: ExecutorServiceDelegate ⇒ delegate.executor
+          case other ⇒ other
+        }
+
+        executor match {
+          case fjp: ForkJoinPool ⇒ collectForkJoinMetrics(fjp)
+          case tpe: ThreadPoolExecutor ⇒ collectExecutorMetrics(tpe)
+          case anything ⇒ (0L, 0L, 0L, 0L)
+        }
+      }
+      case _ ⇒ new DispatcherMetrics(0L, 0L, 0L, 0L)
+    }
+  }
+}
diff --git a/kamon-playground/src/main/resources/application.conf b/kamon-playground/src/main/resources/application.conf
index 60b3eb6a..3d57141a 100644
--- a/kamon-playground/src/main/resources/application.conf
+++ b/kamon-playground/src/main/resources/application.conf
@@ -1,8 +1,6 @@
 akka {
-  loggers = [ "akka.event.slf4j.Slf4jLogger" ]
   loglevel = INFO
 
-  extensions = ["kamon.newrelic.NewRelic", "kamon.statsd.StatsD", "kamon.datadog.Datadog"]
 
   actor {
     debug {
@@ -43,7 +41,7 @@ kamon {
       },
       {
         dispatcher {
-          includes = [ "default-dispatcher" ]
+          includes = [ "akka.actor.default-dispatcher" ]
           excludes = []
         }
       },
@@ -85,4 +83,12 @@ kamon {
       ]
     }
   }
-}
\ No newline at end of file
+
+  weaver {
+      showWeaveInfo = on
+      verbose = off
+      debug = off
+      showWarn = off
+    }
+}
+