From b6d300b500e661f8640c64e2924213f3303d3926 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 20 May 2014 16:34:58 -0300 Subject: + core: added dispatchers metrics instrumentation --- kamon-core/src/main/resources/META-INF/aop.xml | 4 + kamon-core/src/main/resources/reference.conf | 19 +++++ .../akka/instrumentation/DispatcherTracing.scala | 97 ++++++++++++++++++++++ .../scala/kamon/metrics/DispatcherMetrics.scala | 71 ++++++++++++++++ .../dispatcher/DispatcherMetricsCollector.scala | 59 +++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala create mode 100644 kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala create mode 100644 kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala (limited to 'kamon-core/src/main') diff --git a/kamon-core/src/main/resources/META-INF/aop.xml b/kamon-core/src/main/resources/META-INF/aop.xml index 180d905b..3f7dd42d 100644 --- a/kamon-core/src/main/resources/META-INF/aop.xml +++ b/kamon-core/src/main/resources/META-INF/aop.xml @@ -11,6 +11,10 @@ + + + + diff --git a/kamon-core/src/main/resources/reference.conf b/kamon-core/src/main/resources/reference.conf index 9abfd26a..d2830892 100644 --- a/kamon-core/src/main/resources/reference.conf +++ b/kamon-core/src/main/resources/reference.conf @@ -73,6 +73,25 @@ kamon { significant-value-digits = 2 } } + + dispatcher { + maximum-pool-size { + highest-trackable-value = 999999999 + significant-value-digits = 2 + } + running-thread-count { + highest-trackable-value = 999999999 + significant-value-digits = 2 + } + queued-task-count { + highest-trackable-value = 999999999 + significant-value-digits = 2 + } + pool-size { + highest-trackable-value = 999999999 + significant-value-digits = 2 + } + } } } diff --git a/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala b/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala new file mode 100644 index 00000000..63e96fa0 --- /dev/null +++ b/kamon-core/src/main/scala/akka/instrumentation/DispatcherTracing.scala @@ -0,0 +1,97 @@ +/* + * ========================================================================================= + * Copyright © 2013-2014 the kamon project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ========================================================================================= + */ + +package akka.instrumentation + +import org.aspectj.lang.annotation._ +import akka.dispatch.{ Dispatcher, MessageDispatcher } +import kamon.metrics.{ Metrics, DispatcherMetrics } +import kamon.metrics.DispatcherMetrics.DispatcherMetricRecorder +import kamon.Kamon +import akka.actor.{ Cancellable, ActorSystemImpl } +import kamon.metrics.dispatcher.DispatcherMetricsCollector + +@Aspect +class DispatcherTracing { + + private[this] var actorSystem: ActorSystemImpl = _ + + @Pointcut("execution(akka.actor.ActorSystemImpl.new(..)) && this(system)") + def onActorSystemStartup(system: ActorSystemImpl) = {} + + @Before("onActorSystemStartup(system)") + def beforeActorSystemStartup(system: ActorSystemImpl): Unit = { + actorSystem = system + } + + @Pointcut("call(* akka.dispatch.ExecutorServiceFactory.createExecutorService(..))") + def onCreateExecutorService(): Unit = {} + + @Pointcut("cflow((execution(* akka.dispatch.MessageDispatcher.registerForExecution(..)) || execution(* akka.dispatch.MessageDispatcher.executeTask(..))) && this(dispatcher))") + def onCflowMessageDispatcher(dispatcher: Dispatcher): Unit = {} + + @Pointcut("onCreateExecutorService() && onCflowMessageDispatcher(dispatcher)") + def onDispatcherStartup(dispatcher: Dispatcher): Unit = {} + + @After("onDispatcherStartup(dispatcher)") + def afterDispatcherStartup(dispatcher: MessageDispatcher): Unit = { + + val metricsExtension = Kamon(Metrics)(actorSystem) + val metricIdentity = DispatcherMetrics(dispatcher.id) + val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics] + + dispatcherWithMetrics.metricIdentity = metricIdentity + dispatcherWithMetrics.dispatcherMetricsRecorder = metricsExtension.register(metricIdentity, DispatcherMetrics.Factory) + + if (dispatcherWithMetrics.dispatcherMetricsRecorder.isDefined) { + dispatcherWithMetrics.dispatcherCollectorCancellable = metricsExtension.scheduleGaugeRecorder { + dispatcherWithMetrics.dispatcherMetricsRecorder.map { + dm ⇒ + val (maximumPoolSize, runningThreadCount, queueTaskCount, poolSize) = DispatcherMetricsCollector.collect(dispatcher) + + dm.maximumPoolSize.record(maximumPoolSize) + dm.runningThreadCount.record(runningThreadCount) + dm.queueTaskCount.record(queueTaskCount) + dm.poolSize.record(poolSize) + } + } + } + } + + @Pointcut("execution(* akka.dispatch.MessageDispatcher.shutdown(..)) && this(dispatcher)") + def onDispatcherShutdown(dispatcher: MessageDispatcher): Unit = {} + + @After("onDispatcherShutdown(dispatcher)") + def afterDispatcherShutdown(dispatcher: MessageDispatcher): Unit = { + val dispatcherWithMetrics = dispatcher.asInstanceOf[DispatcherMessageMetrics] + dispatcherWithMetrics.dispatcherCollectorCancellable.cancel() + Kamon(Metrics)(actorSystem).unregister(dispatcherWithMetrics.metricIdentity) + } +} + +@Aspect +class DispatcherMetricsMixin { + + @DeclareMixin("akka.dispatch.Dispatcher") + def mixinDispatcherMetricsToMessageDispatcher: DispatcherMessageMetrics = new DispatcherMessageMetrics {} +} + +trait DispatcherMessageMetrics { + var metricIdentity: DispatcherMetrics = _ + var dispatcherMetricsRecorder: Option[DispatcherMetricRecorder] = _ + var dispatcherCollectorCancellable: Cancellable = _ +} + diff --git a/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala b/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala new file mode 100644 index 00000000..77a1c99b --- /dev/null +++ b/kamon-core/src/main/scala/kamon/metrics/DispatcherMetrics.scala @@ -0,0 +1,71 @@ +/* + * ========================================================================================= + * Copyright © 2013-2014 the kamon project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ========================================================================================= + */ + +package kamon.metrics + +import com.typesafe.config.Config +import org.HdrHistogram.HdrRecorder + +case class DispatcherMetrics(name: String) extends MetricGroupIdentity { + val category = DispatcherMetrics +} + +object DispatcherMetrics extends MetricGroupCategory { + val name = "dispatcher" + + case object MaximunPoolSize extends MetricIdentity { val name, tag = "maximum-pool-size" } + case object RunningThreadCount extends MetricIdentity { val name, tag = "running-thread-count" } + case object QueueTaskCount extends MetricIdentity { val name, tag = "queued-task-count" } + case object PoolSize extends MetricIdentity { val name, tag = "pool-size" } + + case class DispatcherMetricRecorder(maximumPoolSize: MetricRecorder, runningThreadCount: MetricRecorder, queueTaskCount: MetricRecorder, poolSize: MetricRecorder) + extends MetricGroupRecorder { + + def collect: MetricGroupSnapshot = { + DispatcherMetricSnapshot(maximumPoolSize.collect(), runningThreadCount.collect(), queueTaskCount.collect(), poolSize.collect()) + } + } + + case class DispatcherMetricSnapshot(maximumPoolSize: MetricSnapshotLike, runningThreadCount: MetricSnapshotLike, queueTaskCount: MetricSnapshotLike, poolSize: MetricSnapshotLike) + extends MetricGroupSnapshot { + + val metrics: Map[MetricIdentity, MetricSnapshotLike] = Map( + (MaximunPoolSize -> maximumPoolSize), + (RunningThreadCount -> runningThreadCount), + (QueueTaskCount -> queueTaskCount), + (PoolSize -> poolSize)) + } + + val Factory = new MetricGroupFactory { + type GroupRecorder = DispatcherMetricRecorder + + def create(config: Config): DispatcherMetricRecorder = { + val settings = config.getConfig("precision.dispatcher") + + val threadCountConfig = extractPrecisionConfig(settings.getConfig("maximum-pool-size")) + val RunningThreadCountConfig = extractPrecisionConfig(settings.getConfig("running-thread-count")) + val QueueTaskCountConfig = extractPrecisionConfig(settings.getConfig("queued-task-count")) + val PoolSizeConfig = extractPrecisionConfig(settings.getConfig("pool-size")) + + new DispatcherMetricRecorder( + HdrRecorder(threadCountConfig.highestTrackableValue, threadCountConfig.significantValueDigits, Scale.Unit), + HdrRecorder(RunningThreadCountConfig.highestTrackableValue, RunningThreadCountConfig.significantValueDigits, Scale.Unit), + HdrRecorder(QueueTaskCountConfig.highestTrackableValue, QueueTaskCountConfig.significantValueDigits, Scale.Unit), + HdrRecorder(PoolSizeConfig.highestTrackableValue, PoolSizeConfig.significantValueDigits, Scale.Unit)) + } + } +} + diff --git a/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala b/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala new file mode 100644 index 00000000..e7edce51 --- /dev/null +++ b/kamon-core/src/main/scala/kamon/metrics/dispatcher/DispatcherMetricsCollector.scala @@ -0,0 +1,59 @@ +package kamon.metrics.dispatcher + +/* + * ========================================================================================= + * Copyright © 2013-2014 the kamon project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + * ========================================================================================= + */ + +import scala.concurrent.forkjoin.ForkJoinPool +import java.util.concurrent.ThreadPoolExecutor +import akka.dispatch.{ExecutorServiceDelegate, Dispatcher, MessageDispatcher} +import java.lang.reflect.Method + +object DispatcherMetricsCollector { + + private[this] type DispatcherMetrics = (Long, Long, Long, Long) + + private[this] def collectForkJoinMetrics(pool: ForkJoinPool):DispatcherMetrics = { + (pool.getParallelism, pool.getActiveThreadCount, (pool.getQueuedTaskCount + pool.getQueuedSubmissionCount), pool.getPoolSize) + } + private[this] def collectExecutorMetrics(pool: ThreadPoolExecutor):DispatcherMetrics = { + (pool.getMaximumPoolSize, pool.getActiveCount, pool.getQueue.size(), pool.getPoolSize) + } + + private[this] val executorServiceMethod: Method = { + // executorService is protected + val method = classOf[Dispatcher].getDeclaredMethod("executorService") + method.setAccessible(true) + method + } + + def collect(dispatcher: MessageDispatcher): (Long, Long, Long, Long) = { + dispatcher match { + case x: Dispatcher ⇒ { + val executor = executorServiceMethod.invoke(x) match { + case delegate: ExecutorServiceDelegate ⇒ delegate.executor + case other ⇒ other + } + + executor match { + case fjp: ForkJoinPool ⇒ collectForkJoinMetrics(fjp) + case tpe: ThreadPoolExecutor ⇒ collectExecutorMetrics(tpe) + case anything ⇒ (0L, 0L, 0L, 0L) + } + } + case _ ⇒ new DispatcherMetrics(0L, 0L, 0L, 0L) + } + } +} -- cgit v1.2.3