From 29068fc70a3e5a17a630c2c7fff951572bb5fa21 Mon Sep 17 00:00:00 2001 From: Ivan Topolnjak Date: Thu, 3 Jul 2014 14:36:42 -0300 Subject: ! all: refactor the core metric recording instruments and accomodate UserMetrics This PR is including several changes to the kamon-core, most notably: - Formalize the interface for Histograms, Counters and MinMaxCounters. Making sure that the interfaces are as clean as possible. - Move away from the all Vector[Measurement] based Histogram snapshot to a new approach in which we use a single long to store both the index in the counts array and the frequency on that bucket. The leftmost 2 bytes of each long are used for storing the counts array index and the remaining 6 bytes are used for the actual count, and everything is put into a simple long array. This way only the buckets that actually have values will be included in the snapshot with the smallest possible memory footprint. - Introduce Gauges. - Reorganize the instrumentation for Akka and Scala and rewrite most of the tests of this components to avoid going through the subscription protocol to test. - Introduce trace tests and fixes on various tests. - Necessary changes on new relic, datadog and statsd modules to compile with the new codebase. Pending: - Finish the upgrade of the new relic to the current model. - Introduce proper limit checks for histograms to ensure that we never pass the 2/6 bytes limits. - More testing, more testing, more testing. - Create the KamonStandalone module. --- .../main/scala/kamon/newrelic/CustomMetrics.scala | 6 ++-- .../scala/kamon/newrelic/MetricTranslator.scala | 2 +- .../src/main/scala/kamon/newrelic/NewRelic.scala | 8 +++--- .../kamon/newrelic/WebTransactionMetrics.scala | 9 ++++-- .../src/main/scala/kamon/newrelic/package.scala | 32 ++++++++++++---------- 5 files changed, 32 insertions(+), 25 deletions(-) (limited to 'kamon-newrelic/src') diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/CustomMetrics.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/CustomMetrics.scala index 08e0add3..44d9c605 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/CustomMetrics.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/CustomMetrics.scala @@ -17,16 +17,16 @@ package kamon.newrelic import akka.actor.Actor -import kamon.metrics._ +import kamon.metric._ trait CustomMetrics { self: Actor ⇒ def collectCustomMetrics(metrics: Map[MetricGroupIdentity, MetricGroupSnapshot]): Seq[NewRelic.Metric] = { metrics.collect { - case (CustomMetric(name), groupSnapshot) ⇒ + case (UserMetrics, groupSnapshot) ⇒ groupSnapshot.metrics collect { - case (_, snapshot) ⇒ toNewRelicMetric(Scale.Unit)(s"Custom/$name", None, snapshot) + case (name, snapshot) ⇒ toNewRelicMetric(Scale.Unit)(s"Custom/$name", None, snapshot) } }.flatten.toSeq } diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/MetricTranslator.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/MetricTranslator.scala index 46e22571..a3bb6311 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/MetricTranslator.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/MetricTranslator.scala @@ -17,7 +17,7 @@ package kamon.newrelic import akka.actor.{ Props, ActorRef, Actor } -import kamon.metrics.Subscriptions.TickMetricSnapshot +import kamon.metric.Subscriptions.TickMetricSnapshot import kamon.newrelic.MetricTranslator.TimeSliceMetrics class MetricTranslator(receiver: ActorRef) extends Actor diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala index c195ed12..85861454 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala @@ -18,8 +18,8 @@ package kamon.newrelic import akka.actor._ import scala.concurrent.duration._ import kamon.Kamon -import kamon.metrics.{ CustomMetric, TickMetricSnapshotBuffer, TraceMetrics, Metrics } -import kamon.metrics.Subscriptions.TickMetricSnapshot +import kamon.metric.{ UserMetrics, TickMetricSnapshotBuffer, TraceMetrics, Metrics } +import kamon.metric.Subscriptions.TickMetricSnapshot import akka.actor import java.util.concurrent.TimeUnit.MILLISECONDS @@ -30,7 +30,7 @@ class NewRelicExtension(system: ExtendedActorSystem) extends Kamon.Extension { val apdexT: Double = config.getDuration("apdexT", MILLISECONDS) / 1E3 // scale to seconds. Kamon(Metrics)(system).subscribe(TraceMetrics, "*", metricsListener, permanently = true) - Kamon(Metrics)(system).subscribe(CustomMetric, "*", metricsListener, permanently = true) + //Kamon(Metrics)(system).subscribe(UserMetrics, "*", metricsListener, permanently = true) } class NewRelicMetricsListener extends Actor with ActorLogging { @@ -50,7 +50,7 @@ object NewRelic extends ExtensionId[NewRelicExtension] with ExtensionIdProvider def createExtension(system: ExtendedActorSystem): NewRelicExtension = new NewRelicExtension(system) case class Metric(name: String, scope: Option[String], callCount: Long, total: Double, totalExclusive: Double, - min: Double, max: Double, sumOfSquares: Double) { + min: Double, max: Double, sumOfSquares: Double) { def merge(that: Metric): Metric = { Metric(name, scope, diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala index 90f1e8a5..38517e10 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala @@ -16,8 +16,8 @@ package kamon.newrelic -import kamon.metrics._ -import kamon.metrics.TraceMetrics.ElapsedTime +import kamon.metric._ +import kamon.metric.TraceMetrics.ElapsedTime import akka.actor.Actor import kamon.Kamon @@ -27,6 +27,7 @@ trait WebTransactionMetrics { def collectWebTransactionMetrics(metrics: Map[MetricGroupIdentity, MetricGroupSnapshot]): Seq[NewRelic.Metric] = { val apdexBuilder = new ApdexBuilder("Apdex", None, Kamon(NewRelic)(context.system).apdexT) + /* // Trace metrics are recorded in nanoseconds. var accumulatedHttpDispatcher: MetricSnapshotLike = MetricSnapshot(InstrumentTypes.Histogram, 0, Scale.Nano, Vector.empty) @@ -46,7 +47,9 @@ trait WebTransactionMetrics { val httpDispatcher = toNewRelicMetric(Scale.Unit)("HttpDispatcher", None, accumulatedHttpDispatcher) val webTransaction = toNewRelicMetric(Scale.Unit)("WebTransaction", None, accumulatedHttpDispatcher) - Seq(httpDispatcher, webTransaction, apdexBuilder.build) ++ webTransactionMetrics.flatten.toSeq + Seq(httpDispatcher, webTransaction, apdexBuilder.build) ++ webTransactionMetrics.flatten.toSeq */ + + ??? } } diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/package.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/package.scala index f6e377c7..89a8b15b 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/package.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/package.scala @@ -16,26 +16,30 @@ package kamon -import kamon.metrics.{ Scale, MetricSnapshotLike } +import kamon.metric.instrument.{ Counter, Histogram } +import kamon.metric.{ MetricSnapshot, Scale } package object newrelic { - def toNewRelicMetric(scale: Scale)(name: String, scope: Option[String], snapshot: MetricSnapshotLike): NewRelic.Metric = { - var total: Double = 0D - var sumOfSquares: Double = 0D + def toNewRelicMetric(scale: Scale)(name: String, scope: Option[String], snapshot: MetricSnapshot): NewRelic.Metric = { + snapshot match { + case hs: Histogram.Snapshot ⇒ + var total: Double = 0D + var sumOfSquares: Double = 0D + val scaledMin = Scale.convert(hs.scale, scale, hs.min) + val scaledMax = Scale.convert(hs.scale, scale, hs.max) - val measurementLevels = snapshot.measurements.iterator - while (measurementLevels.hasNext) { - val level = measurementLevels.next() - val scaledValue = Scale.convert(snapshot.scale, scale, level.value) + hs.recordsIterator.foreach { record ⇒ + val scaledValue = Scale.convert(hs.scale, scale, record.level) - total += scaledValue * level.count - sumOfSquares += (scaledValue * scaledValue) * level.count - } + total += scaledValue * record.count + sumOfSquares += (scaledValue * scaledValue) * record.count + } - val scaledMin = Scale.convert(snapshot.scale, scale, snapshot.min) - val scaledMax = Scale.convert(snapshot.scale, scale, snapshot.max) + NewRelic.Metric(name, scope, hs.numberOfMeasurements, total, total, scaledMin, scaledMax, sumOfSquares) - NewRelic.Metric(name, scope, snapshot.numberOfMeasurements, total, total, scaledMin, scaledMax, sumOfSquares) + case cs: Counter.Snapshot ⇒ + NewRelic.Metric(name, scope, cs.count, cs.count, cs.count, 0, cs.count, cs.count * cs.count) + } } } -- cgit v1.2.3 From 33390af40b39ca22b83e3bced83c9496f6c556ab Mon Sep 17 00:00:00 2001 From: Ivan Topolnjak Date: Sun, 27 Jul 2014 20:33:38 -0300 Subject: = newrelic: upgrade to the latest metrics scheme, closes #54 --- .../src/main/scala/kamon/metric/TraceMetrics.scala | 4 ++-- .../scala/kamon/metric/instrument/Histogram.scala | 11 +++++++++++ kamon-newrelic/src/main/resources/reference.conf | 2 +- .../src/main/scala/kamon/newrelic/NewRelic.scala | 4 ++-- .../kamon/newrelic/WebTransactionMetrics.scala | 21 +++++++++++---------- .../src/main/resources/application.conf | 4 ++-- 6 files changed, 29 insertions(+), 17 deletions(-) (limited to 'kamon-newrelic/src') diff --git a/kamon-core/src/main/scala/kamon/metric/TraceMetrics.scala b/kamon-core/src/main/scala/kamon/metric/TraceMetrics.scala index 187eb07d..c506fe81 100644 --- a/kamon-core/src/main/scala/kamon/metric/TraceMetrics.scala +++ b/kamon-core/src/main/scala/kamon/metric/TraceMetrics.scala @@ -69,8 +69,8 @@ object TraceMetrics extends MetricGroupCategory { val segmentConfig = settings.getConfig("segment") new TraceMetricRecorder( - Histogram.fromConfig(elapsedTimeConfig), - () ⇒ Histogram.fromConfig(segmentConfig)) + Histogram.fromConfig(elapsedTimeConfig, Scale.Nano), + () ⇒ Histogram.fromConfig(segmentConfig, Scale.Nano)) } } diff --git a/kamon-core/src/main/scala/kamon/metric/instrument/Histogram.scala b/kamon-core/src/main/scala/kamon/metric/instrument/Histogram.scala index 9ce11f49..7d8022f7 100644 --- a/kamon-core/src/main/scala/kamon/metric/instrument/Histogram.scala +++ b/kamon-core/src/main/scala/kamon/metric/instrument/Histogram.scala @@ -78,6 +78,17 @@ object Histogram { def recordsIterator: Iterator[Record] def merge(that: Histogram.Snapshot, context: CollectionContext): Histogram.Snapshot } + + object Snapshot { + def empty(targetScale: Scale) = new Snapshot { + override def min: Long = 0L + override def max: Long = 0L + override def recordsIterator: Iterator[Record] = Iterator.empty + override def merge(that: Snapshot, context: CollectionContext): Snapshot = that + override def scale: Scale = targetScale + override def numberOfMeasurements: Long = 0L + } + } } /** diff --git a/kamon-newrelic/src/main/resources/reference.conf b/kamon-newrelic/src/main/resources/reference.conf index 24522d63..13aaca2f 100644 --- a/kamon-newrelic/src/main/resources/reference.conf +++ b/kamon-newrelic/src/main/resources/reference.conf @@ -7,7 +7,7 @@ kamon { apdexT = 1 second app-name = "Kamon[Development]" - license-key = 2e24765acb032cb9e7207013b5ba3e2ab7d2d75c + license-key = e7d350b14228f3d28f35bc3140df2c3e565ea5d5 } } diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala index 85861454..d83e0335 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/NewRelic.scala @@ -26,11 +26,12 @@ import java.util.concurrent.TimeUnit.MILLISECONDS class NewRelicExtension(system: ExtendedActorSystem) extends Kamon.Extension { val config = system.settings.config.getConfig("kamon.newrelic") + val collectionContext = Kamon(Metrics)(system).buildDefaultCollectionContext val metricsListener = system.actorOf(Props[NewRelicMetricsListener], "kamon-newrelic") val apdexT: Double = config.getDuration("apdexT", MILLISECONDS) / 1E3 // scale to seconds. Kamon(Metrics)(system).subscribe(TraceMetrics, "*", metricsListener, permanently = true) - //Kamon(Metrics)(system).subscribe(UserMetrics, "*", metricsListener, permanently = true) + Kamon(Metrics)(system).subscribe(UserMetrics.category, "*", metricsListener, permanently = true) } class NewRelicMetricsListener extends Actor with ActorLogging { @@ -61,6 +62,5 @@ object NewRelic extends ExtensionId[NewRelicExtension] with ExtensionIdProvider math.max(max, that.max), sumOfSquares + that.sumOfSquares) } - } } \ No newline at end of file diff --git a/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala b/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala index 38517e10..a8c54684 100644 --- a/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala +++ b/kamon-newrelic/src/main/scala/kamon/newrelic/WebTransactionMetrics.scala @@ -20,25 +20,28 @@ import kamon.metric._ import kamon.metric.TraceMetrics.ElapsedTime import akka.actor.Actor import kamon.Kamon +import kamon.metric.instrument.Histogram trait WebTransactionMetrics { self: Actor ⇒ def collectWebTransactionMetrics(metrics: Map[MetricGroupIdentity, MetricGroupSnapshot]): Seq[NewRelic.Metric] = { - val apdexBuilder = new ApdexBuilder("Apdex", None, Kamon(NewRelic)(context.system).apdexT) + val newRelicExtension = Kamon(NewRelic)(context.system) + val apdexBuilder = new ApdexBuilder("Apdex", None, newRelicExtension.apdexT) + val collectionContext = newRelicExtension.collectionContext - /* // Trace metrics are recorded in nanoseconds. - var accumulatedHttpDispatcher: MetricSnapshotLike = MetricSnapshot(InstrumentTypes.Histogram, 0, Scale.Nano, Vector.empty) + var accumulatedHttpDispatcher: Histogram.Snapshot = Histogram.Snapshot.empty(Scale.Nano) val webTransactionMetrics = metrics.collect { case (TraceMetrics(name), groupSnapshot) ⇒ groupSnapshot.metrics collect { - case (ElapsedTime, snapshot) ⇒ - accumulatedHttpDispatcher = accumulatedHttpDispatcher.merge(snapshot) - snapshot.measurements.foreach(level ⇒ - apdexBuilder.record(Scale.convert(snapshot.scale, Scale.Unit, level.value), level.count)) + case (ElapsedTime, snapshot: Histogram.Snapshot) ⇒ + accumulatedHttpDispatcher = accumulatedHttpDispatcher.merge(snapshot, collectionContext) + snapshot.recordsIterator.foreach { record ⇒ + apdexBuilder.record(Scale.convert(snapshot.scale, Scale.Unit, record.level), record.count) + } toNewRelicMetric(Scale.Unit)(s"WebTransaction/Custom/$name", None, snapshot) } @@ -47,9 +50,7 @@ trait WebTransactionMetrics { val httpDispatcher = toNewRelicMetric(Scale.Unit)("HttpDispatcher", None, accumulatedHttpDispatcher) val webTransaction = toNewRelicMetric(Scale.Unit)("WebTransaction", None, accumulatedHttpDispatcher) - Seq(httpDispatcher, webTransaction, apdexBuilder.build) ++ webTransactionMetrics.flatten.toSeq */ - - ??? + Seq(httpDispatcher, webTransaction, apdexBuilder.build) ++ webTransactionMetrics.flatten.toSeq } } diff --git a/kamon-playground/src/main/resources/application.conf b/kamon-playground/src/main/resources/application.conf index d827731a..e9f73954 100644 --- a/kamon-playground/src/main/resources/application.conf +++ b/kamon-playground/src/main/resources/application.conf @@ -1,7 +1,7 @@ akka { loglevel = INFO - extensions = ["kamon.statsd.StatsD", "kamon.system.SystemMetrics"] + extensions = ["kamon.newrelic.NewRelic"] actor { debug { @@ -19,7 +19,7 @@ spray.can { kamon { newrelic { app-name = "SimpleRequestProcessor" - license-key = 2e24765acb032cb9e7207013b5ba3e2ab7d2d75c + license-key = e7d350b14228f3d28f35bc3140df2c3e565ea5d5 } } -- cgit v1.2.3