From 1e8b589267b66ea04d3bf15a8f9f28c34d4832d9 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 11 Aug 2015 01:29:56 -0300 Subject: = system-metrics: The sigar not implemented exceptions are listed as warnings. closes #235 --- .../main/scala/kamon/system/sigar/CpuMetrics.scala | 30 ++++++++++++------ .../kamon/system/sigar/FileSystemMetrics.scala | 11 ++++--- .../kamon/system/sigar/LoadAverageMetrics.scala | 11 ++++--- .../scala/kamon/system/sigar/MemoryMetrics.scala | 36 +++++++++++++++------- .../scala/kamon/system/sigar/NetworkMetrics.scala | 11 ++++--- .../kamon/system/sigar/ProcessCpuMetrics.scala | 8 +++-- .../kamon/system/sigar/SigarMetricsUpdater.scala | 36 ++++++++++++++++------ 7 files changed, 98 insertions(+), 45 deletions(-) (limited to 'kamon-system-metrics') diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/CpuMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/CpuMetrics.scala index 0e9a5b53..54b95ab7 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/CpuMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/CpuMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument.InstrumentFactory import org.hyperic.sigar.Sigar @@ -28,7 +29,7 @@ import org.hyperic.sigar.Sigar * - idle: Total percentage of system cpu idle time * - stolen: Total percentage of system cpu involuntary wait time. @see [[https://www.datadoghq.com/2013/08/understanding-aws-stolen-cpu-and-how-it-affects-your-apps/ "Understanding Stolen Cpu"]] */ -class CpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class CpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { val user = histogram("cpu-user") val system = histogram("cpu-system") val Wait = histogram("cpu-wait") @@ -36,18 +37,29 @@ class CpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends Gen val stolen = histogram("cpu-stolen") def update(): Unit = { - val cpuPerc = sigar.getCpuPerc + import SigarSafeRunner._ - user.record((cpuPerc.getUser * 100L).toLong) - system.record((cpuPerc.getSys * 100L).toLong) - Wait.record((cpuPerc.getWait * 100L).toLong) - idle.record((cpuPerc.getIdle * 100L).toLong) - stolen.record((cpuPerc.getStolen * 100L).toLong) + def cpuPerc = { + val cpuPerc = sigar.getCpuPerc + ((cpuPerc.getUser * 100L).toLong, + (cpuPerc.getSys * 100L).toLong, + (cpuPerc.getWait * 100L).toLong, + (cpuPerc.getIdle * 100L).toLong, + (cpuPerc.getStolen * 100L).toLong) + } + + val (cpuUser, cpuSys, cpuWait, cpuIdle, cpuStolen) = runSafe(cpuPerc, (0L, 0L, 0L, 0L, 0L), "cpu", logger) + + user.record(cpuUser) + system.record(cpuSys) + Wait.record(cpuWait) + idle.record(cpuIdle) + stolen.record(cpuStolen) } } object CpuMetrics extends SigarMetricRecorderCompanion("cpu") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): CpuMetrics = - new CpuMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): CpuMetrics = + new CpuMetrics(sigar, instrumentFactory, logger) } diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/FileSystemMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/FileSystemMetrics.scala index d3bfefbe..5e2a27ea 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/FileSystemMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/FileSystemMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument.{ Memory, InstrumentFactory } import org.hyperic.sigar.{ DiskUsage, FileSystem, Sigar } @@ -26,11 +27,13 @@ import scala.util.Try * - readBytes: Total number of physical disk reads. * - writesBytes: Total number of physical disk writes. */ -class FileSystemMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class FileSystemMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { + import kamon.system.sigar.SigarSafeRunner._ + val reads = DiffRecordingHistogram(histogram("file-system-reads", Memory.Bytes)) val writes = DiffRecordingHistogram(histogram("file-system-writes", Memory.Bytes)) - val fileSystems = sigar.getFileSystemList.filter(_.getType == FileSystem.TYPE_LOCAL_DISK).map(_.getDevName).toSet + val fileSystems = runSafe(sigar.getFileSystemList.filter(_.getType == FileSystem.TYPE_LOCAL_DISK).map(_.getDevName).toSet, Set.empty[String], "file-system", logger) def sumOfAllFileSystems(sigar: Sigar, thunk: DiskUsage ⇒ Long): Long = Try { fileSystems.map(i ⇒ thunk(sigar.getDiskUsage(i))).fold(0L)(_ + _) @@ -43,6 +46,6 @@ class FileSystemMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) exte } object FileSystemMetrics extends SigarMetricRecorderCompanion("file-system") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): FileSystemMetrics = - new FileSystemMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): FileSystemMetrics = + new FileSystemMetrics(sigar, instrumentFactory, logger) } diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/LoadAverageMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/LoadAverageMetrics.scala index 8d7bd808..46baeea4 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/LoadAverageMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/LoadAverageMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument.InstrumentFactory import org.hyperic.sigar.Sigar @@ -24,13 +25,15 @@ import org.hyperic.sigar.Sigar * Load Average metrics, as reported by Sigar: * - The system load averages for the past 1, 5, and 15 minutes. */ -class LoadAverageMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class LoadAverageMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { + import SigarSafeRunner._ + val oneMinute = histogram("one-minute") val fiveMinutes = histogram("five-minutes") val fifteenMinutes = histogram("fifteen-minutes") def update(): Unit = { - val loadAverage = sigar.getLoadAverage + val loadAverage = runSafe(sigar.getLoadAverage, Array(0D, 0D, 0D), "load-average", logger) oneMinute.record(loadAverage(0).toLong) fiveMinutes.record(loadAverage(1).toLong) @@ -40,6 +43,6 @@ class LoadAverageMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) ext object LoadAverageMetrics extends SigarMetricRecorderCompanion("load-average") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): LoadAverageMetrics = - new LoadAverageMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): LoadAverageMetrics = + new LoadAverageMetrics(sigar, instrumentFactory, logger) } diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/MemoryMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/MemoryMetrics.scala index 787c9f2f..19fa81d8 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/MemoryMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/MemoryMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument.{ Memory, InstrumentFactory } import org.hyperic.sigar.Sigar @@ -27,7 +28,9 @@ import org.hyperic.sigar.Sigar * - swap-used: Total used system swap. * - swap-free: Total free system swap. */ -class MemoryMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class MemoryMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { + import SigarSafeRunner._ + val used = histogram("memory-used", Memory.Bytes) val cached = histogram("memory-cache-and-buffer", Memory.Bytes) val free = histogram("memory-free", Memory.Bytes) @@ -36,22 +39,33 @@ class MemoryMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends val swapFree = histogram("swap-free", Memory.Bytes) def update(): Unit = { - val mem = sigar.getMem - val swap = sigar.getSwap - val cachedMemory = if (mem.getActualFree > mem.getFree) mem.getActualFree - mem.getFree else 0L + def mem = { + val mem = sigar.getMem + (mem.getActualUsed, mem.getActualFree, mem.getFree, mem.getTotal) + } + + def swap = { + val swap = sigar.getSwap + (swap.getUsed, swap.getFree) + } + + val (memActualUsed, memActualFree, memFree, memTotal) = runSafe(mem, (0L, 0L, 0L, 0L), "memory", logger) + val (memSwapUsed, memSwapFree) = runSafe(swap, (0L, 0L), "swap", logger) + + def cachedMemory = if (memActualFree > memFree) memActualFree - memFree else 0L - used.record(mem.getActualUsed) - free.record(mem.getActualFree) + used.record(memActualUsed) + free.record(memActualFree) cached.record(cachedMemory) - total.record(mem.getTotal) - swapUsed.record(swap.getUsed) - swapFree.record(swap.getFree) + total.record(memTotal) + swapUsed.record(memSwapUsed) + swapFree.record(memSwapFree) } } object MemoryMetrics extends SigarMetricRecorderCompanion("memory") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): MemoryMetrics = - new MemoryMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): MemoryMetrics = + new MemoryMetrics(sigar, instrumentFactory, logger) } diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/NetworkMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/NetworkMetrics.scala index 30575508..ba9f966f 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/NetworkMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/NetworkMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument._ import org.hyperic.sigar.{ NetInterfaceStat, Sigar } @@ -30,7 +31,9 @@ import scala.util.Try * - rxDropped: Total number of incoming packets dropped. * - txDropped: Total number of outgoing packets dropped. */ -class NetworkMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class NetworkMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { + import SigarSafeRunner._ + val receivedBytes = DiffRecordingHistogram(histogram("rx-bytes", Memory.Bytes)) val transmittedBytes = DiffRecordingHistogram(histogram("tx-bytes", Memory.Bytes)) val receiveErrors = DiffRecordingHistogram(histogram("rx-errors")) @@ -38,7 +41,7 @@ class NetworkMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends val receiveDrops = DiffRecordingHistogram(histogram("rx-dropped")) val transmitDrops = DiffRecordingHistogram(histogram("tx-dropped")) - val interfaces = sigar.getNetInterfaceList.toList.filter(_ != "lo") + val interfaces = runSafe(sigar.getNetInterfaceList.toList.filter(_ != "lo"), List.empty[String], "network", logger) def sumOfAllInterfaces(sigar: Sigar, thunk: NetInterfaceStat ⇒ Long): Long = Try { interfaces.map(i ⇒ thunk(sigar.getNetInterfaceStat(i))).fold(0L)(_ + _) @@ -56,6 +59,6 @@ class NetworkMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends } object NetworkMetrics extends SigarMetricRecorderCompanion("network") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): NetworkMetrics = - new NetworkMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): NetworkMetrics = + new NetworkMetrics(sigar, instrumentFactory, logger) } \ No newline at end of file diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/ProcessCpuMetrics.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/ProcessCpuMetrics.scala index 43b6ba83..1283c07d 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/ProcessCpuMetrics.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/ProcessCpuMetrics.scala @@ -16,6 +16,7 @@ package kamon.system.sigar +import akka.event.LoggingAdapter import kamon.metric.GenericEntityRecorder import kamon.metric.instrument.InstrumentFactory import org.hyperic.sigar.{ ProcCpu, Sigar } @@ -28,7 +29,8 @@ import scala.util.Try * - total: Process cpu time (sum of User and Sys). * - system: Process cpu kernel time. */ -class ProcessCpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { +class ProcessCpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter) extends GenericEntityRecorder(instrumentFactory) with SigarMetric { + val processUserCpu = histogram("process-user-cpu") val processSystemCpu = histogram("process-system-cpu") val processTotalCpu = histogram("process-cpu") @@ -74,6 +76,6 @@ class ProcessCpuMetrics(sigar: Sigar, instrumentFactory: InstrumentFactory) exte } object ProcessCpuMetrics extends SigarMetricRecorderCompanion("process-cpu") { - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): ProcessCpuMetrics = - new ProcessCpuMetrics(sigar, instrumentFactory) + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter): ProcessCpuMetrics = + new ProcessCpuMetrics(sigar, instrumentFactory, logger) } diff --git a/kamon-system-metrics/src/main/scala/kamon/system/sigar/SigarMetricsUpdater.scala b/kamon-system-metrics/src/main/scala/kamon/system/sigar/SigarMetricsUpdater.scala index 5e5f7518..69bc00ec 100644 --- a/kamon-system-metrics/src/main/scala/kamon/system/sigar/SigarMetricsUpdater.scala +++ b/kamon-system-metrics/src/main/scala/kamon/system/sigar/SigarMetricsUpdater.scala @@ -17,6 +17,7 @@ package kamon.system.sigar import akka.actor.{ Props, Actor } +import akka.event.{ Logging, NoLogging, LoggingAdapter } import kamon.Kamon import kamon.metric.instrument.InstrumentFactory import kamon.metric._ @@ -28,14 +29,15 @@ import scala.concurrent.duration.FiniteDuration class SigarMetricsUpdater(refreshInterval: FiniteDuration) extends Actor { val sigar = new Sigar val metricsExtension = Kamon.metrics + val logger = Logging(context.system, this) val sigarMetrics = List( - CpuMetrics.register(sigar, metricsExtension), - FileSystemMetrics.register(sigar, metricsExtension), - LoadAverageMetrics.register(sigar, metricsExtension), - MemoryMetrics.register(sigar, metricsExtension), - NetworkMetrics.register(sigar, metricsExtension), - ProcessCpuMetrics.register(sigar, metricsExtension)) + CpuMetrics.register(sigar, metricsExtension, logger), + FileSystemMetrics.register(sigar, metricsExtension, logger), + LoadAverageMetrics.register(sigar, metricsExtension, logger), + MemoryMetrics.register(sigar, metricsExtension, logger), + NetworkMetrics.register(sigar, metricsExtension, logger), + ProcessCpuMetrics.register(sigar, metricsExtension, logger)) val refreshSchedule = context.system.scheduler.schedule(refreshInterval, refreshInterval, self, UpdateSigarMetrics)(context.dispatcher) @@ -55,7 +57,7 @@ class SigarMetricsUpdater(refreshInterval: FiniteDuration) extends Actor { object SigarMetricsUpdater { def props(refreshInterval: FiniteDuration): Props = - Props(new SigarMetricsUpdater((refreshInterval))) + Props(new SigarMetricsUpdater(refreshInterval)) case object UpdateSigarMetrics } @@ -64,10 +66,24 @@ trait SigarMetric extends EntityRecorder { def update(): Unit } +object SigarSafeRunner { + private val errorLogged = scala.collection.mutable.Set[String]() + + def runSafe[T](thunk: ⇒ T, defaultValue: ⇒ T, error: String, logger: LoggingAdapter): T = { + try thunk catch { + case e: Exception ⇒ + if (!errorLogged.contains(error)) { + errorLogged += error + logger.warning("Couldn't get the metric [{}]. Due to [{}]", error, e.getMessage) + } + defaultValue + } + } +} + abstract class SigarMetricRecorderCompanion(metricName: String) { - def register(sigar: Sigar, metricsExtension: MetricsModule): SigarMetric = + def register(sigar: Sigar, metricsExtension: MetricsModule, logger: LoggingAdapter = NoLogging): SigarMetric = metricsExtension.entity(EntityRecorderFactory("system-metric", apply(sigar, _)), metricName) - def apply(sigar: Sigar, instrumentFactory: InstrumentFactory): SigarMetric + def apply(sigar: Sigar, instrumentFactory: InstrumentFactory, logger: LoggingAdapter = NoLogging): SigarMetric } - -- cgit v1.2.3