diff options
Diffstat (limited to 'test')
3 files changed, 61 insertions, 59 deletions
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md index 07e72f09a1..370d610bc4 100644 --- a/test/benchmarks/README.md +++ b/test/benchmarks/README.md @@ -3,9 +3,11 @@ This directory is a standalone SBT project, within the Scala project, that makes use of the [SBT plugin](https://github.com/ktoso/sbt-jmh) for [JMH](http://openjdk.java.net/projects/code-tools/jmh/). -## running a benchmark +## Running a benchmark -The benchmarks require first building Scala into `../../build/pack`, using Ant. +The benchmarks require first building Scala into `../../build/pack` with `ant`. +If you want to build with `sbt dist/mkPack` instead, +you'll need to change `scalaHome` in this project. You'll then need to know the fully-qualified name of the benchmark runner class. The benchmarking classes are organized under `src/main/scala`, @@ -23,14 +25,14 @@ The JMH results can be found under `target/jmh-results/`. `target` gets deleted on an SBT `clean`, so you should copy these files out of `target` if you wish to preserve them. -## creating a benchmark and runner +## Creating a benchmark and runner The benchmarking classes use the same package hierarchy as the classes that they test in order to make it easy to expose, in package scope, members of the class under test, should that be necessary for benchmarking. There are two types of classes in the source directory: -those suffixed "`Benchmark`" and those suffixed "`Runner`". +those suffixed `Benchmark` and those suffixed `Runner`. The former are benchmarks that can be run directly using `jmh:run`; however, they are normally run from a corresponding class of the latter type, which is run using `jmh:runMain` (as described above). @@ -41,39 +43,45 @@ The `benchmark.JmhRunner` trait should be woven into any runner class, for the s This includes creating output files in a subdirectory of `target/jmh-results` derived from the fully-qualified package name of the `Runner` class. -## some useful HotSpot options +## Some useful HotSpot options Adding these to the `jmh:run` or `jmh:runMain` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler. They require prefixing with `-jvmArgs`. See [the Java documentation](http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html) for more options. -### viewing JIT compilation events +### Viewing JIT compilation events Adding `-XX:+PrintCompilation` shows when Java methods are being compiled or deoptimized. At the most basic level, these messages will tell you whether the code that you're measuring is still being tuned, so that you know whether you're running enough warm-up iterations. See [Kris Mok's notes](https://gist.github.com/rednaxelafx/1165804#file-notes-md) to interpret the output in detail. -### consider GC events +### Consider GC events If you're not explicitly performing `System.gc()` calls outside of your benchmarking code, you should add the JVM option `-verbose:gc` to understand the effect that GCs may be having on your tests. -### "diagnostic" options +### "Diagnostic" options These require the `-XX:+UnlockDiagnosticVMOptions` JVM option. -#### viewing inlining events +#### Viewing inlining events Add `-XX:+PrintInlining`. -#### viewing the disassembled code +#### Viewing the disassembled code +If you're running OpenJDK or Oracle JVM, +you may need to install the disassembler library (`hsdis-amd64.so` for the `amd64` architecture). +In Debian, this is available in +<a href="https://packages.debian.org/search?keywords=libhsdis0-fcml">the `libhsdis0-fcml` package</a>. +For an Oracle (or other compatible) JVM not set up by your distribution, +you may also need to copy or link the disassembler library +to the `jre/lib/`_`architecture`_ directory inside your JVM installation directory. + To show the assembly code corresponding to the code generated by the JIT compiler for specific methods, add `-XX:CompileCommand=print,scala.collection.mutable.OpenHashMap::*`, for example, to show all of the methods in the `scala.collection.mutable.OpenHashMap` class. -If you're running OpenJDK, you may need to install the disassembler library (`hsdis-amd64.so` for the `amd64` architecture). -In Debian, this is available in <a href="https://packages.debian.org/search?keywords=libhsdis0-fcml">the `libhsdis0-fcml` package</a>. To show it for _all_ methods, add `-XX:+PrintAssembly`. (This is usually excessive.) -## useful reading +## Useful reading * [OpenJDK advice on microbenchmarks](https://wiki.openjdk.java.net/display/HotSpot/MicroBenchmarks) * Brian Goetz's "Java theory and practice" articles: * "[Dynamic compilation and performance measurement](http://www.ibm.com/developerworks/java/library/j-jtp12214/)" @@ -81,7 +89,7 @@ To show it for _all_ methods, add `-XX:+PrintAssembly`. * [Doug Lea's JSR 166 benchmarks](http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/src/test/loops/) * "[Measuring performance](http://docs.scala-lang.org/overviews/parallel-collections/performance.html)" of Scala parallel collections -## legacy frameworks +## Legacy frameworks An older version of the benchmarking framework is still present in this directory, in the following locations: diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala index 78e160a713..26e26b3065 100644 --- a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala +++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala @@ -170,7 +170,7 @@ private object OpenHashMapBenchmark { class OpenHashMapBenchmark { import OpenHashMapBenchmark._ - @Param(Array("50", "100", "250", "1000", "2500", "10000", "25000", "100000", "250000", "1000000", "2500000", + @Param(Array("25", "50", "100", "250", "1000", "2500", "10000", "25000", "100000", "250000", "1000000", "2500000", "5000000", "7500000", "10000000", "25000000")) var size: Int = _ diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala index c139c55933..1a58b18ee9 100644 --- a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala +++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala @@ -14,6 +14,7 @@ import org.openjdk.jmh.runner.options.Options import benchmark.JmhRunner import org.openjdk.jmh.runner.options.OptionsBuilder import org.openjdk.jmh.runner.options.VerboseMode +import org.openjdk.jmh.results.Result /** Replacement JMH application that runs the [[OpenHashMap]] benchmark. * @@ -24,27 +25,35 @@ object OpenHashMapRunner extends JmhRunner { private[this] val outputFile = new File(outputDirectory, "OpenHashMap.dat") /** Qualifier to add to the name of a memory usage data set. */ - private[this] val memoryDatasetQualifer = " memory" + private[this] val memoryDatasetQualifier = "-memory" - /** Name of the JMH parameter for the number of map entries per invocation. */ - private[this] val sizeParamName = "size" + private[this] implicit class MyRunResult(r: RunResult) { + /** Return the dataset label. */ + def label = r.getPrimaryResult.getLabel - /** Name of the JMH auxiliary counter that collects operation counts. */ - private[this] val operationsAuxCounterName = "operations" + /** Return the value of the JMH parameter for the number of map entries per invocation. */ + def size: String = r.getParams.getParam("size") - /** Name of the JMH auxiliary counter that collects memory usage. */ - private[this] val memoryAuxCounterName = "memory" + /** Return the operation counts. */ + def operations = Option(r.getSecondaryResults.get("operations")) + + /** Return the number of map entries. */ + def entries = r.getSecondaryResults.get("mapEntries") + + /** Return the memory usage. */ + def memory = Option(r.getSecondaryResults.get("memory")) + } + + /** Return the statistics of the given result as a string. */ + private[this] def stats(r: Result[_]) = r.getScore + " " + r.getStatistics.getStandardDeviation - /** Name of the JMH auxiliary counter that collects the number of map entries. */ - private[this] val entriesAuxCounterName = "mapEntries" def main(args: Array[String]) { import scala.collection.JavaConversions._ import scala.language.existentials val opts = new CommandLineOptions(args: _*) - var builder = new OptionsBuilder().parent(opts) - .jvmArgsPrepend("-Xmx6000m") + var builder = new OptionsBuilder().parent(opts).jvmArgsPrepend("-Xmx6000m") if (!opts.verbosity.hasValue) builder = builder.verbosity(VerboseMode.SILENT) val results = new Runner(builder.build).run() @@ -55,18 +64,17 @@ object OpenHashMapRunner extends JmhRunner { val datasetByName = Map.empty[String, Set[RunResult]] /** Ordering for the results within a data set. Orders by increasing number of map entries. */ - val ordering = Ordering.by[RunResult, Int](_.getParams.getParam(sizeParamName).toInt) + val ordering = Ordering.by[RunResult, Int](_.size.toInt) - def addToDataset(result: RunResult, key: String): Unit = - datasetByName.get(key) - .getOrElse({ val d = SortedSet.empty(ordering); datasetByName.put(key, d); d }) += result + def addToDataset(key: String, result: RunResult): Unit = + datasetByName.getOrElseUpdate(key, SortedSet.empty(ordering)) += result - results.foreach { result: RunResult ⇒ - addToDataset(result, result.getPrimaryResult.getLabel) + results.foreach { result => + addToDataset(result.label, result) // Create another data set for trials that track memory usage - if (result.getSecondaryResults.containsKey(memoryAuxCounterName)) - addToDataset(result, result.getPrimaryResult.getLabel + memoryDatasetQualifer) + if (result.memory.isDefined) + addToDataset(result.label + memoryDatasetQualifier, result) } //TODO Write out test parameters @@ -75,31 +83,17 @@ object OpenHashMapRunner extends JmhRunner { val f = new PrintWriter(outputFile, "UTF-8") try { - datasetByName.foreach(_ match { case (label: String, dataset: Iterable[RunResult]) ⇒ { - f.format("# [%s]\n", label) - - val isMemoryUsageDataset = label.contains(memoryDatasetQualifer) - dataset.foreach { result ⇒ - val size = result.getParams.getParam(sizeParamName) - val secondaryResults = result.getSecondaryResults - if (isMemoryUsageDataset) { - val memoryResult = secondaryResults.get(memoryAuxCounterName) - val entriesResult = secondaryResults.get(entriesAuxCounterName) - f.format("%s %f %f %f %f\n", size, - Double.box(entriesResult.getScore), Double.box(entriesResult.getStatistics.getStandardDeviation), - Double.box(memoryResult.getScore), Double.box(memoryResult.getStatistics.getStandardDeviation)) - } - else { - if (secondaryResults.containsKey(operationsAuxCounterName)) { - val operationsResult = secondaryResults.get(operationsAuxCounterName) - f.format("%s %f %f\n", size, - Double.box(operationsResult.getScore), Double.box(operationsResult.getStatistics.getStandardDeviation)) - } else { - val primary = result.getPrimaryResult - f.format("%s %f %f\n", size, - Double.box(primary.getScore), Double.box(primary.getStatistics.getStandardDeviation)) - } - } + datasetByName.foreach(_ match { case (label: String, dataset: Iterable[RunResult]) => { + f.println(s"# [$label]") + + val isMemoryUsageDataset = label.endsWith(memoryDatasetQualifier) + dataset.foreach { r => + f.println(r.size + " " + ( + if (isMemoryUsageDataset) + stats(r.entries) + " " + stats(r.memory.get) + else + stats(r.operations getOrElse r.getPrimaryResult) + )) } f.println(); f.println() // data set separator |