From 56c5d92236daf8a8094429072ec70cf830fd10ac Mon Sep 17 00:00:00 2001 From: Performant Data LLC Date: Mon, 21 Mar 2016 13:21:19 -0700 Subject: Add JMH to the benchmark framework. Add an example benchmark for OpenHashMap. --- test/benchmarks/.gitignore | 6 ++ test/benchmarks/README.md | 65 ++++++++++++++++++ test/benchmarks/build.sbt | 8 +++ test/benchmarks/project/plugins.sbt | 1 + .../collection/mutable/OpenHashMapBenchmark.scala | 76 ++++++++++++++++++++++ 5 files changed, 156 insertions(+) create mode 100644 test/benchmarks/.gitignore create mode 100644 test/benchmarks/README.md create mode 100644 test/benchmarks/build.sbt create mode 100644 test/benchmarks/project/plugins.sbt create mode 100644 test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala diff --git a/test/benchmarks/.gitignore b/test/benchmarks/.gitignore new file mode 100644 index 0000000000..6e3ddad6d2 --- /dev/null +++ b/test/benchmarks/.gitignore @@ -0,0 +1,6 @@ +/project/project/ +/project/target/ +/target/ + +# what appears to be a Scala IDE-generated file +.cache-main diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md new file mode 100644 index 0000000000..99b358dd99 --- /dev/null +++ b/test/benchmarks/README.md @@ -0,0 +1,65 @@ +# Scala library benchmarks + +This directory is a standalone SBT project +that makes use of the [SBT plugin for JMH](https://github.com/ktoso/sbt-jmh), +with the usual directory structure: +source code for the benchmarks, which utilize [JMH](http://openjdk.java.net/projects/code-tools/jmh/), +should be placed in `src/main/scala`. + +The benchmarks require first building Scala into `../../build/pack`. +They can then be (built and) run from `sbt` with "`jmh:run`". +"`jmh:run -h`" displays the usual JMH options available. + +## some useful HotSpot options +Adding these to the `jmh:run` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler. +They require prefixing with `-jvmArgs`. +See [the Java documentation](http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html) for more options. + +### viewing JIT compilation events +Adding `-XX:+PrintCompilation` shows when Java methods are being compiled or deoptimized. +At the most basic level, +these messages will tell you whether the code that you're measuring is still being tuned, +so that you know whether you're running enough warm-up iterations. +See [Kris Mok's notes](https://gist.github.com/rednaxelafx/1165804#file-notes-md) to interpret the output in detail. + +### consider GC events +If you're not explicitly performing `System.gc()` calls outside of your benchmarking code, +you should add the JVM option `-verbose:gc` to understand the effect that GCs may be having on your tests. + +### "diagnostic" options +These require the `-XX:+UnlockDiagnosticVMOptions` JVM option. + +#### viewing inlining events +Add `-XX:+PrintInlining`. + +#### viewing the disassembled code +To show the assembly code corresponding to the code generated by the JIT compiler for specific methods, +add `-XX:CompileCommand=print,scala.collection.mutable.OpenHashMap::*`, +for example, to show all of the methods in the `scala.collection.mutable.OpenHashMap` class. +If you're running OpenJDK, you may need to install the disassembler library (`hsdis-amd64.so` for the `amd64` architecture). +In Debian, this is available in the `libhsdis0-fcml` package. + +To show it for _all_ methods, add `-XX:+PrintAssembly`. +(This is usually excessive.) + +## useful reading +* [OpenJDK advice on microbenchmarks](https://wiki.openjdk.java.net/display/HotSpot/MicroBenchmarks) +* "[Measuring performance](http://docs.scala-lang.org/overviews/parallel-collections/performance.html)" of Scala parallel collections +* Brian Goetz's "Java theory and practice" articles: + * "[Dynamic compilation and performance measurement](http://www.ibm.com/developerworks/java/library/j-jtp12214/)" + * "[Anatomy of a flawed benchmark](http://www.ibm.com/developerworks/java/library/j-jtp02225/)" + +## legacy frameworks + +An older version of the benchmarking framework is still present in this directory, in the following locations: + +
+
bench
+
A script to run the old benchmarks.
+
source.list
+
A temporary file used by bench.
+
src/scala/
+
The older benchmarks, including the previous framework.
+
+ +Another, older set of benchmarks is present in `../benchmarking/`. diff --git a/test/benchmarks/build.sbt b/test/benchmarks/build.sbt new file mode 100644 index 0000000000..92a5fce177 --- /dev/null +++ b/test/benchmarks/build.sbt @@ -0,0 +1,8 @@ +scalaHome := Some(file("../../build/pack")) + +lazy val root = (project in file(".")). + enablePlugins(JmhPlugin). + settings( + name := "test-benchmarks", + version := "0.0.1" + ) diff --git a/test/benchmarks/project/plugins.sbt b/test/benchmarks/project/plugins.sbt new file mode 100644 index 0000000000..f5319fb187 --- /dev/null +++ b/test/benchmarks/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.6") diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala new file mode 100644 index 0000000000..eeea8f6508 --- /dev/null +++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala @@ -0,0 +1,76 @@ +package scala.collection.mutable; + +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations._ + +private object OpenHashMapBenchmark { + /** State container for the `put()` bulk calling tests. + * + * Provides a thread-scoped map, so that allocation for the hash table will be done + * in the first warm-up iteration, not during measurement. + * + * Performs a GC after every invocation, so that only the GCs caused by the invocation + * contribute to the measurement. + */ + @State(Scope.Thread) + class BulkPutState { + val map = new OpenHashMap[Int,Int].empty + + @TearDown(Level.Invocation) + def teardown { map.clear(); System.gc() } + } +} + +/** Benchmark for the library's [[OpenHashMap]]. + * + * The `put()` calls are tested by looping to the size desired for the map; + * instead of using the JMH harness, which iterates for a fixed length of time. + */ +@BenchmarkMode(Array(Mode.AverageTime)) +@Threads(1) +@Fork(1) +@Warmup(iterations = 20) +@Measurement(iterations = 20) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +class OpenHashMapBenchmark { + import OpenHashMapBenchmark._ + + @Param(Array("100", "250", "1000", "2500", "10000", "25000", "100000", "250000", "1000000", "2500000", + "5000000", "7500000", "10000000", "25000000")) + var size: Int = _ + + /** Put elements into the given map. */ + private[this] def put_Int(map: OpenHashMap[Int,Int], from: Int, to: Int) { + var i = from + while (i <= to) { // using a `for` expression instead adds significant overhead + map.put(i, i) + i += 1 + } + } + + /** Test putting elements to a map of `Int` to `Int`. */ + @Benchmark + def put_Int(state: BulkPutState) { put_Int(state.map, 1, size) } + + /** Test putting and removing elements to a growing map of `Int` to `Int`. */ + @Benchmark + def put_remove_Int(state: BulkPutState) { + val blocks = 50 // should be a factor of `size` + val totalPuts = 2 * size // add twice as many, because we remove half of them + val blockSize: Int = totalPuts / blocks + var base = 0 + while (base < totalPuts) { + put_Int(state.map, base + 1, base + blockSize) + + // remove every other entry + var i = base + 1 + while (i <= base + blockSize) { + state.map.remove(i) + i += 2 + } + + base += blockSize + } + } +} -- cgit v1.2.3