Add JMH to the benchmark framework.

Add an example benchmark for OpenHashMap.

5 files changed, 156 insertions, 0 deletions

diff --git a/test/benchmarks/.gitignore b/test/benchmarks/.gitignore
new file mode 100644
index 0000000000..6e3ddad6d2
--- /dev/null
+++ b/test/benchmarks/.gitignore
@@ -0,0 +1,6 @@
+/project/project/
+/project/target/
+/target/
+
+# what appears to be a Scala IDE-generated file
+.cache-main
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md
new file mode 100644
index 0000000000..99b358dd99
--- /dev/null
+++ b/test/benchmarks/README.md
@@ -0,0 +1,65 @@
+# Scala library benchmarks
+
+This directory is a standalone SBT project
+that makes use of the [SBT plugin for JMH](https://github.com/ktoso/sbt-jmh),
+with the usual directory structure:
+source code for the benchmarks, which utilize [JMH](http://openjdk.java.net/projects/code-tools/jmh/),
+should be placed in `src/main/scala`.
+
+The benchmarks require first building Scala into `../../build/pack`.
+They can then be (built and) run from `sbt` with "`jmh:run`".
+"`jmh:run -h`" displays the usual JMH options available.
+
+## some useful HotSpot options
+Adding these to the `jmh:run` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler.
+They require prefixing with `-jvmArgs`.
+See [the Java documentation](http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html) for more options. 
+
+### viewing JIT compilation events
+Adding `-XX:+PrintCompilation` shows when Java methods are being compiled or deoptimized.
+At the most basic level,
+these messages will tell you whether the code that you're measuring is still being tuned,
+so that you know whether you're running enough warm-up iterations.
+See [Kris Mok's notes](https://gist.github.com/rednaxelafx/1165804#file-notes-md) to interpret the output in detail.
+
+### consider GC events
+If you're not explicitly performing `System.gc()` calls outside of your benchmarking code,
+you should add the JVM option `-verbose:gc` to understand the effect that GCs may be having on your tests.
+
+### "diagnostic" options
+These require the `-XX:+UnlockDiagnosticVMOptions` JVM option.
+
+#### viewing inlining events
+Add `-XX:+PrintInlining`.
+
+#### viewing the disassembled code
+To show the assembly code corresponding to the code generated by the JIT compiler for specific methods,
+add `-XX:CompileCommand=print,scala.collection.mutable.OpenHashMap::*`,
+for example, to show all of the methods in the `scala.collection.mutable.OpenHashMap` class.
+If you're running OpenJDK, you may need to install the disassembler library (`hsdis-amd64.so` for the `amd64` architecture).
+In Debian, this is available in <a href="https://packages.debian.org/search?keywords=libhsdis0-fcml">the `libhsdis0-fcml` package</a>.
+
+To show it for _all_ methods, add `-XX:+PrintAssembly`.
+(This is usually excessive.)
+
+## useful reading
+* [OpenJDK advice on microbenchmarks](https://wiki.openjdk.java.net/display/HotSpot/MicroBenchmarks)
+* "[Measuring performance](http://docs.scala-lang.org/overviews/parallel-collections/performance.html)" of Scala parallel collections
+* Brian Goetz's "Java theory and practice" articles:
+  * "[Dynamic compilation and performance measurement](http://www.ibm.com/developerworks/java/library/j-jtp12214/)"
+  * "[Anatomy of a flawed benchmark](http://www.ibm.com/developerworks/java/library/j-jtp02225/)"
+
+## legacy frameworks
+
+An older version of the benchmarking framework is still present in this directory, in the following locations:
+
+<dl>
+<dt><code>bench</code></dt>
+<dd>A script to run the old benchmarks.</dd>
+<dt><code>source.list</code></dt>
+<dd>A temporary file used by <code>bench</code>.</dd>
+<dt><code>src/scala/</code></dt>
+<dd>The older benchmarks, including the previous framework.</dd>
+</dl>
+
+Another, older set of benchmarks is present in `../benchmarking/`.
diff --git a/test/benchmarks/build.sbt b/test/benchmarks/build.sbt
new file mode 100644
index 0000000000..92a5fce177
--- /dev/null
+++ b/test/benchmarks/build.sbt
@@ -0,0 +1,8 @@
+scalaHome := Some(file("../../build/pack"))
+
+lazy val root = (project in file(".")).
+  enablePlugins(JmhPlugin).
+  settings(
+    name := "test-benchmarks",
+    version := "0.0.1"
+  )
diff --git a/test/benchmarks/project/plugins.sbt b/test/benchmarks/project/plugins.sbt
new file mode 100644
index 0000000000..f5319fb187
--- /dev/null
+++ b/test/benchmarks/project/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.6")
diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala
new file mode 100644
index 0000000000..eeea8f6508
--- /dev/null
+++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapBenchmark.scala
@@ -0,0 +1,76 @@
+package scala.collection.mutable;
+
+import java.util.concurrent.TimeUnit
+import org.openjdk.jmh.annotations._
+
+private object OpenHashMapBenchmark {
+  /** State container for the `put()` bulk calling tests.
+    * 
+    * Provides a thread-scoped map, so that allocation for the hash table will be done
+    * in the first warm-up iteration, not during measurement.
+    * 
+    * Performs a GC after every invocation, so that only the GCs caused by the invocation
+    * contribute to the measurement.
+    */
+  @State(Scope.Thread)
+  class BulkPutState {
+    val map = new OpenHashMap[Int,Int].empty
+  
+    @TearDown(Level.Invocation)
+    def teardown { map.clear(); System.gc() }
+  }
+}
+
+/** Benchmark for the library's [[OpenHashMap]].
+  * 
+  * The `put()` calls are tested by looping to the size desired for the map;
+  * instead of using the JMH harness, which iterates for a fixed length of time.
+  */
+@BenchmarkMode(Array(Mode.AverageTime))
+@Threads(1)
+@Fork(1)
+@Warmup(iterations = 20)
+@Measurement(iterations = 20)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+class OpenHashMapBenchmark {
+  import OpenHashMapBenchmark._
+
+  @Param(Array("100", "250", "1000", "2500", "10000", "25000", "100000", "250000", "1000000", "2500000",
+               "5000000", "7500000", "10000000", "25000000"))
+  var size: Int = _
+
+  /** Put elements into the given map. */
+  private[this] def put_Int(map: OpenHashMap[Int,Int], from: Int, to: Int) {
+    var i = from
+    while (i <= to) {  // using a `for` expression instead adds significant overhead
+      map.put(i, i)
+      i += 1
+    }
+  }
+
+  /** Test putting elements to a map of `Int` to `Int`. */
+  @Benchmark
+  def put_Int(state: BulkPutState) { put_Int(state.map, 1, size) }
+
+  /** Test putting and removing elements to a growing map of `Int` to `Int`. */
+  @Benchmark
+  def put_remove_Int(state: BulkPutState) {
+    val blocks = 50  // should be a factor of `size`
+    val totalPuts = 2 * size  // add twice as many, because we remove half of them
+    val blockSize: Int = totalPuts / blocks
+    var base = 0
+    while (base < totalPuts) {
+      put_Int(state.map, base + 1, base + blockSize)
+
+      // remove every other entry
+      var i = base + 1
+      while (i <= base + blockSize) {
+        state.map.remove(i)
+        i += 2
+      }
+
+      base += blockSize
+    }
+  }
+}