Add a JMH runner class to the library benchmark framework.

3 files changed, 167 insertions, 9 deletions

diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md
index aea72e90ed..07e72f09a1 100644
--- a/test/benchmarks/README.md
+++ b/test/benchmarks/README.md
@@ -1,17 +1,48 @@
 # Scala library benchmarks
 
-This directory is a standalone SBT project
-that makes use of the [SBT plugin for JMH](https://github.com/ktoso/sbt-jmh),
-with the usual directory structure:
-source code for the benchmarks, which utilize [JMH](http://openjdk.java.net/projects/code-tools/jmh/),
-should be placed in `src/main/scala`.
+This directory is a standalone SBT project, within the Scala project,
+that makes use of the [SBT plugin](https://github.com/ktoso/sbt-jmh) for [JMH](http://openjdk.java.net/projects/code-tools/jmh/).
 
-The benchmarks require first building Scala into `../../build/pack`.
-They can then be (built and) run from `sbt` with "`jmh:run`".
-"`jmh:run -h`" displays the usual JMH options available.
+## running a benchmark
+
+The benchmarks require first building Scala into `../../build/pack`, using Ant.
+
+You'll then need to know the fully-qualified name of the benchmark runner class.
+The benchmarking classes are organized under `src/main/scala`,
+in the same package hierarchy as the classes that they test.
+Assuming that we're benchmarking `scala.collection.mutable.OpenHashMap`,
+the benchmark runner would likely be named `scala.collection.mutable.OpenHashMapRunner`.
+Using this example, one would simply run
+
+    jmh:runMain scala.collection.mutable.OpenHashMapRunner
+
+in SBT.
+SBT should be run _from this directory_.
+
+The JMH results can be found under `target/jmh-results/`.
+`target` gets deleted on an SBT `clean`,
+so you should copy these files out of `target` if you wish to preserve them.
+
+## creating a benchmark and runner
+
+The benchmarking classes use the same package hierarchy as the classes that they test
+in order to make it easy to expose, in package scope, members of the class under test,
+should that be necessary for benchmarking.
+
+There are two types of classes in the source directory:
+those suffixed "`Benchmark`" and those suffixed "`Runner`".
+The former are benchmarks that can be run directly using `jmh:run`;
+however, they are normally run from a corresponding class of the latter type,
+which is run using `jmh:runMain` (as described above).
+This …`Runner` class is useful for setting appropriate JMH command options,
+and for processing the JMH results into files that can be read by other tools, such as Gnuplot.
+
+The `benchmark.JmhRunner` trait should be woven into any runner class, for the standard behavior that it provides.
+This includes creating output files in a subdirectory of `target/jmh-results`
+derived from the fully-qualified package name of the `Runner` class.
 
 ## some useful HotSpot options
-Adding these to the `jmh:run` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler.
+Adding these to the `jmh:run` or `jmh:runMain` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler.
 They require prefixing with `-jvmArgs`.
 See [the Java documentation](http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html) for more options. 
 
diff --git a/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala b/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala
new file mode 100644
index 0000000000..cc75be529d
--- /dev/null
+++ b/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala
@@ -0,0 +1,16 @@
+package benchmark
+
+import java.io.File
+
+/** Common code for JMH runner objects. */
+trait JmhRunner {
+  private[this] val parentDirectory = new File("target", "jmh-results")
+
+  /** Return the output directory for this class, creating the directory if necessary. */
+  protected def outputDirectory: File = {
+    val subdir = getClass.getPackage.getName.replace('.', File.separatorChar)
+    val dir = new File(parentDirectory, subdir)
+    if (!dir.isDirectory) dir.mkdirs()
+    dir
+  }
+}
diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala
new file mode 100644
index 0000000000..c139c55933
--- /dev/null
+++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala
@@ -0,0 +1,111 @@
+package scala.collection.mutable
+
+import java.io.BufferedWriter
+import java.io.File
+import java.io.FileOutputStream
+import java.io.OutputStreamWriter
+import java.io.PrintWriter
+import scala.collection.JavaConversions
+import scala.language.existentials
+import org.openjdk.jmh.results.RunResult
+import org.openjdk.jmh.runner.Runner
+import org.openjdk.jmh.runner.options.CommandLineOptions
+import org.openjdk.jmh.runner.options.Options
+import benchmark.JmhRunner
+import org.openjdk.jmh.runner.options.OptionsBuilder
+import org.openjdk.jmh.runner.options.VerboseMode
+
+/** Replacement JMH application that runs the [[OpenHashMap]] benchmark.
+  * 
+  * Outputs the results in a form consumable by a Gnuplot script.
+  */
+object OpenHashMapRunner extends JmhRunner {
+  /** File that will be created for the output data set. */
+  private[this] val outputFile = new File(outputDirectory, "OpenHashMap.dat")
+
+  /** Qualifier to add to the name of a memory usage data set. */
+  private[this] val memoryDatasetQualifer = " memory"
+
+  /** Name of the JMH parameter for the number of map entries per invocation. */
+  private[this] val sizeParamName = "size"
+
+  /** Name of the JMH auxiliary counter that collects operation counts. */
+  private[this] val operationsAuxCounterName = "operations"
+
+  /** Name of the JMH auxiliary counter that collects memory usage. */
+  private[this] val memoryAuxCounterName = "memory"
+
+  /** Name of the JMH auxiliary counter that collects the number of map entries. */
+  private[this] val entriesAuxCounterName = "mapEntries"
+
+  def main(args: Array[String]) {
+    import scala.collection.JavaConversions._
+    import scala.language.existentials
+  
+    val opts = new CommandLineOptions(args: _*)
+    var builder = new OptionsBuilder().parent(opts)
+      .jvmArgsPrepend("-Xmx6000m")
+    if (!opts.verbosity.hasValue)  builder = builder.verbosity(VerboseMode.SILENT)
+
+    val results = new Runner(builder.build).run()
+
+    // Sort the results
+
+    /** Map from data set name to data set. */
+    val datasetByName = Map.empty[String, Set[RunResult]]
+
+    /** Ordering for the results within a data set. Orders by increasing number of map entries. */
+    val ordering = Ordering.by[RunResult, Int](_.getParams.getParam(sizeParamName).toInt)
+
+    def addToDataset(result: RunResult, key: String): Unit =
+      datasetByName.get(key)
+        .getOrElse({ val d = SortedSet.empty(ordering); datasetByName.put(key, d); d }) += result
+
+    results.foreach { result: RunResult ⇒
+      addToDataset(result, result.getPrimaryResult.getLabel)
+
+      // Create another data set for trials that track memory usage
+      if (result.getSecondaryResults.containsKey(memoryAuxCounterName))
+        addToDataset(result, result.getPrimaryResult.getLabel + memoryDatasetQualifer)
+    }
+
+    //TODO Write out test parameters
+    //    val jvm = params.getJvm
+    //    val jvmArgs = params.getJvmArgs.mkString(" ")
+
+    val f = new PrintWriter(outputFile, "UTF-8")
+    try {
+      datasetByName.foreach(_ match { case (label: String, dataset: Iterable[RunResult]) ⇒ {
+        f.format("# [%s]\n", label)
+
+        val isMemoryUsageDataset = label.contains(memoryDatasetQualifer)
+        dataset.foreach { result ⇒
+          val size = result.getParams.getParam(sizeParamName)
+          val secondaryResults = result.getSecondaryResults
+          if (isMemoryUsageDataset) {
+            val memoryResult = secondaryResults.get(memoryAuxCounterName)
+            val entriesResult = secondaryResults.get(entriesAuxCounterName)
+            f.format("%s %f %f %f %f\n", size,
+              Double.box(entriesResult.getScore), Double.box(entriesResult.getStatistics.getStandardDeviation),
+              Double.box(memoryResult.getScore), Double.box(memoryResult.getStatistics.getStandardDeviation))
+          }
+          else {
+            if (secondaryResults.containsKey(operationsAuxCounterName)) {
+              val operationsResult = secondaryResults.get(operationsAuxCounterName)
+              f.format("%s %f %f\n", size,
+                Double.box(operationsResult.getScore), Double.box(operationsResult.getStatistics.getStandardDeviation))
+            } else {
+              val primary = result.getPrimaryResult
+              f.format("%s %f %f\n", size,
+                Double.box(primary.getScore), Double.box(primary.getStatistics.getStandardDeviation))
+            }
+          }
+        }
+
+        f.println(); f.println()  // data set separator
+      }})
+    } finally {
+      f.close()
+    }
+  }
+}