diff options
author | Performant Data LLC <performantdata@users.noreply.github.com> | 2016-03-27 11:59:13 -0700 |
---|---|---|
committer | Performant Data LLC <performantdata@users.noreply.github.com> | 2016-05-03 11:46:58 -0700 |
commit | 2e8fb12f6d92e6021131461285b9c28909584d04 (patch) | |
tree | 47bd39afcae9944039eb975c83a8e36d4cc7e4a8 /test | |
parent | b88933eb84f1f1f5215b0feb43f4ecfc12c8847d (diff) | |
download | scala-2e8fb12f6d92e6021131461285b9c28909584d04.tar.gz scala-2e8fb12f6d92e6021131461285b9c28909584d04.tar.bz2 scala-2e8fb12f6d92e6021131461285b9c28909584d04.zip |
Add a JMH runner class to the library benchmark framework.
Diffstat (limited to 'test')
-rw-r--r-- | test/benchmarks/README.md | 49 | ||||
-rw-r--r-- | test/benchmarks/src/main/scala/benchmark/JmhRunner.scala | 16 | ||||
-rw-r--r-- | test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala | 111 |
3 files changed, 167 insertions, 9 deletions
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md index aea72e90ed..07e72f09a1 100644 --- a/test/benchmarks/README.md +++ b/test/benchmarks/README.md @@ -1,17 +1,48 @@ # Scala library benchmarks -This directory is a standalone SBT project -that makes use of the [SBT plugin for JMH](https://github.com/ktoso/sbt-jmh), -with the usual directory structure: -source code for the benchmarks, which utilize [JMH](http://openjdk.java.net/projects/code-tools/jmh/), -should be placed in `src/main/scala`. +This directory is a standalone SBT project, within the Scala project, +that makes use of the [SBT plugin](https://github.com/ktoso/sbt-jmh) for [JMH](http://openjdk.java.net/projects/code-tools/jmh/). -The benchmarks require first building Scala into `../../build/pack`. -They can then be (built and) run from `sbt` with "`jmh:run`". -"`jmh:run -h`" displays the usual JMH options available. +## running a benchmark + +The benchmarks require first building Scala into `../../build/pack`, using Ant. + +You'll then need to know the fully-qualified name of the benchmark runner class. +The benchmarking classes are organized under `src/main/scala`, +in the same package hierarchy as the classes that they test. +Assuming that we're benchmarking `scala.collection.mutable.OpenHashMap`, +the benchmark runner would likely be named `scala.collection.mutable.OpenHashMapRunner`. +Using this example, one would simply run + + jmh:runMain scala.collection.mutable.OpenHashMapRunner + +in SBT. +SBT should be run _from this directory_. + +The JMH results can be found under `target/jmh-results/`. +`target` gets deleted on an SBT `clean`, +so you should copy these files out of `target` if you wish to preserve them. + +## creating a benchmark and runner + +The benchmarking classes use the same package hierarchy as the classes that they test +in order to make it easy to expose, in package scope, members of the class under test, +should that be necessary for benchmarking. + +There are two types of classes in the source directory: +those suffixed "`Benchmark`" and those suffixed "`Runner`". +The former are benchmarks that can be run directly using `jmh:run`; +however, they are normally run from a corresponding class of the latter type, +which is run using `jmh:runMain` (as described above). +This …`Runner` class is useful for setting appropriate JMH command options, +and for processing the JMH results into files that can be read by other tools, such as Gnuplot. + +The `benchmark.JmhRunner` trait should be woven into any runner class, for the standard behavior that it provides. +This includes creating output files in a subdirectory of `target/jmh-results` +derived from the fully-qualified package name of the `Runner` class. ## some useful HotSpot options -Adding these to the `jmh:run` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler. +Adding these to the `jmh:run` or `jmh:runMain` command line may help if you're using the HotSpot (Oracle, OpenJDK) compiler. They require prefixing with `-jvmArgs`. See [the Java documentation](http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html) for more options. diff --git a/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala b/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala new file mode 100644 index 0000000000..cc75be529d --- /dev/null +++ b/test/benchmarks/src/main/scala/benchmark/JmhRunner.scala @@ -0,0 +1,16 @@ +package benchmark + +import java.io.File + +/** Common code for JMH runner objects. */ +trait JmhRunner { + private[this] val parentDirectory = new File("target", "jmh-results") + + /** Return the output directory for this class, creating the directory if necessary. */ + protected def outputDirectory: File = { + val subdir = getClass.getPackage.getName.replace('.', File.separatorChar) + val dir = new File(parentDirectory, subdir) + if (!dir.isDirectory) dir.mkdirs() + dir + } +} diff --git a/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala new file mode 100644 index 0000000000..c139c55933 --- /dev/null +++ b/test/benchmarks/src/main/scala/scala/collection/mutable/OpenHashMapRunner.scala @@ -0,0 +1,111 @@ +package scala.collection.mutable + +import java.io.BufferedWriter +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStreamWriter +import java.io.PrintWriter +import scala.collection.JavaConversions +import scala.language.existentials +import org.openjdk.jmh.results.RunResult +import org.openjdk.jmh.runner.Runner +import org.openjdk.jmh.runner.options.CommandLineOptions +import org.openjdk.jmh.runner.options.Options +import benchmark.JmhRunner +import org.openjdk.jmh.runner.options.OptionsBuilder +import org.openjdk.jmh.runner.options.VerboseMode + +/** Replacement JMH application that runs the [[OpenHashMap]] benchmark. + * + * Outputs the results in a form consumable by a Gnuplot script. + */ +object OpenHashMapRunner extends JmhRunner { + /** File that will be created for the output data set. */ + private[this] val outputFile = new File(outputDirectory, "OpenHashMap.dat") + + /** Qualifier to add to the name of a memory usage data set. */ + private[this] val memoryDatasetQualifer = " memory" + + /** Name of the JMH parameter for the number of map entries per invocation. */ + private[this] val sizeParamName = "size" + + /** Name of the JMH auxiliary counter that collects operation counts. */ + private[this] val operationsAuxCounterName = "operations" + + /** Name of the JMH auxiliary counter that collects memory usage. */ + private[this] val memoryAuxCounterName = "memory" + + /** Name of the JMH auxiliary counter that collects the number of map entries. */ + private[this] val entriesAuxCounterName = "mapEntries" + + def main(args: Array[String]) { + import scala.collection.JavaConversions._ + import scala.language.existentials + + val opts = new CommandLineOptions(args: _*) + var builder = new OptionsBuilder().parent(opts) + .jvmArgsPrepend("-Xmx6000m") + if (!opts.verbosity.hasValue) builder = builder.verbosity(VerboseMode.SILENT) + + val results = new Runner(builder.build).run() + + // Sort the results + + /** Map from data set name to data set. */ + val datasetByName = Map.empty[String, Set[RunResult]] + + /** Ordering for the results within a data set. Orders by increasing number of map entries. */ + val ordering = Ordering.by[RunResult, Int](_.getParams.getParam(sizeParamName).toInt) + + def addToDataset(result: RunResult, key: String): Unit = + datasetByName.get(key) + .getOrElse({ val d = SortedSet.empty(ordering); datasetByName.put(key, d); d }) += result + + results.foreach { result: RunResult ⇒ + addToDataset(result, result.getPrimaryResult.getLabel) + + // Create another data set for trials that track memory usage + if (result.getSecondaryResults.containsKey(memoryAuxCounterName)) + addToDataset(result, result.getPrimaryResult.getLabel + memoryDatasetQualifer) + } + + //TODO Write out test parameters + // val jvm = params.getJvm + // val jvmArgs = params.getJvmArgs.mkString(" ") + + val f = new PrintWriter(outputFile, "UTF-8") + try { + datasetByName.foreach(_ match { case (label: String, dataset: Iterable[RunResult]) ⇒ { + f.format("# [%s]\n", label) + + val isMemoryUsageDataset = label.contains(memoryDatasetQualifer) + dataset.foreach { result ⇒ + val size = result.getParams.getParam(sizeParamName) + val secondaryResults = result.getSecondaryResults + if (isMemoryUsageDataset) { + val memoryResult = secondaryResults.get(memoryAuxCounterName) + val entriesResult = secondaryResults.get(entriesAuxCounterName) + f.format("%s %f %f %f %f\n", size, + Double.box(entriesResult.getScore), Double.box(entriesResult.getStatistics.getStandardDeviation), + Double.box(memoryResult.getScore), Double.box(memoryResult.getStatistics.getStandardDeviation)) + } + else { + if (secondaryResults.containsKey(operationsAuxCounterName)) { + val operationsResult = secondaryResults.get(operationsAuxCounterName) + f.format("%s %f %f\n", size, + Double.box(operationsResult.getScore), Double.box(operationsResult.getStatistics.getStandardDeviation)) + } else { + val primary = result.getPrimaryResult + f.format("%s %f %f\n", size, + Double.box(primary.getScore), Double.box(primary.getStatistics.getStandardDeviation)) + } + } + } + + f.println(); f.println() // data set separator + }}) + } finally { + f.close() + } + } +} |