Add benchmarks to compare recursive and flat cp representations

The goal of these changes is to add possibility to:
- compare an efficiency and a content of both cp implementations
(ClassPathImplComparator)
- examine the memory consumption by creating a lot of globals using
a specified classpath (ClassPathMemoryConsumptionTester) - it can be
considered as e.g. some approximation of ScalaPresentationCompilers
in Scala IDE when working with many projects

ClassPathMemoryConsumptionTester is placed in main (I mean not test)
sources so thanks to that it has properly, out of the box configured
boot classpath etc. and it's easy to use it, e.g.:

scala scala.tools.nsc.ClassPathMemoryConsumptionTester
-YclasspathImpl:<implementation_to_test> -cp <some_cp>
-sourcepath <some_sp> -requiredInstances 50 SomeFileToCompile.scala

At the end it waits for the "exit" command so there can be used some
profiler like JProfiler to look how the given implementation behaves.

Also flat classpath implementation is set as a default one to test it
on Jenkins. This particular change must be reverted when all tests
will pass because for now it's not desirable to make it permanently
the default representation.

3 files changed, 221 insertions, 1 deletions

diff --git a/src/compiler/scala/tools/nsc/ClassPathMemoryConsumptionTester.scala b/src/compiler/scala/tools/nsc/ClassPathMemoryConsumptionTester.scala
new file mode 100644
index 0000000000..2faf6c6272
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/ClassPathMemoryConsumptionTester.scala
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc
+
+import scala.io.StdIn.readLine
+
+/**
+ * Simple application to check out amount of memory used by chosen classpath representation.
+ * It allows us to create many scalac-like calls based on specified parameters, where each main retains Global.
+ * And we need additional tool (e.g. profiler) to measure memory consumption itself.
+ */
+object ClassPathMemoryConsumptionTester {
+
+  private class TestSettings extends Settings {
+    val requiredInstances = IntSetting("-requiredInstances",
+      "Determine how many times classpath should be loaded", 10, Some((1, 10000)), (_: String) => None)
+  }
+
+  private class MainRetainsGlobal extends scala.tools.nsc.MainClass {
+    var retainedGlobal: Global = _
+    override def doCompile(compiler: Global) {
+      retainedGlobal = compiler
+      super.doCompile(compiler)
+    }
+  }
+
+  def main(args: Array[String]): Unit = {
+    if (args contains "-help") usage()
+    else doTest(args)
+  }
+
+  private def doTest(args: Array[String]) = {
+    val settings = loadSettings(args.toList)
+
+    val mains = (1 to settings.requiredInstances.value) map (_ => new MainRetainsGlobal)
+
+    // we need original settings without additional params to be able to use them later
+    val baseArgs = argsWithoutRequiredInstances(args)
+
+    println(s"Loading classpath ${settings.requiredInstances.value} times")
+    val startTime = System.currentTimeMillis()
+
+    mains map (_.process(baseArgs))
+
+    val elapsed = System.currentTimeMillis() - startTime
+    println(s"Operation finished - elapsed $elapsed ms")
+    println("Memory consumption can be now measured")
+
+    var textFromStdIn = ""
+    while (textFromStdIn.toLowerCase != "exit")
+      textFromStdIn = readLine("Type 'exit' to close application: ")
+  }
+
+  /**
+   * Prints usage information
+   */
+  private def usage(): Unit =
+    println( """Use classpath and sourcepath options like in the case of e.g. 'scala' command.
+               | There's also one additional option:
+               | -requiredInstances <int value> Determine how many times classpath should be loaded
+             """.stripMargin.trim)
+
+  private def loadSettings(args: List[String]) = {
+    val settings = new TestSettings()
+    settings.processArguments(args, processAll = true)
+    if (settings.classpath.isDefault)
+      settings.classpath.value = sys.props("java.class.path")
+    settings
+  }
+
+  private def argsWithoutRequiredInstances(args: Array[String]) = {
+    val instancesIndex = args.indexOf("-requiredInstances")
+    if (instancesIndex == -1) args
+    else args.dropRight(args.length - instancesIndex) ++ args.drop(instancesIndex + 2)
+  }
+}
diff --git a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala
index 18e639b81c..c599b7b443 100644
--- a/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala
+++ b/src/compiler/scala/tools/nsc/settings/ScalaSettings.scala
@@ -202,7 +202,7 @@ trait ScalaSettings extends AbsScalaSettings
   val YmethodInfer    = BooleanSetting    ("-Yinfer-argument-types", "Infer types for arguments of overriden methods.")
   val etaExpandKeepsStar = BooleanSetting ("-Yeta-expand-keeps-star", "Eta-expand varargs methods to T* rather than Seq[T].  This is a temporary option to ease transition.").withDeprecationMessage(removalIn212)
   val inferByName     = BooleanSetting    ("-Yinfer-by-name", "Allow inference of by-name types. This is a temporary option to ease transition. See SI-7899.").withDeprecationMessage(removalIn212)
-  val YclasspathImpl  = ChoiceSetting     ("-YclasspathImpl", "implementation", "Choose classpath scanning method.", List(ClassPathRepresentationType.Recursive, ClassPathRepresentationType.Flat), ClassPathRepresentationType.Recursive)
+  val YclasspathImpl  = ChoiceSetting     ("-YclasspathImpl", "implementation", "Choose classpath scanning method.", List(ClassPathRepresentationType.Recursive, ClassPathRepresentationType.Flat), ClassPathRepresentationType.Flat)
   val YdisableFlatCpCaching  = BooleanSetting    ("-YdisableFlatCpCaching", "Do not cache flat classpath representation of classpath elements from jars across compiler instances.")
 
   val YvirtClasses    = false // too embryonic to even expose as a -Y //BooleanSetting    ("-Yvirtual-classes", "Support virtual classes")
diff --git a/test/junit/scala/tools/nsc/util/ClassPathImplComparator.scala b/test/junit/scala/tools/nsc/util/ClassPathImplComparator.scala
new file mode 100644
index 0000000000..f2926e3e17
--- /dev/null
+++ b/test/junit/scala/tools/nsc/util/ClassPathImplComparator.scala
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc.util
+
+import scala.reflect.io.AbstractFile
+import scala.tools.nsc.Settings
+import scala.tools.nsc.settings.ClassPathRepresentationType
+import scala.tools.util.PathResolverFactory
+
+/**
+ * Simple application to compare efficiency of the recursive and the flat classpath representations
+ */
+object ClassPathImplComparator {
+
+  private class TestSettings extends Settings {
+    val checkClasses = PathSetting("-checkClasses", "Specify names of classes which should be found separated with ;", "")
+    val requiredIterations = IntSetting("-requiredIterations",
+      "Repeat tests specified number of times (to check e.g. impact of caches)", 1, Some((1, Int.MaxValue)), (_: String) => None)
+    val cpCreationRepetitions = IntSetting("-cpCreationRepetitions",
+      "Repeat tests specified number of times (to check e.g. impact of caches)", 1, Some((1, Int.MaxValue)), (_: String) => None)
+    val cpLookupRepetitions = IntSetting("-cpLookupRepetitions",
+      "Repeat tests specified number of times (to check e.g. impact of caches)", 1, Some((1, Int.MaxValue)), (_: String) => None)
+  }
+
+  private class DurationStats(name: String) {
+    private var sum = 0L
+    private var iterations = 0
+
+    def noteMeasuredTime(millis: Long): Unit = {
+      sum += millis
+      iterations += 1
+    }
+
+    def printResults(): Unit = {
+      val avg = if (iterations == 0) 0 else sum.toDouble / iterations
+      println(s"$name - total duration: $sum ms; iterations: $iterations; avg: $avg ms")
+    }
+  }
+
+  private lazy val defaultClassesToFind = List(
+    "scala.collection.immutable.List",
+    "scala.Option",
+    "scala.Int",
+    "scala.collection.immutable.Vector",
+    "scala.util.hashing.MurmurHash3"
+  )
+
+  private val oldCpCreationStats = new DurationStats("Old classpath - create")
+  private val oldCpSearchingStats = new DurationStats("Old classpath - search")
+
+  private val flatCpCreationStats = new DurationStats("Flat classpath - create")
+  private val flatCpSearchingStats = new DurationStats("Flat classpath - search")
+
+  def main(args: Array[String]): Unit = {
+
+    if (args contains "-help")
+      usage()
+    else {
+      val oldCpSettings = loadSettings(args.toList, ClassPathRepresentationType.Recursive)
+      val flatCpSettings = loadSettings(args.toList, ClassPathRepresentationType.Flat)
+
+      val classesToCheck = oldCpSettings.checkClasses.value
+      val classesToFind =
+        if (classesToCheck.isEmpty) defaultClassesToFind
+        else classesToCheck.split(";").toList
+
+      def doTest(classPath: => ClassFileLookup[AbstractFile], cpCreationStats: DurationStats, cpSearchingStats: DurationStats,
+                 cpCreationRepetitions: Int, cpLookupRepetitions: Int)= {
+
+        def createClassPaths() = (1 to cpCreationRepetitions).map(_ => classPath).last
+        def testClassLookup(cp: ClassFileLookup[AbstractFile]): Boolean = (1 to cpCreationRepetitions).foldLeft(true) {
+          case (a, _) => a && checkExistenceOfClasses(classesToFind)(cp)
+        }
+
+        val cp = withMeasuredTime("Creating classpath", createClassPaths(), cpCreationStats)
+        val result = withMeasuredTime("Searching for specified classes", testClassLookup(cp), cpSearchingStats)
+        println(s"The end of the test case. All expected classes found = $result \n")
+      }
+
+      (1 to oldCpSettings.requiredIterations.value) foreach { iteration =>
+        if (oldCpSettings.requiredIterations.value > 1)
+          println(s"Iteration no $iteration")
+
+        println("Recursive (old) classpath representation:")
+        doTest(PathResolverFactory.create(oldCpSettings).result, oldCpCreationStats, oldCpSearchingStats,
+          oldCpSettings.cpCreationRepetitions.value, oldCpSettings.cpLookupRepetitions.value)
+
+        println("Flat classpath representation:")
+        doTest(PathResolverFactory.create(flatCpSettings).result, flatCpCreationStats, flatCpSearchingStats,
+          flatCpSettings.cpCreationRepetitions.value, flatCpSettings.cpLookupRepetitions.value)
+      }
+
+      if (oldCpSettings.requiredIterations.value > 1) {
+        println("\nOld classpath - summary")
+        oldCpCreationStats.printResults()
+        oldCpSearchingStats.printResults()
+
+        println("\nFlat classpath - summary")
+        flatCpCreationStats.printResults()
+        flatCpSearchingStats.printResults()
+      }
+    }
+  }
+
+  /**
+   * Prints usage information
+   */
+  private def usage(): Unit =
+    println("""Use classpath and sourcepath options like in the case of e.g. 'scala' command.
+              | There are also two additional options:
+              | -checkClasses <semicolon separated class names> Specify names of classes which should be found
+              | -requiredIterations <int value>                 Repeat tests specified count of times (to check e.g. impact of caches)
+              | Note: Option -YclasspathImpl will be set automatically for each case.
+            """.stripMargin.trim)
+
+  private def loadSettings(args: List[String], implType: String) = {
+    val settings = new TestSettings()
+    settings.processArguments(args, processAll = true)
+    settings.YclasspathImpl.value = implType
+    if (settings.classpath.isDefault)
+      settings.classpath.value = sys.props("java.class.path")
+    settings
+  }
+
+  private def withMeasuredTime[T](operationName: String, f: => T, durationStats: DurationStats): T = {
+    val startTime = System.currentTimeMillis()
+    val res = f
+    val elapsed = System.currentTimeMillis() - startTime
+    durationStats.noteMeasuredTime(elapsed)
+    println(s"$operationName - elapsed $elapsed ms")
+    res
+  }
+
+  private def checkExistenceOfClasses(classesToCheck: Seq[String])(classPath: ClassFileLookup[AbstractFile]): Boolean =
+    classesToCheck.foldLeft(true) {
+      case (res, classToCheck) =>
+        val found = classPath.findClass(classToCheck).isDefined
+        if (!found)
+          println(s"Class $classToCheck not found") // of course in this case the measured time will be affected by IO operation
+        found
+    }
+}