Partest: add `instrumented` test category.

---
(taken from README)

Tests in `instrumented/` directory are executed the same way as in `run/` but
they have additional byte-code instrumentation performed for profiling. You
should put your tests in `instrumented/` directory if you are interested in
method call counts. Examples include tests for specialization (you want to
count boxing and unboxing method calls) or high-level tests for optimizer
where you are interested if methods are successfuly inlined (so they should
not be called at runtime) or closures are eliminated (so no constructors
of closures are called).

Check `scala.tools.partest.instrumented.Instrumentation` to learn how to
use the instrumentation infrastructure.

The instrumentation itself is achieved by attaching a Java agent to the
forked VM process that injects calls to profiler. Check
`scala.tools.partest.javaagent.ProfilingAgent` for details.

---

A few notes on low-level details of this change:

  * Partest now depends on asm library for byte-code instrumentation
    (`build.xml`)
  * Build additional jar called `scala-partest-javaagent.jar` that is
    used with `-javaagent:` option. (`build.xml`)
  * Set `-javaagent:` option for all tests in `instrumented/` directory.
    (`RunnerManger.scala`)
  * Introduce a new category of tests called `instrumented`.
  * Add one instrumented test to demonstrate usage and test new
    infrastructure itself. (`InstrumentationTest.scala`)

Review by @phaller.

13 files changed, 306 insertions, 2 deletions

diff --git a/src/partest/scala/tools/partest/PartestTask.scala b/src/partest/scala/tools/partest/PartestTask.scala
index b4c685b70c..959d682872 100644
--- a/src/partest/scala/tools/partest/PartestTask.scala
+++ b/src/partest/scala/tools/partest/PartestTask.scala
@@ -48,6 +48,7 @@ import org.apache.tools.ant.types.Commandline.Argument
  *  - `scalaptests`,
  *  - `scalachecktests`,
  *  - `specializedtests`,
+ *  - `instrumentedtests`,
  *  - `presentationtests`,
  *  - `scripttests`.
  *
@@ -99,6 +100,10 @@ class PartestTask extends Task with CompilationPathProperty {
     specializedFiles = Some(input)
   }
 
+  def addConfiguredInstrumentedTests(input: FileSet) {
+    instrumentedFiles = Some(input)
+  }
+
   def addConfiguredPresentationTests(input: FileSet) {
     presentationFiles = Some(input)
   }
@@ -189,6 +194,7 @@ class PartestTask extends Task with CompilationPathProperty {
   private var shootoutFiles: Option[FileSet] = None
   private var scalapFiles: Option[FileSet] = None
   private var specializedFiles: Option[FileSet] = None
+  private var instrumentedFiles: Option[FileSet] = None
   private var presentationFiles: Option[FileSet] = None
   private var antFiles: Option[FileSet] = None
   private var errorOnFailed: Boolean = false
@@ -245,6 +251,7 @@ class PartestTask extends Task with CompilationPathProperty {
   private def getShootoutFiles     = getFiles(shootoutFiles)
   private def getScalapFiles       = getFiles(scalapFiles)
   private def getSpecializedFiles  = getFiles(specializedFiles)
+  private def getInstrumentedFiles = getFilesAndDirs(instrumentedFiles)
   private def getPresentationFiles = getDirs(presentationFiles)
   private def getAntFiles          = getFiles(antFiles)
 
@@ -375,6 +382,7 @@ class PartestTask extends Task with CompilationPathProperty {
       (getShootoutFiles, "shootout", "Running shootout tests"),
       (getScalapFiles, "scalap", "Running scalap tests"),
       (getSpecializedFiles, "specialized", "Running specialized files"),
+      (getInstrumentedFiles, "instrumented", "Running instrumented files"),
       (getPresentationFiles, "presentation", "Running presentation compiler test files"),
       (getAntFiles, "ant", "Running ant task tests")
     )
diff --git a/src/partest/scala/tools/partest/instrumented/Instrumentation.scala b/src/partest/scala/tools/partest/instrumented/Instrumentation.scala
new file mode 100644
index 0000000000..f29a7f90fd
--- /dev/null
+++ b/src/partest/scala/tools/partest/instrumented/Instrumentation.scala
@@ -0,0 +1,86 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2012 LAMP/EPFL
+ * @author Grzegorz Kossakowski
+ */
+
+package scala.tools.partest.instrumented
+
+import scala.collection.JavaConverters._
+
+case class MethodCallTrace(className: String, methodName: String, methodDescriptor: String) {
+  override def toString(): String = className + "." + methodName + methodDescriptor
+}
+object MethodCallTrace {
+  implicit val ordering: Ordering[MethodCallTrace] = Ordering.by(x => (x.className, x.methodName, x.methodDescriptor))
+}
+
+/**
+ * An object that controls profiling of instrumented byte-code. The instrumentation is achieved
+ * by using `java.lang.instrument` package. The instrumentation agent can be found in
+ * `scala.tools.partest.javaagent` package.
+ *
+ * At the moment the following classes are being instrumented:
+ *   * all classes with empty package
+ *   * all classes from scala package (except for classes responsible for instrumentation)
+ *
+ * The canonical way of using instrumentation is have a test-case in `files/instrumented` directory.
+ * The following code in main:
+ *
+ * {{{
+ * import scala.tools.partest.instrumented.Instrumentation._
+ * def main(args: Array[String]): Unit = {
+ *   startProfiling()
+ *   // should box the boolean
+    println(true)
+    stopProfiling()
+    printStatistics()
+ * }
+ * }}}
+ *
+ *
+ * should print:
+ *
+ * {{{
+ * true
+ * Method call statistics:
+ * scala/Predef$.println(Ljava/lang/Object;)V: 1
+ * scala/runtime/BoxesRunTime.boxToBoolean(Z)Ljava/lang/Boolean;: 1
+ * }}}
+ */
+object Instrumentation {
+
+  type Statistics = Map[MethodCallTrace, Int]
+
+  def startProfiling(): Unit = Profiler.startProfiling()
+  def stopProfiling(): Unit = Profiler.stopProfiling()
+  def resetProfiling(): Unit = Profiler.resetProfiling()
+
+  def getStatistics: Statistics = {
+    Profiler.stopProfiling()
+    val stats = Profiler.getStatistics().asScala.toSeq.map {
+      case (trace, count) => MethodCallTrace(trace.className, trace.methodName, trace.methodDescriptor) -> count.intValue
+    }
+    val res = Map(stats: _*)
+    Profiler.startProfiling()
+    res
+  }
+
+  val standardFilter: MethodCallTrace => Boolean = t => {
+    // ignore all calls to Console trigger by printing
+    t.className != "scala/Console$" &&
+    // console accesses DynamicVariable, let's discard it too
+    !t.className.startsWith("scala/util/DynamicVariable")
+  }
+
+  def printStatistics(stats: Statistics = getStatistics, filter: MethodCallTrace => Boolean = standardFilter): Unit = {
+    val stats = getStatistics
+    println("Method call statistics:")
+    val toBePrinted = stats.toSeq.filter(p => filter(p._1)).sortBy(_._1)
+    // <count> <trace>
+    val format = "%5d  %s\n"
+    toBePrinted foreach {
+      case (trace, count) => printf(format, count, trace)
+    }
+  }
+
+}
diff --git a/src/partest/scala/tools/partest/instrumented/Profiler.java b/src/partest/scala/tools/partest/instrumented/Profiler.java
new file mode 100644
index 0000000000..215bdbba08
--- /dev/null
+++ b/src/partest/scala/tools/partest/instrumented/Profiler.java
@@ -0,0 +1,78 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2012 LAMP/EPFL
+ * @author Grzegorz Kossakowski
+ */
+
+package scala.tools.partest.instrumented;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A simple profiler class that counts method invocations. It is being used in byte-code instrumentation by inserting
+ * call to {@link Profiler#methodCalled(String, String, String)} at the beginning of every instrumented class.
+ *
+ * WARANING: This class is INTERNAL implementation detail and should never be used directly. It's made public only
+ * because it must be universally accessible for instrumentation needs. If you want to profile your test use
+ * {@link Instrumentation} instead.
+ */
+public class Profiler {
+	
+	private static boolean isProfiling = false;
+	private static Map<MethodCallTrace, Integer> counts = new HashMap<MethodCallTrace, Integer>();
+	
+	static public class MethodCallTrace {
+	  final String className;
+	  final String methodName;
+	  final String methodDescriptor;
+	  
+	  public MethodCallTrace(final String className, final String methodName, final String methodDescriptor) {
+	    this.className = className;
+	    this.methodName = methodName;
+	    this.methodDescriptor = methodDescriptor;
+	  }
+	  
+	  @Override
+	  public boolean equals(Object obj) {
+	    if (!(obj instanceof MethodCallTrace)) {
+	      return false;
+	    } else {
+	      MethodCallTrace that = (MethodCallTrace) obj;
+	      return that.className.equals(className) && that.methodName.equals(methodName) && that.methodDescriptor.equals(methodDescriptor);
+	    }
+	  }
+	  @Override
+	  public int hashCode() {
+	    return className.hashCode() ^ methodName.hashCode() ^ methodDescriptor.hashCode();
+	  }
+	}
+	
+	public static void startProfiling() {
+		isProfiling = true;
+	}
+	
+	public static void stopProfiling() {
+		isProfiling = false;
+	}
+	
+	public static void resetProfiling() {
+	  counts = new HashMap<MethodCallTrace, Integer>();
+	}
+	
+	public static void methodCalled(final String className, final String methodName, final String methodDescriptor) {
+		if (isProfiling) {
+		  MethodCallTrace trace = new MethodCallTrace(className, methodName, methodDescriptor);
+			Integer counter = counts.get(trace);
+			if (counter == null) {
+				counts.put(trace, 1);
+			} else {
+				counts.put(trace, counter+1);
+			}
+		}
+	}
+	
+	public static Map<MethodCallTrace, Integer> getStatistics() {
+	  return new HashMap<MethodCallTrace, Integer>(counts);
+	}
+
+}
diff --git a/src/partest/scala/tools/partest/javaagent/ASMTransformer.java b/src/partest/scala/tools/partest/javaagent/ASMTransformer.java
new file mode 100644
index 0000000000..643c683002
--- /dev/null
+++ b/src/partest/scala/tools/partest/javaagent/ASMTransformer.java
@@ -0,0 +1,38 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2012 LAMP/EPFL
+ * @author Grzegorz Kossakowski
+ */
+
+package scala.tools.partest.javaagent;
+
+import java.lang.instrument.ClassFileTransformer;
+import java.security.ProtectionDomain;
+
+import scala.tools.asm.ClassReader;
+import scala.tools.asm.ClassWriter;
+
+public class ASMTransformer implements ClassFileTransformer {
+  
+  private boolean shouldTransform(String className) {
+    return 
+        // do not instrument instrumentation logic (in order to avoid infinite recursion)
+        !className.startsWith("scala/tools/partest/instrumented/") &&
+        !className.startsWith("scala/tools/partest/javaagent/") &&
+        // we instrument all classes from empty package
+        (!className.contains("/") ||
+        // we instrument all classes from scala package
+        className.startsWith("scala/"));
+  }
+	
+	public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) {
+	  if (shouldTransform(className)) {
+  		ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS);
+  		ProfilerVisitor visitor = new ProfilerVisitor(writer);
+  		ClassReader reader = new ClassReader(classfileBuffer);
+  		reader.accept(visitor, 0);
+  		return writer.toByteArray();
+	  } else {
+	    return classfileBuffer;
+	  }
+	}
+}
diff --git a/src/partest/scala/tools/partest/javaagent/MANIFEST.MF b/src/partest/scala/tools/partest/javaagent/MANIFEST.MF
new file mode 100644
index 0000000000..be0fee46a2
--- /dev/null
+++ b/src/partest/scala/tools/partest/javaagent/MANIFEST.MF
@@ -0,0 +1 @@
+Premain-Class: scala.tools.partest.javaagent.ProfilingAgent
diff --git a/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java b/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java
new file mode 100644
index 0000000000..f3a25e87d9
--- /dev/null
+++ b/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java
@@ -0,0 +1,46 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2012 LAMP/EPFL
+ * @author Grzegorz Kossakowski
+ */
+
+package scala.tools.partest.javaagent;
+
+import scala.tools.asm.ClassVisitor;
+import scala.tools.asm.MethodVisitor;
+import scala.tools.asm.Opcodes;
+
+public class ProfilerVisitor extends ClassVisitor implements Opcodes {
+  
+  private static String profilerClass = "scala/tools/partest/instrumented/Profiler";
+
+  public ProfilerVisitor(final ClassVisitor cv) {
+    super(ASM4, cv);
+  }
+
+  private String className = null;
+
+  @Override
+  public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+    className = name;
+    super.visit(version, access, name, signature, superName, interfaces);
+  }
+
+  public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+    // delegate the method call to the next
+    // chained visitor
+    MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions);
+    if (!profilerClass.equals(className)) {
+      // only instrument non-abstract methods
+      if((access & ACC_ABSTRACT) == 0) {
+        assert(className != null);
+        mv.visitLdcInsn(className);
+        mv.visitLdcInsn(name);
+        mv.visitLdcInsn(desc);
+        mv.visitMethodInsn(INVOKESTATIC, profilerClass, "methodCalled",
+            "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V");
+      }
+    }
+    return mv; 
+  }
+
+}
diff --git a/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java b/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java
new file mode 100644
index 0000000000..15efd73d94
--- /dev/null
+++ b/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java
@@ -0,0 +1,25 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2012 LAMP/EPFL
+ * @author Grzegorz Kossakowski
+ */
+
+package scala.tools.partest.javaagent;
+
+import java.lang.instrument.Instrumentation;
+import java.lang.instrument.UnmodifiableClassException;
+
+/**
+ * Profiling agent that instruments byte-code to insert calls to
+ * {@link scala.tools.partest.instrumented.Profiler#methodCalled(String, String, String)}
+ * by using ASM library for byte-code manipulation.
+ */
+public class ProfilingAgent {
+	public static void premain(String args, Instrumentation inst) throws UnmodifiableClassException {
+	  // NOTE: we are adding transformer that won't be applied to classes that are already loaded
+	  // This should be ok because premain should be executed before main is executed so Scala library
+	  // and the test-case itself won't be loaded yet. We rely here on the fact that ASMTransformer does
+	  // not depend on Scala library. In case our assumptions are wrong we can always insert call to
+	  // inst.retransformClasses.
+		inst.addTransformer(new ASMTransformer(), true);
+	}
+}
diff --git a/src/partest/scala/tools/partest/nest/CompileManager.scala b/src/partest/scala/tools/partest/nest/CompileManager.scala
index c674e21482..6e6c767117 100644
--- a/src/partest/scala/tools/partest/nest/CompileManager.scala
+++ b/src/partest/scala/tools/partest/nest/CompileManager.scala
@@ -112,6 +112,7 @@ class DirectCompiler(val fileManager: FileManager) extends SimpleCompiler {
       case "scalap"       => ScalapTestFile.apply
       case "scalacheck"   => ScalaCheckTestFile.apply
       case "specialized"  => SpecializedTestFile.apply
+      case "instrumented" => InstrumentedTestFile.apply
       case "presentation" => PresentationTestFile.apply
       case "ant"          => AntTestFile.apply
     }
diff --git a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
index 962520844c..20e6f47802 100644
--- a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
+++ b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
@@ -40,6 +40,7 @@ class ConsoleRunner extends DirectRunner {
       TestSet("scalacheck", stdFilter, "Testing ScalaCheck tests"),
       TestSet("scalap", _.isDirectory, "Run scalap decompiler tests"),
       TestSet("specialized", stdFilter, "Testing specialized tests"),
+      TestSet("instrumented", stdFilter, "Testing instrumented tests"),
       TestSet("presentation", _.isDirectory, "Testing presentation compiler tests."),
       TestSet("ant", antFilter, "Run Ant task tests.")
     )
diff --git a/src/partest/scala/tools/partest/nest/NestUI.scala b/src/partest/scala/tools/partest/nest/NestUI.scala
index 7fd503e769..54e99a7ddf 100644
--- a/src/partest/scala/tools/partest/nest/NestUI.scala
+++ b/src/partest/scala/tools/partest/nest/NestUI.scala
@@ -80,6 +80,7 @@ object NestUI {
     println("    --scalacheck    run ScalaCheck tests")
     println("    --script        run script runner tests")
     println("    --shootout      run shootout tests")
+    println("    --instrumented  run instrumented tests")
     println("    --presentation  run presentation compiler tests")
     println("    --grep <expr>    run all tests whose source file contains <expr>")
     println
diff --git a/src/partest/scala/tools/partest/nest/PathSettings.scala b/src/partest/scala/tools/partest/nest/PathSettings.scala
index ac04c64c33..b409a29165 100644
--- a/src/partest/scala/tools/partest/nest/PathSettings.scala
+++ b/src/partest/scala/tools/partest/nest/PathSettings.scala
@@ -49,6 +49,12 @@ object PathSettings {
       getOrElse sys.error("No code.jar found in %s".format(srcCodeLibDir))
   )
 
+  lazy val instrumentationAgentLib: File = {
+    findJar(buildPackLibDir.files, "scala-partest-javaagent") getOrElse {
+      sys.error("No partest-javaagent jar found in '%s' or '%s'".format(buildPackLibDir, srcLibDir))
+    }
+  }
+
   // Directory <root>/build
   lazy val buildDir: Directory = {
     val bases      = testRoot :: testRoot.parents
diff --git a/src/partest/scala/tools/partest/nest/RunnerManager.scala b/src/partest/scala/tools/partest/nest/RunnerManager.scala
index feca40a159..dc15d4475b 100644
--- a/src/partest/scala/tools/partest/nest/RunnerManager.scala
+++ b/src/partest/scala/tools/partest/nest/RunnerManager.scala
@@ -185,7 +185,7 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP
       outDir
     }
 
-    private def execTest(outDir: File, logFile: File, classpathPrefix: String = ""): Boolean = {
+    private def execTest(outDir: File, logFile: File, classpathPrefix: String = "", javaOpts: String = ""): Boolean = {
       // check whether there is a ".javaopts" file
       val argsFile  = new File(logFile.getParentFile, fileBase + ".javaopts")
       val argString = file2String(argsFile)
@@ -228,7 +228,7 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP
 
       val classpath = if (classpathPrefix != "") join(classpathPrefix, CLASSPATH) else CLASSPATH
       val cmd = javaCmd +: (
-        (JAVA_OPTS.split(' ') ++ argString.split(' ')).map(_.trim).filter(_ != "") ++ Seq(
+        (JAVA_OPTS.split(' ') ++ javaOpts.split(' ') ++ argString.split(' ')).map(_.trim).filter(_ != "") ++ Seq(
           "-classpath",
           join(outDir.toString, classpath)
         ) ++ propertyOptions ++ Seq(
@@ -426,6 +426,15 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP
         )
       })
 
+    def runInstrumentedTest(file: File): (Boolean, LogContext) =
+      runTestCommon(file, expectFailure = false)((logFile, outDir) => {
+        val dir       = file.getParentFile
+
+        // adding the javagent option with path to instrumentation agent
+        execTest(outDir, logFile, javaOpts = "-javaagent:"+PathSettings.instrumentationAgentLib) &&
+        diffCheck(file, compareOutput(dir, logFile))
+      })
+
     def processSingleFile(file: File): (Boolean, LogContext) = kind match {
       case "scalacheck" =>
         val succFn: (File, File) => Boolean = { (logFile, outDir) =>
@@ -475,6 +484,9 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP
       case "specialized" =>
         runSpecializedTest(file)
 
+      case "instrumented" =>
+        runInstrumentedTest(file)
+
       case "presentation" =>
         runJvmTest(file) // for the moment, it's exactly the same as for a run test
 
diff --git a/src/partest/scala/tools/partest/nest/TestFile.scala b/src/partest/scala/tools/partest/nest/TestFile.scala
index 1aa0a7baf6..52af16274e 100644
--- a/src/partest/scala/tools/partest/nest/TestFile.scala
+++ b/src/partest/scala/tools/partest/nest/TestFile.scala
@@ -78,3 +78,4 @@ case class SpecializedTestFile(file: JFile, fileManager: FileManager) extends Te
 }
 case class PresentationTestFile(file: JFile, fileManager: FileManager) extends TestFile("presentation")
 case class AntTestFile(file: JFile, fileManager: FileManager) extends TestFile("ant")
+case class InstrumentedTestFile(file: JFile, fileManager: FileManager) extends TestFile("instrumented")