diff options
17 files changed, 355 insertions, 5 deletions
@@ -1337,7 +1337,7 @@ QUICK BUILD (QUICK) <stopwatch name="quick.scalap.timer" action="total"/> </target> - <target name="quick.pre-partest" depends="quick.scalap"> + <target name="quick.pre-partest" depends="quick.scalap, asm.done"> <uptodate property="quick.partest.available" targetfile="${build-quick.dir}/partest.complete"> <srcfiles dir="${src.dir}/partest"/> </uptodate> @@ -1356,6 +1356,7 @@ QUICK BUILD (QUICK) <pathelement location="${build-quick.dir}/classes/compiler"/> <pathelement location="${build-quick.dir}/classes/scalap"/> <pathelement location="${build-quick.dir}/classes/partest"/> + <path refid="asm.classpath"/> </classpath> <include name="**/*.java"/> <compilerarg line="${javac.args}"/> @@ -1575,7 +1576,14 @@ PACKED QUICK BUILD (PACK) <target name="pack.partest" depends="pack.pre-partest" unless="pack.partest.available"> <mkdir dir="${build-pack.dir}/lib"/> <jar destfile="${build-pack.dir}/lib/scala-partest.jar"> - <fileset dir="${build-quick.dir}/classes/partest"/> + <fileset dir="${build-quick.dir}/classes/partest"> + <exclude name="scala/tools/partest/javaagent/**"/> + </fileset> + </jar> + <jar destfile="${build-pack.dir}/lib/scala-partest-javaagent.jar" manifest="${src.dir}/partest/scala/tools/partest/javaagent/MANIFEST.MF"> + <fileset dir="${build-quick.dir}/classes/partest"> + <include name="scala/tools/partest/javaagent/**"/> + </fileset> </jar> </target> @@ -1974,7 +1982,7 @@ BOOTSTRAPPING BUILD (STRAP) <stopwatch name="strap.scalap.timer" action="total"/> </target> - <target name="strap.pre-partest" depends="strap.scalap"> + <target name="strap.pre-partest" depends="strap.scalap, asm.done"> <uptodate property="strap.partest.available" targetfile="${build-strap.dir}/partest.complete"> <srcfiles dir="${src.dir}/partest"/> </uptodate> @@ -1993,6 +2001,7 @@ BOOTSTRAPPING BUILD (STRAP) <pathelement location="${build-strap.dir}/classes/compiler"/> <pathelement location="${build-strap.dir}/classes/scalap"/> <pathelement location="${build-strap.dir}/classes/partest"/> + <path refid="asm.classpath"/> </classpath> <include name="**/*.java"/> <compilerarg line="${javac.args}"/> @@ -2012,6 +2021,7 @@ BOOTSTRAPPING BUILD (STRAP) <pathelement location="${build-strap.dir}/classes/partest"/> <pathelement location="${ant.jar}"/> <path refid="forkjoin.classpath"/> + <path refid="asm.classpath"/> <pathelement location="${scalacheck.jar}"/> </compilationpath> </scalacfork> @@ -2393,6 +2403,9 @@ BOOTRAPING TEST AND TEST SUITE <specializedtests dir="${partest.dir}/${partest.srcdir}/specialized"> <include name="*.scala"/> </specializedtests> + <instrumentedtests dir="${partest.dir}/${partest.srcdir}/instrumented"> + <include name="*.scala"/> + </instrumentedtests> <presentationtests dir="${partest.dir}/${partest.srcdir}/presentation"> <include name="*/*.scala"/> </presentationtests> diff --git a/src/partest/scala/tools/partest/PartestTask.scala b/src/partest/scala/tools/partest/PartestTask.scala index b4c685b70c..959d682872 100644 --- a/src/partest/scala/tools/partest/PartestTask.scala +++ b/src/partest/scala/tools/partest/PartestTask.scala @@ -48,6 +48,7 @@ import org.apache.tools.ant.types.Commandline.Argument * - `scalaptests`, * - `scalachecktests`, * - `specializedtests`, + * - `instrumentedtests`, * - `presentationtests`, * - `scripttests`. * @@ -99,6 +100,10 @@ class PartestTask extends Task with CompilationPathProperty { specializedFiles = Some(input) } + def addConfiguredInstrumentedTests(input: FileSet) { + instrumentedFiles = Some(input) + } + def addConfiguredPresentationTests(input: FileSet) { presentationFiles = Some(input) } @@ -189,6 +194,7 @@ class PartestTask extends Task with CompilationPathProperty { private var shootoutFiles: Option[FileSet] = None private var scalapFiles: Option[FileSet] = None private var specializedFiles: Option[FileSet] = None + private var instrumentedFiles: Option[FileSet] = None private var presentationFiles: Option[FileSet] = None private var antFiles: Option[FileSet] = None private var errorOnFailed: Boolean = false @@ -245,6 +251,7 @@ class PartestTask extends Task with CompilationPathProperty { private def getShootoutFiles = getFiles(shootoutFiles) private def getScalapFiles = getFiles(scalapFiles) private def getSpecializedFiles = getFiles(specializedFiles) + private def getInstrumentedFiles = getFilesAndDirs(instrumentedFiles) private def getPresentationFiles = getDirs(presentationFiles) private def getAntFiles = getFiles(antFiles) @@ -375,6 +382,7 @@ class PartestTask extends Task with CompilationPathProperty { (getShootoutFiles, "shootout", "Running shootout tests"), (getScalapFiles, "scalap", "Running scalap tests"), (getSpecializedFiles, "specialized", "Running specialized files"), + (getInstrumentedFiles, "instrumented", "Running instrumented files"), (getPresentationFiles, "presentation", "Running presentation compiler test files"), (getAntFiles, "ant", "Running ant task tests") ) diff --git a/src/partest/scala/tools/partest/instrumented/Instrumentation.scala b/src/partest/scala/tools/partest/instrumented/Instrumentation.scala new file mode 100644 index 0000000000..f29a7f90fd --- /dev/null +++ b/src/partest/scala/tools/partest/instrumented/Instrumentation.scala @@ -0,0 +1,86 @@ +/* NEST (New Scala Test) + * Copyright 2007-2012 LAMP/EPFL + * @author Grzegorz Kossakowski + */ + +package scala.tools.partest.instrumented + +import scala.collection.JavaConverters._ + +case class MethodCallTrace(className: String, methodName: String, methodDescriptor: String) { + override def toString(): String = className + "." + methodName + methodDescriptor +} +object MethodCallTrace { + implicit val ordering: Ordering[MethodCallTrace] = Ordering.by(x => (x.className, x.methodName, x.methodDescriptor)) +} + +/** + * An object that controls profiling of instrumented byte-code. The instrumentation is achieved + * by using `java.lang.instrument` package. The instrumentation agent can be found in + * `scala.tools.partest.javaagent` package. + * + * At the moment the following classes are being instrumented: + * * all classes with empty package + * * all classes from scala package (except for classes responsible for instrumentation) + * + * The canonical way of using instrumentation is have a test-case in `files/instrumented` directory. + * The following code in main: + * + * {{{ + * import scala.tools.partest.instrumented.Instrumentation._ + * def main(args: Array[String]): Unit = { + * startProfiling() + * // should box the boolean + println(true) + stopProfiling() + printStatistics() + * } + * }}} + * + * + * should print: + * + * {{{ + * true + * Method call statistics: + * scala/Predef$.println(Ljava/lang/Object;)V: 1 + * scala/runtime/BoxesRunTime.boxToBoolean(Z)Ljava/lang/Boolean;: 1 + * }}} + */ +object Instrumentation { + + type Statistics = Map[MethodCallTrace, Int] + + def startProfiling(): Unit = Profiler.startProfiling() + def stopProfiling(): Unit = Profiler.stopProfiling() + def resetProfiling(): Unit = Profiler.resetProfiling() + + def getStatistics: Statistics = { + Profiler.stopProfiling() + val stats = Profiler.getStatistics().asScala.toSeq.map { + case (trace, count) => MethodCallTrace(trace.className, trace.methodName, trace.methodDescriptor) -> count.intValue + } + val res = Map(stats: _*) + Profiler.startProfiling() + res + } + + val standardFilter: MethodCallTrace => Boolean = t => { + // ignore all calls to Console trigger by printing + t.className != "scala/Console$" && + // console accesses DynamicVariable, let's discard it too + !t.className.startsWith("scala/util/DynamicVariable") + } + + def printStatistics(stats: Statistics = getStatistics, filter: MethodCallTrace => Boolean = standardFilter): Unit = { + val stats = getStatistics + println("Method call statistics:") + val toBePrinted = stats.toSeq.filter(p => filter(p._1)).sortBy(_._1) + // <count> <trace> + val format = "%5d %s\n" + toBePrinted foreach { + case (trace, count) => printf(format, count, trace) + } + } + +} diff --git a/src/partest/scala/tools/partest/instrumented/Profiler.java b/src/partest/scala/tools/partest/instrumented/Profiler.java new file mode 100644 index 0000000000..215bdbba08 --- /dev/null +++ b/src/partest/scala/tools/partest/instrumented/Profiler.java @@ -0,0 +1,78 @@ +/* NEST (New Scala Test) + * Copyright 2007-2012 LAMP/EPFL + * @author Grzegorz Kossakowski + */ + +package scala.tools.partest.instrumented; + +import java.util.HashMap; +import java.util.Map; + +/** + * A simple profiler class that counts method invocations. It is being used in byte-code instrumentation by inserting + * call to {@link Profiler#methodCalled(String, String, String)} at the beginning of every instrumented class. + * + * WARANING: This class is INTERNAL implementation detail and should never be used directly. It's made public only + * because it must be universally accessible for instrumentation needs. If you want to profile your test use + * {@link Instrumentation} instead. + */ +public class Profiler { + + private static boolean isProfiling = false; + private static Map<MethodCallTrace, Integer> counts = new HashMap<MethodCallTrace, Integer>(); + + static public class MethodCallTrace { + final String className; + final String methodName; + final String methodDescriptor; + + public MethodCallTrace(final String className, final String methodName, final String methodDescriptor) { + this.className = className; + this.methodName = methodName; + this.methodDescriptor = methodDescriptor; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof MethodCallTrace)) { + return false; + } else { + MethodCallTrace that = (MethodCallTrace) obj; + return that.className.equals(className) && that.methodName.equals(methodName) && that.methodDescriptor.equals(methodDescriptor); + } + } + @Override + public int hashCode() { + return className.hashCode() ^ methodName.hashCode() ^ methodDescriptor.hashCode(); + } + } + + public static void startProfiling() { + isProfiling = true; + } + + public static void stopProfiling() { + isProfiling = false; + } + + public static void resetProfiling() { + counts = new HashMap<MethodCallTrace, Integer>(); + } + + public static void methodCalled(final String className, final String methodName, final String methodDescriptor) { + if (isProfiling) { + MethodCallTrace trace = new MethodCallTrace(className, methodName, methodDescriptor); + Integer counter = counts.get(trace); + if (counter == null) { + counts.put(trace, 1); + } else { + counts.put(trace, counter+1); + } + } + } + + public static Map<MethodCallTrace, Integer> getStatistics() { + return new HashMap<MethodCallTrace, Integer>(counts); + } + +} diff --git a/src/partest/scala/tools/partest/javaagent/ASMTransformer.java b/src/partest/scala/tools/partest/javaagent/ASMTransformer.java new file mode 100644 index 0000000000..643c683002 --- /dev/null +++ b/src/partest/scala/tools/partest/javaagent/ASMTransformer.java @@ -0,0 +1,38 @@ +/* NEST (New Scala Test) + * Copyright 2007-2012 LAMP/EPFL + * @author Grzegorz Kossakowski + */ + +package scala.tools.partest.javaagent; + +import java.lang.instrument.ClassFileTransformer; +import java.security.ProtectionDomain; + +import scala.tools.asm.ClassReader; +import scala.tools.asm.ClassWriter; + +public class ASMTransformer implements ClassFileTransformer { + + private boolean shouldTransform(String className) { + return + // do not instrument instrumentation logic (in order to avoid infinite recursion) + !className.startsWith("scala/tools/partest/instrumented/") && + !className.startsWith("scala/tools/partest/javaagent/") && + // we instrument all classes from empty package + (!className.contains("/") || + // we instrument all classes from scala package + className.startsWith("scala/")); + } + + public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) { + if (shouldTransform(className)) { + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); + ProfilerVisitor visitor = new ProfilerVisitor(writer); + ClassReader reader = new ClassReader(classfileBuffer); + reader.accept(visitor, 0); + return writer.toByteArray(); + } else { + return classfileBuffer; + } + } +} diff --git a/src/partest/scala/tools/partest/javaagent/MANIFEST.MF b/src/partest/scala/tools/partest/javaagent/MANIFEST.MF new file mode 100644 index 0000000000..be0fee46a2 --- /dev/null +++ b/src/partest/scala/tools/partest/javaagent/MANIFEST.MF @@ -0,0 +1 @@ +Premain-Class: scala.tools.partest.javaagent.ProfilingAgent diff --git a/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java b/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java new file mode 100644 index 0000000000..f3a25e87d9 --- /dev/null +++ b/src/partest/scala/tools/partest/javaagent/ProfilerVisitor.java @@ -0,0 +1,46 @@ +/* NEST (New Scala Test) + * Copyright 2007-2012 LAMP/EPFL + * @author Grzegorz Kossakowski + */ + +package scala.tools.partest.javaagent; + +import scala.tools.asm.ClassVisitor; +import scala.tools.asm.MethodVisitor; +import scala.tools.asm.Opcodes; + +public class ProfilerVisitor extends ClassVisitor implements Opcodes { + + private static String profilerClass = "scala/tools/partest/instrumented/Profiler"; + + public ProfilerVisitor(final ClassVisitor cv) { + super(ASM4, cv); + } + + private String className = null; + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + className = name; + super.visit(version, access, name, signature, superName, interfaces); + } + + public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { + // delegate the method call to the next + // chained visitor + MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); + if (!profilerClass.equals(className)) { + // only instrument non-abstract methods + if((access & ACC_ABSTRACT) == 0) { + assert(className != null); + mv.visitLdcInsn(className); + mv.visitLdcInsn(name); + mv.visitLdcInsn(desc); + mv.visitMethodInsn(INVOKESTATIC, profilerClass, "methodCalled", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); + } + } + return mv; + } + +} diff --git a/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java b/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java new file mode 100644 index 0000000000..15efd73d94 --- /dev/null +++ b/src/partest/scala/tools/partest/javaagent/ProfilingAgent.java @@ -0,0 +1,25 @@ +/* NEST (New Scala Test) + * Copyright 2007-2012 LAMP/EPFL + * @author Grzegorz Kossakowski + */ + +package scala.tools.partest.javaagent; + +import java.lang.instrument.Instrumentation; +import java.lang.instrument.UnmodifiableClassException; + +/** + * Profiling agent that instruments byte-code to insert calls to + * {@link scala.tools.partest.instrumented.Profiler#methodCalled(String, String, String)} + * by using ASM library for byte-code manipulation. + */ +public class ProfilingAgent { + public static void premain(String args, Instrumentation inst) throws UnmodifiableClassException { + // NOTE: we are adding transformer that won't be applied to classes that are already loaded + // This should be ok because premain should be executed before main is executed so Scala library + // and the test-case itself won't be loaded yet. We rely here on the fact that ASMTransformer does + // not depend on Scala library. In case our assumptions are wrong we can always insert call to + // inst.retransformClasses. + inst.addTransformer(new ASMTransformer(), true); + } +} diff --git a/src/partest/scala/tools/partest/nest/CompileManager.scala b/src/partest/scala/tools/partest/nest/CompileManager.scala index c674e21482..6e6c767117 100644 --- a/src/partest/scala/tools/partest/nest/CompileManager.scala +++ b/src/partest/scala/tools/partest/nest/CompileManager.scala @@ -112,6 +112,7 @@ class DirectCompiler(val fileManager: FileManager) extends SimpleCompiler { case "scalap" => ScalapTestFile.apply case "scalacheck" => ScalaCheckTestFile.apply case "specialized" => SpecializedTestFile.apply + case "instrumented" => InstrumentedTestFile.apply case "presentation" => PresentationTestFile.apply case "ant" => AntTestFile.apply } diff --git a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala index 962520844c..20e6f47802 100644 --- a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala +++ b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala @@ -40,6 +40,7 @@ class ConsoleRunner extends DirectRunner { TestSet("scalacheck", stdFilter, "Testing ScalaCheck tests"), TestSet("scalap", _.isDirectory, "Run scalap decompiler tests"), TestSet("specialized", stdFilter, "Testing specialized tests"), + TestSet("instrumented", stdFilter, "Testing instrumented tests"), TestSet("presentation", _.isDirectory, "Testing presentation compiler tests."), TestSet("ant", antFilter, "Run Ant task tests.") ) diff --git a/src/partest/scala/tools/partest/nest/NestUI.scala b/src/partest/scala/tools/partest/nest/NestUI.scala index 7fd503e769..54e99a7ddf 100644 --- a/src/partest/scala/tools/partest/nest/NestUI.scala +++ b/src/partest/scala/tools/partest/nest/NestUI.scala @@ -80,6 +80,7 @@ object NestUI { println(" --scalacheck run ScalaCheck tests") println(" --script run script runner tests") println(" --shootout run shootout tests") + println(" --instrumented run instrumented tests") println(" --presentation run presentation compiler tests") println(" --grep <expr> run all tests whose source file contains <expr>") println diff --git a/src/partest/scala/tools/partest/nest/PathSettings.scala b/src/partest/scala/tools/partest/nest/PathSettings.scala index ac04c64c33..b409a29165 100644 --- a/src/partest/scala/tools/partest/nest/PathSettings.scala +++ b/src/partest/scala/tools/partest/nest/PathSettings.scala @@ -49,6 +49,12 @@ object PathSettings { getOrElse sys.error("No code.jar found in %s".format(srcCodeLibDir)) ) + lazy val instrumentationAgentLib: File = { + findJar(buildPackLibDir.files, "scala-partest-javaagent") getOrElse { + sys.error("No partest-javaagent jar found in '%s' or '%s'".format(buildPackLibDir, srcLibDir)) + } + } + // Directory <root>/build lazy val buildDir: Directory = { val bases = testRoot :: testRoot.parents diff --git a/src/partest/scala/tools/partest/nest/RunnerManager.scala b/src/partest/scala/tools/partest/nest/RunnerManager.scala index feca40a159..dc15d4475b 100644 --- a/src/partest/scala/tools/partest/nest/RunnerManager.scala +++ b/src/partest/scala/tools/partest/nest/RunnerManager.scala @@ -185,7 +185,7 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP outDir } - private def execTest(outDir: File, logFile: File, classpathPrefix: String = ""): Boolean = { + private def execTest(outDir: File, logFile: File, classpathPrefix: String = "", javaOpts: String = ""): Boolean = { // check whether there is a ".javaopts" file val argsFile = new File(logFile.getParentFile, fileBase + ".javaopts") val argString = file2String(argsFile) @@ -228,7 +228,7 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP val classpath = if (classpathPrefix != "") join(classpathPrefix, CLASSPATH) else CLASSPATH val cmd = javaCmd +: ( - (JAVA_OPTS.split(' ') ++ argString.split(' ')).map(_.trim).filter(_ != "") ++ Seq( + (JAVA_OPTS.split(' ') ++ javaOpts.split(' ') ++ argString.split(' ')).map(_.trim).filter(_ != "") ++ Seq( "-classpath", join(outDir.toString, classpath) ) ++ propertyOptions ++ Seq( @@ -426,6 +426,15 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP ) }) + def runInstrumentedTest(file: File): (Boolean, LogContext) = + runTestCommon(file, expectFailure = false)((logFile, outDir) => { + val dir = file.getParentFile + + // adding the javagent option with path to instrumentation agent + execTest(outDir, logFile, javaOpts = "-javaagent:"+PathSettings.instrumentationAgentLib) && + diffCheck(file, compareOutput(dir, logFile)) + }) + def processSingleFile(file: File): (Boolean, LogContext) = kind match { case "scalacheck" => val succFn: (File, File) => Boolean = { (logFile, outDir) => @@ -475,6 +484,9 @@ class RunnerManager(kind: String, val fileManager: FileManager, params: TestRunP case "specialized" => runSpecializedTest(file) + case "instrumented" => + runInstrumentedTest(file) + case "presentation" => runJvmTest(file) // for the moment, it's exactly the same as for a run test diff --git a/src/partest/scala/tools/partest/nest/TestFile.scala b/src/partest/scala/tools/partest/nest/TestFile.scala index 1aa0a7baf6..52af16274e 100644 --- a/src/partest/scala/tools/partest/nest/TestFile.scala +++ b/src/partest/scala/tools/partest/nest/TestFile.scala @@ -78,3 +78,4 @@ case class SpecializedTestFile(file: JFile, fileManager: FileManager) extends Te } case class PresentationTestFile(file: JFile, fileManager: FileManager) extends TestFile("presentation") case class AntTestFile(file: JFile, fileManager: FileManager) extends TestFile("ant") +case class InstrumentedTestFile(file: JFile, fileManager: FileManager) extends TestFile("instrumented") diff --git a/test/files/instrumented/InstrumentationTest.check b/test/files/instrumented/InstrumentationTest.check new file mode 100644 index 0000000000..3652df270a --- /dev/null +++ b/test/files/instrumented/InstrumentationTest.check @@ -0,0 +1,4 @@ +true +Method call statistics: + 1 scala/Predef$.println(Ljava/lang/Object;)V + 1 scala/runtime/BoxesRunTime.boxToBoolean(Z)Ljava/lang/Boolean; diff --git a/test/files/instrumented/InstrumentationTest.scala b/test/files/instrumented/InstrumentationTest.scala new file mode 100644 index 0000000000..ec5314c624 --- /dev/null +++ b/test/files/instrumented/InstrumentationTest.scala @@ -0,0 +1,14 @@ +import scala.tools.partest.instrumented.Instrumentation._ + +/** Tests if instrumentation itself works correctly */ +object Test { + def main(args: Array[String]) { + // force predef initialization before profiling + Predef + startProfiling() + // should box the boolean + println(true) + stopProfiling() + printStatistics() + } +} diff --git a/test/files/instrumented/README b/test/files/instrumented/README new file mode 100644 index 0000000000..32d0ef2da5 --- /dev/null +++ b/test/files/instrumented/README @@ -0,0 +1,15 @@ +Tests in `instrumented` directory are executed the same way as in `run` but +they have additional byte-code instrumentation performed for profiling. You +should put your tests in `instrumented` directory if you are interested in +method call counts. Examples include tests for specialization (you want to +count boxing and unboxing method calls) or high-level tests for optimizer +where you are interested if methods are successfuly inlined (so they should +not be called at runtime) or closures are eliminated (so no constructors +of closures are called). + +Check `scala.tools.partest.instrumented.Instrumentation` to learn how to +use the instrumentation infrastructure. + +The instrumentation itself is achieved by attaching a Java agent to the forked +VM process that injects calls to profiler. Check +`scala.tools.partest.instrumented.Instrumentation`. |