diff options
Diffstat (limited to 'compiler/test/dotty/tools/vulpix')
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/ChildJVMMain.java | 34 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/ParallelTesting.scala | 1156 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala | 196 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/Status.scala | 7 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/SummaryReport.scala | 145 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/TestConfiguration.scala | 67 | ||||
| -rw-r--r-- | compiler/test/dotty/tools/vulpix/VulpixTests.scala | 76 |
7 files changed, 1681 insertions, 0 deletions
diff --git a/compiler/test/dotty/tools/vulpix/ChildJVMMain.java b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java new file mode 100644 index 000000000..90b795898 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java @@ -0,0 +1,34 @@ +package dotty.tools.vulpix; + +import java.io.File; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.lang.reflect.Method; + +public class ChildJVMMain { + static final String MessageEnd = "##THIS IS THE END FOR ME, GOODBYE##"; + + private static void runMain(String dir) throws Exception { + ArrayList<URL> cp = new ArrayList<>(); + for (String path : dir.split(":")) + cp.add(new File(path).toURI().toURL()); + + URLClassLoader ucl = new URLClassLoader(cp.toArray(new URL[cp.size()])); + Class<?> cls = ucl.loadClass("Test"); + Method meth = cls.getMethod("main", String[].class); + Object[] args = new Object[]{ new String[]{ "jvm" } }; + meth.invoke(null, args); + } + + public static void main(String[] args) throws Exception { + BufferedReader stdin = new BufferedReader(new InputStreamReader(System.in)); + + while (true) { + runMain(stdin.readLine()); + System.out.println(MessageEnd); + } + } +} diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala new file mode 100644 index 000000000..b0312523d --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -0,0 +1,1156 @@ +package dotty +package tools +package vulpix + +import java.io.{ File => JFile } +import java.text.SimpleDateFormat +import java.util.HashMap +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.nio.file.{ Files, Path, Paths, NoSuchFileException } +import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } + +import scala.io.Source +import scala.util.control.NonFatal +import scala.util.Try +import scala.collection.mutable +import scala.util.matching.Regex +import scala.util.Random + +import dotc.core.Contexts._ +import dotc.reporting.{ Reporter, TestReporter } +import dotc.reporting.diagnostic.MessageContainer +import dotc.interfaces.Diagnostic.ERROR +import dotc.util.DiffUtil +import dotc.{ Driver, Compiler } + +/** A parallel testing suite whose goal is to integrate nicely with JUnit + * + * This trait can be mixed in to offer parallel testing to compile runs. When + * using this, you should be running your JUnit tests **sequentially**, as the + * test suite itself runs with a high level of concurrency. + */ +trait ParallelTesting extends RunnerOrchestration { self => + + import ParallelTesting._ + + /** If the running environment supports an interactive terminal, each `Test` + * will be run with a progress bar and real time feedback + */ + def isInteractive: Boolean + + /** A string which is used to filter which tests to run, if `None` will run + * all tests. All absolute paths that contain the substring `testFilter` + * will be run + */ + def testFilter: Option[String] + + /** A test source whose files or directory of files is to be compiled + * in a specific way defined by the `Test` + */ + private sealed trait TestSource { self => + def name: String + def outDir: JFile + def flags: Array[String] + + def classPath: String = + outDir.getAbsolutePath + + flags + .dropWhile(_ != "-classpath") + .drop(1) + .headOption + .map(":" + _) + .getOrElse("") + + + def title: String = self match { + case self: JointCompilationSource => + if (self.files.length > 1) name + else self.files.head.getPath + + case self: SeparateCompilationSource => + self.dir.getPath + } + + /** Adds the flags specified in `newFlags0` if they do not already exist */ + def withFlags(newFlags0: String*) = { + val newFlags = newFlags0.toArray + if (!flags.containsSlice(newFlags)) self match { + case self: JointCompilationSource => + self.copy(flags = flags ++ newFlags) + case self: SeparateCompilationSource => + self.copy(flags = flags ++ newFlags) + } + else self + } + + /** Generate the instructions to redo the test from the command line */ + def buildInstructions(errors: Int, warnings: Int): String = { + val sb = new StringBuilder + val maxLen = 80 + var lineLen = 0 + + sb.append( + s"""| + |Test '$title' compiled with $errors error(s) and $warnings warning(s), + |the test can be reproduced by running:""".stripMargin + ) + sb.append("\n\n./bin/dotc ") + flags.foreach { arg => + if (lineLen > maxLen) { + sb.append(" \\\n ") + lineLen = 4 + } + sb.append(arg) + lineLen += arg.length + sb += ' ' + } + + self match { + case JointCompilationSource(_, files, _, _) => { + files.map(_.getAbsolutePath).foreach { path => + sb.append("\\\n ") + sb.append(path) + sb += ' ' + } + sb.toString + "\n\n" + } + case self: SeparateCompilationSource => { + val command = sb.toString + val fsb = new StringBuilder(command) + self.compilationGroups.foreach { files => + files.map(_.getPath).foreach { path => + fsb.append("\\\n ") + lineLen = 8 + fsb.append(path) + fsb += ' ' + } + fsb.append("\n\n") + fsb.append(command) + } + fsb.toString + "\n\n" + } + } + } + } + + /** A group of files that may all be compiled together, with the same flags + * and output directory + */ + private final case class JointCompilationSource( + name: String, + files: Array[JFile], + flags: Array[String], + outDir: JFile + ) extends TestSource { + def sourceFiles: Array[JFile] = files.filter(isSourceFile) + + override def toString() = outDir.toString + } + + /** A test source whose files will be compiled separately according to their + * suffix `_X` + */ + private final case class SeparateCompilationSource( + name: String, + dir: JFile, + flags: Array[String], + outDir: JFile + ) extends TestSource { + + /** Get the files grouped by `_X` as a list of groups, files missing this + * suffix will be put into the same group + * + * Filters out all none source files + */ + def compilationGroups: List[Array[JFile]] = + dir + .listFiles + .groupBy { file => + val name = file.getName + Try { + val potentialNumber = name + .substring(0, name.lastIndexOf('.')) + .reverse.takeWhile(_ != '_').reverse + + potentialNumber.toInt.toString + } + .toOption + .getOrElse("") + } + .toList.sortBy(_._1).map(_._2.filter(isSourceFile)) + } + + /** Each `Test` takes the `testSources` and performs the compilation and assertions + * according to the implementing class "neg", "run" or "pos". + */ + private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit val summaryReport: SummaryReporting) { test => + + import summaryReport._ + + protected final val realStdout = System.out + protected final val realStderr = System.err + + /** A runnable that logs its contents in a buffer */ + trait LoggedRunnable extends Runnable { + /** Instances of `LoggedRunnable` implement this method instead of the + * `run` method + */ + def checkTestSource(): Unit + + private[this] val logBuffer = mutable.ArrayBuffer.empty[String] + def log(msg: String): Unit = logBuffer.append(msg) + + def logReporterContents(reporter: TestReporter): Unit = + reporter.messages.foreach(log) + + def echo(msg: String): Unit = { + log(msg) + test.echo(msg) + } + + final def run(): Unit = { + checkTestSource() + summaryReport.echoToLog(logBuffer.iterator) + } + } + + /** Actual compilation run logic, the test behaviour is defined here */ + protected def encapsulatedCompilation(testSource: TestSource): LoggedRunnable + + /** All testSources left after filtering out */ + private val filteredSources = + if (!testFilter.isDefined) testSources + else testSources.filter { + case JointCompilationSource(_, files, _, _) => + files.exists(file => file.getAbsolutePath.contains(testFilter.get)) + case SeparateCompilationSource(_, dir, _, _) => + dir.getAbsolutePath.contains(testFilter.get) + } + + /** Total amount of test sources being compiled by this test */ + val sourceCount = filteredSources.length + + private[this] var _errorCount = 0 + def errorCount: Int = _errorCount + + private[this] var _testSourcesCompleted = 0 + private def testSourcesCompleted: Int = _testSourcesCompleted + + /** Complete the current compilation with the amount of errors encountered */ + protected final def registerCompletion(errors: Int) = synchronized { + _testSourcesCompleted += 1 + _errorCount += errors + } + + private[this] var _failed = false + /** Fail the current test */ + protected[this] final def fail(): Unit = synchronized { _failed = true } + def didFail: Boolean = _failed + + protected def logBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { + val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) + addFailureInstruction(errorMsg) + failTestSource(testSource) + } + + /** Instructions on how to reproduce failed test source compilations */ + private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String] + protected final def addFailureInstruction(ins: String): Unit = + synchronized { reproduceInstructions.append(ins) } + + /** The test sources that failed according to the implementing subclass */ + private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] + protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized { + val extra = reason.map(" with reason: " + _).getOrElse("") + failedTestSources.append(testSource.title + s" failed" + extra) + fail() + } + + /** Prints to `System.err` if we're not suppressing all output */ + protected def echo(msg: String): Unit = if (!suppressAllOutput) { + // pad right so that output is at least as large as progress bar line + val paddingRight = " " * math.max(0, 80 - msg.length) + realStderr.println(msg + paddingRight) + } + + /** A single `Runnable` that prints a progress bar for the curent `Test` */ + private def createProgressMonitor: Runnable = new Runnable { + def run(): Unit = { + val start = System.currentTimeMillis + var tCompiled = testSourcesCompleted + while (tCompiled < sourceCount) { + val timestamp = (System.currentTimeMillis - start) / 1000 + val progress = (tCompiled.toDouble / sourceCount * 40).toInt + + realStdout.print( + "[" + ("=" * (math.max(progress - 1, 0))) + + (if (progress > 0) ">" else "") + + (" " * (39 - progress)) + + s"] completed ($tCompiled/$sourceCount, ${timestamp}s)\r" + ) + + Thread.sleep(100) + tCompiled = testSourcesCompleted + } + // println, otherwise no newline and cursor at start of line + realStdout.println( + s"[=======================================] completed ($sourceCount/$sourceCount, " + + s"${(System.currentTimeMillis - start) / 1000}s) " + ) + } + } + + /** Wrapper function to make sure that the compiler itself did not crash - + * if it did, the test should automatically fail. + */ + protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = + try { + val testing = s"Testing ${testSource.title}" + summaryReport.echoToLog(testing) + if (!isInteractive) realStdout.println(testing) + op + } catch { + case NonFatal(e) => { + // if an exception is thrown during compilation, the complete test + // run should fail + failTestSource(testSource) + e.printStackTrace() + registerCompletion(1) + throw e + } + } + + protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = { + + val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath) + + def flattenFiles(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flattenFiles) + else Array(f) + + val files: Array[JFile] = files0.flatMap(flattenFiles) + + def addOutDir(xs: Array[String]): Array[String] = { + val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") + if (cpAndAfter.nonEmpty) { + val (cp :: cpArg :: rest) = cpAndAfter + (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray + } + else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray + } + + def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { + val fullArgs = Array( + "javac", + "-classpath", + s".:${Jars.scalaLibraryFromRuntime}:${targetDir.getAbsolutePath}" + ) ++ flags.takeRight(2) ++ fs + + Runtime.getRuntime.exec(fullArgs).waitFor() == 0 + } else true + + val reporter = + TestReporter.reporter(realStdout, logLevel = + if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) + + val driver = + if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } + else new Driver { + def newCompiler(implicit ctx: Context) = new Compiler + + private def ntimes(n: Int)(op: Int => Reporter): Reporter = + (emptyReporter /: (1 to n)) ((_, i) => op(i)) + + override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) = + ntimes(times) { run => + val start = System.nanoTime() + val rep = super.doCompile(comp, files) + ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms") + rep + } + } + + val allArgs = addOutDir(flags) + + // Compile with a try to catch any StackTrace generated by the compiler: + try { + driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) + + val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) + assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") + } + catch { + case NonFatal(ex) => reporter.logStackTrace(ex) + } + + reporter + } + + private[ParallelTesting] def executeTestSuite(): this.type = { + assert(_testSourcesCompleted == 0, "not allowed to re-use a `CompileRun`") + + if (filteredSources.nonEmpty) { + val pool = threadLimit match { + case Some(i) => JExecutors.newWorkStealingPool(i) + case None => JExecutors.newWorkStealingPool() + } + + if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) + + filteredSources.foreach { target => + pool.submit(encapsulatedCompilation(target)) + } + + pool.shutdown() + if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { + pool.shutdownNow() + System.setOut(realStdout) + System.setErr(realStderr) + throw new TimeoutException("Compiling targets timed out") + } + + if (didFail) { + reportFailed() + failedTestSources.toSet.foreach(addFailedTest) + reproduceInstructions.iterator.foreach(addReproduceInstruction) + } + else reportPassed() + } + else echo { + testFilter + .map(r => s"""No files matched "$r" in test""") + .getOrElse("No tests available under target - erroneous test?") + } + + this + } + } + + private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { + testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + registerCompletion(reporter.errorCount) + + if (reporter.compilerCrashed || reporter.errorCount > 0) { + logReporterContents(reporter) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) + val errorCount = reporters.foldLeft(0) { (acc, reporter) => + if (reporter.errorCount > 0) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + acc + reporter.errorCount + } + + def warningCount = reporters.foldLeft(0)(_ + _.warningCount) + + registerCompletion(errorCount) + + if (compilerCrashed || errorCount > 0) { + reporters.foreach(logReporterContents) + logBuildInstructions(reporters.head, testSource, errorCount, warningCount) + } + } + } + } + } + } + + private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + private[this] var didAddNoRunWarning = false + private[this] def addNoRunWarning() = if (!didAddNoRunWarning) { + didAddNoRunWarning = true + summaryReport.addStartingMessage { + """|WARNING + |------- + |Run tests were only compiled, not run - this is due to the `dotty.tests.norun` + |property being set + |""".stripMargin + } + } + + private def verifyOutput(checkFile: Option[JFile], dir: JFile, testSource: TestSource, warnings: Int) = { + if (Properties.testsNoRun) addNoRunWarning() + else runMain(testSource.classPath) match { + case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success! + case Success(output) => { + val outputLines = output.lines.toArray + val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines.toArray + val sourceTitle = testSource.title + + def linesMatch = + outputLines + .zip(checkLines) + .forall { case (x, y) => x == y } + + if (outputLines.length != checkLines.length || !linesMatch) { + // Print diff to files and summary: + val diff = outputLines.zip(checkLines).map { case (act, exp) => + DiffUtil.mkColoredLineDiff(exp, act) + }.mkString("\n") + + val msg = + s"""|Output from '$sourceTitle' did not match check file. + |Diff ('e' is expected, 'a' is actual): + |""".stripMargin + diff + "\n" + echo(msg) + addFailureInstruction(msg) + + // Print build instructions to file and summary: + val buildInstr = testSource.buildInstructions(0, warnings) + addFailureInstruction(buildInstr) + + // Fail target: + failTestSource(testSource) + } + } + + case Failure(output) => + echo(s"Test '${testSource.title}' failed with output:") + echo(output) + failTestSource(testSource) + + case Timeout => + echo("failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + } + } + + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { + val (compilerCrashed, errorCount, warningCount, verifier: Function0[Unit]) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val checkFile = files.flatMap { file => + if (file.isDirectory) Nil + else { + val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check" + val checkFile = new JFile(fname) + if (checkFile.exists) List(checkFile) + else Nil + } + }.headOption + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + + if (reporter.compilerCrashed || reporter.errorCount > 0) { + logReporterContents(reporter) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } + + (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, () => verifyOutput(checkFile, outDir, testSource, reporter.warningCount)) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") + val reporters = testSource.compilationGroups.map(compile(_, flags, false, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) + val (errorCount, warningCount) = + reporters.foldLeft((0,0)) { case ((errors, warnings), reporter) => + if (reporter.errorCount > 0) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + (errors + reporter.errorCount, warnings + reporter.warningCount) + } + + if (errorCount > 0) { + reporters.foreach(logReporterContents) + logBuildInstructions(reporters.head, testSource, errorCount, warningCount) + } + + (compilerCrashed, errorCount, warningCount, () => verifyOutput(Some(checkFile), outDir, testSource, warningCount)) + } + } + + if (!compilerCrashed && errorCount == 0) verifier() + else { + echo(s" Compilation failed for: '${testSource.title}' ") + val buildInstr = testSource.buildInstructions(errorCount, warningCount) + addFailureInstruction(buildInstr) + failTestSource(testSource) + } + registerCompletion(errorCount) + } + } + } + + private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { + // In neg-tests we allow two types of error annotations, + // "nopos-error" which doesn't care about position and "error" which + // has to be annotated on the correct line number. + // + // We collect these in a map `"file:row" -> numberOfErrors`, for + // nopos errors we save them in `"file" -> numberOfNoPosErrors` + def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = { + val errorMap = new HashMap[String, Integer]() + var expectedErrors = 0 + files.filter(_.getName.endsWith(".scala")).foreach { file => + Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) => + val errors = line.sliding("// error".length).count(_.mkString == "// error") + if (errors > 0) + errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors) + + val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error") + if (noposErrors > 0) { + val nopos = errorMap.get("nopos") + val existing: Integer = if (nopos eq null) 0 else nopos + errorMap.put("nopos", noposErrors + existing) + } + + expectedErrors += noposErrors + errors + } + } + + (errorMap, expectedErrors) + } + + def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error => + val key = if (error.pos.exists) { + val fileName = error.pos.source.file.toString + s"$fileName:${error.pos.line}" + + } else "nopos" + + val errors = errorMap.get(key) + + if (errors ne null) { + if (errors == 1) errorMap.remove(key) + else errorMap.put(key, errors - 1) + true + } + else { + echo { + s"Error reported in ${error.pos.source}, but no annotation found" + } + false + } + } + + val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val sourceFiles = testSource.sourceFiles + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) + val reporter = compile(sourceFiles, flags, true, outDir) + val actualErrors = reporter.errorCount + + if (reporter.compilerCrashed || actualErrors > 0) + logReporterContents(reporter) + + (reporter.compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val compilationGroups = testSource.compilationGroups + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) + val reporters = compilationGroups.map(compile(_, flags, true, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) + val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) + val errors = reporters.iterator.flatMap(_.errors) + + if (actualErrors > 0) + reporters.foreach(logReporterContents) + + (compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) + } + } + + if (compilerCrashed) { + echo(s"Compiler crashed when compiling: ${testSource.title}") + failTestSource(testSource) + } + else if (expectedErrors != actualErrors) { + echo { + s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" + } + failTestSource(testSource) + } + else if (hasMissingAnnotations()) { + echo { + s"\nErrors found on incorrect row numbers when compiling $testSource" + } + failTestSource(testSource) + } + else if (!errorMap.isEmpty) { + echo { + s"\nExpected error(s) have {<error position>=<unreported error>}: $errorMap" + } + failTestSource(testSource) + } + + registerCompletion(actualErrors) + } + } + } + + /** The `CompilationTest` is the main interface to `ParallelTesting`, it + * can be instantiated via one of the following methods: + * + * - `compileFile` + * - `compileDir` + * - `compileList` + * - `compileFilesInDir` + * - `compileShallowFilesInDir` + * + * Each compilation test can then be turned into either a "pos", "neg" or + * "run" test: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).pos() + * ``` + * + * These tests can be customized before calling one of the execution + * methods, for instance: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos() + * ``` + * + * Which would compile `i1103.scala` twice with the verbose flag as a "pos" + * test. + * + * pos tests + * ========= + * Pos tests verify that the compiler is able to compile the given + * `TestSource`s and that they generate no errors or exceptions during + * compilation + * + * neg tests + * ========= + * Neg tests are expected to generate a certain amount of errors - but not + * crash the compiler. In each `.scala` file, you specifiy the line on which + * the error will be generated, e.g: + * + * ``` + * val x: String = 1 // error + * ``` + * + * if a line generates multiple errors, you need to annotate it multiple + * times. For a line that generates two errors: + * + * ``` + * val y: String = { val y1: String = 1; 2 } // error // error + * ``` + * + * Certain errors have no position, if you need to check these annotate the + * file anywhere with `// nopos-error` + * + * run tests + * ========= + * Run tests are a superset of pos tests, they both verify compilation and + * that the compiler does not crash. In addition, run tests verify that the + * tests are able to run as expected. + * + * Run tests need to have the following form: + * + * ``` + * object Test { + * def main(args: Array[String]): Unit = () + * } + * ``` + * + * This is because the runner instantiates the `Test` class and calls the + * main method. + * + * Other definitions are allowed in the same file, but the file needs to at + * least have the `Test` object with a `main` method. + * + * To verify output you may use `.check` files. These files should share the + * name of the file or directory that they are testing. For instance: + * + * ```none + * . + * └── tests + * ├── i1513.scala + * └── i1513.check + * ``` + * + * If you are testing a directory under separate compilation, you would + * have: + * + * ```none + * . + * └── tests + * ├── myTestDir + * │ ├── T_1.scala + * │ ├── T_2.scala + * │ └── T_3.scala + * └── myTestDir.check + * ``` + * + * In the above example, `i1513.scala` and one of the files `T_X.scala` + * would contain a `Test` object with a main method. + * + * Composing tests + * =============== + * Since this is a parallel test suite, it is essential to be able to + * compose tests to take advantage of the concurrency. This is done using + * the `+` function. This function will make sure that tests being combined + * are compatible according to the `require`s in `+`. + */ + final class CompilationTest private ( + private[ParallelTesting] val targets: List[TestSource], + private[ParallelTesting] val times: Int, + private[ParallelTesting] val shouldDelete: Boolean, + private[ParallelTesting] val threadLimit: Option[Int], + private[ParallelTesting] val shouldFail: Boolean + ) { + import org.junit.Assert.fail + + private[ParallelTesting] def this(target: TestSource) = + this(List(target), 1, true, None, false) + + private[ParallelTesting] def this(targets: List[TestSource]) = + this(targets, 1, true, None, false) + + /** Compose test targets from `this` with `other` + * + * It does this, only if the two tests are compatible. Otherwise it throws + * an `IllegalArgumentException`. + * + * Grouping tests together like this allows us to take advantage of the + * concurrency offered by this test suite as each call to an executing + * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the + * maximum allowed level of concurrency. Doing this for only a few targets + * does not yield any real benefit over sequential compilation. + * + * As such, each `CompilationTest` should contain as many targets as + * possible. + */ + def +(other: CompilationTest) = { + require(other.times == times, "can't combine tests that are meant to be benchmark compiled") + require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output") + require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome") + new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail) + } + + /** Creates a "pos" test run, which makes sure that all tests pass + * compilation without generating errors and that they do not crash the + * compiler + */ + def checkCompile()(implicit summaryReport: SummaryReporting): this.type = { + val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail(s"Expected no errors when compiling, but found: ${test.errorCount}") + } + else if (shouldFail && !test.didFail) { + fail("Pos test should have failed, but didn't") + } + + cleanup() + } + + /** Creates a "neg" test run, which makes sure that each test generates the + * correct amount of errors at the correct positions. It also makes sure + * that none of these tests crash the compiler + */ + def checkExpectedErrors()(implicit summaryReport: SummaryReporting): this.type = { + val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Neg test shouldn't have failed, but did") + } + else if (shouldFail && !test.didFail) { + fail("Neg test should have failed, but did not") + } + + cleanup() + } + + /** Creates a "run" test run, which is a superset of "pos". In addition to + * making sure that all tests pass compilation and that they do not crash + * the compiler; it also makes sure that all tests can run with the + * expected output + */ + def checkRuns()(implicit summaryReport: SummaryReporting): this.type = { + val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Run test failed, but should not") + } + else if (shouldFail && !test.didFail) { + fail("Run test should have failed, but did not") + } + + cleanup() + } + + /** Deletes output directories and files */ + private def cleanup(): this.type = { + if (shouldDelete) delete() + this + } + + /** Copies `file` to `dir` - taking into account if `file` is a directory, + * and if so copying recursively + */ + private def copyToDir(dir: JFile, file: JFile): JFile = { + val target = Paths.get(dir.getAbsolutePath, file.getName) + Files.copy(file.toPath, target, REPLACE_EXISTING) + if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _)) + target.toFile + } + + /** Builds a new `CompilationTest` where we have copied the target files to + * the out directory. This is needed for tests that modify the original + * source, such as `-rewrite` tests + */ + def copyToTarget(): CompilationTest = new CompilationTest ( + targets.map { + case target @ JointCompilationSource(_, files, _, outDir) => + target.copy(files = files.map(copyToDir(outDir,_))) + case target @ SeparateCompilationSource(_, dir, _, outDir) => + target.copy(dir = copyToDir(outDir, dir)) + }, + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which performs the compilation `i` times on + * each target + */ + def times(i: Int): CompilationTest = + new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail) + + /** Builds a `Compilationtest` which passes the verbose flag and logs the + * classpath + */ + def verbose: CompilationTest = new CompilationTest( + targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")), + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which keeps the generated output files + * + * This is needed for tests like `tastyBootstrap` which relies on first + * compiling a certain part of the project and then compiling a second + * part which depends on the first + */ + def keepOutput: CompilationTest = + new CompilationTest(targets, times, false, threadLimit, shouldFail) + + /** Builds a `CompilationTest` with a limited level of concurrency with + * maximum `i` threads + */ + def limitThreads(i: Int): CompilationTest = + new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail) + + /** Builds a `CompilationTest` where the executed test is expected to fail + * + * This behaviour is mainly needed for the tests that test the test suite. + */ + def expectFailure: CompilationTest = + new CompilationTest(targets, times, shouldDelete, threadLimit, true) + + /** Delete all output files generated by this `CompilationTest` */ + def delete(): Unit = targets.foreach(t => delete(t.outDir)) + + private def delete(file: JFile): Unit = { + if (file.isDirectory) file.listFiles.foreach(delete) + try Files.delete(file.toPath) + catch { + case _: NoSuchFileException => // already deleted, everything's fine + } + } + } + + /** Create out directory for directory `d` */ + private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = { + val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}") + targetDir.mkdirs() + targetDir + } + + /** Create out directory for `file` */ + private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = { + val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.')) + val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir") + targetDir.mkdirs() + targetDir + } + + /** Make sure that directory string is as expected */ + private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = { + require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`") + require(outDir.last == '/', "please specify an `outDir` with a trailing slash") + } + + /** Separates directories from files and returns them as `(dirs, files)` */ + private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) = + sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) => + if (f.isDirectory) (f :: dirs, files) + else if (isSourceFile(f)) (dirs, f :: files) + else (dirs, files) + } + + /** Gets the name of the calling method via reflection. + * + * It does this in a way that needs to work both with the bootstrapped dotty + * and the non-bootstrapped version. Since the two compilers generate + * different bridges, we first need to filter out methods with the same name + * (bridges) - and then find the `@Test` method in our extending class + */ + private def getCallingMethod(): String = { + val seen = mutable.Set.empty[String] + Thread.currentThread.getStackTrace + .filter { elem => + if (seen.contains(elem.getMethodName)) false + else { seen += elem.getMethodName; true } + } + .find { elem => + val callingClass = Class.forName(elem.getClassName) + classOf[ParallelTesting].isAssignableFrom(callingClass) && + elem.getFileName != "ParallelTesting.scala" + } + .map(_.getMethodName) + .getOrElse { + throw new IllegalStateException("Unable to reflectively find calling method") + } + .takeWhile(_ != '$') + } + + /** Compiles a single file from the string path `f` using the supplied flags */ + def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val sourceFile = new JFile(f) + val parent = sourceFile.getParentFile + val outDir = + outDirectory + callingMethod + "/" + + sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/" + + require( + sourceFile.exists && !sourceFile.isDirectory && + (parent ne null) && parent.exists && parent.isDirectory, + s"Source file: $f, didn't exist" + ) + + val target = JointCompilationSource( + callingMethod, + Array(sourceFile), + flags, + createOutputDirsForFile(sourceFile, parent, outDir) + ) + new CompilationTest(target) + } + + /** Compiles a directory `f` using the supplied `flags`. This method does + * deep compilation, that is - it compiles all files and subdirectories + * contained within the directory `f`. + * + * By default, files are compiled in alphabetical order. An optional seed + * can be used for randomization. + */ + def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + def flatten(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flatten) + else Array(f) + + // Sort files either alphabetically or randomly using the provided seed: + val sortedFiles = flatten(sourceDir).sorted + val randomized = randomOrder match { + case None => sortedFiles + case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray + } + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/") + targetDir.mkdirs() + + val target = JointCompilationSource(s"compiling '$f' in test '$callingMethod'", randomized, flags, targetDir) + new CompilationTest(target) + } + + /** Compiles all `files` together as a single compilation run. It is given a + * `testName` since files can be in separate directories and or be otherwise + * dissociated + */ + def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + testName + "/" + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir) + targetDir.mkdirs() + assert(targetDir.exists, s"couldn't create target directory: $targetDir") + + val target = JointCompilationSource(s"$testName from $callingMethod", files.map(new JFile(_)).toArray, flags, targetDir) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(target) + } + + /** This function compiles the files and folders contained within directory + * `f` in a specific way. + * + * - Each file is compiled separately as a single compilation run + * - Each directory is compiled as a `SeparateCompilationTaret`, in this + * target all files are grouped according to the file suffix `_X` where `X` + * is a number. These groups are then ordered in ascending order based on + * the value of `X` and each group is compiled one after the other. + * + * For this function to work as expected, we use the same convention for + * directory layout as the old partest. That is: + * + * - Single files can have an associated check-file with the same name (but + * with file extension `.check`) + * - Directories can have an associated check-file, where the check file has + * the same name as the directory (with the file extension `.check`) + */ + def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (dirs, files) = compilationTargets(sourceDir) + + val targets = + files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++ + dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir))) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } + + /** This function behaves similar to `compileFilesInDir` but it ignores + * sub-directories and as such, does **not** perform separate compilation + * tests. + */ + def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (_, files) = compilationTargets(sourceDir) + + val targets = files.map { file => + JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir)) + } + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } +} + +object ParallelTesting { + def isSourceFile(f: JFile): Boolean = { + val name = f.getName + name.endsWith(".scala") || name.endsWith(".java") + } +} diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala new file mode 100644 index 000000000..ad068e9ef --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -0,0 +1,196 @@ +package dotty +package tools +package vulpix + +import java.io.{ File => JFile, InputStreamReader, BufferedReader, PrintStream } +import java.util.concurrent.TimeoutException + +import scala.concurrent.duration.Duration +import scala.concurrent.{ Await, Future } +import scala.concurrent.ExecutionContext.Implicits.global +import scala.collection.mutable + +/** Vulpix spawns JVM subprocesses (`numberOfSlaves`) in order to run tests + * without compromising the main JVM + * + * These need to be orchestrated in a safe manner with a simple protocol. This + * interface provides just that. + * + * The protocol is defined as: + * + * - master sends classpath to for which to run `Test#main` and waits for + * `maxDuration` + * - slave invokes the method and waits until completion + * - upon completion it sends back a `RunComplete` message + * - the master checks if the child is still alive + * - child is still alive, the output was valid + * - child is dead, the output is the failure message + * + * If this whole chain of events is not completed within `maxDuration`, the + * child process is destroyed and a new child is spawned. + */ +trait RunnerOrchestration { + + /** The maximum amount of active runners, which contain a child JVM */ + def numberOfSlaves: Int + + /** The maximum duration the child process is allowed to consume before + * getting destroyed + */ + def maxDuration: Duration + + /** Destroy and respawn process after each test */ + def safeMode: Boolean + + /** Running a `Test` class's main method from the specified `dir` */ + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = + monitor.runMain(classPath) + + private[this] val monitor = new RunnerMonitor + + /** The runner monitor object keeps track of child JVM processes by keeping + * them in two structures - one for free, and one for busy children. + * + * When a user calls `runMain` the monitor makes takes a free JVM and blocks + * until the run is complete - or `maxDuration` has passed. It then performs + * cleanup by returning the used JVM to the free list, or respawning it if + * it died + */ + private class RunnerMonitor { + + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = + withRunner(_.runMain(classPath)) + + private class Runner(private var process: Process) { + private[this] var childStdout: BufferedReader = _ + private[this] var childStdin: PrintStream = _ + + /** Checks if `process` is still alive + * + * When `process.exitValue()` is called on an active process the caught + * exception is thrown. As such we can know if the subprocess exited or + * not. + */ + def isAlive: Boolean = + try { process.exitValue(); false } + catch { case _: IllegalThreadStateException => true } + + /** Destroys the underlying process and kills IO streams */ + def kill(): Unit = { + if (process ne null) process.destroy() + process = null + childStdout = null + childStdin = null + } + + /** Did add hook to kill the child VMs? */ + private[this] var didAddCleanupCallback = false + + /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = { + if (!didAddCleanupCallback) { + // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we + // need to clean up ourselves. + summaryReport.addCleanup(killAll) + } + assert(process ne null, + "Runner was killed and then reused without setting a new process") + + // Makes the encapsulating RunnerMonitor spawn a new runner + def respawn(): Unit = { + process.destroy() + process = createProcess + childStdout = null + childStdin = null + } + + if (childStdin eq null) + childStdin = new PrintStream(process.getOutputStream, /* autoFlush = */ true) + + // pass file to running process + childStdin.println(classPath) + + // Create a future reading the object: + val readOutput = Future { + val sb = new StringBuilder + + if (childStdout eq null) + childStdout = new BufferedReader(new InputStreamReader(process.getInputStream)) + + var childOutput = childStdout.readLine() + while (childOutput != ChildJVMMain.MessageEnd && childOutput != null) { + sb.append(childOutput) + sb += '\n' + childOutput = childStdout.readLine() + } + + if (process.isAlive && childOutput != null) Success(sb.toString) + else Failure(sb.toString) + } + + // Await result for `maxDuration` and then timout and destroy the + // process: + val status = + try Await.result(readOutput, maxDuration) + catch { case _: TimeoutException => Timeout } + + // Handle failure of the VM: + status match { + case _: Success if safeMode => respawn() + case _: Success => // no need to respawn sub process + case _: Failure => respawn() + case Timeout => respawn() + } + status + } + } + + /** Create a process which has the classpath of the `ChildJVMMain` and the + * scala library. + */ + private def createProcess: Process = { + val sep = sys.props("file.separator") + val cp = + classOf[ChildJVMMain].getProtectionDomain.getCodeSource.getLocation.getFile + ":" + + Jars.scalaLibraryFromRuntime + val javaBin = sys.props("java.home") + sep + "bin" + sep + "java" + new ProcessBuilder(javaBin, "-cp", cp, "dotty.tools.vulpix.ChildJVMMain") + .redirectErrorStream(true) + .redirectInput(ProcessBuilder.Redirect.PIPE) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .start() + } + + private[this] val allRunners = List.fill(numberOfSlaves)(new Runner(createProcess)) + private[this] val freeRunners = mutable.Queue(allRunners: _*) + private[this] val busyRunners = mutable.Set.empty[Runner] + + private def getRunner(): Runner = synchronized { + while (freeRunners.isEmpty) wait() + + val runner = freeRunners.dequeue() + busyRunners += runner + + notify() + runner + } + + private def freeRunner(runner: Runner): Unit = synchronized { + freeRunners.enqueue(runner) + busyRunners -= runner + notify() + } + + private def withRunner[T](op: Runner => T): T = { + val runner = getRunner() + val result = op(runner) + freeRunner(runner) + result + } + + private def killAll(): Unit = allRunners.foreach(_.kill()) + + // On shutdown, we need to kill all runners: + sys.addShutdownHook(killAll()) + } +} diff --git a/compiler/test/dotty/tools/vulpix/Status.scala b/compiler/test/dotty/tools/vulpix/Status.scala new file mode 100644 index 000000000..3de7aff2b --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/Status.scala @@ -0,0 +1,7 @@ +package dotty.tools +package vulpix + +sealed trait Status +final case class Success(output: String) extends Status +final case class Failure(output: String) extends Status +final case object Timeout extends Status diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.scala b/compiler/test/dotty/tools/vulpix/SummaryReport.scala new file mode 100644 index 000000000..678d88809 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.scala @@ -0,0 +1,145 @@ +package dotty +package tools +package vulpix + +import scala.collection.mutable +import dotc.reporting.TestReporter + +/** `SummaryReporting` can be used by unit tests by utilizing `@AfterClass` to + * call `echoSummary` + * + * This is used in vulpix by passing the companion object's `SummaryReporting` + * to each test, the `@AfterClass def` then calls the `SummaryReport`'s + * `echoSummary` method in order to dump the summary to both stdout and a log + * file + */ +trait SummaryReporting { + /** Report a failed test */ + def reportFailed(): Unit + + /** Report a test as passing */ + def reportPassed(): Unit + + /** Add the name of the failed test */ + def addFailedTest(msg: String): Unit + + /** Add instructions to reproduce the error */ + def addReproduceInstruction(instr: String): Unit + + /** Add a message that will be issued in the beginning of the summary */ + def addStartingMessage(msg: String): Unit + + /** Add a cleanup hook to be run upon completion */ + def addCleanup(f: () => Unit): Unit + + /** Echo the summary report to the appropriate locations */ + def echoSummary(): Unit + + /** Echoes *immediately* to file */ + def echoToLog(msg: String): Unit + + /** Echoes contents of `it` to file *immediately* then flushes */ + def echoToLog(it: Iterator[String]): Unit +} + +/** A summary report that doesn't do anything */ +final class NoSummaryReport extends SummaryReporting { + def reportFailed(): Unit = () + def reportPassed(): Unit = () + def addFailedTest(msg: String): Unit = () + def addReproduceInstruction(instr: String): Unit = () + def addStartingMessage(msg: String): Unit = () + def addCleanup(f: () => Unit): Unit = () + def echoSummary(): Unit = () + def echoToLog(msg: String): Unit = () + def echoToLog(it: Iterator[String]): Unit = () +} + +/** A summary report that logs to both stdout and the `TestReporter.logWriter` + * which outputs to a log file in `./testlogs/` + */ +final class SummaryReport extends SummaryReporting { + + private val startingMessages = mutable.ArrayBuffer.empty[String] + private val failedTests = mutable.ArrayBuffer.empty[String] + private val reproduceInstructions = mutable.ArrayBuffer.empty[String] + private val cleanUps = mutable.ArrayBuffer.empty[() => Unit] + + private[this] var passed = 0 + private[this] var failed = 0 + + def reportFailed(): Unit = + failed += 1 + + def reportPassed(): Unit = + passed += 1 + + def addFailedTest(msg: String): Unit = + failedTests.append(msg) + + def addReproduceInstruction(instr: String): Unit = + reproduceInstructions.append(instr) + + def addStartingMessage(msg: String): Unit = + startingMessages.append(msg) + + def addCleanup(f: () => Unit): Unit = + cleanUps.append(f) + + /** Both echoes the summary to stdout and prints to file */ + def echoSummary(): Unit = { + import SummaryReport._ + + val rep = new StringBuilder + rep.append( + s"""| + |================================================================================ + |Test Report + |================================================================================ + | + |$passed passed, $failed failed, ${passed + failed} total + |""".stripMargin + ) + + startingMessages.foreach(rep.append) + + failedTests.map(x => s" $x\n").foreach(rep.append) + + // If we're compiling locally, we don't need instructions on how to + // reproduce failures + if (isInteractive) { + println(rep.toString) + if (failed > 0) println { + s"""| + |-------------------------------------------------------------------------------- + |Note - reproduction instructions have been dumped to log file: + | ${TestReporter.logPath} + |--------------------------------------------------------------------------------""".stripMargin + } + } + + rep += '\n' + + reproduceInstructions.foreach(rep.append) + + // If we're on the CI, we want everything + if (!isInteractive) println(rep.toString) + + TestReporter.logPrintln(rep.toString) + + // Perform cleanup callback: + if (cleanUps.nonEmpty) cleanUps.foreach(_.apply()) + } + + def echoToLog(msg: String): Unit = + TestReporter.logPrintln(msg) + + def echoToLog(it: Iterator[String]): Unit = { + it.foreach(TestReporter.logPrint) + TestReporter.logFlush() + } +} + +object SummaryReport { + val isInteractive = Properties.testsInteractive && !Properties.isRunByDrone +} diff --git a/compiler/test/dotty/tools/vulpix/TestConfiguration.scala b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala new file mode 100644 index 000000000..dcf3fbaf0 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala @@ -0,0 +1,67 @@ +package dotty +package tools +package vulpix + +object TestConfiguration { + implicit val defaultOutputDir: String = "../out/" + + implicit class RichStringArray(val xs: Array[String]) extends AnyVal { + def and(args: String*): Array[String] = { + val argsArr: Array[String] = args.toArray + xs ++ argsArr + } + } + + val noCheckOptions = Array( + "-pagewidth", "120", + "-color:never" + ) + + val checkOptions = Array( + "-Yno-deep-subtypes", + "-Yno-double-bindings", + "-Yforce-sbt-phases" + ) + + val classPath = { + val paths = Jars.dottyTestDeps map { p => + val file = new java.io.File(p) + assert( + file.exists, + s"""|File "$p" couldn't be found. Run `packageAll` from build tool before + |testing. + | + |If running without sbt, test paths need to be setup environment variables: + | + | - DOTTY_LIBRARY + | - DOTTY_COMPILER + | - DOTTY_INTERFACES + | - DOTTY_EXTRAS + | + |Where these all contain locations, except extras which is a colon + |separated list of jars. + | + |When compiling with eclipse, you need the sbt-interfaces jar, put + |it in extras.""" + ) + file.getAbsolutePath + } mkString (":") + + Array("-classpath", paths) + } + + private val yCheckOptions = Array("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") + + val defaultOptions = noCheckOptions ++ checkOptions ++ yCheckOptions ++ classPath + val allowDeepSubtypes = defaultOptions diff Array("-Yno-deep-subtypes") + val allowDoubleBindings = defaultOptions diff Array("-Yno-double-bindings") + val picklingOptions = defaultOptions ++ Array( + "-Xprint-types", + "-Ytest-pickler", + "-Ystop-after:pickler", + "-Yprintpos" + ) + val scala2Mode = defaultOptions ++ Array("-language:Scala2") + val explicitUTF8 = defaultOptions ++ Array("-encoding", "UTF8") + val explicitUTF16 = defaultOptions ++ Array("-encoding", "UTF16") +} diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala new file mode 100644 index 000000000..f875e7c13 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -0,0 +1,76 @@ +package dotty.tools +package vulpix + +import org.junit.Assert._ +import org.junit.Test + +import scala.concurrent.duration._ +import scala.util.control.NonFatal + +/** Meta tests for the Vulpix test suite */ +class VulpixTests extends ParallelTesting { + import TestConfiguration._ + + implicit val _: SummaryReporting = new NoSummaryReport + + def maxDuration = 3.seconds + def numberOfSlaves = 5 + def safeMode = sys.env.get("SAFEMODE").isDefined + def isInteractive = !sys.env.contains("DRONE") + def testFilter = None + + @Test def missingFile: Unit = + try { + compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors() + fail("didn't fail properly") + } + catch { + case _: IllegalArgumentException => // pass! + case NonFatal(_) => fail("wrong exception thrown") + } + + @Test def pos1Error: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile() + + @Test def negMissingAnnot: Unit = + compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negAnnotWrongLine: Unit = + compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negTooManyAnnots: Unit = + compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negNoPositionAnnot: Unit = + compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def runCompileFail: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput1: Unit = + compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput2: Unit = + compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runDiffOutput1: Unit = + compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runStackOverflow: Unit = + compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runOutRedirects: Unit = + compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiteNonRec: Unit = + compileFile("../tests/partest-test/infinite.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiteTailRec: Unit = + compileFile("../tests/partest-test/infiniteTail.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiniteAlloc: Unit = + compileFile("../tests/partest-test/infiniteAlloc.scala", defaultOptions).expectFailure.checkRuns() + + @Test def deadlock: Unit = + compileFile("../tests/partest-test/deadlock.scala", defaultOptions).expectFailure.checkRuns() +} |