aboutsummaryrefslogtreecommitdiff
path: root/compiler/test/dotty/tools/vulpix/ParallelTesting.scala
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/test/dotty/tools/vulpix/ParallelTesting.scala')
-rw-r--r--compiler/test/dotty/tools/vulpix/ParallelTesting.scala1156
1 files changed, 1156 insertions, 0 deletions
diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala
new file mode 100644
index 000000000..b0312523d
--- /dev/null
+++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala
@@ -0,0 +1,1156 @@
+package dotty
+package tools
+package vulpix
+
+import java.io.{ File => JFile }
+import java.text.SimpleDateFormat
+import java.util.HashMap
+import java.nio.file.StandardCopyOption.REPLACE_EXISTING
+import java.nio.file.{ Files, Path, Paths, NoSuchFileException }
+import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException }
+
+import scala.io.Source
+import scala.util.control.NonFatal
+import scala.util.Try
+import scala.collection.mutable
+import scala.util.matching.Regex
+import scala.util.Random
+
+import dotc.core.Contexts._
+import dotc.reporting.{ Reporter, TestReporter }
+import dotc.reporting.diagnostic.MessageContainer
+import dotc.interfaces.Diagnostic.ERROR
+import dotc.util.DiffUtil
+import dotc.{ Driver, Compiler }
+
+/** A parallel testing suite whose goal is to integrate nicely with JUnit
+ *
+ * This trait can be mixed in to offer parallel testing to compile runs. When
+ * using this, you should be running your JUnit tests **sequentially**, as the
+ * test suite itself runs with a high level of concurrency.
+ */
+trait ParallelTesting extends RunnerOrchestration { self =>
+
+ import ParallelTesting._
+
+ /** If the running environment supports an interactive terminal, each `Test`
+ * will be run with a progress bar and real time feedback
+ */
+ def isInteractive: Boolean
+
+ /** A string which is used to filter which tests to run, if `None` will run
+ * all tests. All absolute paths that contain the substring `testFilter`
+ * will be run
+ */
+ def testFilter: Option[String]
+
+ /** A test source whose files or directory of files is to be compiled
+ * in a specific way defined by the `Test`
+ */
+ private sealed trait TestSource { self =>
+ def name: String
+ def outDir: JFile
+ def flags: Array[String]
+
+ def classPath: String =
+ outDir.getAbsolutePath +
+ flags
+ .dropWhile(_ != "-classpath")
+ .drop(1)
+ .headOption
+ .map(":" + _)
+ .getOrElse("")
+
+
+ def title: String = self match {
+ case self: JointCompilationSource =>
+ if (self.files.length > 1) name
+ else self.files.head.getPath
+
+ case self: SeparateCompilationSource =>
+ self.dir.getPath
+ }
+
+ /** Adds the flags specified in `newFlags0` if they do not already exist */
+ def withFlags(newFlags0: String*) = {
+ val newFlags = newFlags0.toArray
+ if (!flags.containsSlice(newFlags)) self match {
+ case self: JointCompilationSource =>
+ self.copy(flags = flags ++ newFlags)
+ case self: SeparateCompilationSource =>
+ self.copy(flags = flags ++ newFlags)
+ }
+ else self
+ }
+
+ /** Generate the instructions to redo the test from the command line */
+ def buildInstructions(errors: Int, warnings: Int): String = {
+ val sb = new StringBuilder
+ val maxLen = 80
+ var lineLen = 0
+
+ sb.append(
+ s"""|
+ |Test '$title' compiled with $errors error(s) and $warnings warning(s),
+ |the test can be reproduced by running:""".stripMargin
+ )
+ sb.append("\n\n./bin/dotc ")
+ flags.foreach { arg =>
+ if (lineLen > maxLen) {
+ sb.append(" \\\n ")
+ lineLen = 4
+ }
+ sb.append(arg)
+ lineLen += arg.length
+ sb += ' '
+ }
+
+ self match {
+ case JointCompilationSource(_, files, _, _) => {
+ files.map(_.getAbsolutePath).foreach { path =>
+ sb.append("\\\n ")
+ sb.append(path)
+ sb += ' '
+ }
+ sb.toString + "\n\n"
+ }
+ case self: SeparateCompilationSource => {
+ val command = sb.toString
+ val fsb = new StringBuilder(command)
+ self.compilationGroups.foreach { files =>
+ files.map(_.getPath).foreach { path =>
+ fsb.append("\\\n ")
+ lineLen = 8
+ fsb.append(path)
+ fsb += ' '
+ }
+ fsb.append("\n\n")
+ fsb.append(command)
+ }
+ fsb.toString + "\n\n"
+ }
+ }
+ }
+ }
+
+ /** A group of files that may all be compiled together, with the same flags
+ * and output directory
+ */
+ private final case class JointCompilationSource(
+ name: String,
+ files: Array[JFile],
+ flags: Array[String],
+ outDir: JFile
+ ) extends TestSource {
+ def sourceFiles: Array[JFile] = files.filter(isSourceFile)
+
+ override def toString() = outDir.toString
+ }
+
+ /** A test source whose files will be compiled separately according to their
+ * suffix `_X`
+ */
+ private final case class SeparateCompilationSource(
+ name: String,
+ dir: JFile,
+ flags: Array[String],
+ outDir: JFile
+ ) extends TestSource {
+
+ /** Get the files grouped by `_X` as a list of groups, files missing this
+ * suffix will be put into the same group
+ *
+ * Filters out all none source files
+ */
+ def compilationGroups: List[Array[JFile]] =
+ dir
+ .listFiles
+ .groupBy { file =>
+ val name = file.getName
+ Try {
+ val potentialNumber = name
+ .substring(0, name.lastIndexOf('.'))
+ .reverse.takeWhile(_ != '_').reverse
+
+ potentialNumber.toInt.toString
+ }
+ .toOption
+ .getOrElse("")
+ }
+ .toList.sortBy(_._1).map(_._2.filter(isSourceFile))
+ }
+
+ /** Each `Test` takes the `testSources` and performs the compilation and assertions
+ * according to the implementing class "neg", "run" or "pos".
+ */
+ private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit val summaryReport: SummaryReporting) { test =>
+
+ import summaryReport._
+
+ protected final val realStdout = System.out
+ protected final val realStderr = System.err
+
+ /** A runnable that logs its contents in a buffer */
+ trait LoggedRunnable extends Runnable {
+ /** Instances of `LoggedRunnable` implement this method instead of the
+ * `run` method
+ */
+ def checkTestSource(): Unit
+
+ private[this] val logBuffer = mutable.ArrayBuffer.empty[String]
+ def log(msg: String): Unit = logBuffer.append(msg)
+
+ def logReporterContents(reporter: TestReporter): Unit =
+ reporter.messages.foreach(log)
+
+ def echo(msg: String): Unit = {
+ log(msg)
+ test.echo(msg)
+ }
+
+ final def run(): Unit = {
+ checkTestSource()
+ summaryReport.echoToLog(logBuffer.iterator)
+ }
+ }
+
+ /** Actual compilation run logic, the test behaviour is defined here */
+ protected def encapsulatedCompilation(testSource: TestSource): LoggedRunnable
+
+ /** All testSources left after filtering out */
+ private val filteredSources =
+ if (!testFilter.isDefined) testSources
+ else testSources.filter {
+ case JointCompilationSource(_, files, _, _) =>
+ files.exists(file => file.getAbsolutePath.contains(testFilter.get))
+ case SeparateCompilationSource(_, dir, _, _) =>
+ dir.getAbsolutePath.contains(testFilter.get)
+ }
+
+ /** Total amount of test sources being compiled by this test */
+ val sourceCount = filteredSources.length
+
+ private[this] var _errorCount = 0
+ def errorCount: Int = _errorCount
+
+ private[this] var _testSourcesCompleted = 0
+ private def testSourcesCompleted: Int = _testSourcesCompleted
+
+ /** Complete the current compilation with the amount of errors encountered */
+ protected final def registerCompletion(errors: Int) = synchronized {
+ _testSourcesCompleted += 1
+ _errorCount += errors
+ }
+
+ private[this] var _failed = false
+ /** Fail the current test */
+ protected[this] final def fail(): Unit = synchronized { _failed = true }
+ def didFail: Boolean = _failed
+
+ protected def logBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = {
+ val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount)
+ addFailureInstruction(errorMsg)
+ failTestSource(testSource)
+ }
+
+ /** Instructions on how to reproduce failed test source compilations */
+ private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String]
+ protected final def addFailureInstruction(ins: String): Unit =
+ synchronized { reproduceInstructions.append(ins) }
+
+ /** The test sources that failed according to the implementing subclass */
+ private[this] val failedTestSources = mutable.ArrayBuffer.empty[String]
+ protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized {
+ val extra = reason.map(" with reason: " + _).getOrElse("")
+ failedTestSources.append(testSource.title + s" failed" + extra)
+ fail()
+ }
+
+ /** Prints to `System.err` if we're not suppressing all output */
+ protected def echo(msg: String): Unit = if (!suppressAllOutput) {
+ // pad right so that output is at least as large as progress bar line
+ val paddingRight = " " * math.max(0, 80 - msg.length)
+ realStderr.println(msg + paddingRight)
+ }
+
+ /** A single `Runnable` that prints a progress bar for the curent `Test` */
+ private def createProgressMonitor: Runnable = new Runnable {
+ def run(): Unit = {
+ val start = System.currentTimeMillis
+ var tCompiled = testSourcesCompleted
+ while (tCompiled < sourceCount) {
+ val timestamp = (System.currentTimeMillis - start) / 1000
+ val progress = (tCompiled.toDouble / sourceCount * 40).toInt
+
+ realStdout.print(
+ "[" + ("=" * (math.max(progress - 1, 0))) +
+ (if (progress > 0) ">" else "") +
+ (" " * (39 - progress)) +
+ s"] completed ($tCompiled/$sourceCount, ${timestamp}s)\r"
+ )
+
+ Thread.sleep(100)
+ tCompiled = testSourcesCompleted
+ }
+ // println, otherwise no newline and cursor at start of line
+ realStdout.println(
+ s"[=======================================] completed ($sourceCount/$sourceCount, " +
+ s"${(System.currentTimeMillis - start) / 1000}s) "
+ )
+ }
+ }
+
+ /** Wrapper function to make sure that the compiler itself did not crash -
+ * if it did, the test should automatically fail.
+ */
+ protected def tryCompile(testSource: TestSource)(op: => Unit): Unit =
+ try {
+ val testing = s"Testing ${testSource.title}"
+ summaryReport.echoToLog(testing)
+ if (!isInteractive) realStdout.println(testing)
+ op
+ } catch {
+ case NonFatal(e) => {
+ // if an exception is thrown during compilation, the complete test
+ // run should fail
+ failTestSource(testSource)
+ e.printStackTrace()
+ registerCompletion(1)
+ throw e
+ }
+ }
+
+ protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = {
+
+ val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath)
+
+ def flattenFiles(f: JFile): Array[JFile] =
+ if (f.isDirectory) f.listFiles.flatMap(flattenFiles)
+ else Array(f)
+
+ val files: Array[JFile] = files0.flatMap(flattenFiles)
+
+ def addOutDir(xs: Array[String]): Array[String] = {
+ val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath")
+ if (cpAndAfter.nonEmpty) {
+ val (cp :: cpArg :: rest) = cpAndAfter
+ (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray
+ }
+ else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray
+ }
+
+ def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) {
+ val fullArgs = Array(
+ "javac",
+ "-classpath",
+ s".:${Jars.scalaLibraryFromRuntime}:${targetDir.getAbsolutePath}"
+ ) ++ flags.takeRight(2) ++ fs
+
+ Runtime.getRuntime.exec(fullArgs).waitFor() == 0
+ } else true
+
+ val reporter =
+ TestReporter.reporter(realStdout, logLevel =
+ if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR)
+
+ val driver =
+ if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler }
+ else new Driver {
+ def newCompiler(implicit ctx: Context) = new Compiler
+
+ private def ntimes(n: Int)(op: Int => Reporter): Reporter =
+ (emptyReporter /: (1 to n)) ((_, i) => op(i))
+
+ override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) =
+ ntimes(times) { run =>
+ val start = System.nanoTime()
+ val rep = super.doCompile(comp, files)
+ ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms")
+ rep
+ }
+ }
+
+ val allArgs = addOutDir(flags)
+
+ // Compile with a try to catch any StackTrace generated by the compiler:
+ try {
+ driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter)
+
+ val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath)
+ assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}")
+ }
+ catch {
+ case NonFatal(ex) => reporter.logStackTrace(ex)
+ }
+
+ reporter
+ }
+
+ private[ParallelTesting] def executeTestSuite(): this.type = {
+ assert(_testSourcesCompleted == 0, "not allowed to re-use a `CompileRun`")
+
+ if (filteredSources.nonEmpty) {
+ val pool = threadLimit match {
+ case Some(i) => JExecutors.newWorkStealingPool(i)
+ case None => JExecutors.newWorkStealingPool()
+ }
+
+ if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor)
+
+ filteredSources.foreach { target =>
+ pool.submit(encapsulatedCompilation(target))
+ }
+
+ pool.shutdown()
+ if (!pool.awaitTermination(20, TimeUnit.MINUTES)) {
+ pool.shutdownNow()
+ System.setOut(realStdout)
+ System.setErr(realStderr)
+ throw new TimeoutException("Compiling targets timed out")
+ }
+
+ if (didFail) {
+ reportFailed()
+ failedTestSources.toSet.foreach(addFailedTest)
+ reproduceInstructions.iterator.foreach(addReproduceInstruction)
+ }
+ else reportPassed()
+ }
+ else echo {
+ testFilter
+ .map(r => s"""No files matched "$r" in test""")
+ .getOrElse("No tests available under target - erroneous test?")
+ }
+
+ this
+ }
+ }
+
+ private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting)
+ extends Test(testSources, times, threadLimit, suppressAllOutput) {
+ protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
+ def checkTestSource(): Unit = tryCompile(testSource) {
+ testSource match {
+ case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+ val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+ registerCompletion(reporter.errorCount)
+
+ if (reporter.compilerCrashed || reporter.errorCount > 0) {
+ logReporterContents(reporter)
+ logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+ }
+ }
+
+ case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+ val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir))
+ val compilerCrashed = reporters.exists(_.compilerCrashed)
+ val errorCount = reporters.foldLeft(0) { (acc, reporter) =>
+ if (reporter.errorCount > 0)
+ logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+
+ acc + reporter.errorCount
+ }
+
+ def warningCount = reporters.foldLeft(0)(_ + _.warningCount)
+
+ registerCompletion(errorCount)
+
+ if (compilerCrashed || errorCount > 0) {
+ reporters.foreach(logReporterContents)
+ logBuildInstructions(reporters.head, testSource, errorCount, warningCount)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting)
+ extends Test(testSources, times, threadLimit, suppressAllOutput) {
+ private[this] var didAddNoRunWarning = false
+ private[this] def addNoRunWarning() = if (!didAddNoRunWarning) {
+ didAddNoRunWarning = true
+ summaryReport.addStartingMessage {
+ """|WARNING
+ |-------
+ |Run tests were only compiled, not run - this is due to the `dotty.tests.norun`
+ |property being set
+ |""".stripMargin
+ }
+ }
+
+ private def verifyOutput(checkFile: Option[JFile], dir: JFile, testSource: TestSource, warnings: Int) = {
+ if (Properties.testsNoRun) addNoRunWarning()
+ else runMain(testSource.classPath) match {
+ case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success!
+ case Success(output) => {
+ val outputLines = output.lines.toArray
+ val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines.toArray
+ val sourceTitle = testSource.title
+
+ def linesMatch =
+ outputLines
+ .zip(checkLines)
+ .forall { case (x, y) => x == y }
+
+ if (outputLines.length != checkLines.length || !linesMatch) {
+ // Print diff to files and summary:
+ val diff = outputLines.zip(checkLines).map { case (act, exp) =>
+ DiffUtil.mkColoredLineDiff(exp, act)
+ }.mkString("\n")
+
+ val msg =
+ s"""|Output from '$sourceTitle' did not match check file.
+ |Diff ('e' is expected, 'a' is actual):
+ |""".stripMargin + diff + "\n"
+ echo(msg)
+ addFailureInstruction(msg)
+
+ // Print build instructions to file and summary:
+ val buildInstr = testSource.buildInstructions(0, warnings)
+ addFailureInstruction(buildInstr)
+
+ // Fail target:
+ failTestSource(testSource)
+ }
+ }
+
+ case Failure(output) =>
+ echo(s"Test '${testSource.title}' failed with output:")
+ echo(output)
+ failTestSource(testSource)
+
+ case Timeout =>
+ echo("failed because test " + testSource.title + " timed out")
+ failTestSource(testSource, Some("test timed out"))
+ }
+ }
+
+ protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
+ def checkTestSource(): Unit = tryCompile(testSource) {
+ val (compilerCrashed, errorCount, warningCount, verifier: Function0[Unit]) = testSource match {
+ case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+ val checkFile = files.flatMap { file =>
+ if (file.isDirectory) Nil
+ else {
+ val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check"
+ val checkFile = new JFile(fname)
+ if (checkFile.exists) List(checkFile)
+ else Nil
+ }
+ }.headOption
+ val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+
+ if (reporter.compilerCrashed || reporter.errorCount > 0) {
+ logReporterContents(reporter)
+ logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+ }
+
+ (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, () => verifyOutput(checkFile, outDir, testSource, reporter.warningCount))
+ }
+
+ case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+ val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check")
+ val reporters = testSource.compilationGroups.map(compile(_, flags, false, outDir))
+ val compilerCrashed = reporters.exists(_.compilerCrashed)
+ val (errorCount, warningCount) =
+ reporters.foldLeft((0,0)) { case ((errors, warnings), reporter) =>
+ if (reporter.errorCount > 0)
+ logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+
+ (errors + reporter.errorCount, warnings + reporter.warningCount)
+ }
+
+ if (errorCount > 0) {
+ reporters.foreach(logReporterContents)
+ logBuildInstructions(reporters.head, testSource, errorCount, warningCount)
+ }
+
+ (compilerCrashed, errorCount, warningCount, () => verifyOutput(Some(checkFile), outDir, testSource, warningCount))
+ }
+ }
+
+ if (!compilerCrashed && errorCount == 0) verifier()
+ else {
+ echo(s" Compilation failed for: '${testSource.title}' ")
+ val buildInstr = testSource.buildInstructions(errorCount, warningCount)
+ addFailureInstruction(buildInstr)
+ failTestSource(testSource)
+ }
+ registerCompletion(errorCount)
+ }
+ }
+ }
+
+ private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting)
+ extends Test(testSources, times, threadLimit, suppressAllOutput) {
+ protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
+ def checkTestSource(): Unit = tryCompile(testSource) {
+ // In neg-tests we allow two types of error annotations,
+ // "nopos-error" which doesn't care about position and "error" which
+ // has to be annotated on the correct line number.
+ //
+ // We collect these in a map `"file:row" -> numberOfErrors`, for
+ // nopos errors we save them in `"file" -> numberOfNoPosErrors`
+ def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = {
+ val errorMap = new HashMap[String, Integer]()
+ var expectedErrors = 0
+ files.filter(_.getName.endsWith(".scala")).foreach { file =>
+ Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) =>
+ val errors = line.sliding("// error".length).count(_.mkString == "// error")
+ if (errors > 0)
+ errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors)
+
+ val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error")
+ if (noposErrors > 0) {
+ val nopos = errorMap.get("nopos")
+ val existing: Integer = if (nopos eq null) 0 else nopos
+ errorMap.put("nopos", noposErrors + existing)
+ }
+
+ expectedErrors += noposErrors + errors
+ }
+ }
+
+ (errorMap, expectedErrors)
+ }
+
+ def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error =>
+ val key = if (error.pos.exists) {
+ val fileName = error.pos.source.file.toString
+ s"$fileName:${error.pos.line}"
+
+ } else "nopos"
+
+ val errors = errorMap.get(key)
+
+ if (errors ne null) {
+ if (errors == 1) errorMap.remove(key)
+ else errorMap.put(key, errors - 1)
+ true
+ }
+ else {
+ echo {
+ s"Error reported in ${error.pos.source}, but no annotation found"
+ }
+ false
+ }
+ }
+
+ val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
+ case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+ val sourceFiles = testSource.sourceFiles
+ val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles)
+ val reporter = compile(sourceFiles, flags, true, outDir)
+ val actualErrors = reporter.errorCount
+
+ if (reporter.compilerCrashed || actualErrors > 0)
+ logReporterContents(reporter)
+
+ (reporter.compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap)
+ }
+
+ case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+ val compilationGroups = testSource.compilationGroups
+ val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten)
+ val reporters = compilationGroups.map(compile(_, flags, true, outDir))
+ val compilerCrashed = reporters.exists(_.compilerCrashed)
+ val actualErrors = reporters.foldLeft(0)(_ + _.errorCount)
+ val errors = reporters.iterator.flatMap(_.errors)
+
+ if (actualErrors > 0)
+ reporters.foreach(logReporterContents)
+
+ (compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap)
+ }
+ }
+
+ if (compilerCrashed) {
+ echo(s"Compiler crashed when compiling: ${testSource.title}")
+ failTestSource(testSource)
+ }
+ else if (expectedErrors != actualErrors) {
+ echo {
+ s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n"
+ }
+ failTestSource(testSource)
+ }
+ else if (hasMissingAnnotations()) {
+ echo {
+ s"\nErrors found on incorrect row numbers when compiling $testSource"
+ }
+ failTestSource(testSource)
+ }
+ else if (!errorMap.isEmpty) {
+ echo {
+ s"\nExpected error(s) have {<error position>=<unreported error>}: $errorMap"
+ }
+ failTestSource(testSource)
+ }
+
+ registerCompletion(actualErrors)
+ }
+ }
+ }
+
+ /** The `CompilationTest` is the main interface to `ParallelTesting`, it
+ * can be instantiated via one of the following methods:
+ *
+ * - `compileFile`
+ * - `compileDir`
+ * - `compileList`
+ * - `compileFilesInDir`
+ * - `compileShallowFilesInDir`
+ *
+ * Each compilation test can then be turned into either a "pos", "neg" or
+ * "run" test:
+ *
+ * ```
+ * compileFile("../tests/pos/i1103.scala", opts).pos()
+ * ```
+ *
+ * These tests can be customized before calling one of the execution
+ * methods, for instance:
+ *
+ * ```
+ * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos()
+ * ```
+ *
+ * Which would compile `i1103.scala` twice with the verbose flag as a "pos"
+ * test.
+ *
+ * pos tests
+ * =========
+ * Pos tests verify that the compiler is able to compile the given
+ * `TestSource`s and that they generate no errors or exceptions during
+ * compilation
+ *
+ * neg tests
+ * =========
+ * Neg tests are expected to generate a certain amount of errors - but not
+ * crash the compiler. In each `.scala` file, you specifiy the line on which
+ * the error will be generated, e.g:
+ *
+ * ```
+ * val x: String = 1 // error
+ * ```
+ *
+ * if a line generates multiple errors, you need to annotate it multiple
+ * times. For a line that generates two errors:
+ *
+ * ```
+ * val y: String = { val y1: String = 1; 2 } // error // error
+ * ```
+ *
+ * Certain errors have no position, if you need to check these annotate the
+ * file anywhere with `// nopos-error`
+ *
+ * run tests
+ * =========
+ * Run tests are a superset of pos tests, they both verify compilation and
+ * that the compiler does not crash. In addition, run tests verify that the
+ * tests are able to run as expected.
+ *
+ * Run tests need to have the following form:
+ *
+ * ```
+ * object Test {
+ * def main(args: Array[String]): Unit = ()
+ * }
+ * ```
+ *
+ * This is because the runner instantiates the `Test` class and calls the
+ * main method.
+ *
+ * Other definitions are allowed in the same file, but the file needs to at
+ * least have the `Test` object with a `main` method.
+ *
+ * To verify output you may use `.check` files. These files should share the
+ * name of the file or directory that they are testing. For instance:
+ *
+ * ```none
+ * .
+ * └── tests
+ * ├── i1513.scala
+ * └── i1513.check
+ * ```
+ *
+ * If you are testing a directory under separate compilation, you would
+ * have:
+ *
+ * ```none
+ * .
+ * └── tests
+ * ├── myTestDir
+ * │ ├── T_1.scala
+ * │ ├── T_2.scala
+ * │ └── T_3.scala
+ * └── myTestDir.check
+ * ```
+ *
+ * In the above example, `i1513.scala` and one of the files `T_X.scala`
+ * would contain a `Test` object with a main method.
+ *
+ * Composing tests
+ * ===============
+ * Since this is a parallel test suite, it is essential to be able to
+ * compose tests to take advantage of the concurrency. This is done using
+ * the `+` function. This function will make sure that tests being combined
+ * are compatible according to the `require`s in `+`.
+ */
+ final class CompilationTest private (
+ private[ParallelTesting] val targets: List[TestSource],
+ private[ParallelTesting] val times: Int,
+ private[ParallelTesting] val shouldDelete: Boolean,
+ private[ParallelTesting] val threadLimit: Option[Int],
+ private[ParallelTesting] val shouldFail: Boolean
+ ) {
+ import org.junit.Assert.fail
+
+ private[ParallelTesting] def this(target: TestSource) =
+ this(List(target), 1, true, None, false)
+
+ private[ParallelTesting] def this(targets: List[TestSource]) =
+ this(targets, 1, true, None, false)
+
+ /** Compose test targets from `this` with `other`
+ *
+ * It does this, only if the two tests are compatible. Otherwise it throws
+ * an `IllegalArgumentException`.
+ *
+ * Grouping tests together like this allows us to take advantage of the
+ * concurrency offered by this test suite as each call to an executing
+ * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the
+ * maximum allowed level of concurrency. Doing this for only a few targets
+ * does not yield any real benefit over sequential compilation.
+ *
+ * As such, each `CompilationTest` should contain as many targets as
+ * possible.
+ */
+ def +(other: CompilationTest) = {
+ require(other.times == times, "can't combine tests that are meant to be benchmark compiled")
+ require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output")
+ require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome")
+ new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail)
+ }
+
+ /** Creates a "pos" test run, which makes sure that all tests pass
+ * compilation without generating errors and that they do not crash the
+ * compiler
+ */
+ def checkCompile()(implicit summaryReport: SummaryReporting): this.type = {
+ val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+ if (!shouldFail && test.didFail) {
+ fail(s"Expected no errors when compiling, but found: ${test.errorCount}")
+ }
+ else if (shouldFail && !test.didFail) {
+ fail("Pos test should have failed, but didn't")
+ }
+
+ cleanup()
+ }
+
+ /** Creates a "neg" test run, which makes sure that each test generates the
+ * correct amount of errors at the correct positions. It also makes sure
+ * that none of these tests crash the compiler
+ */
+ def checkExpectedErrors()(implicit summaryReport: SummaryReporting): this.type = {
+ val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+ if (!shouldFail && test.didFail) {
+ fail("Neg test shouldn't have failed, but did")
+ }
+ else if (shouldFail && !test.didFail) {
+ fail("Neg test should have failed, but did not")
+ }
+
+ cleanup()
+ }
+
+ /** Creates a "run" test run, which is a superset of "pos". In addition to
+ * making sure that all tests pass compilation and that they do not crash
+ * the compiler; it also makes sure that all tests can run with the
+ * expected output
+ */
+ def checkRuns()(implicit summaryReport: SummaryReporting): this.type = {
+ val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+ if (!shouldFail && test.didFail) {
+ fail("Run test failed, but should not")
+ }
+ else if (shouldFail && !test.didFail) {
+ fail("Run test should have failed, but did not")
+ }
+
+ cleanup()
+ }
+
+ /** Deletes output directories and files */
+ private def cleanup(): this.type = {
+ if (shouldDelete) delete()
+ this
+ }
+
+ /** Copies `file` to `dir` - taking into account if `file` is a directory,
+ * and if so copying recursively
+ */
+ private def copyToDir(dir: JFile, file: JFile): JFile = {
+ val target = Paths.get(dir.getAbsolutePath, file.getName)
+ Files.copy(file.toPath, target, REPLACE_EXISTING)
+ if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _))
+ target.toFile
+ }
+
+ /** Builds a new `CompilationTest` where we have copied the target files to
+ * the out directory. This is needed for tests that modify the original
+ * source, such as `-rewrite` tests
+ */
+ def copyToTarget(): CompilationTest = new CompilationTest (
+ targets.map {
+ case target @ JointCompilationSource(_, files, _, outDir) =>
+ target.copy(files = files.map(copyToDir(outDir,_)))
+ case target @ SeparateCompilationSource(_, dir, _, outDir) =>
+ target.copy(dir = copyToDir(outDir, dir))
+ },
+ times, shouldDelete, threadLimit, shouldFail
+ )
+
+ /** Builds a `CompilationTest` which performs the compilation `i` times on
+ * each target
+ */
+ def times(i: Int): CompilationTest =
+ new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail)
+
+ /** Builds a `Compilationtest` which passes the verbose flag and logs the
+ * classpath
+ */
+ def verbose: CompilationTest = new CompilationTest(
+ targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")),
+ times, shouldDelete, threadLimit, shouldFail
+ )
+
+ /** Builds a `CompilationTest` which keeps the generated output files
+ *
+ * This is needed for tests like `tastyBootstrap` which relies on first
+ * compiling a certain part of the project and then compiling a second
+ * part which depends on the first
+ */
+ def keepOutput: CompilationTest =
+ new CompilationTest(targets, times, false, threadLimit, shouldFail)
+
+ /** Builds a `CompilationTest` with a limited level of concurrency with
+ * maximum `i` threads
+ */
+ def limitThreads(i: Int): CompilationTest =
+ new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail)
+
+ /** Builds a `CompilationTest` where the executed test is expected to fail
+ *
+ * This behaviour is mainly needed for the tests that test the test suite.
+ */
+ def expectFailure: CompilationTest =
+ new CompilationTest(targets, times, shouldDelete, threadLimit, true)
+
+ /** Delete all output files generated by this `CompilationTest` */
+ def delete(): Unit = targets.foreach(t => delete(t.outDir))
+
+ private def delete(file: JFile): Unit = {
+ if (file.isDirectory) file.listFiles.foreach(delete)
+ try Files.delete(file.toPath)
+ catch {
+ case _: NoSuchFileException => // already deleted, everything's fine
+ }
+ }
+ }
+
+ /** Create out directory for directory `d` */
+ private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = {
+ val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}")
+ targetDir.mkdirs()
+ targetDir
+ }
+
+ /** Create out directory for `file` */
+ private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = {
+ val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.'))
+ val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir")
+ targetDir.mkdirs()
+ targetDir
+ }
+
+ /** Make sure that directory string is as expected */
+ private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = {
+ require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`")
+ require(outDir.last == '/', "please specify an `outDir` with a trailing slash")
+ }
+
+ /** Separates directories from files and returns them as `(dirs, files)` */
+ private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) =
+ sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) =>
+ if (f.isDirectory) (f :: dirs, files)
+ else if (isSourceFile(f)) (dirs, f :: files)
+ else (dirs, files)
+ }
+
+ /** Gets the name of the calling method via reflection.
+ *
+ * It does this in a way that needs to work both with the bootstrapped dotty
+ * and the non-bootstrapped version. Since the two compilers generate
+ * different bridges, we first need to filter out methods with the same name
+ * (bridges) - and then find the `@Test` method in our extending class
+ */
+ private def getCallingMethod(): String = {
+ val seen = mutable.Set.empty[String]
+ Thread.currentThread.getStackTrace
+ .filter { elem =>
+ if (seen.contains(elem.getMethodName)) false
+ else { seen += elem.getMethodName; true }
+ }
+ .find { elem =>
+ val callingClass = Class.forName(elem.getClassName)
+ classOf[ParallelTesting].isAssignableFrom(callingClass) &&
+ elem.getFileName != "ParallelTesting.scala"
+ }
+ .map(_.getMethodName)
+ .getOrElse {
+ throw new IllegalStateException("Unable to reflectively find calling method")
+ }
+ .takeWhile(_ != '$')
+ }
+
+ /** Compiles a single file from the string path `f` using the supplied flags */
+ def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+ val callingMethod = getCallingMethod
+ val sourceFile = new JFile(f)
+ val parent = sourceFile.getParentFile
+ val outDir =
+ outDirectory + callingMethod + "/" +
+ sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/"
+
+ require(
+ sourceFile.exists && !sourceFile.isDirectory &&
+ (parent ne null) && parent.exists && parent.isDirectory,
+ s"Source file: $f, didn't exist"
+ )
+
+ val target = JointCompilationSource(
+ callingMethod,
+ Array(sourceFile),
+ flags,
+ createOutputDirsForFile(sourceFile, parent, outDir)
+ )
+ new CompilationTest(target)
+ }
+
+ /** Compiles a directory `f` using the supplied `flags`. This method does
+ * deep compilation, that is - it compiles all files and subdirectories
+ * contained within the directory `f`.
+ *
+ * By default, files are compiled in alphabetical order. An optional seed
+ * can be used for randomization.
+ */
+ def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = {
+ val callingMethod = getCallingMethod
+ val outDir = outDirectory + callingMethod + "/"
+ val sourceDir = new JFile(f)
+ checkRequirements(f, sourceDir, outDir)
+
+ def flatten(f: JFile): Array[JFile] =
+ if (f.isDirectory) f.listFiles.flatMap(flatten)
+ else Array(f)
+
+ // Sort files either alphabetically or randomly using the provided seed:
+ val sortedFiles = flatten(sourceDir).sorted
+ val randomized = randomOrder match {
+ case None => sortedFiles
+ case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray
+ }
+
+ // Directories in which to compile all containing files with `flags`:
+ val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/")
+ targetDir.mkdirs()
+
+ val target = JointCompilationSource(s"compiling '$f' in test '$callingMethod'", randomized, flags, targetDir)
+ new CompilationTest(target)
+ }
+
+ /** Compiles all `files` together as a single compilation run. It is given a
+ * `testName` since files can be in separate directories and or be otherwise
+ * dissociated
+ */
+ def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+ val callingMethod = getCallingMethod
+ val outDir = outDirectory + callingMethod + "/" + testName + "/"
+
+ // Directories in which to compile all containing files with `flags`:
+ val targetDir = new JFile(outDir)
+ targetDir.mkdirs()
+ assert(targetDir.exists, s"couldn't create target directory: $targetDir")
+
+ val target = JointCompilationSource(s"$testName from $callingMethod", files.map(new JFile(_)).toArray, flags, targetDir)
+
+ // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+ new CompilationTest(target)
+ }
+
+ /** This function compiles the files and folders contained within directory
+ * `f` in a specific way.
+ *
+ * - Each file is compiled separately as a single compilation run
+ * - Each directory is compiled as a `SeparateCompilationTaret`, in this
+ * target all files are grouped according to the file suffix `_X` where `X`
+ * is a number. These groups are then ordered in ascending order based on
+ * the value of `X` and each group is compiled one after the other.
+ *
+ * For this function to work as expected, we use the same convention for
+ * directory layout as the old partest. That is:
+ *
+ * - Single files can have an associated check-file with the same name (but
+ * with file extension `.check`)
+ * - Directories can have an associated check-file, where the check file has
+ * the same name as the directory (with the file extension `.check`)
+ */
+ def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+ val callingMethod = getCallingMethod
+ val outDir = outDirectory + callingMethod + "/"
+ val sourceDir = new JFile(f)
+ checkRequirements(f, sourceDir, outDir)
+
+ val (dirs, files) = compilationTargets(sourceDir)
+
+ val targets =
+ files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++
+ dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir)))
+
+ // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+ new CompilationTest(targets)
+ }
+
+ /** This function behaves similar to `compileFilesInDir` but it ignores
+ * sub-directories and as such, does **not** perform separate compilation
+ * tests.
+ */
+ def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+ val callingMethod = getCallingMethod
+ val outDir = outDirectory + callingMethod + "/"
+ val sourceDir = new JFile(f)
+ checkRequirements(f, sourceDir, outDir)
+
+ val (_, files) = compilationTargets(sourceDir)
+
+ val targets = files.map { file =>
+ JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir))
+ }
+
+ // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+ new CompilationTest(targets)
+ }
+}
+
+object ParallelTesting {
+ def isSourceFile(f: JFile): Boolean = {
+ val name = f.getName
+ name.endsWith(".scala") || name.endsWith(".java")
+ }
+}