From d65e4eadaf5c9fbed379680de591bb3ef42c6b3d Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 3 Apr 2017 17:04:46 +0200 Subject: Rename test suite to "vulpix" --- .drone.yml | 8 +- compiler/foo | 0 compiler/test/dotc/comptest.scala | 2 +- .../test/dotty/tools/dotc/CompilationTests.scala | 8 +- .../dotty/tools/dotc/ParallelSummaryReport.java | 73 -- .../test/dotty/tools/dotc/ParallelTestTests.scala | 58 - .../test/dotty/tools/dotc/ParallelTesting.scala | 1122 ------------------- .../dotty/tools/dotc/vulpix/ParallelTesting.scala | 1124 ++++++++++++++++++++ .../dotty/tools/dotc/vulpix/SummaryReport.java | 75 ++ .../test/dotty/tools/dotc/vulpix/VulpixTests.scala | 59 + 10 files changed, 1267 insertions(+), 1262 deletions(-) delete mode 100644 compiler/foo delete mode 100644 compiler/test/dotty/tools/dotc/ParallelSummaryReport.java delete mode 100644 compiler/test/dotty/tools/dotc/ParallelTestTests.scala delete mode 100644 compiler/test/dotty/tools/dotc/ParallelTesting.scala create mode 100644 compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala create mode 100644 compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java create mode 100644 compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala diff --git a/.drone.yml b/.drone.yml index 98e34cc4e..ed772eb32 100644 --- a/.drone.yml +++ b/.drone.yml @@ -40,9 +40,9 @@ pipeline: matrix: TEST: - - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;test ;dotty-bin-tests/test - - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;publishLocal ;dotty-bootstrapped/test - - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only-no-bootstrap --show-diff --verbose - - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only --show-diff --verbose + - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;test ;dotty-bin-tests/test + - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;publishLocal ;dotty-bootstrapped/test + - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;partest-only-no-bootstrap --show-diff --verbose + - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;partest-only --show-diff --verbose - ;dotty-compiler/testOnly dotty.tools.dotc.CompilationTests - ;publishLocal ;dotty-bootstrapped/testOnly dotty.tools.dotc.CompilationTests diff --git a/compiler/foo b/compiler/foo deleted file mode 100644 index e69de29bb..000000000 diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala index dce002c81..d283f4ef0 100644 --- a/compiler/test/dotc/comptest.scala +++ b/compiler/test/dotc/comptest.scala @@ -1,6 +1,6 @@ package dotc -import dotty.tools.dotc.ParallelTesting +import dotty.tools.dotc.vulpix.ParallelTesting object comptest extends ParallelTesting { diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 742b93fae..a2d632b19 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -3,16 +3,16 @@ package tools package dotc import org.junit.Test -import java.io.{ File => JFile } import org.junit.experimental.categories.Category import scala.util.matching.Regex +import vulpix.{ ParallelTesting, SummaryReport } @Category(Array(classOf[ParallelTesting])) -class CompilationTests extends ParallelSummaryReport with ParallelTesting { +class CompilationTests extends SummaryReport with ParallelTesting { import CompilationTests._ - def isInteractive: Boolean = ParallelSummaryReport.isInteractive + def isInteractive: Boolean = SummaryReport.isInteractive def testFilter: Option[Regex] = sys.props.get("dotty.partest.filter").map(r => new Regex(r)) @@ -270,7 +270,7 @@ object CompilationTests { val classPath = { val paths = Jars.dottyTestDeps map { p => - val file = new JFile(p) + val file = new java.io.File(p) assert( file.exists, s"""|File "$p" couldn't be found. Run `packageAll` from build tool before diff --git a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java deleted file mode 100644 index 5608b3656..000000000 --- a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java +++ /dev/null @@ -1,73 +0,0 @@ -package dotty.tools.dotc; - -import org.junit.BeforeClass; -import org.junit.AfterClass; -import java.util.ArrayDeque; - -import dotty.tools.dotc.reporting.TestReporter; -import dotty.tools.dotc.reporting.TestReporter$; - -/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with - * this class - */ -public class ParallelSummaryReport { - public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); - - private static TestReporter rep = TestReporter.reporter(System.out, -1); - private static ArrayDeque failedTests = new ArrayDeque<>(); - private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); - private static int passed; - private static int failed; - - public final static void reportFailed() { - failed++; - } - - public final static void reportPassed() { - passed++; - } - - public final static void addFailedTest(String msg) { - failedTests.offer(msg); - } - - public final static void addReproduceInstruction(String msg) { - reproduceInstructions.offer(msg); - } - - @BeforeClass public final static void setup() { - rep = TestReporter.reporter(System.out, -1); - failedTests = new ArrayDeque<>(); - reproduceInstructions = new ArrayDeque<>(); - } - - @AfterClass public final static void teardown() { - rep.echo( - "\n================================================================================" + - "\nTest Report" + - "\n================================================================================" + - "\n" + - passed + " passed, " + failed + " failed, " + (passed + failed) + " total" + - "\n" - ); - - failedTests - .stream() - .map(x -> " " + x) - .forEach(rep::echo); - - // If we're compiling locally, we don't need reproduce instructions - if (isInteractive) rep.flushToStdErr(); - - rep.echo(""); - - reproduceInstructions - .stream() - .forEach(rep::echo); - - // If we're on the CI, we want everything - if (!isInteractive) rep.flushToStdErr(); - - if (failed > 0) rep.flushToFile(); - } -} diff --git a/compiler/test/dotty/tools/dotc/ParallelTestTests.scala b/compiler/test/dotty/tools/dotc/ParallelTestTests.scala deleted file mode 100644 index cfb108ea7..000000000 --- a/compiler/test/dotty/tools/dotc/ParallelTestTests.scala +++ /dev/null @@ -1,58 +0,0 @@ -package dotty -package tools -package dotc - -import org.junit.Assert._ -import org.junit.Test - -import scala.util.control.NonFatal - -class ParallelTestTests extends ParallelTesting { - import CompilationTests._ - - def isInteractive = !sys.env.contains("DRONE") - def testFilter = None - - @Test def missingFile: Unit = - try { - compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors() - fail("didn't fail properly") - } - catch { - case _: IllegalArgumentException => // pass! - case NonFatal(_) => fail("wrong exception thrown") - } - - @Test def pos1Error: Unit = - compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile() - - @Test def negMissingAnnot: Unit = - compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negAnnotWrongLine: Unit = - compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negTooManyAnnots: Unit = - compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negNoPositionAnnot: Unit = - compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def runCompileFail: Unit = - compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runWrongOutput1: Unit = - compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runWrongOutput2: Unit = - compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runDiffOutput1: Unit = - compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runStackOverflow: Unit = - compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runOutRedirects: Unit = - compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() -} diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala deleted file mode 100644 index 80c56808b..000000000 --- a/compiler/test/dotty/tools/dotc/ParallelTesting.scala +++ /dev/null @@ -1,1122 +0,0 @@ -package dotty -package tools -package dotc - -import java.io.{ File => JFile } -import java.text.SimpleDateFormat -import java.util.HashMap -import java.lang.reflect.InvocationTargetException -import java.nio.file.StandardCopyOption.REPLACE_EXISTING -import java.nio.file.{ Files, Path, Paths, NoSuchFileException } -import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } - -import scala.io.Source -import scala.util.control.NonFatal -import scala.util.Try -import scala.collection.mutable -import scala.util.matching.Regex -import scala.util.Random - -import core.Contexts._ -import reporting.{ Reporter, TestReporter } -import reporting.diagnostic.MessageContainer -import interfaces.Diagnostic.ERROR -import dotc.util.DiffUtil - -/** A parallel testing suite whose goal is to integrate nicely with JUnit - * - * This trait can be mixed in to offer parallel testing to compile runs. When - * using this, you should be running your JUnit tests **sequentially**, as the - * test suite itself runs with a high level of concurrency. - */ -trait ParallelTesting { self => - - import ParallelTesting._ - import ParallelSummaryReport._ - - /** If the running environment supports an interactive terminal, each `Test` - * will be run with a progress bar and real time feedback - */ - def isInteractive: Boolean - - /** A regex which is used to filter which tests to run, if `None` will run - * all tests - */ - def testFilter: Option[Regex] - - /** A test source whose files or directory of files is to be compiled - * in a specific way defined by the `Test` - */ - private sealed trait TestSource { self => - def name: String - def outDir: JFile - def flags: Array[String] - - - def title: String = self match { - case self: JointCompilationSource => - if (self.files.length > 1) name - else self.files.head.getPath - - case self: SeparateCompilationSource => - self.dir.getPath - } - - /** Adds the flags specified in `newFlags0` if they do not already exist */ - def withFlags(newFlags0: String*) = { - val newFlags = newFlags0.toArray - if (!flags.containsSlice(newFlags)) self match { - case self: JointCompilationSource => - self.copy(flags = flags ++ newFlags) - case self: SeparateCompilationSource => - self.copy(flags = flags ++ newFlags) - } - else self - } - - /** Generate the instructions to redo the test from the command line */ - def buildInstructions(errors: Int, warnings: Int): String = { - val sb = new StringBuilder - val maxLen = 80 - var lineLen = 0 - - sb.append( - s"""| - |Test '$title' compiled with $errors error(s) and $warnings warning(s), - |the test can be reproduced by running:""".stripMargin - ) - sb.append("\n\n./bin/dotc ") - flags.foreach { arg => - if (lineLen > maxLen) { - sb.append(" \\\n ") - lineLen = 4 - } - sb.append(arg) - lineLen += arg.length - sb += ' ' - } - - self match { - case JointCompilationSource(_, files, _, _) => { - files.map(_.getAbsolutePath).foreach { path => - sb.append("\\\n ") - sb.append(path) - sb += ' ' - } - sb.toString + "\n\n" - } - case self: SeparateCompilationSource => { - val command = sb.toString - val fsb = new StringBuilder(command) - self.compilationGroups.foreach { files => - files.map(_.getPath).foreach { path => - fsb.append("\\\n ") - lineLen = 8 - fsb.append(path) - fsb += ' ' - } - fsb.append("\n\n") - fsb.append(command) - } - fsb.toString + "\n\n" - } - } - } - } - - /** A group of files that may all be compiled together, with the same flags - * and output directory - */ - private final case class JointCompilationSource( - name: String, - files: Array[JFile], - flags: Array[String], - outDir: JFile - ) extends TestSource { - def sourceFiles: Array[JFile] = files.filter(isSourceFile) - - override def toString() = outDir.toString - } - - /** A test source whose files will be compiled separately according to their - * suffix `_X` - */ - private final case class SeparateCompilationSource( - name: String, - dir: JFile, - flags: Array[String], - outDir: JFile - ) extends TestSource { - - /** Get the files grouped by `_X` as a list of groups, files missing this - * suffix will be put into the same group - * - * Filters out all none source files - */ - def compilationGroups: List[Array[JFile]] = - dir - .listFiles - .groupBy { file => - val name = file.getName - Try { - val potentialNumber = name - .substring(0, name.lastIndexOf('.')) - .reverse.takeWhile(_ != '_').reverse - - potentialNumber.toInt.toString - } - .toOption - .getOrElse("") - } - .toList.sortBy(_._1).map(_._2.filter(isSourceFile)) - } - - /** Each `Test` takes the `testSources` and performs the compilation and assertions - * according to the implementing class "neg", "run" or "pos". - */ - private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { - protected final val realStdout = System.out - protected final val realStderr = System.err - - /** Actual compilation run logic, the test behaviour is defined here */ - protected def compilationRunnable(testSource: TestSource): Runnable - - /** All testSources left after filtering out */ - private val filteredSources = - if (!testFilter.isDefined) testSources - else testSources.filter { - case JointCompilationSource(_, files, _, _) => - files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined) - case SeparateCompilationSource(_, dir, _, _) => - testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined - } - - /** Total amount of test sources being compiled by this test */ - val sourceCount = filteredSources.length - - private[this] var _errorCount = 0 - def errorCount: Int = _errorCount - - private[this] var _testSourcesCompiled = 0 - private def testSourcesCompiled: Int = _testSourcesCompiled - - /** Complete the current compilation with the amount of errors encountered */ - protected final def registerCompilation(errors: Int) = synchronized { - _testSourcesCompiled += 1 - _errorCount += errors - } - - private[this] var _failed = false - /** Fail the current test */ - protected[this] final def fail(): Unit = synchronized { _failed = true } - def didFail: Boolean = _failed - - protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { - val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) - addFailureInstruction(errorMsg) - failTestSource(testSource) - } - - /** Instructions on how to reproduce failed test source compilations */ - private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String] - protected final def addFailureInstruction(ins: String): Unit = - synchronized { reproduceInstructions.append(ins) } - - /** The test sources that failed according to the implementing subclass */ - private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] - protected final def failTestSource(testSource: TestSource) = synchronized { - failedTestSources.append(testSource.name + " failed") - fail() - } - - /** Prints to `System.err` if we're not suppressing all output */ - protected def echo(msg: String): Unit = - if (!suppressAllOutput) realStderr.println(msg) - - /** A single `Runnable` that prints a progress bar for the curent `Test` */ - private def createProgressMonitor: Runnable = new Runnable { - def run(): Unit = { - val start = System.currentTimeMillis - var tCompiled = testSourcesCompiled - while (tCompiled < sourceCount) { - val timestamp = (System.currentTimeMillis - start) / 1000 - val progress = (tCompiled.toDouble / sourceCount * 40).toInt - - realStdout.print( - "[" + ("=" * (math.max(progress - 1, 0))) + - (if (progress > 0) ">" else "") + - (" " * (39 - progress)) + - s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" - ) - - Thread.sleep(100) - tCompiled = testSourcesCompiled - } - // println, otherwise no newline and cursor at start of line - realStdout.println( - s"[=======================================] compiled ($sourceCount/$sourceCount, " + - s"${(System.currentTimeMillis - start) / 1000}s) " - ) - } - } - - /** Wrapper function to make sure that the compiler itself did not crash - - * if it did, the test should automatically fail. - */ - protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = - try { - if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") - op - } catch { - case NonFatal(e) => { - // if an exception is thrown during compilation, the complete test - // run should fail - failTestSource(testSource) - e.printStackTrace() - registerCompilation(1) - throw e - } - } - - protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = { - - val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath) - - def flattenFiles(f: JFile): Array[JFile] = - if (f.isDirectory) f.listFiles.flatMap(flattenFiles) - else Array(f) - - val files: Array[JFile] = files0.flatMap(flattenFiles) - - def findJarFromRuntime(partialName: String) = { - val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) - urls.find(_.contains(partialName)).getOrElse { - throw new java.io.FileNotFoundException( - s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" - ) - } - } - - def addOutDir(xs: Array[String]): Array[String] = { - val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") - if (cpAndAfter.nonEmpty) { - val (cp :: cpArg :: rest) = cpAndAfter - (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray - } - else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray - } - - def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { - val scalaLib = findJarFromRuntime("scala-library-2.") - val fullArgs = Array( - "javac", - "-classpath", - s".:$scalaLib:${targetDir.getAbsolutePath}" - ) ++ flags.takeRight(2) ++ fs - - Runtime.getRuntime.exec(fullArgs).waitFor() == 0 - } else true - - val reporter = - TestReporter.reporter(realStdout, logLevel = - if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) - - val driver = - if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } - else new Driver { - def newCompiler(implicit ctx: Context) = new Compiler - - private def ntimes(n: Int)(op: Int => Reporter): Reporter = - (emptyReporter /: (1 to n)) ((_, i) => op(i)) - - override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) = - ntimes(times) { run => - val start = System.nanoTime() - val rep = super.doCompile(comp, files) - ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms") - rep - } - } - - val allArgs = addOutDir(flags) - driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) - - val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) - assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") - - reporter - } - - private[ParallelTesting] def executeTestSuite(): this.type = { - assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`") - - if (filteredSources.nonEmpty) { - val pool = threadLimit match { - case Some(i) => JExecutors.newWorkStealingPool(i) - case None => JExecutors.newWorkStealingPool() - } - - if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) - - filteredSources.foreach { target => - pool.submit(compilationRunnable(target)) - } - - pool.shutdown() - if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { - pool.shutdownNow() - System.setOut(realStdout) - System.setErr(realStderr) - throw new TimeoutException("Compiling targets timed out") - } - - if (didFail) { - reportFailed() - failedTestSources.toSet.foreach(addFailedTest) - reproduceInstructions.iterator.foreach(addReproduceInstruction) - } - else reportPassed() - } - else echo { - testFilter - .map(r => s"""No files matched regex "$r" in test""") - .getOrElse("No tests available under target - erroneous test?") - } - - this - } - } - - private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val reporter = compile(testSource.sourceFiles, flags, false, outDir) - registerCompilation(reporter.errorCount) - - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) - val errorCount = reporters.foldLeft(0) { (acc, reporter) => - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - acc + reporter.errorCount - } - - registerCompilation(errorCount) - - if (errorCount > 0) failTestSource(testSource) - } - } - - } - } - } - - private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - private def runMain(dir: JFile, testSource: TestSource): Array[String] = { - def renderStackTrace(ex: Throwable): String = - if (ex == null) "" - else ex.getStackTrace - .takeWhile(_.getMethodName != "invoke0") - .mkString(" ", "\n ", "") - - import java.io.{ ByteArrayOutputStream, PrintStream } - import java.net.{ URL, URLClassLoader } - - val printStream = new ByteArrayOutputStream - - try { - // Do classloading magic and running here: - val ucl = new URLClassLoader(Array(dir.toURI.toURL)) - val cls = ucl.loadClass("Test") - val meth = cls.getMethod("main", classOf[Array[String]]) - - synchronized { - try { - val ps = new PrintStream(printStream) - System.setOut(ps) - System.setErr(ps) - Console.withOut(printStream) { - Console.withErr(printStream) { - meth.invoke(null, Array("jvm")) // partest passes at least "jvm" as an arg - } - } - System.setOut(realStdout) - System.setErr(realStderr) - } catch { - case t: Throwable => - System.setOut(realStdout) - System.setErr(realStderr) - throw t - } - } - } - catch { - case ex: NoSuchMethodException => - echo(s"test in '$dir' did not contain method: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") - failTestSource(testSource) - - case ex: ClassNotFoundException => - echo(s"test in '$dir' did not contain class: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") - failTestSource(testSource) - - case ex: InvocationTargetException => - echo(s"An exception ocurred when running main: ${ex.getCause}\n${renderStackTrace(ex.getCause)}") - failTestSource(testSource) - } - printStream.toString("utf-8").lines.toArray - } - - private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - val outputLines = runMain(dir, testSource) - val checkLines = Source.fromFile(checkFile).getLines.toArray - val sourceTitle = testSource.title - - def linesMatch = - outputLines - .zip(checkLines) - .forall { case (x, y) => x == y } - - if (outputLines.length != checkLines.length || !linesMatch) { - // Print diff to files and summary: - val diff = outputLines.zip(checkLines).map { case (act, exp) => - DiffUtil.mkColoredLineDiff(exp, act) - }.mkString("\n") - - val msg = - s"""|Output from '$sourceTitle' did not match check file. - |Diff ('e' is expected, 'a' is actual): - |""".stripMargin + diff + "\n" - echo(msg) - addFailureInstruction(msg) - - // Print build instructions to file and summary: - val buildInstr = testSource.buildInstructions(0, warnings) - addFailureInstruction(buildInstr) - - // Fail target: - failTestSource(testSource) - } - } - - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val checkFile = files.flatMap { file => - if (file.isDirectory) Nil - else { - val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check" - val checkFile = new JFile(fname) - if (checkFile.exists) List(checkFile) - else Nil - } - }.headOption - val reporter = compile(testSource.sourceFiles, flags, false, outDir) - - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - registerCompilation(reporter.errorCount) - (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") - val (errorCount, warningCount) = - testSource - .compilationGroups - .map(compile(_, flags, false, outDir)) - .foldLeft((0,0)) { case ((errors, warnings), reporter) => - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - (errors + reporter.errorCount, warnings + reporter.warningCount) - } - - if (errorCount > 0) fail() - - registerCompilation(errorCount) - (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) - } - } - - if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.outDir, testSource) - else if (errorCount > 0) { - echo(s"\nCompilation failed for: '$testSource'") - val buildInstr = testSource.buildInstructions(errorCount, warningCount) - addFailureInstruction(buildInstr) - failTestSource(testSource) - } - } - } - } - - private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - // In neg-tests we allow two types of error annotations, - // "nopos-error" which doesn't care about position and "error" which - // has to be annotated on the correct line number. - // - // We collect these in a map `"file:row" -> numberOfErrors`, for - // nopos errors we save them in `"file" -> numberOfNoPosErrors` - def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = { - val errorMap = new HashMap[String, Integer]() - var expectedErrors = 0 - files.filter(_.getName.endsWith(".scala")).foreach { file => - Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) => - val errors = line.sliding("// error".length).count(_.mkString == "// error") - if (errors > 0) - errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors) - - val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error") - if (noposErrors > 0) { - val nopos = errorMap.get("nopos") - val existing: Integer = if (nopos eq null) 0 else nopos - errorMap.put("nopos", noposErrors + existing) - } - - expectedErrors += noposErrors + errors - } - } - - (errorMap, expectedErrors) - } - - def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error => - val key = if (error.pos.exists) { - val fileName = error.pos.source.file.toString - s"$fileName:${error.pos.line}" - - } else "nopos" - - val errors = errorMap.get(key) - - if (errors ne null) { - if (errors == 1) errorMap.remove(key) - else errorMap.put(key, errors - 1) - true - } - else { - echo { - s"Error reported in ${error.pos.source}, but no annotation found" - } - false - } - } - - val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val sourceFiles = testSource.sourceFiles - val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) - val reporter = compile(sourceFiles, flags, true, outDir) - val actualErrors = reporter.errorCount - - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val compilationGroups = testSource.compilationGroups - val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) - val reporters = compilationGroups.map(compile(_, flags, true, outDir)) - val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) - val errors = reporters.iterator.flatMap(_.errors) - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) - } - } - - if (expectedErrors != actualErrors) { - echo { - s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" - } - failTestSource(testSource) - } - else if (hasMissingAnnotations()) { - echo { - s"\nErrors found on incorrect row numbers when compiling $testSource" - } - failTestSource(testSource) - } - else if (!errorMap.isEmpty) { - echo { - s"\nExpected error(s) have {=}: $errorMap" - } - failTestSource(testSource) - } - - registerCompilation(actualErrors) - } - } - } - - /** The `CompilationTest` is the main interface to `ParallelTesting`, it - * can be instantiated via one of the following methods: - * - * - `compileFile` - * - `compileDir` - * - `compileList` - * - `compileFilesInDir` - * - `compileShallowFilesInDir` - * - * Each compilation test can then be turned into either a "pos", "neg" or - * "run" test: - * - * ``` - * compileFile("../tests/pos/i1103.scala", opts).pos() - * ``` - * - * These tests can be customized before calling one of the execution - * methods, for instance: - * - * ``` - * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos() - * ``` - * - * Which would compile `i1103.scala` twice with the verbose flag as a "pos" - * test. - * - * pos tests - * ========= - * Pos tests verify that the compiler is able to compile the given - * `TestSource`s and that they generate no errors or exceptions during - * compilation - * - * neg tests - * ========= - * Neg tests are expected to generate a certain amount of errors - but not - * crash the compiler. In each `.scala` file, you specifiy the line on which - * the error will be generated, e.g: - * - * ``` - * val x: String = 1 // error - * ``` - * - * if a line generates multiple errors, you need to annotate it multiple - * times. For a line that generates two errors: - * - * ``` - * val y: String = { val y1: String = 1; 2 } // error // error - * ``` - * - * Certain errors have no position, if you need to check these annotate the - * file anywhere with `// nopos-error` - * - * run tests - * ========= - * Run tests are a superset of pos tests, they both verify compilation and - * that the compiler does not crash. In addition, run tests verify that the - * tests are able to run as expected. - * - * Run tests need to have the following form: - * - * ``` - * object Test { - * def main(args: Array[String]): Unit = () - * } - * ``` - * - * This is because the runner instantiates the `Test` class and calls the - * main method. - * - * Other definitions are allowed in the same file, but the file needs to at - * least have the `Test` object with a `main` method. - * - * To verify output you may use `.check` files. These files should share the - * name of the file or directory that they are testing. For instance: - * - * ```none - * . - * └── tests - * ├── i1513.scala - * └── i1513.check - * ``` - * - * If you are testing a directory under separate compilation, you would - * have: - * - * ```none - * . - * └── tests - * ├── myTestDir - * │ ├── T_1.scala - * │ ├── T_2.scala - * │ └── T_3.scala - * └── myTestDir.check - * ``` - * - * In the above example, `i1513.scala` and one of the files `T_X.scala` - * would contain a `Test` object with a main method. - * - * Composing tests - * =============== - * Since this is a parallel test suite, it is essential to be able to - * compose tests to take advantage of the concurrency. This is done using - * the `+` function. This function will make sure that tests being combined - * are compatible according to the `require`s in `+`. - */ - final class CompilationTest private ( - private[ParallelTesting] val targets: List[TestSource], - private[ParallelTesting] val times: Int, - private[ParallelTesting] val shouldDelete: Boolean, - private[ParallelTesting] val threadLimit: Option[Int], - private[ParallelTesting] val shouldFail: Boolean - ) { - import org.junit.Assert.fail - - private[ParallelTesting] def this(target: TestSource) = - this(List(target), 1, true, None, false) - - private[ParallelTesting] def this(targets: List[TestSource]) = - this(targets, 1, true, None, false) - - /** Compose test targets from `this` with `other` - * - * It does this, only if the two tests are compatible. Otherwise it throws - * an `IllegalArgumentException`. - * - * Grouping tests together like this allows us to take advantage of the - * concurrency offered by this test suite as each call to an executing - * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the - * maximum allowed level of concurrency. Doing this for only a few targets - * does not yield any real benefit over sequential compilation. - * - * As such, each `CompilationTest` should contain as many targets as - * possible. - */ - def +(other: CompilationTest) = { - require(other.times == times, "can't combine tests that are meant to be benchmark compiled") - require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output") - require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome") - new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail) - } - - /** Creates a "pos" test run, which makes sure that all tests pass - * compilation without generating errors and that they do not crash the - * compiler - */ - def checkCompile(): this.type = { - val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail(s"Expected no errors when compiling, but found: ${test.errorCount}") - } - else if (shouldFail && !test.didFail) { - fail("Pos test should have failed, but didn't") - } - - cleanup() - } - - /** Creates a "neg" test run, which makes sure that each test generates the - * correct amount of errors at the correct positions. It also makes sure - * that none of these tests crash the compiler - */ - def checkExpectedErrors(): this.type = { - val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail("Neg test shouldn't have failed, but did") - } - else if (shouldFail && !test.didFail) { - fail("Neg test should have failed, but did not") - } - - cleanup() - } - - /** Creates a "run" test run, which is a superset of "pos". In addition to - * making sure that all tests pass compilation and that they do not crash - * the compiler; it also makes sure that all tests can run with the - * expected output - */ - def checkRuns(): this.type = { - val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail("Run test failed, but should not") - } - else if (shouldFail && !test.didFail) { - fail("Run test should have failed, but did not") - } - - cleanup() - } - - /** Deletes output directories and files */ - private def cleanup(): this.type = { - if (shouldDelete) delete() - this - } - - /** Copies `file` to `dir` - taking into account if `file` is a directory, - * and if so copying recursively - */ - private def copyToDir(dir: JFile, file: JFile): JFile = { - val target = Paths.get(dir.getAbsolutePath, file.getName) - Files.copy(file.toPath, target, REPLACE_EXISTING) - if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _)) - target.toFile - } - - /** Builds a new `CompilationTest` where we have copied the target files to - * the out directory. This is needed for tests that modify the original - * source, such as `-rewrite` tests - */ - def copyToTarget(): CompilationTest = new CompilationTest ( - targets.map { - case target @ JointCompilationSource(_, files, _, outDir) => - target.copy(files = files.map(copyToDir(outDir,_))) - case target @ SeparateCompilationSource(_, dir, _, outDir) => - target.copy(dir = copyToDir(outDir, dir)) - }, - times, shouldDelete, threadLimit, shouldFail - ) - - /** Builds a `CompilationTest` which performs the compilation `i` times on - * each target - */ - def times(i: Int): CompilationTest = - new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail) - - /** Builds a `Compilationtest` which passes the verbose flag and logs the - * classpath - */ - def verbose: CompilationTest = new CompilationTest( - targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")), - times, shouldDelete, threadLimit, shouldFail - ) - - /** Builds a `CompilationTest` which keeps the generated output files - * - * This is needed for tests like `tastyBootstrap` which relies on first - * compiling a certain part of the project and then compiling a second - * part which depends on the first - */ - def keepOutput: CompilationTest = - new CompilationTest(targets, times, false, threadLimit, shouldFail) - - /** Builds a `CompilationTest` with a limited level of concurrency with - * maximum `i` threads - */ - def limitThreads(i: Int): CompilationTest = - new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail) - - /** Builds a `CompilationTest` where the executed test is expected to fail - * - * This behaviour is mainly needed for the tests that test the test suite. - */ - def expectFailure: CompilationTest = - new CompilationTest(targets, times, shouldDelete, threadLimit, true) - - /** Delete all output files generated by this `CompilationTest` */ - def delete(): Unit = targets.foreach(t => delete(t.outDir)) - - private def delete(file: JFile): Unit = { - if (file.isDirectory) file.listFiles.foreach(delete) - try Files.delete(file.toPath) - catch { - case _: NoSuchFileException => // already deleted, everything's fine - } - } - } - - /** Create out directory for directory `d` */ - private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = { - val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}") - targetDir.mkdirs() - targetDir - } - - /** Create out directory for `file` */ - private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = { - val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.')) - val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir") - targetDir.mkdirs() - targetDir - } - - /** Make sure that directory string is as expected */ - private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = { - require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`") - require(outDir.last == '/', "please specify an `outDir` with a trailing slash") - } - - /** Separates directories from files and returns them as `(dirs, files)` */ - private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) = - sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) => - if (f.isDirectory) (f :: dirs, files) - else if (isSourceFile(f)) (dirs, f :: files) - else (dirs, files) - } - - /** Gets the name of the calling method via reflection. - * - * It does this in a way that needs to work both with the bootstrapped dotty - * and the non-bootstrapped version. Since the two compilers generate - * different bridges, we first need to filter out methods with the same name - * (bridges) - and then find the `@Test` method in our extending class - */ - private def getCallingMethod(): String = { - val seen = mutable.Set.empty[String] - Thread.currentThread.getStackTrace - .filter { elem => - if (seen.contains(elem.getMethodName)) false - else { seen += elem.getMethodName; true } - } - .find { elem => - val callingClass = Class.forName(elem.getClassName) - classOf[ParallelTesting].isAssignableFrom(callingClass) && - elem.getFileName != "ParallelTesting.scala" - } - .map(_.getMethodName) - .getOrElse { - throw new IllegalStateException("Unable to reflectively find calling method") - } - } - - /** Compiles a single file from the string path `f` using the supplied flags */ - def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val sourceFile = new JFile(f) - val parent = sourceFile.getParentFile - val outDir = - outDirectory + callingMethod + "/" + - sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/" - - require( - sourceFile.exists && !sourceFile.isDirectory && - (parent ne null) && parent.exists && parent.isDirectory, - s"Source file: $f, didn't exist" - ) - - val target = JointCompilationSource( - callingMethod, - Array(sourceFile), - flags, - createOutputDirsForFile(sourceFile, parent, outDir) - ) - new CompilationTest(target) - } - - /** Compiles a directory `f` using the supplied `flags`. This method does - * deep compilation, that is - it compiles all files and subdirectories - * contained within the directory `f`. - * - * By default, files are compiled in alphabetical order. An optional seed - * can be used for randomization. - */ - def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - def flatten(f: JFile): Array[JFile] = - if (f.isDirectory) f.listFiles.flatMap(flatten) - else Array(f) - - // Sort files either alphabetically or randomly using the provided seed: - val sortedFiles = flatten(sourceDir).sorted - val randomized = randomOrder match { - case None => sortedFiles - case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray - } - - // Directories in which to compile all containing files with `flags`: - val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/") - targetDir.mkdirs() - - val target = JointCompilationSource(callingMethod, randomized, flags, targetDir) - new CompilationTest(target) - } - - /** Compiles all `files` together as a single compilation run. It is given a - * `testName` since files can be in separate directories and or be otherwise - * dissociated - */ - def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" + testName + "/" - - // Directories in which to compile all containing files with `flags`: - val targetDir = new JFile(outDir) - targetDir.mkdirs() - assert(targetDir.exists, s"couldn't create target directory: $targetDir") - - val target = JointCompilationSource(callingMethod, files.map(new JFile(_)).toArray, flags, targetDir) - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(target) - } - - /** This function compiles the files and folders contained within directory - * `f` in a specific way. - * - * - Each file is compiled separately as a single compilation run - * - Each directory is compiled as a `SeparateCompilationTaret`, in this - * target all files are grouped according to the file suffix `_X` where `X` - * is a number. These groups are then ordered in ascending order based on - * the value of `X` and each group is compiled one after the other. - * - * For this function to work as expected, we use the same convention for - * directory layout as the old partest. That is: - * - * - Single files can have an associated check-file with the same name (but - * with file extension `.check`) - * - Directories can have an associated check-file, where the check file has - * the same name as the directory (with the file extension `.check`) - */ - def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - val (dirs, files) = compilationTargets(sourceDir) - - val targets = - files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++ - dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir))) - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(targets) - } - - /** This function behaves similar to `compileFilesInDir` but it ignores - * sub-directories and as such, does **not** perform separate compilation - * tests. - */ - def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - val (_, files) = compilationTargets(sourceDir) - - val targets = files.map { file => - JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir)) - } - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(targets) - } -} - -object ParallelTesting { - def isSourceFile(f: JFile): Boolean = { - val name = f.getName - name.endsWith(".scala") || name.endsWith(".java") - } -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala new file mode 100644 index 000000000..674f9e563 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala @@ -0,0 +1,1124 @@ +package dotty +package tools +package dotc +package vulpix + +import java.io.{ File => JFile } +import java.text.SimpleDateFormat +import java.util.HashMap +import java.lang.reflect.InvocationTargetException +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.nio.file.{ Files, Path, Paths, NoSuchFileException } +import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } + +import scala.io.Source +import scala.util.control.NonFatal +import scala.util.Try +import scala.collection.mutable +import scala.util.matching.Regex +import scala.util.Random + +import core.Contexts._ +import reporting.{ Reporter, TestReporter } +import reporting.diagnostic.MessageContainer +import interfaces.Diagnostic.ERROR +import dotc.util.DiffUtil + +/** A parallel testing suite whose goal is to integrate nicely with JUnit + * + * This trait can be mixed in to offer parallel testing to compile runs. When + * using this, you should be running your JUnit tests **sequentially**, as the + * test suite itself runs with a high level of concurrency. + */ +trait ParallelTesting { self => + + import ParallelTesting._ + import SummaryReport._ + + /** If the running environment supports an interactive terminal, each `Test` + * will be run with a progress bar and real time feedback + */ + def isInteractive: Boolean + + /** A regex which is used to filter which tests to run, if `None` will run + * all tests + */ + def testFilter: Option[Regex] + + /** A test source whose files or directory of files is to be compiled + * in a specific way defined by the `Test` + */ + private sealed trait TestSource { self => + def name: String + def outDir: JFile + def flags: Array[String] + + + def title: String = self match { + case self: JointCompilationSource => + if (self.files.length > 1) name + else self.files.head.getPath + + case self: SeparateCompilationSource => + self.dir.getPath + } + + /** Adds the flags specified in `newFlags0` if they do not already exist */ + def withFlags(newFlags0: String*) = { + val newFlags = newFlags0.toArray + if (!flags.containsSlice(newFlags)) self match { + case self: JointCompilationSource => + self.copy(flags = flags ++ newFlags) + case self: SeparateCompilationSource => + self.copy(flags = flags ++ newFlags) + } + else self + } + + /** Generate the instructions to redo the test from the command line */ + def buildInstructions(errors: Int, warnings: Int): String = { + val sb = new StringBuilder + val maxLen = 80 + var lineLen = 0 + + sb.append( + s"""| + |Test '$title' compiled with $errors error(s) and $warnings warning(s), + |the test can be reproduced by running:""".stripMargin + ) + sb.append("\n\n./bin/dotc ") + flags.foreach { arg => + if (lineLen > maxLen) { + sb.append(" \\\n ") + lineLen = 4 + } + sb.append(arg) + lineLen += arg.length + sb += ' ' + } + + self match { + case JointCompilationSource(_, files, _, _) => { + files.map(_.getAbsolutePath).foreach { path => + sb.append("\\\n ") + sb.append(path) + sb += ' ' + } + sb.toString + "\n\n" + } + case self: SeparateCompilationSource => { + val command = sb.toString + val fsb = new StringBuilder(command) + self.compilationGroups.foreach { files => + files.map(_.getPath).foreach { path => + fsb.append("\\\n ") + lineLen = 8 + fsb.append(path) + fsb += ' ' + } + fsb.append("\n\n") + fsb.append(command) + } + fsb.toString + "\n\n" + } + } + } + } + + /** A group of files that may all be compiled together, with the same flags + * and output directory + */ + private final case class JointCompilationSource( + name: String, + files: Array[JFile], + flags: Array[String], + outDir: JFile + ) extends TestSource { + def sourceFiles: Array[JFile] = files.filter(isSourceFile) + + override def toString() = outDir.toString + } + + /** A test source whose files will be compiled separately according to their + * suffix `_X` + */ + private final case class SeparateCompilationSource( + name: String, + dir: JFile, + flags: Array[String], + outDir: JFile + ) extends TestSource { + + /** Get the files grouped by `_X` as a list of groups, files missing this + * suffix will be put into the same group + * + * Filters out all none source files + */ + def compilationGroups: List[Array[JFile]] = + dir + .listFiles + .groupBy { file => + val name = file.getName + Try { + val potentialNumber = name + .substring(0, name.lastIndexOf('.')) + .reverse.takeWhile(_ != '_').reverse + + potentialNumber.toInt.toString + } + .toOption + .getOrElse("") + } + .toList.sortBy(_._1).map(_._2.filter(isSourceFile)) + } + + /** Each `Test` takes the `testSources` and performs the compilation and assertions + * according to the implementing class "neg", "run" or "pos". + */ + private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { + protected final val realStdout = System.out + protected final val realStderr = System.err + + /** Actual compilation run logic, the test behaviour is defined here */ + protected def compilationRunnable(testSource: TestSource): Runnable + + /** All testSources left after filtering out */ + private val filteredSources = + if (!testFilter.isDefined) testSources + else testSources.filter { + case JointCompilationSource(_, files, _, _) => + files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined) + case SeparateCompilationSource(_, dir, _, _) => + testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined + } + + /** Total amount of test sources being compiled by this test */ + val sourceCount = filteredSources.length + + private[this] var _errorCount = 0 + def errorCount: Int = _errorCount + + private[this] var _testSourcesCompiled = 0 + private def testSourcesCompiled: Int = _testSourcesCompiled + + /** Complete the current compilation with the amount of errors encountered */ + protected final def registerCompilation(errors: Int) = synchronized { + _testSourcesCompiled += 1 + _errorCount += errors + } + + private[this] var _failed = false + /** Fail the current test */ + protected[this] final def fail(): Unit = synchronized { _failed = true } + def didFail: Boolean = _failed + + protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { + val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) + addFailureInstruction(errorMsg) + failTestSource(testSource) + } + + /** Instructions on how to reproduce failed test source compilations */ + private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String] + protected final def addFailureInstruction(ins: String): Unit = + synchronized { reproduceInstructions.append(ins) } + + /** The test sources that failed according to the implementing subclass */ + private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] + protected final def failTestSource(testSource: TestSource) = synchronized { + failedTestSources.append(testSource.name + " failed") + fail() + } + + /** Prints to `System.err` if we're not suppressing all output */ + protected def echo(msg: String): Unit = + if (!suppressAllOutput) realStderr.println(msg) + + /** A single `Runnable` that prints a progress bar for the curent `Test` */ + private def createProgressMonitor: Runnable = new Runnable { + def run(): Unit = { + val start = System.currentTimeMillis + var tCompiled = testSourcesCompiled + while (tCompiled < sourceCount) { + val timestamp = (System.currentTimeMillis - start) / 1000 + val progress = (tCompiled.toDouble / sourceCount * 40).toInt + + realStdout.print( + "[" + ("=" * (math.max(progress - 1, 0))) + + (if (progress > 0) ">" else "") + + (" " * (39 - progress)) + + s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" + ) + + Thread.sleep(100) + tCompiled = testSourcesCompiled + } + // println, otherwise no newline and cursor at start of line + realStdout.println( + s"[=======================================] compiled ($sourceCount/$sourceCount, " + + s"${(System.currentTimeMillis - start) / 1000}s) " + ) + } + } + + /** Wrapper function to make sure that the compiler itself did not crash - + * if it did, the test should automatically fail. + */ + protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = + try { + if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") + op + } catch { + case NonFatal(e) => { + // if an exception is thrown during compilation, the complete test + // run should fail + failTestSource(testSource) + e.printStackTrace() + registerCompilation(1) + throw e + } + } + + protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = { + + val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath) + + def flattenFiles(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flattenFiles) + else Array(f) + + val files: Array[JFile] = files0.flatMap(flattenFiles) + + def findJarFromRuntime(partialName: String) = { + val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) + urls.find(_.contains(partialName)).getOrElse { + throw new java.io.FileNotFoundException( + s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" + ) + } + } + + def addOutDir(xs: Array[String]): Array[String] = { + val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") + if (cpAndAfter.nonEmpty) { + val (cp :: cpArg :: rest) = cpAndAfter + (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray + } + else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray + } + + def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { + val scalaLib = findJarFromRuntime("scala-library-2.") + val fullArgs = Array( + "javac", + "-classpath", + s".:$scalaLib:${targetDir.getAbsolutePath}" + ) ++ flags.takeRight(2) ++ fs + + Runtime.getRuntime.exec(fullArgs).waitFor() == 0 + } else true + + val reporter = + TestReporter.reporter(realStdout, logLevel = + if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) + + val driver = + if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } + else new Driver { + def newCompiler(implicit ctx: Context) = new Compiler + + private def ntimes(n: Int)(op: Int => Reporter): Reporter = + (emptyReporter /: (1 to n)) ((_, i) => op(i)) + + override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) = + ntimes(times) { run => + val start = System.nanoTime() + val rep = super.doCompile(comp, files) + ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms") + rep + } + } + + val allArgs = addOutDir(flags) + driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) + + val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) + assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") + + reporter + } + + private[ParallelTesting] def executeTestSuite(): this.type = { + assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`") + + if (filteredSources.nonEmpty) { + val pool = threadLimit match { + case Some(i) => JExecutors.newWorkStealingPool(i) + case None => JExecutors.newWorkStealingPool() + } + + if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) + + filteredSources.foreach { target => + pool.submit(compilationRunnable(target)) + } + + pool.shutdown() + if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { + pool.shutdownNow() + System.setOut(realStdout) + System.setErr(realStderr) + throw new TimeoutException("Compiling targets timed out") + } + + if (didFail) { + reportFailed() + failedTestSources.toSet.foreach(addFailedTest) + reproduceInstructions.iterator.foreach(addReproduceInstruction) + } + else reportPassed() + } + else echo { + testFilter + .map(r => s"""No files matched regex "$r" in test""") + .getOrElse("No tests available under target - erroneous test?") + } + + this + } + } + + private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + registerCompilation(reporter.errorCount) + + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) + val errorCount = reporters.foldLeft(0) { (acc, reporter) => + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + acc + reporter.errorCount + } + + registerCompilation(errorCount) + + if (errorCount > 0) + failTestSource(testSource) + } + } + + } + } + } + + private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + private def runMain(dir: JFile, testSource: TestSource): Array[String] = { + def renderStackTrace(ex: Throwable): String = + if (ex == null) "" + else ex.getStackTrace + .takeWhile(_.getMethodName != "invoke0") + .mkString(" ", "\n ", "") + + import java.io.{ ByteArrayOutputStream, PrintStream } + import java.net.{ URL, URLClassLoader } + + val printStream = new ByteArrayOutputStream + + try { + // Do classloading magic and running here: + val ucl = new URLClassLoader(Array(dir.toURI.toURL)) + val cls = ucl.loadClass("Test") + val meth = cls.getMethod("main", classOf[Array[String]]) + + synchronized { + try { + val ps = new PrintStream(printStream) + System.setOut(ps) + System.setErr(ps) + Console.withOut(printStream) { + Console.withErr(printStream) { + meth.invoke(null, Array("jvm")) // partest passes at least "jvm" as an arg + } + } + System.setOut(realStdout) + System.setErr(realStderr) + } catch { + case t: Throwable => + System.setOut(realStdout) + System.setErr(realStderr) + throw t + } + } + } + catch { + case ex: NoSuchMethodException => + echo(s"test in '$dir' did not contain method: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") + failTestSource(testSource) + + case ex: ClassNotFoundException => + echo(s"test in '$dir' did not contain class: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") + failTestSource(testSource) + + case ex: InvocationTargetException => + echo(s"An exception ocurred when running main: ${ex.getCause}\n${renderStackTrace(ex.getCause)}") + failTestSource(testSource) + } + printStream.toString("utf-8").lines.toArray + } + + private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { + val outputLines = runMain(dir, testSource) + val checkLines = Source.fromFile(checkFile).getLines.toArray + val sourceTitle = testSource.title + + def linesMatch = + outputLines + .zip(checkLines) + .forall { case (x, y) => x == y } + + if (outputLines.length != checkLines.length || !linesMatch) { + // Print diff to files and summary: + val diff = outputLines.zip(checkLines).map { case (act, exp) => + DiffUtil.mkColoredLineDiff(exp, act) + }.mkString("\n") + + val msg = + s"""|Output from '$sourceTitle' did not match check file. + |Diff ('e' is expected, 'a' is actual): + |""".stripMargin + diff + "\n" + echo(msg) + addFailureInstruction(msg) + + // Print build instructions to file and summary: + val buildInstr = testSource.buildInstructions(0, warnings) + addFailureInstruction(buildInstr) + + // Fail target: + failTestSource(testSource) + } + } + + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val checkFile = files.flatMap { file => + if (file.isDirectory) Nil + else { + val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check" + val checkFile = new JFile(fname) + if (checkFile.exists) List(checkFile) + else Nil + } + }.headOption + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + registerCompilation(reporter.errorCount) + (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") + val (errorCount, warningCount) = + testSource + .compilationGroups + .map(compile(_, flags, false, outDir)) + .foldLeft((0,0)) { case ((errors, warnings), reporter) => + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + (errors + reporter.errorCount, warnings + reporter.warningCount) + } + + if (errorCount > 0) fail() + + registerCompilation(errorCount) + (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) + } + } + + if (errorCount == 0 && hasCheckFile) verifier() + else if (errorCount == 0) runMain(testSource.outDir, testSource) + else if (errorCount > 0) { + echo(s"\nCompilation failed for: '$testSource'") + val buildInstr = testSource.buildInstructions(errorCount, warningCount) + addFailureInstruction(buildInstr) + failTestSource(testSource) + } + } + } + } + + private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + // In neg-tests we allow two types of error annotations, + // "nopos-error" which doesn't care about position and "error" which + // has to be annotated on the correct line number. + // + // We collect these in a map `"file:row" -> numberOfErrors`, for + // nopos errors we save them in `"file" -> numberOfNoPosErrors` + def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = { + val errorMap = new HashMap[String, Integer]() + var expectedErrors = 0 + files.filter(_.getName.endsWith(".scala")).foreach { file => + Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) => + val errors = line.sliding("// error".length).count(_.mkString == "// error") + if (errors > 0) + errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors) + + val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error") + if (noposErrors > 0) { + val nopos = errorMap.get("nopos") + val existing: Integer = if (nopos eq null) 0 else nopos + errorMap.put("nopos", noposErrors + existing) + } + + expectedErrors += noposErrors + errors + } + } + + (errorMap, expectedErrors) + } + + def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error => + val key = if (error.pos.exists) { + val fileName = error.pos.source.file.toString + s"$fileName:${error.pos.line}" + + } else "nopos" + + val errors = errorMap.get(key) + + if (errors ne null) { + if (errors == 1) errorMap.remove(key) + else errorMap.put(key, errors - 1) + true + } + else { + echo { + s"Error reported in ${error.pos.source}, but no annotation found" + } + false + } + } + + val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val sourceFiles = testSource.sourceFiles + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) + val reporter = compile(sourceFiles, flags, true, outDir) + val actualErrors = reporter.errorCount + + (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val compilationGroups = testSource.compilationGroups + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) + val reporters = compilationGroups.map(compile(_, flags, true, outDir)) + val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) + val errors = reporters.iterator.flatMap(_.errors) + (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) + } + } + + if (expectedErrors != actualErrors) { + echo { + s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" + } + failTestSource(testSource) + } + else if (hasMissingAnnotations()) { + echo { + s"\nErrors found on incorrect row numbers when compiling $testSource" + } + failTestSource(testSource) + } + else if (!errorMap.isEmpty) { + echo { + s"\nExpected error(s) have {=}: $errorMap" + } + failTestSource(testSource) + } + + registerCompilation(actualErrors) + } + } + } + + /** The `CompilationTest` is the main interface to `ParallelTesting`, it + * can be instantiated via one of the following methods: + * + * - `compileFile` + * - `compileDir` + * - `compileList` + * - `compileFilesInDir` + * - `compileShallowFilesInDir` + * + * Each compilation test can then be turned into either a "pos", "neg" or + * "run" test: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).pos() + * ``` + * + * These tests can be customized before calling one of the execution + * methods, for instance: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos() + * ``` + * + * Which would compile `i1103.scala` twice with the verbose flag as a "pos" + * test. + * + * pos tests + * ========= + * Pos tests verify that the compiler is able to compile the given + * `TestSource`s and that they generate no errors or exceptions during + * compilation + * + * neg tests + * ========= + * Neg tests are expected to generate a certain amount of errors - but not + * crash the compiler. In each `.scala` file, you specifiy the line on which + * the error will be generated, e.g: + * + * ``` + * val x: String = 1 // error + * ``` + * + * if a line generates multiple errors, you need to annotate it multiple + * times. For a line that generates two errors: + * + * ``` + * val y: String = { val y1: String = 1; 2 } // error // error + * ``` + * + * Certain errors have no position, if you need to check these annotate the + * file anywhere with `// nopos-error` + * + * run tests + * ========= + * Run tests are a superset of pos tests, they both verify compilation and + * that the compiler does not crash. In addition, run tests verify that the + * tests are able to run as expected. + * + * Run tests need to have the following form: + * + * ``` + * object Test { + * def main(args: Array[String]): Unit = () + * } + * ``` + * + * This is because the runner instantiates the `Test` class and calls the + * main method. + * + * Other definitions are allowed in the same file, but the file needs to at + * least have the `Test` object with a `main` method. + * + * To verify output you may use `.check` files. These files should share the + * name of the file or directory that they are testing. For instance: + * + * ```none + * . + * └── tests + * ├── i1513.scala + * └── i1513.check + * ``` + * + * If you are testing a directory under separate compilation, you would + * have: + * + * ```none + * . + * └── tests + * ├── myTestDir + * │ ├── T_1.scala + * │ ├── T_2.scala + * │ └── T_3.scala + * └── myTestDir.check + * ``` + * + * In the above example, `i1513.scala` and one of the files `T_X.scala` + * would contain a `Test` object with a main method. + * + * Composing tests + * =============== + * Since this is a parallel test suite, it is essential to be able to + * compose tests to take advantage of the concurrency. This is done using + * the `+` function. This function will make sure that tests being combined + * are compatible according to the `require`s in `+`. + */ + final class CompilationTest private ( + private[ParallelTesting] val targets: List[TestSource], + private[ParallelTesting] val times: Int, + private[ParallelTesting] val shouldDelete: Boolean, + private[ParallelTesting] val threadLimit: Option[Int], + private[ParallelTesting] val shouldFail: Boolean + ) { + import org.junit.Assert.fail + + private[ParallelTesting] def this(target: TestSource) = + this(List(target), 1, true, None, false) + + private[ParallelTesting] def this(targets: List[TestSource]) = + this(targets, 1, true, None, false) + + /** Compose test targets from `this` with `other` + * + * It does this, only if the two tests are compatible. Otherwise it throws + * an `IllegalArgumentException`. + * + * Grouping tests together like this allows us to take advantage of the + * concurrency offered by this test suite as each call to an executing + * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the + * maximum allowed level of concurrency. Doing this for only a few targets + * does not yield any real benefit over sequential compilation. + * + * As such, each `CompilationTest` should contain as many targets as + * possible. + */ + def +(other: CompilationTest) = { + require(other.times == times, "can't combine tests that are meant to be benchmark compiled") + require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output") + require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome") + new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail) + } + + /** Creates a "pos" test run, which makes sure that all tests pass + * compilation without generating errors and that they do not crash the + * compiler + */ + def checkCompile(): this.type = { + val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail(s"Expected no errors when compiling, but found: ${test.errorCount}") + } + else if (shouldFail && !test.didFail) { + fail("Pos test should have failed, but didn't") + } + + cleanup() + } + + /** Creates a "neg" test run, which makes sure that each test generates the + * correct amount of errors at the correct positions. It also makes sure + * that none of these tests crash the compiler + */ + def checkExpectedErrors(): this.type = { + val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Neg test shouldn't have failed, but did") + } + else if (shouldFail && !test.didFail) { + fail("Neg test should have failed, but did not") + } + + cleanup() + } + + /** Creates a "run" test run, which is a superset of "pos". In addition to + * making sure that all tests pass compilation and that they do not crash + * the compiler; it also makes sure that all tests can run with the + * expected output + */ + def checkRuns(): this.type = { + val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Run test failed, but should not") + } + else if (shouldFail && !test.didFail) { + fail("Run test should have failed, but did not") + } + + cleanup() + } + + /** Deletes output directories and files */ + private def cleanup(): this.type = { + if (shouldDelete) delete() + this + } + + /** Copies `file` to `dir` - taking into account if `file` is a directory, + * and if so copying recursively + */ + private def copyToDir(dir: JFile, file: JFile): JFile = { + val target = Paths.get(dir.getAbsolutePath, file.getName) + Files.copy(file.toPath, target, REPLACE_EXISTING) + if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _)) + target.toFile + } + + /** Builds a new `CompilationTest` where we have copied the target files to + * the out directory. This is needed for tests that modify the original + * source, such as `-rewrite` tests + */ + def copyToTarget(): CompilationTest = new CompilationTest ( + targets.map { + case target @ JointCompilationSource(_, files, _, outDir) => + target.copy(files = files.map(copyToDir(outDir,_))) + case target @ SeparateCompilationSource(_, dir, _, outDir) => + target.copy(dir = copyToDir(outDir, dir)) + }, + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which performs the compilation `i` times on + * each target + */ + def times(i: Int): CompilationTest = + new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail) + + /** Builds a `Compilationtest` which passes the verbose flag and logs the + * classpath + */ + def verbose: CompilationTest = new CompilationTest( + targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")), + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which keeps the generated output files + * + * This is needed for tests like `tastyBootstrap` which relies on first + * compiling a certain part of the project and then compiling a second + * part which depends on the first + */ + def keepOutput: CompilationTest = + new CompilationTest(targets, times, false, threadLimit, shouldFail) + + /** Builds a `CompilationTest` with a limited level of concurrency with + * maximum `i` threads + */ + def limitThreads(i: Int): CompilationTest = + new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail) + + /** Builds a `CompilationTest` where the executed test is expected to fail + * + * This behaviour is mainly needed for the tests that test the test suite. + */ + def expectFailure: CompilationTest = + new CompilationTest(targets, times, shouldDelete, threadLimit, true) + + /** Delete all output files generated by this `CompilationTest` */ + def delete(): Unit = targets.foreach(t => delete(t.outDir)) + + private def delete(file: JFile): Unit = { + if (file.isDirectory) file.listFiles.foreach(delete) + try Files.delete(file.toPath) + catch { + case _: NoSuchFileException => // already deleted, everything's fine + } + } + } + + /** Create out directory for directory `d` */ + private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = { + val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}") + targetDir.mkdirs() + targetDir + } + + /** Create out directory for `file` */ + private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = { + val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.')) + val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir") + targetDir.mkdirs() + targetDir + } + + /** Make sure that directory string is as expected */ + private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = { + require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`") + require(outDir.last == '/', "please specify an `outDir` with a trailing slash") + } + + /** Separates directories from files and returns them as `(dirs, files)` */ + private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) = + sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) => + if (f.isDirectory) (f :: dirs, files) + else if (isSourceFile(f)) (dirs, f :: files) + else (dirs, files) + } + + /** Gets the name of the calling method via reflection. + * + * It does this in a way that needs to work both with the bootstrapped dotty + * and the non-bootstrapped version. Since the two compilers generate + * different bridges, we first need to filter out methods with the same name + * (bridges) - and then find the `@Test` method in our extending class + */ + private def getCallingMethod(): String = { + val seen = mutable.Set.empty[String] + Thread.currentThread.getStackTrace + .filter { elem => + if (seen.contains(elem.getMethodName)) false + else { seen += elem.getMethodName; true } + } + .find { elem => + val callingClass = Class.forName(elem.getClassName) + classOf[ParallelTesting].isAssignableFrom(callingClass) && + elem.getFileName != "ParallelTesting.scala" + } + .map(_.getMethodName) + .getOrElse { + throw new IllegalStateException("Unable to reflectively find calling method") + } + } + + /** Compiles a single file from the string path `f` using the supplied flags */ + def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val sourceFile = new JFile(f) + val parent = sourceFile.getParentFile + val outDir = + outDirectory + callingMethod + "/" + + sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/" + + require( + sourceFile.exists && !sourceFile.isDirectory && + (parent ne null) && parent.exists && parent.isDirectory, + s"Source file: $f, didn't exist" + ) + + val target = JointCompilationSource( + callingMethod, + Array(sourceFile), + flags, + createOutputDirsForFile(sourceFile, parent, outDir) + ) + new CompilationTest(target) + } + + /** Compiles a directory `f` using the supplied `flags`. This method does + * deep compilation, that is - it compiles all files and subdirectories + * contained within the directory `f`. + * + * By default, files are compiled in alphabetical order. An optional seed + * can be used for randomization. + */ + def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + def flatten(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flatten) + else Array(f) + + // Sort files either alphabetically or randomly using the provided seed: + val sortedFiles = flatten(sourceDir).sorted + val randomized = randomOrder match { + case None => sortedFiles + case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray + } + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/") + targetDir.mkdirs() + + val target = JointCompilationSource(callingMethod, randomized, flags, targetDir) + new CompilationTest(target) + } + + /** Compiles all `files` together as a single compilation run. It is given a + * `testName` since files can be in separate directories and or be otherwise + * dissociated + */ + def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + testName + "/" + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir) + targetDir.mkdirs() + assert(targetDir.exists, s"couldn't create target directory: $targetDir") + + val target = JointCompilationSource(callingMethod, files.map(new JFile(_)).toArray, flags, targetDir) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(target) + } + + /** This function compiles the files and folders contained within directory + * `f` in a specific way. + * + * - Each file is compiled separately as a single compilation run + * - Each directory is compiled as a `SeparateCompilationTaret`, in this + * target all files are grouped according to the file suffix `_X` where `X` + * is a number. These groups are then ordered in ascending order based on + * the value of `X` and each group is compiled one after the other. + * + * For this function to work as expected, we use the same convention for + * directory layout as the old partest. That is: + * + * - Single files can have an associated check-file with the same name (but + * with file extension `.check`) + * - Directories can have an associated check-file, where the check file has + * the same name as the directory (with the file extension `.check`) + */ + def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (dirs, files) = compilationTargets(sourceDir) + + val targets = + files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++ + dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir))) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } + + /** This function behaves similar to `compileFilesInDir` but it ignores + * sub-directories and as such, does **not** perform separate compilation + * tests. + */ + def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (_, files) = compilationTargets(sourceDir) + + val targets = files.map { file => + JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir)) + } + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } +} + +object ParallelTesting { + def isSourceFile(f: JFile): Boolean = { + val name = f.getName + name.endsWith(".scala") || name.endsWith(".java") + } +} diff --git a/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java b/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java new file mode 100644 index 000000000..1d900e777 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java @@ -0,0 +1,75 @@ +package dotty.tools.dotc.vulpix; + +import org.junit.BeforeClass; +import org.junit.AfterClass; +import java.util.ArrayDeque; +import java.util.function.Supplier; +import scala.Function0; +import scala.Unit; + +import dotty.tools.dotc.reporting.TestReporter; + +/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with + * this class + */ +public class SummaryReport { + public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); + + private static TestReporter rep = TestReporter.reporter(System.out, -1); + private static ArrayDeque failedTests = new ArrayDeque<>(); + private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); + private static int passed; + private static int failed; + + public final static void reportFailed() { + failed++; + } + + public final static void reportPassed() { + passed++; + } + + public final static void addFailedTest(String msg) { + failedTests.offer(msg); + } + + public final static void addReproduceInstruction(String msg) { + reproduceInstructions.offer(msg); + } + + @BeforeClass public final static void setup() { + rep = TestReporter.reporter(System.out, -1); + failedTests = new ArrayDeque<>(); + reproduceInstructions = new ArrayDeque<>(); + } + + @AfterClass public final static void teardown() { + rep.echo( + "\n================================================================================" + + "\nTest Report" + + "\n================================================================================" + + "\n" + + passed + " passed, " + failed + " failed, " + (passed + failed) + " total" + + "\n" + ); + + failedTests + .stream() + .map(x -> " " + x) + .forEach(rep::echo); + + // If we're compiling locally, we don't need reproduce instructions + if (isInteractive) rep.flushToStdErr(); + + rep.echo(""); + + reproduceInstructions + .stream() + .forEach(rep::echo); + + // If we're on the CI, we want everything + if (!isInteractive) rep.flushToStdErr(); + + if (failed > 0) rep.flushToFile(); + } +} diff --git a/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala new file mode 100644 index 000000000..dea5aa777 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala @@ -0,0 +1,59 @@ +package dotty +package tools +package dotc +package vulpix + +import org.junit.Assert._ +import org.junit.Test + +import scala.util.control.NonFatal + +class VulpixTests extends ParallelTesting { + import CompilationTests._ + + def isInteractive = !sys.env.contains("DRONE") + def testFilter = None + + @Test def missingFile: Unit = + try { + compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors() + fail("didn't fail properly") + } + catch { + case _: IllegalArgumentException => // pass! + case NonFatal(_) => fail("wrong exception thrown") + } + + @Test def pos1Error: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile() + + @Test def negMissingAnnot: Unit = + compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negAnnotWrongLine: Unit = + compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negTooManyAnnots: Unit = + compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negNoPositionAnnot: Unit = + compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def runCompileFail: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput1: Unit = + compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput2: Unit = + compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runDiffOutput1: Unit = + compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runStackOverflow: Unit = + compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runOutRedirects: Unit = + compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() +} -- cgit v1.2.3 From 5c573d3b5b9f294920bdd7a142219a967c926bb4 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 3 Apr 2017 17:10:57 +0200 Subject: Add reproduction instr for pos separate compilation --- compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala index 674f9e563..0b5d41329 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala @@ -410,10 +410,12 @@ trait ParallelTesting { self => acc + reporter.errorCount } + def warningCount = reporters.foldLeft(0)(_ + _.warningCount) + registerCompilation(errorCount) if (errorCount > 0) - failTestSource(testSource) + echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) } } -- cgit v1.2.3 From c15b83be52ec2db7369dbdfa3db0044b3de9ff76 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 3 Apr 2017 17:12:10 +0200 Subject: Add cleanup hooks to SummaryReporter --- .../test/dotty/tools/dotc/vulpix/SummaryReport.java | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java b/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java index 1d900e777..4f2b6350b 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java +++ b/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java @@ -18,6 +18,7 @@ public class SummaryReport { private static TestReporter rep = TestReporter.reporter(System.out, -1); private static ArrayDeque failedTests = new ArrayDeque<>(); private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); + private static Supplier cleanup; private static int passed; private static int failed; @@ -37,6 +38,23 @@ public class SummaryReport { reproduceInstructions.offer(msg); } + public final static void addCleanup(Function0 func) { + // Wow, look at how neatly we - compose cleanup callbacks: + if (cleanup == null) { + cleanup = () -> { + func.apply(); + return null; + }; + } else { + Supplier oldCleanup = cleanup; + cleanup = () -> { + oldCleanup.get(); + func.apply(); + return null; + }; + } + } + @BeforeClass public final static void setup() { rep = TestReporter.reporter(System.out, -1); failedTests = new ArrayDeque<>(); @@ -71,5 +89,8 @@ public class SummaryReport { if (!isInteractive) rep.flushToStdErr(); if (failed > 0) rep.flushToFile(); + + // Perform cleanup callback: + if (cleanup != null) cleanup.get(); } } -- cgit v1.2.3 From 29691015c75daf955414aef07124d385e3f36404 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 3 Apr 2017 17:28:21 +0200 Subject: Add initial RunnerOrchestration interface --- .../tools/dotc/vulpix/RunnerOrchestration.scala | 81 ++++++++++++++++++++++ compiler/test/dotty/tools/dotc/vulpix/Status.scala | 10 +++ 2 files changed, 91 insertions(+) create mode 100644 compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala create mode 100644 compiler/test/dotty/tools/dotc/vulpix/Status.scala diff --git a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala new file mode 100644 index 000000000..64eed1035 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala @@ -0,0 +1,81 @@ +package dotty +package tools +package dotc +package vulpix + +import java.io.{ + File => JFile, + InputStream, ObjectInputStream, + OutputStream, ObjectOutputStream +} + +import scala.concurrent.duration.Duration +import scala.collection.mutable + +trait RunnerOrchestration { + + /** The maximum amount of active runners, which contain a child JVM */ + val numberOfSlaves: Int + + /** The maximum duration the child process is allowed to consume before + * getting destroyed + */ + val maxDuration: Duration + + /** Destroy and respawn process after each test */ + val safeMode: Boolean + + /** Running a `Test` class's main method from the specified `dir` */ + def runMain(dir: JFile): Status = monitor.runMain(dir) + + private[this] val monitor = new RunnerMonitor + + private class RunnerMonitor { + + def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) + + private class Runner(private var process: Process) { + def kill(): Unit = ??? + def runMain(dir: JFile): Status = ??? + } + + private def createProcess: Process = ??? + + private[this] val allRunners = + List.fill(numberOfSlaves)(new Runner(createProcess)) + + private[this] val freeRunners = mutable.Queue(allRunners: _*) + private[this] val busyRunners = mutable.Set.empty[Runner] + + private def getRunner(): Runner = synchronized { + while (freeRunners.isEmpty) wait() + + val runner = freeRunners.dequeue() + busyRunners += runner + + notify() + runner + } + + private def freeRunner(runner: Runner): Unit = synchronized { + freeRunners.enqueue(runner) + busyRunners -= runner + notify() + } + + private def withRunner[T](op: Runner => T): T = { + val runner = getRunner() + val result = op(runner) + freeRunner(runner) + result + } + + private def killAll(): Unit = allRunners.foreach(_.kill()) + + // On shutdown, we need to kill all runners: + sys.addShutdownHook(killAll()) + // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we + // need to clean up ourselves. + SummaryReport.addCleanup(killAll) + } +} diff --git a/compiler/test/dotty/tools/dotc/vulpix/Status.scala b/compiler/test/dotty/tools/dotc/vulpix/Status.scala new file mode 100644 index 000000000..34ddc1e3f --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/Status.scala @@ -0,0 +1,10 @@ +package dotty +package tools +package dotc +package vulpix + +/** The status of each call to `main` in the test applications */ +sealed trait Status extends Serializable +final case class Success(output: String) extends Status +final case class Failure(output: String) extends Status +final case object Timeout extends Status -- cgit v1.2.3 From 429066b6dacdd7670d1fce4b445d7bedc93df14d Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 3 Apr 2017 17:30:31 +0200 Subject: Implement inter-VM communication logic --- .../tools/dotc/vulpix/RunnerOrchestration.scala | 46 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala index 64eed1035..8a5d16b83 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala @@ -8,8 +8,11 @@ import java.io.{ InputStream, ObjectInputStream, OutputStream, ObjectOutputStream } +import java.util.concurrent.TimeoutException import scala.concurrent.duration.Duration +import scala.concurrent.{ Await, Future } +import scala.concurrent.ExecutionContext.Implicits.global import scala.collection.mutable trait RunnerOrchestration { @@ -35,8 +38,47 @@ trait RunnerOrchestration { def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) private class Runner(private var process: Process) { - def kill(): Unit = ??? - def runMain(dir: JFile): Status = ??? + private[this] val ois = new ObjectInputStream(process.getInputStream) + private[this] val oos = new ObjectOutputStream(process.getOutputStream) + + def kill(): Unit = { + if (process ne null) process.destroy() + process = null + } + + def runMain(dir: JFile): Status = { + assert(process ne null, + "Runner was killed and then reused without setting a new process") + + // Makes the encapsulating RunnerMonitor spawn a new runner + def respawn(): Unit = { + process.destroy() + process = createProcess + } + + // pass file to running process + oos.writeObject(dir) + + // Create a future reading the object: + val readObject = Future(ois.readObject().asInstanceOf[Status]) + + // Await result for `maxDuration` and then timout and destroy the + // process: + val status = + try Await.result(readObject, maxDuration) + catch { case _: TimeoutException => { Timeout } } + + // Handle failure of the VM: + status match { + case _ if safeMode => respawn() + case status: Failure => respawn() + case Timeout => respawn() + case _ => () + } + + // return run status: + status + } } private def createProcess: Process = ??? -- cgit v1.2.3 From a3078ee21bab6da72fa0077fe140173785e17c6a Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 4 Apr 2017 16:43:51 +0200 Subject: Fix settings having uninitialized values --- compiler/test/dotc/comptest.scala | 5 +++++ compiler/test/dotty/tools/dotc/CompilationTests.scala | 9 +++++++-- .../test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala | 10 ++++------ compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala | 4 ++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala index d283f4ef0..ab55e0271 100644 --- a/compiler/test/dotc/comptest.scala +++ b/compiler/test/dotc/comptest.scala @@ -2,8 +2,13 @@ package dotc import dotty.tools.dotc.vulpix.ParallelTesting +import scala.concurrent.duration._ + object comptest extends ParallelTesting { + def maxDuration = 3.seconds + def numberOfSlaves = 5 + def safeMode = false def isInteractive = true def testFilter = None diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index a2d632b19..241714f8b 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -6,15 +6,20 @@ import org.junit.Test import org.junit.experimental.categories.Category import scala.util.matching.Regex +import scala.concurrent.duration._ import vulpix.{ ParallelTesting, SummaryReport } @Category(Array(classOf[ParallelTesting])) class CompilationTests extends SummaryReport with ParallelTesting { import CompilationTests._ - def isInteractive: Boolean = SummaryReport.isInteractive + // Test suite configuration -------------------------------------------------- - def testFilter: Option[Regex] = sys.props.get("dotty.partest.filter").map(r => new Regex(r)) + def maxDuration = 3.seconds + def numberOfSlaves = 5 + def safeMode = sys.env.get("SAFEMODE").isDefined + def isInteractive = SummaryReport.isInteractive + def testFilter = sys.props.get("dotty.partest.filter").map(r => new Regex(r)) // Positive tests ------------------------------------------------------------ diff --git a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala index 8a5d16b83..a7da752bb 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala @@ -1,6 +1,4 @@ -package dotty -package tools -package dotc +package dotty.tools.dotc package vulpix import java.io.{ @@ -18,15 +16,15 @@ import scala.collection.mutable trait RunnerOrchestration { /** The maximum amount of active runners, which contain a child JVM */ - val numberOfSlaves: Int + def numberOfSlaves: Int /** The maximum duration the child process is allowed to consume before * getting destroyed */ - val maxDuration: Duration + def maxDuration: Duration /** Destroy and respawn process after each test */ - val safeMode: Boolean + def safeMode: Boolean /** Running a `Test` class's main method from the specified `dir` */ def runMain(dir: JFile): Status = monitor.runMain(dir) diff --git a/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala index dea5aa777..1a1775995 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala @@ -6,11 +6,15 @@ package vulpix import org.junit.Assert._ import org.junit.Test +import scala.concurrent.duration._ import scala.util.control.NonFatal class VulpixTests extends ParallelTesting { import CompilationTests._ + def maxDuration = 3.seconds + def numberOfSlaves = 5 + def safeMode = sys.env.get("SAFEMODE").isDefined def isInteractive = !sys.env.contains("DRONE") def testFilter = None -- cgit v1.2.3 From a5b9d0763583210b71220a0e2cf68a3792c5062b Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 4 Apr 2017 17:01:39 +0200 Subject: Complete subprocess communication protocol --- .../tools/dotc/vulpix/RunnerOrchestration.scala | 56 +++++++++++++++++----- compiler/test/dotty/tools/dotc/vulpix/Status.scala | 10 ---- .../test/dotty/tools/dotc/vulpix/Statuses.java | 20 ++++++++ 3 files changed, 64 insertions(+), 22 deletions(-) delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/Status.scala create mode 100644 compiler/test/dotty/tools/dotc/vulpix/Statuses.java diff --git a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala index a7da752bb..9dc808af7 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala @@ -13,6 +13,8 @@ import scala.concurrent.{ Await, Future } import scala.concurrent.ExecutionContext.Implicits.global import scala.collection.mutable +import vulpix.Statuses._ + trait RunnerOrchestration { /** The maximum amount of active runners, which contain a child JVM */ @@ -36,14 +38,30 @@ trait RunnerOrchestration { def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) private class Runner(private var process: Process) { - private[this] val ois = new ObjectInputStream(process.getInputStream) - private[this] val oos = new ObjectOutputStream(process.getOutputStream) - + private[this] var ois: ObjectInputStream = _ + private[this] var oos: ObjectOutputStream = _ + + /** Checks if `process` is still alive + * + * When `process.exitValue()` is called on an active process the caught + * exception is thrown. As such we can know if the subprocess exited or + * not. + * + * @note used for debug + */ + def isAlive: Boolean = + try { process.exitValue(); false } + catch { case _: IllegalThreadStateException => true } + + /** Destroys the underlying process and kills IO streams */ def kill(): Unit = { if (process ne null) process.destroy() process = null + ois = null + oos = null } + /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ def runMain(dir: JFile): Status = { assert(process ne null, "Runner was killed and then reused without setting a new process") @@ -52,37 +70,51 @@ trait RunnerOrchestration { def respawn(): Unit = { process.destroy() process = createProcess + ois = null + oos = null } + if (oos eq null) oos = new ObjectOutputStream(process.getOutputStream) + // pass file to running process oos.writeObject(dir) + oos.flush() // Create a future reading the object: - val readObject = Future(ois.readObject().asInstanceOf[Status]) + val readObject = Future { + if (ois eq null) ois = new ObjectInputStream(process.getInputStream) + ois.readObject().asInstanceOf[Status] + } // Await result for `maxDuration` and then timout and destroy the // process: val status = try Await.result(readObject, maxDuration) - catch { case _: TimeoutException => { Timeout } } + catch { case _: TimeoutException => new Timeout() } // Handle failure of the VM: status match { case _ if safeMode => respawn() - case status: Failure => respawn() - case Timeout => respawn() + case _: Failure => respawn() + case _: Timeout => respawn() case _ => () } - - // return run status: status } } - private def createProcess: Process = ??? + private def createProcess: Process = { + val sep = sys.props("file.separator") + val cp = sys.props("java.class.path") + val java = sys.props("java.home") + sep + "bin" + sep + "java" + new ProcessBuilder(java, "-cp", cp, "dotty.tools.dotc.vulpix.ChildMain")//classOf[ChildMain].getName) + .redirectErrorStream(true) + .redirectInput(ProcessBuilder.Redirect.PIPE) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .start() + } - private[this] val allRunners = - List.fill(numberOfSlaves)(new Runner(createProcess)) + private[this] val allRunners = List.fill(numberOfSlaves)(new Runner(createProcess)) private[this] val freeRunners = mutable.Queue(allRunners: _*) private[this] val busyRunners = mutable.Set.empty[Runner] diff --git a/compiler/test/dotty/tools/dotc/vulpix/Status.scala b/compiler/test/dotty/tools/dotc/vulpix/Status.scala deleted file mode 100644 index 34ddc1e3f..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/Status.scala +++ /dev/null @@ -1,10 +0,0 @@ -package dotty -package tools -package dotc -package vulpix - -/** The status of each call to `main` in the test applications */ -sealed trait Status extends Serializable -final case class Success(output: String) extends Status -final case class Failure(output: String) extends Status -final case object Timeout extends Status diff --git a/compiler/test/dotty/tools/dotc/vulpix/Statuses.java b/compiler/test/dotty/tools/dotc/vulpix/Statuses.java new file mode 100644 index 000000000..d5d801c8c --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/Statuses.java @@ -0,0 +1,20 @@ +package dotty.tools.dotc.vulpix; + +import java.io.Serializable; + +/** The status of each call to `main` in the test applications */ +public class Statuses { + interface Status {} + + static class Success implements Status, Serializable { + public final String output; + public Success(String output) { this.output = output; } + } + + static class Failure implements Status, Serializable { + public final String output; + public Failure(String output) { this.output = output; } + } + + static class Timeout implements Status, Serializable {} +} -- cgit v1.2.3 From eff50df8308e3e1989dcbc509c5efbc5c0d087ac Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 4 Apr 2017 21:23:29 +0200 Subject: Add `RunnerOrchestration` to `ParallelTesting` trait --- .../test/dotty/tools/dotc/CompilationTests.scala | 2 +- .../test/dotty/tools/dotc/vulpix/ChildMain.scala | 83 ++++++++++++ .../dotty/tools/dotc/vulpix/ParallelTesting.scala | 143 +++++++++------------ .../test/dotty/tools/dotc/vulpix/Statuses.java | 9 +- 4 files changed, 149 insertions(+), 88 deletions(-) create mode 100644 compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 241714f8b..fc7ab8a8d 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -15,7 +15,7 @@ class CompilationTests extends SummaryReport with ParallelTesting { // Test suite configuration -------------------------------------------------- - def maxDuration = 3.seconds + def maxDuration = 180.seconds def numberOfSlaves = 5 def safeMode = sys.env.get("SAFEMODE").isDefined def isInteractive = SummaryReport.isInteractive diff --git a/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala b/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala new file mode 100644 index 000000000..fdd602379 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala @@ -0,0 +1,83 @@ +package dotty.tools.dotc +package vulpix + +import java.io.{ + File => JFile, + InputStream, ObjectInputStream, + OutputStream, ObjectOutputStream, + ByteArrayOutputStream, PrintStream +} +import java.lang.reflect.InvocationTargetException + +import dotty.tools.dotc.vulpix.Statuses._ + +object ChildMain { + val realStdin = System.in + val realStderr = System.err + val realStdout = System.out + + private def runMain(dir: JFile): Status = { + def renderStackTrace(ex: Throwable): String = + ex.getStackTrace + .takeWhile(_.getMethodName != "invoke0") + .mkString(" ", "\n ", "") + + def resetOutDescriptors(): Unit = { + System.setOut(realStdout) + System.setErr(realStderr) + } + + import java.net.{ URL, URLClassLoader } + + val printStream = new ByteArrayOutputStream + + try { + // Do classloading magic and running here: + val ucl = new URLClassLoader(Array(dir.toURI.toURL)) + val cls = ucl.loadClass("Test") + val meth = cls.getMethod("main", classOf[Array[String]]) + + try { + val ps = new PrintStream(printStream) + System.setOut(ps) + System.setErr(ps) + Console.withOut(printStream) { + Console.withErr(printStream) { + // invoke main with "jvm" as arg + meth.invoke(null, Array("jvm")) + } + } + resetOutDescriptors() + } catch { + case t: Throwable => + resetOutDescriptors() + throw t + } + new Success(printStream.toString("utf-8")) + } + catch { + case ex: NoSuchMethodException => + val msg = s"test in '$dir' did not contain method: ${ex.getMessage}" + new Failure(msg, renderStackTrace(ex.getCause)) + + case ex: ClassNotFoundException => + val msg = s"test in '$dir' did not contain class: ${ex.getMessage}" + new Failure(msg, renderStackTrace(ex.getCause)) + + case ex: InvocationTargetException => + val msg = s"An exception ocurred when running main: ${ex.getCause}" + new Failure(msg, renderStackTrace(ex.getCause)) + } + } + + def main(args: Array[String]): Unit = { + val stdin = new ObjectInputStream(System.in); + val stdout = new ObjectOutputStream(System.out); + + while (true) { + val dir = stdin.readObject().asInstanceOf[JFile] + stdout.writeObject(runMain(dir)) + stdout.flush() + } + } +} diff --git a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala index 0b5d41329..28c2ea22b 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala @@ -6,7 +6,6 @@ package vulpix import java.io.{ File => JFile } import java.text.SimpleDateFormat import java.util.HashMap -import java.lang.reflect.InvocationTargetException import java.nio.file.StandardCopyOption.REPLACE_EXISTING import java.nio.file.{ Files, Path, Paths, NoSuchFileException } import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } @@ -24,13 +23,15 @@ import reporting.diagnostic.MessageContainer import interfaces.Diagnostic.ERROR import dotc.util.DiffUtil +import vulpix.Statuses._ + /** A parallel testing suite whose goal is to integrate nicely with JUnit * * This trait can be mixed in to offer parallel testing to compile runs. When * using this, you should be running your JUnit tests **sequentially**, as the * test suite itself runs with a high level of concurrency. */ -trait ParallelTesting { self => +trait ParallelTesting extends RunnerOrchestration { self => import ParallelTesting._ import SummaryReport._ @@ -225,14 +226,18 @@ trait ParallelTesting { self => /** The test sources that failed according to the implementing subclass */ private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] - protected final def failTestSource(testSource: TestSource) = synchronized { - failedTestSources.append(testSource.name + " failed") + protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized { + val extra = reason.map(" with reason: " + _).getOrElse("") + failedTestSources.append(testSource.name + " failed" + extra) fail() } /** Prints to `System.err` if we're not suppressing all output */ - protected def echo(msg: String): Unit = - if (!suppressAllOutput) realStderr.println(msg) + protected def echo(msg: String): Unit = if (!suppressAllOutput) { + // pad right so that output is at least as large as progress bar line + val paddingRight = " " * math.max(0, 80 - msg.length) + realStderr.println(msg + paddingRight) + } /** A single `Runnable` that prints a progress bar for the curent `Test` */ private def createProgressMonitor: Runnable = new Runnable { @@ -418,98 +423,58 @@ trait ParallelTesting { self => echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) } } - } } } private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { - private def runMain(dir: JFile, testSource: TestSource): Array[String] = { - def renderStackTrace(ex: Throwable): String = - if (ex == null) "" - else ex.getStackTrace - .takeWhile(_.getMethodName != "invoke0") - .mkString(" ", "\n ", "") - - import java.io.{ ByteArrayOutputStream, PrintStream } - import java.net.{ URL, URLClassLoader } - - val printStream = new ByteArrayOutputStream - - try { - // Do classloading magic and running here: - val ucl = new URLClassLoader(Array(dir.toURI.toURL)) - val cls = ucl.loadClass("Test") - val meth = cls.getMethod("main", classOf[Array[String]]) - - synchronized { - try { - val ps = new PrintStream(printStream) - System.setOut(ps) - System.setErr(ps) - Console.withOut(printStream) { - Console.withErr(printStream) { - meth.invoke(null, Array("jvm")) // partest passes at least "jvm" as an arg - } - } - System.setOut(realStdout) - System.setErr(realStderr) - } catch { - case t: Throwable => - System.setOut(realStdout) - System.setErr(realStderr) - throw t + private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { + runMain(dir) match { + case success: Success => { + val outputLines = success.output.lines.toArray + val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray + val sourceTitle = testSource.title + + def linesMatch = + outputLines + .zip(checkLines) + .forall { case (x, y) => x == y } + + if (outputLines.length != checkLines.length || !linesMatch) { + // Print diff to files and summary: + val diff = outputLines.zip(checkLines).map { case (act, exp) => + DiffUtil.mkColoredLineDiff(exp, act) + }.mkString("\n") + + val msg = + s"""|Output from '$sourceTitle' did not match check file. + |Diff ('e' is expected, 'a' is actual): + |""".stripMargin + diff + "\n" + echo(msg) + addFailureInstruction(msg) + + // Print build instructions to file and summary: + val buildInstr = testSource.buildInstructions(0, warnings) + addFailureInstruction(buildInstr) + + // Fail target: + failTestSource(testSource) } } - } - catch { - case ex: NoSuchMethodException => - echo(s"test in '$dir' did not contain method: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") - failTestSource(testSource) - case ex: ClassNotFoundException => - echo(s"test in '$dir' did not contain class: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}") + case failure: Failure => + echo(renderFailure(failure)) failTestSource(testSource) - case ex: InvocationTargetException => - echo(s"An exception ocurred when running main: ${ex.getCause}\n${renderStackTrace(ex.getCause)}") - failTestSource(testSource) + case _: Timeout => + echo("failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) } - printStream.toString("utf-8").lines.toArray } - private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - val outputLines = runMain(dir, testSource) - val checkLines = Source.fromFile(checkFile).getLines.toArray - val sourceTitle = testSource.title - - def linesMatch = - outputLines - .zip(checkLines) - .forall { case (x, y) => x == y } - - if (outputLines.length != checkLines.length || !linesMatch) { - // Print diff to files and summary: - val diff = outputLines.zip(checkLines).map { case (act, exp) => - DiffUtil.mkColoredLineDiff(exp, act) - }.mkString("\n") - - val msg = - s"""|Output from '$sourceTitle' did not match check file. - |Diff ('e' is expected, 'a' is actual): - |""".stripMargin + diff + "\n" - echo(msg) - addFailureInstruction(msg) - - // Print build instructions to file and summary: - val buildInstr = testSource.buildInstructions(0, warnings) - addFailureInstruction(buildInstr) - - // Fail target: - failTestSource(testSource) - } - } + private def renderFailure(failure: Failure): String = + failure.message + "\n" + failure.stacktrace protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { def run(): Unit = tryCompile(testSource) { @@ -554,7 +519,15 @@ trait ParallelTesting { self => } if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.outDir, testSource) + else if (errorCount == 0) runMain(testSource.outDir) match { + case status: Failure => + echo(renderFailure(status)) + failTestSource(testSource) + case _: Timeout => + echo("failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + case _: Success => // success! + } else if (errorCount > 0) { echo(s"\nCompilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) diff --git a/compiler/test/dotty/tools/dotc/vulpix/Statuses.java b/compiler/test/dotty/tools/dotc/vulpix/Statuses.java index d5d801c8c..bec687d01 100644 --- a/compiler/test/dotty/tools/dotc/vulpix/Statuses.java +++ b/compiler/test/dotty/tools/dotc/vulpix/Statuses.java @@ -12,8 +12,13 @@ public class Statuses { } static class Failure implements Status, Serializable { - public final String output; - public Failure(String output) { this.output = output; } + public final String message; + public final String stacktrace; + + public Failure(String message, String stacktrace) { + this.message = message; + this.stacktrace = stacktrace; + } } static class Timeout implements Status, Serializable {} -- cgit v1.2.3 From 2f1a542034ddbc6cccf321e068e5161d32839f07 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 4 Apr 2017 23:13:41 +0200 Subject: Remove partest, keeping `dotc.tests` for now --- compiler/test/dotc/tests.scala | 17 +- compiler/test/dotty/partest/DPConfig.scala | 40 -- compiler/test/dotty/partest/DPConsoleRunner.scala | 411 --------------------- compiler/test/dotty/partest/DPDirectCompiler.scala | 36 -- .../test/dotty/tools/dotc/CompilationTests.scala | 3 +- compiler/test/dotty/tools/dotc/CompilerTest.scala | 231 ++---------- project/Build.scala | 92 +---- 7 files changed, 49 insertions(+), 781 deletions(-) delete mode 100644 compiler/test/dotty/partest/DPConfig.scala delete mode 100644 compiler/test/dotty/partest/DPConsoleRunner.scala delete mode 100644 compiler/test/dotty/partest/DPDirectCompiler.scala diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index af2c88e1a..8e7337524 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -3,6 +3,7 @@ package dotc import dotty.Jars import dotty.tools.dotc.CompilerTest import dotty.tools.StdLibSources +import org.junit.experimental.categories.Category import org.junit.{Before, Test} import org.junit.Assert._ @@ -10,10 +11,20 @@ import java.io.{ File => JFile } import scala.reflect.io.Directory import scala.io.Source -// tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be executed as benchmarks. +/** Marker class to indicate sequential unit tests */ +class SequentialUnitTests + +/** WARNING + * ======= + * These are legacy, do not add tests here, see `CompilationTests.scala` + */ +@Category(Array(classOf[SequentialUnitTests])) class tests extends CompilerTest { - def isRunByJenkins: Boolean = sys.props.isDefinedAt("dotty.jenkins.build") + // tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be + // executed as benchmarks. + + def isRunByDrone: Boolean = sys.props.isDefinedAt("DRONE") val defaultOutputDir = "../out/" @@ -62,7 +73,7 @@ class tests extends CompilerTest { } implicit val defaultOptions: List[String] = noCheckOptions ++ { - if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 + if (isRunByDrone) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") } ++ checkOptions ++ classPath diff --git a/compiler/test/dotty/partest/DPConfig.scala b/compiler/test/dotty/partest/DPConfig.scala deleted file mode 100644 index 5c493f465..000000000 --- a/compiler/test/dotty/partest/DPConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -package dotty.partest - -import scala.collection.JavaConversions._ -import scala.reflect.io.Path -import java.io.File - -import scala.tools.partest.PartestDefaults - - -/** Dotty Partest runs all tests in the provided testDirs located under - * testRoot. There can be several directories with pos resp. neg tests, as - * long as the prefix is pos/neg. - * - * Each testDir can also have a __defaultFlags.flags file, which provides - * compiler flags and is used unless there's a specific flags file (e.g. for - * test pos/A.scala, if there's a pos/A.flags file those flags are used, - * otherwise pos/__defaultFlags.flags are used if the file exists). - */ -object DPConfig { - /** Options used for _running_ the run tests. - * Note that this is different from the options used when _compiling_ tests, - * those are determined by the sbt configuration. - */ - val runJVMOpts = s"-Xms64M -Xmx1024M ${PartestDefaults.javaOpts}" - - val testRoot = (Path("..") / Path("tests") / Path("partest-generated")).toString - val genLog = Path(testRoot) / Path("gen.log") - - lazy val testDirs = { - val root = new File(testRoot) - val dirs = if (!root.exists) Array.empty[String] else root.listFiles.filter(_.isDirectory).map(_.getName) - if (dirs.isEmpty) - throw new Exception("Partest did not detect any generated sources") - dirs - } - - // Tests finish faster when running in parallel, but console output is - // out of order and sometimes the compiler crashes - val runTestsInParallel = true -} diff --git a/compiler/test/dotty/partest/DPConsoleRunner.scala b/compiler/test/dotty/partest/DPConsoleRunner.scala deleted file mode 100644 index 3362d7a59..000000000 --- a/compiler/test/dotty/partest/DPConsoleRunner.scala +++ /dev/null @@ -1,411 +0,0 @@ -/* NOTE: Adapted from ScalaJSPartest.scala in - * https://github.com/scala-js/scala-js/ - * TODO make partest configurable */ - -package dotty.partest - -import dotty.tools.FatalError -import scala.reflect.io.AbstractFile -import scala.tools.partest._ -import scala.tools.partest.nest._ -import TestState.{ Pass, Fail, Crash, Uninitialized, Updated } -import ClassPath.{ join, split } -import FileManager.{ compareFiles, compareContents, joinPaths, withTempFile } -import scala.util.matching.Regex -import tools.nsc.io.{ File => NSCFile } -import java.io.{ File, PrintStream, FileOutputStream, PrintWriter, FileWriter } -import java.net.URLClassLoader - -/** Runs dotty partest from the Console, discovering test sources in - * DPConfig.testRoot that have been generated automatically by - * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are - * required by some run tests, add them to partestDeps in the sbt Build.scala. - */ -object DPConsoleRunner { - def main(args: Array[String]): Unit = { - // unfortunately sbt runTask passes args as single string - // extra jars for run tests are passed with -dottyJars ... - val jarFinder = """-dottyJars (\d*) (.*)""".r - val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined) - val (extraJars, moreArgs) = jarList match { - case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars *\".") - case jarFinder(nr, jarString) :: Nil => - val jars = jarString.split(" ").toList - val count = nr.toInt - if (jars.length < count) - sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr) - else (jars.take(count), jars.drop(count)) - case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list) - } - new DPConsoleRunner((otherArgs ::: moreArgs) mkString (" "), extraJars).runPartest - } -} - -// console runner has a suite runner which creates a test runner for each test -class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) { - override val suiteRunner = new DPSuiteRunner ( - testSourcePath = optSourcePath getOrElse DPConfig.testRoot, - fileManager = new DottyFileManager(extraJars), - updateCheck = optUpdateCheck, - failed = optFailed, - consoleArgs = args) - - override def run = {} - def runPartest = super.run -} - -class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) { - lazy val extraJarList = extraJars.map(NSCFile(_)) - override lazy val libraryUnderTest = Path(extraJars.find(_.contains("scala-library")).getOrElse("")) - override lazy val reflectUnderTest = Path(extraJars.find(_.contains("scala-reflect")).getOrElse("")) - override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse("")) -} - -class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending" - fileManager: DottyFileManager, - updateCheck: Boolean, - failed: Boolean, - consoleArgs: String, - javaCmdPath: String = PartestDefaults.javaCmd, - javacCmdPath: String = PartestDefaults.javacCmd, - scalacExtraArgs: Seq[String] = Seq.empty, - javaOpts: String = DPConfig.runJVMOpts) -extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs, javaOpts) { - - if (!DPConfig.runTestsInParallel) - sys.props("partest.threads") = "1" - - sys.props("partest.root") = "." - - // override to provide Dotty banner - override def banner: String = { - s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString} - |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main - |Generated test sources: ${PathSettings.srcDir}${File.separator} - |Test directories: ${DPConfig.testDirs.toList.mkString(", ")} - |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj - |Parallel: ${DPConfig.runTestsInParallel} - |Options: (use partest --help for usage information) ${consoleArgs} - """.stripMargin - } - - /** Some tests require a limitation of resources, tests which are compiled - * with one or more of the flags in this list will be run with - * `limitedThreads`. This is necessary because some test flags require a lot - * of memory when running the compiler and may exhaust the available memory - * when run in parallel with too many other tests. - * - * This number could be increased on the CI, but might fail locally if - * scaled too extreme - override with: - * - * ``` - * -Ddotty.tests.limitedThreads=X - * ``` - */ - def limitResourceFlags = List("-Ytest-pickler") - private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2") - - override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = { - val (limitResourceTests, parallelTests) = - kindFiles partition { kindFile => - val flags = kindFile.changeExtension("flags").fileContents - limitResourceFlags.exists(seqFlag => flags.contains(seqFlag)) - } - - val seqResults = - if (!limitResourceTests.isEmpty) { - val savedThreads = sys.props("partest.threads") - sys.props("partest.threads") = { - assert( - savedThreads == null || limitedThreads.toInt <= savedThreads.toInt, - """|Should not use more threads than the default, when the point - |is to limit the amount of resources""".stripMargin - ) - limitedThreads - } - - NestUI.echo(s"## we will run ${limitResourceTests.length} tests using ${PartestDefaults.numThreads} thread(s) in parallel") - val res = super.runTestsForFiles(limitResourceTests, kind) - - if (savedThreads != null) - sys.props("partest.threads") = savedThreads - else - sys.props.remove("partest.threads") - - res - } else Array[TestState]() - - val parResults = - if (!parallelTests.isEmpty) { - NestUI.echo(s"## we will run ${parallelTests.length} tests in parallel using ${PartestDefaults.numThreads} thread(s)") - super.runTestsForFiles(parallelTests, kind) - } else Array[TestState]() - - seqResults ++ parResults - } - - // override for DPTestRunner and redirecting compilation output to test.clog - override def runTest(testFile: File): TestState = { - val runner = new DPTestRunner(testFile, this) - - val state = - try { - runner.run match { - // Append compiler output to transcript if compilation failed, - // printed with --verbose option - case TestState.Fail(f, r@"compilation failed", transcript) => - TestState.Fail(f, r, transcript ++ runner.cLogFile.fileLines.dropWhile(_ == "")) - case res => res - } - } catch { - case t: Throwable => throw new RuntimeException(s"Error running $testFile", t) - } - reportTest(state) - runner.cleanup() - - onFinishTest(testFile, state) - } - - // override NestUI.reportTest because --show-diff doesn't work. The diff used - // seems to add each line to transcript separately, whereas NestUI assumes - // that the diff string was added as one entry in the transcript - def reportTest(state: TestState) = { - import NestUI._ - import NestUI.color._ - - if (isTerse && state.isOk) { - NestUI.reportTest(state) - } else { - echo(statusLine(state)) - if (!state.isOk && isDiffy) { - val differ = bold(red("% ")) + "diff " - state.transcript.dropWhile(s => !(s startsWith differ)) foreach (echo(_)) - // state.transcript find (_ startsWith differ) foreach (echo(_)) // original - } - } - } -} - -class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runner(testFile, suiteRunner) { - val cLogFile = SFile(logFile).changeExtension("clog") - - // override to provide DottyCompiler - override def newCompiler = new dotty.partest.DPDirectCompiler(this) - - // Adapted from nest.Runner#javac because: - // - Our classpath handling is different and we need to pass extraClassPath - // to java to get the scala-library which is required for some java tests - // - The compiler output should be redirected to cLogFile, like the output of - // dotty itself - override def javac(files: List[File]): TestState = { - import fileManager._ - import suiteRunner._ - import FileManager.joinPaths - // compile using command-line javac compiler - val args = Seq( - suiteRunner.javacCmdPath, // FIXME: Dotty deviation just writing "javacCmdPath" doesn't work - "-d", - outDir.getAbsolutePath, - "-classpath", - joinPaths(outDir :: extraClasspath ++ testClassPath) - ) ++ files.map(_.getAbsolutePath) - - pushTranscript(args mkString " ") - - val captured = StreamCapture(runCommand(args, cLogFile)) - if (captured.result) genPass() else { - cLogFile appendAll captured.stderr - cLogFile appendAll captured.stdout - genFail("java compilation failed") - } - } - - // Overriden in order to recursively get all sources that should be handed to - // the compiler. Otherwise only sources in the top dir is compiled - works - // because the compiler is on the classpath. - override def sources(file: File): List[File] = - if (file.isDirectory) - file.listFiles.toList.flatMap { f => - if (f.isDirectory) sources(f) - else if (f.isJavaOrScala) List(f) - else Nil - } - else List(file) - - // Enable me to "fix" the depth issue - remove once completed - //override def compilationRounds(file: File): List[CompileRound] = { - // val srcs = sources(file) match { - // case Nil => - // System.err.println { - // s"""|================================================================================ - // |Warning! You attempted to compile sources from: - // | $file - // |but partest was unable to find any sources - uncomment DPConsoleRunner#sources - // |================================================================================""".stripMargin - // } - // List(new File("./tests/pos/HelloWorld.scala")) // "just compile some crap" - Guillaume - // case xs => - // xs - // } - // (groupedFiles(srcs) map mixedCompileGroup).flatten - //} - - // FIXME: This is copy-pasted from nest.Runner where it is private - // Remove this once https://github.com/scala/scala-partest/pull/61 is merged - /** Runs command redirecting standard out and - * error out to output file. - */ - def runCommand(args: Seq[String], outFile: File): Boolean = { - import scala.sys.process.{ Process, ProcessLogger } - //(Process(args) #> outFile !) == 0 or (Process(args) ! pl) == 0 - val pl = ProcessLogger(outFile) - val nonzero = 17 // rounding down from 17.3 - def run: Int = { - val p = Process(args) run pl - try p.exitValue - catch { - case e: InterruptedException => - NestUI verbose s"Interrupted waiting for command to finish (${args mkString " "})" - p.destroy - nonzero - case t: Throwable => - NestUI verbose s"Exception waiting for command to finish: $t (${args mkString " "})" - p.destroy - throw t - } - finally pl.close() - } - (pl buffer run) == 0 - } - - // override to provide default dotty flags from file in directory - override def flagsForCompilation(sources: List[File]): List[String] = { - val specificFlags = super.flagsForCompilation(sources) - if (specificFlags.isEmpty) defaultFlags - else specificFlags - } - - val defaultFlags = { - val defaultFile = parentFile.listFiles.toList.find(_.getName == "__defaultFlags.flags") - defaultFile.map({ file => - SFile(file).safeSlurp.map({ content => words(content).filter(_.nonEmpty) }).getOrElse(Nil) - }).getOrElse(Nil) - } - - // override to add the check for nr of compilation errors if there's a - // target.nerr file - override def runNegTest() = runInContext { - sealed abstract class NegTestState - // Don't get confused, the neg test passes when compilation fails for at - // least one round (optionally checking the number of compiler errors and - // compiler console output) - case object CompFailed extends NegTestState - // the neg test fails when all rounds return either of these: - case class CompFailedButWrongNErr(expected: String, found: String) extends NegTestState - case object CompFailedButWrongDiff extends NegTestState - case object CompSucceeded extends NegTestState - - def nerrIsOk(reason: String) = { - val nerrFinder = """compilation failed with (\d+) errors""".r - reason match { - case nerrFinder(found) => - SFile(FileOps(testFile) changeExtension "nerr").safeSlurp match { - case Some(exp) if (exp != found) => CompFailedButWrongNErr(exp, found) - case _ => CompFailed - } - case _ => CompFailed - } - } - - // we keep the partest semantics where only one round needs to fail - // compilation, not all - val compFailingRounds = - compilationRounds(testFile) - .map { round => - val ok = round.isOk - setLastState(if (ok) genPass else genFail("compilation failed")) - (round.result, ok) - } - .filter { case (_, ok) => !ok } - - val failureStates = compFailingRounds.map({ case (result, _) => result match { - // or, OK, we'll let you crash the compiler with a FatalError if you supply a check file - case Crash(_, t, _) if !checkFile.canRead || !t.isInstanceOf[FatalError] => CompSucceeded - case Fail(_, reason, _) => if (diffIsOk) nerrIsOk(reason) else CompFailedButWrongDiff - case _ => if (diffIsOk) CompFailed else CompFailedButWrongDiff - }}) - - if (failureStates.exists({ case CompFailed => true; case _ => false })) { - true - } else { - val existsNerr = failureStates.exists({ - case CompFailedButWrongNErr(exp, found) => - nextTestActionFailing(s"wrong number of compilation errors, expected: $exp, found: $found") - true - case _ => - false - }) - - if (existsNerr) false - else { - val existsDiff = failureStates.exists({ - case CompFailedButWrongDiff => - nextTestActionFailing(s"output differs") - true - case _ => - false - }) - if (existsDiff) false - else nextTestActionFailing("expected compilation failure") - } - } - } - - // override to change check file updating to original file, not generated - override def diffIsOk: Boolean = { - // always normalize the log first - normalizeLog() - val diff = currentDiff - // if diff is not empty, is update needed? - val updating: Option[Boolean] = ( - if (diff == "") None - else Some(suiteRunner.updateCheck) - ) - pushTranscript(s"diff $logFile $checkFile") - nextTestAction(updating) { - case Some(true) => - val origCheck = SFile(checkFile.changeExtension("checksrc").fileLines(1)) - NestUI.echo("Updating original checkfile " + origCheck) - origCheck writeAll file2String(logFile) - genUpdated() - case Some(false) => - // Get a word-highlighted diff from git if we can find it - val bestDiff = if (updating.isEmpty) "" else { - if (checkFile.canRead) - gitDiff(logFile, checkFile) getOrElse { - s"diff $logFile $checkFile\n$diff" - } - else diff - } - pushTranscript(bestDiff) - genFail("output differs") - case None => genPass() // redundant default case - } getOrElse true - } - - // override to add dotty and scala jars to classpath - override def extraClasspath = - suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath - - - // FIXME: Dotty deviation: error if return type is omitted: - // overriding method cleanup in class Runner of type ()Unit; - // method cleanup of type => Boolean | Unit has incompatible type - - // override to keep class files if failed and delete clog if ok - override def cleanup: Unit = if (lastState.isOk) { - logFile.delete - cLogFile.delete - Directory(outDir).deleteRecursively - } -} diff --git a/compiler/test/dotty/partest/DPDirectCompiler.scala b/compiler/test/dotty/partest/DPDirectCompiler.scala deleted file mode 100644 index 410dac338..000000000 --- a/compiler/test/dotty/partest/DPDirectCompiler.scala +++ /dev/null @@ -1,36 +0,0 @@ -package dotty.partest - -import dotty.tools.dotc.reporting.ConsoleReporter -import scala.tools.partest.{ TestState, nest } -import java.io.{ File, PrintWriter, FileWriter } - - -/* NOTE: Adapted from partest.DirectCompiler */ -class DPDirectCompiler(runner: DPTestRunner) extends nest.DirectCompiler(runner) { - - override def compile(opts0: List[String], sources: List[File]): TestState = { - val clogFWriter = new FileWriter(runner.cLogFile.jfile, true) - val clogWriter = new PrintWriter(clogFWriter, true) - clogWriter.println("\ncompiling " + sources.mkString(" ") + "\noptions: " + opts0.mkString(" ")) - - try { - val processor = - if (opts0.exists(_.startsWith("#"))) dotty.tools.dotc.Bench else dotty.tools.dotc.Main - val clogger = new ConsoleReporter(writer = clogWriter) - val reporter = processor.process((sources.map(_.toString) ::: opts0).toArray, clogger) - if (!reporter.hasErrors) runner.genPass() - else { - clogWriter.println(reporter.summary) - runner.genFail(s"compilation failed with ${reporter.errorCount} errors") - } - } catch { - case t: Throwable => - t.printStackTrace - t.printStackTrace(clogWriter) - runner.genCrash(t) - } finally { - clogFWriter.close - clogWriter.close - } - } -} diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index fc7ab8a8d..4d5ab6963 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -3,13 +3,12 @@ package tools package dotc import org.junit.Test -import org.junit.experimental.categories.Category import scala.util.matching.Regex import scala.concurrent.duration._ + import vulpix.{ ParallelTesting, SummaryReport } -@Category(Array(classOf[ParallelTesting])) class CompilationTests extends SummaryReport with ParallelTesting { import CompilationTests._ diff --git a/compiler/test/dotty/tools/dotc/CompilerTest.scala b/compiler/test/dotty/tools/dotc/CompilerTest.scala index f35f9f919..c5234ccca 100644 --- a/compiler/test/dotty/tools/dotc/CompilerTest.scala +++ b/compiler/test/dotty/tools/dotc/CompilerTest.scala @@ -2,7 +2,6 @@ package dotty.tools.dotc import repl.TestREPL import core.Contexts._ -import dotty.partest.DPConfig import interfaces.Diagnostic.ERROR import reporting._ import diagnostic.MessageContainer @@ -11,33 +10,11 @@ import config.CompilerCommand import dotty.tools.io.PlainFile import scala.collection.mutable.ListBuffer import scala.reflect.io.{ Path, Directory, File => SFile, AbstractFile } -import scala.tools.partest.nest.{ FileManager, NestUI } import scala.annotation.tailrec import java.io.{ RandomAccessFile, File => JFile } -/** This class has two modes: it can directly run compiler tests, or it can - * generate the necessary file structure for partest in the directory - * DPConfig.testRoot. Both modes are regular JUnit tests. Which mode is used - * depends on the existence of the tests/locks/partest-ppid.lock file which is - * created by sbt to trigger partest generation. Sbt will then run partest on - * the generated sources. - * - * Through overriding the partestableXX methods, tests can always be run as - * JUnit compiler tests. Run tests cannot be run by JUnit, only by partest. - * - * A test can either be a file or a directory. Partest will generate a - * -.log file with output of failed tests. Partest reads compiler - * flags and the number of errors expected from a neg test from .flags - * and .nerr files (also generated). The test is in a parent directory - * that determines the kind of test: - * - pos: checks that compilation succeeds - * - neg: checks that compilation fails with the given number of errors - * - run: compilation succeeds, partest: test run generates the output in - * .check. Run tests always need to be: - * object Test { def main(args: Array[String]): Unit = ... } - * Classpath jars can be added to partestDeps in the sbt Build.scala. - */ +/** Legacy compiler tests that run single threaded */ abstract class CompilerTest { /** Override with output dir of test so it can be patched. Partest expects @@ -49,32 +26,9 @@ abstract class CompilerTest { def partestableDir(prefix: String, dirName: String, args: List[String]) = true def partestableList(testName: String, files: List[String], args: List[String]) = true - val generatePartestFiles = { - /* Because we fork in test, the JVM in which this JUnit test runs has a - * different pid from the one that started the partest. But the forked VM - * receives the pid of the parent as system property. If the lock file - * exists, the parent is requesting partest generation. This mechanism - * allows one sbt instance to run test (JUnit only) and another partest. - * We cannot run two instances of partest at the same time, because they're - * writing to the same directories. The sbt lock file generation prevents - * this. - */ - val pid = System.getProperty("partestParentID") - if (pid == null) - false - else - new JFile(".." + JFile.separator + "tests" + JFile.separator + "locks" + JFile.separator + s"partest-$pid.lock").exists - } - - // Delete generated files from previous run and create new log - val logFile = if (!generatePartestFiles) None else Some(CompilerTest.init) - /** Always run with JUnit. */ - def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = { - if (generatePartestFiles) - log("WARNING: compileLine will always run with JUnit, no partest files generated.") + def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = compileArgs(cmdLine.split("\n"), Nil) - } /** Compiles the given code file. * @@ -88,36 +42,22 @@ abstract class CompilerTest { (implicit defaultOptions: List[String]): Unit = { val filePath = s"$prefix$fileName$extension" val expErrors = expectedErrors(filePath) - if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions)) { - if (runTest) - log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension") - if (args.contains("-rewrite")) { - val file = new PlainFile(filePath) - val data = file.toByteArray - // compile with rewrite - compileArgs((filePath :: args).toArray, expErrors) - // compile again, check that file now compiles without -language:Scala2 - val plainArgs = args.filter(arg => arg != "-rewrite" && arg != "-language:Scala2") - compileFile(prefix, fileName, plainArgs, extension, runTest) - // restore original test file - val out = file.output - out.write(data) - out.close() - } - else compileArgs((filePath :: args).toArray, expErrors) - } else { - val kind = testKind(prefix, runTest) - log(s"generating partest files for test file: $prefix$fileName$extension of kind $kind") - - val sourceFile = new JFile(prefix + fileName + extension) - if (sourceFile.exists) { - val firstDest = SFile(DPConfig.testRoot + JFile.separator + kind + JFile.separator + fileName + extension) - val xerrors = expErrors.map(_.totalErrors).sum - computeDestAndCopyFiles(sourceFile, firstDest, kind, args ++ defaultOptions, xerrors.toString) - } else { - throw new java.io.FileNotFoundException(s"Unable to locate test file $prefix$fileName") - } + if (runTest) + log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension") + if (args.contains("-rewrite")) { + val file = new PlainFile(filePath) + val data = file.toByteArray + // compile with rewrite + compileArgs((filePath :: args).toArray, expErrors) + // compile again, check that file now compiles without -language:Scala2 + val plainArgs = args.filter(arg => arg != "-rewrite" && arg != "-language:Scala2") + compileFile(prefix, fileName, plainArgs, extension, runTest) + // restore original test file + val out = file.output + out.write(data) + out.close() } + else compileArgs((filePath :: args).toArray, expErrors) } def runFile(prefix: String, fileName: String, args: List[String] = Nil, extension: String = ".scala") (implicit defaultOptions: List[String]): Unit = { @@ -167,33 +107,11 @@ abstract class CompilerTest { val expErrors = expectedErrors(filePaths.toList) (filePaths, javaFilePaths, normArgs, expErrors) } - if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions)) { - if (runTest) - log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName") - val (filePaths, javaFilePaths, normArgs, expErrors) = computeFilePathsAndExpErrors - compileWithJavac(javaFilePaths, Array.empty) // javac needs to run first on dotty-library - compileArgs(javaFilePaths ++ filePaths ++ normArgs, expErrors) - } else { - val (sourceDir, flags, deep) = args match { - case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep") - case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow") - } - val kind = testKind(prefix, runTest) - log(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind") - - if (sourceDir.exists) { - val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName) - val xerrors = if (isNegTest(prefix)) { - val (_, _, _, expErrors) = computeFilePathsAndExpErrors - expErrors.map(_.totalErrors).sum - } else 0 - computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString) - if (deep == "deep") - Directory(sourceDir).deleteRecursively - } else { - throw new java.io.FileNotFoundException(s"Unable to locate test dir $prefix$dirName") - } - } + if (runTest) + log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName") + val (filePaths, javaFilePaths, normArgs, expErrors) = computeFilePathsAndExpErrors + compileWithJavac(javaFilePaths, Array.empty) // javac needs to run first on dotty-library + compileArgs(javaFilePaths ++ filePaths ++ normArgs, expErrors) } def runDir(prefix: String, dirName: String, args: List[String] = Nil) (implicit defaultOptions: List[String]): Unit = @@ -222,19 +140,8 @@ abstract class CompilerTest { /** Compiles the given list of code files. */ def compileList(testName: String, files: List[String], args: List[String] = Nil) (implicit defaultOptions: List[String]): Unit = { - if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions)) { - val expErrors = expectedErrors(files) - compileArgs((files ++ args).toArray, expErrors) - } else { - val destDir = Directory(DPConfig.testRoot + JFile.separator + testName) - files.foreach({ file => - val sourceFile = new JFile(file) - val destFile = destDir / (if (file.startsWith("../")) file.substring(3) else file) - recCopyFiles(sourceFile, destFile) - }) - compileDir(DPConfig.testRoot + JFile.separator, testName, args) - destDir.deleteRecursively - } + val expErrors = expectedErrors(files) + compileArgs((files ++ args).toArray, expErrors) } // ========== HELPERS ============= @@ -425,60 +332,6 @@ abstract class CompilerTest { } import Difference._ - /** The same source might be used for several partest test cases (e.g. with - * different flags). Detects existing versions and computes the path to be - * used for this version, e.g. testname_v1 for the first alternative. */ - private def computeDestAndCopyFiles(source: JFile, dest: Path, kind: String, oldFlags: List[String], nerr: String, - nr: Int = 0, oldOutput: String = defaultOutputDir): Unit = { - - val partestOutput = dest.jfile.getParentFile + JFile.separator + dest.stripExtension + "-" + kind + ".obj" - - val altOutput = - source.getParentFile.getAbsolutePath.map(x => if (x == JFile.separatorChar) '_' else x) - - val (beforeCp, remaining) = oldFlags - .map(f => if (f == oldOutput) partestOutput else f) - .span(_ != "-classpath") - val flags = beforeCp ++ List("-classpath", (partestOutput :: remaining.drop(1)).mkString(":")) - - val difference = getExisting(dest).isDifferent(source, flags, nerr) - difference match { - case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind) - case ExistsSame => // nothing else to do - case ExistsDifferent => - val nextDest = dest.parent / (dest match { - case d: Directory => - val newVersion = replaceVersion(d.name, nr).getOrElse(altOutput) - Directory(newVersion) - case f => - val newVersion = replaceVersion(f.stripExtension, nr).getOrElse(altOutput) - SFile(newVersion).addExtension(f.extension) - }) - computeDestAndCopyFiles(source, nextDest, kind, flags, nerr, nr + 1, partestOutput) - } - } - - /** Copies the test sources. Creates flags, nerr, check and output files. */ - private def copyFiles(sourceFile: Path, dest: Path, partestOutput: String, flags: List[String], nerr: String, kind: String) = { - recCopyFiles(sourceFile, dest) - - new JFile(partestOutput).mkdirs - - if (flags.nonEmpty) - dest.changeExtension("flags").createFile(true).writeAll(flags.mkString(" ")) - if (nerr != "0") - dest.changeExtension("nerr").createFile(true).writeAll(nerr) - sourceFile.changeExtension("check").ifFile({ check => - if (kind == "run") { - FileManager.copyFile(check.jfile, dest.changeExtension("check").jfile) - dest.changeExtension("checksrc").createFile(true).writeAll("check file generated from source:\n" + check.toString) - } else { - log(s"WARNING: ignoring $check for test kind $kind") - } - }) - - } - /** Recursively copy over source files and directories, excluding extensions * that aren't in extensionsToCopy. */ private def recCopyFiles(sourceFile: Path, dest: Path): Unit = { @@ -576,38 +429,6 @@ abstract class CompilerTest { } } - /** Creates a temporary directory and copies all (deep) files over, thus - * flattening the directory structure. */ - private def flattenDir(prefix: String, dirName: String): JFile = { - val destDir = Directory(DPConfig.testRoot + JFile.separator + "_temp") - Directory(prefix + dirName).deepFiles.foreach(source => recCopyFiles(source, destDir / source.name)) - destDir.jfile - } - - /** Write either to console (JUnit) or log file (partest). */ - private def log(msg: String) = logFile.map(_.appendAll(msg + "\n")).getOrElse(println(msg)) -} - -object CompilerTest extends App { - - /** Deletes generated partest sources from a previous run, recreates - * directory and returns the freshly created log file. */ - lazy val init: SFile = { - scala.reflect.io.Directory(DPConfig.testRoot).deleteRecursively - new JFile(DPConfig.testRoot).mkdirs - val log = DPConfig.genLog.createFile(true) - println(s"CompilerTest is generating tests for partest, log: $log") - log - } - -// val dotcDir = "/Users/odersky/workspace/dotty/src/dotty/" - -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "CompilationUnit") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Compiler") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Driver") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Main") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Run") - -// new CompilerTest().compileDir(dotcDir + "tools/dotc") - // new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Run") + /** Write either to console */ + private def log(msg: String) = println(msg) } diff --git a/project/Build.scala b/project/Build.scala index bb02416cc..391799ef7 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -134,27 +134,7 @@ object Build { settings( triggeredMessage in ThisBuild := Watched.clearWhenTriggered, - addCommandAlias("run", "dotty-compiler/run") ++ - addCommandAlias( - "partest", - ";publishLocal" + // Non-bootstrapped dotty needs to be published first - ";dotty-compiler-bootstrapped/lockPartestFile" + - ";dotty-compiler-bootstrapped/test:test" + - ";dotty-compiler-bootstrapped/runPartestRunner" - ) ++ - addCommandAlias( - "partest-only", - ";publishLocal" + // Non-bootstrapped dotty needs to be published first - ";dotty-compiler-bootstrapped/lockPartestFile" + - ";dotty-compiler-bootstrapped/test:test-only dotc.tests" + - ";dotty-compiler-bootstrapped/runPartestRunner" - ) ++ - addCommandAlias( - "partest-only-no-bootstrap", - ";dotty-compiler/lockPartestFile" + - ";dotty-compiler/test:test-only dotc.tests" + - ";dotty-compiler/runPartestRunner" - ) + addCommandAlias("run", "dotty-compiler/run") ). settings(publishing) @@ -287,43 +267,6 @@ object Build { "org.scala-lang" % "scala-reflect" % scalacVersion, "org.scala-lang" % "scala-library" % scalacVersion % "test"), - // start partest specific settings: - libraryDependencies += "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test", - testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }), - // this option is needed so that partest doesn't run - partestDeps := Seq( - scalaCompiler, - "org.scala-lang" % "scala-reflect" % scalacVersion, - "org.scala-lang" % "scala-library" % scalacVersion % "test" - ), - lockPartestFile := { - // When this file is present, running `test` generates the files for - // partest. Otherwise it just executes the tests directly. - val lockDir = partestLockFile.getParentFile - lockDir.mkdirs - // Cannot have concurrent partests as they write to the same directory. - if (lockDir.list.size > 0) - throw new RuntimeException("ERROR: sbt partest: another partest is already running, pid in lock file: " + lockDir.list.toList.mkString(" ")) - partestLockFile.createNewFile - partestLockFile.deleteOnExit - }, - runPartestRunner := Def.inputTaskDyn { - // Magic! This is both an input task and a dynamic task. Apparently - // command line arguments get passed to the last task in an aliased - // sequence (see partest alias below), so this works. - val args = Def.spaceDelimited("").parsed - val jars = List( - (packageBin in Compile).value.getAbsolutePath, - packageAll.value("dotty-library"), - packageAll.value("dotty-interfaces") - ) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome) - val dottyJars = - s"""-dottyJars ${jars.length + 2} dotty.jar dotty-lib.jar ${jars.mkString(" ")}""" - // Provide the jars required on the classpath of run tests - runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars + " " + args.mkString(" ")) - }.evaluated, - // end partest specific settings - // enable improved incremental compilation algorithm incOptions := incOptions.value.withNameHashing(true), @@ -389,6 +332,12 @@ object Build { "--run-listener=dotty.tools.ContextEscapeDetector" ), + // Ignore old sequential unit tests when running `test` in sbt + testOptions in Test += Tests.Argument( + TestFrameworks.JUnit, + "--exclude-categories=dotc.SequentialUnitTests" + ), + /* Add the sources of scalajs-ir. * To guarantee that dotty can bootstrap without depending on a version * of scalajs-ir built with a different Scala compiler, we add its @@ -471,35 +420,10 @@ object Build { "-Ddotty.tests.classes.compiler=" + pA("dotty-compiler") ) - ("-DpartestParentID=" + pid) :: jars ::: tuning ::: agentOptions ::: ci_build ::: path.toList + jars ::: tuning ::: agentOptions ::: ci_build ::: path.toList } ) - // Partest tasks - lazy val partestDeps = - SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies") - lazy val runPartestRunner = - InputKey[Unit]("runPartestRunner", "Runs partest") - lazy val lockPartestFile = - TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-.lock") - lazy val partestLockFile = - new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock") - - def pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0)) - - def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match { - case Some(home) => - modules.map({ module => - val file = Path(home) / Path("cache") / - Path(module.organization) / Path(module.name) / Path("jars") / - Path(module.name + "-" + module.revision + ".jar") - if (!file.isFile) throw new RuntimeException("ERROR: sbt getJarPaths: dependency jar not found: " + file) - else file.jfile.getAbsolutePath - }) - case None => throw new RuntimeException("ERROR: sbt getJarPaths: ivyHome not defined") - } - // end partest tasks - lazy val `dotty-compiler` = project.in(file("compiler")). dependsOn(`dotty-interfaces`). dependsOn(`dotty-library`). -- cgit v1.2.3 From 7dcfbd71b1c05b4bc5e8c7e1da94fd99600e740f Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 10:50:40 +0200 Subject: Rename filter shorthand to vulpix and update `drone.yml` --- .drone.yml | 10 ++++------ .drone.yml.sig | 2 +- compiler/test/dotc/tests.scala | 5 +---- project/Build.scala | 28 +++++++++++++--------------- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/.drone.yml b/.drone.yml index ed772eb32..c6e4f53e4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -40,9 +40,7 @@ pipeline: matrix: TEST: - - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;test ;dotty-bin-tests/test - - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;publishLocal ;dotty-bootstrapped/test - - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;partest-only-no-bootstrap --show-diff --verbose - - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.hydra.ParallelTesting") ;partest-only --show-diff --verbose - - ;dotty-compiler/testOnly dotty.tools.dotc.CompilationTests - - ;publishLocal ;dotty-bootstrapped/testOnly dotty.tools.dotc.CompilationTests + - test + - ;publishLocal ;dotty-bootstrapped/test + - legacyTests + - dotty-bin-tests/test diff --git a/.drone.yml.sig b/.drone.yml.sig index 046907fff..830de1a92 100644 --- a/.drone.yml.sig +++ b/.drone.yml.sig @@ -1 +1 @@ -eyJhbGciOiJIUzI1NiJ9.IyBBZnRlciB1cGRhdGluZyB0aGlzIGZpbGUsIHlvdSBuZWVkIHRvIHJlLXNpZ24gaXQ6CiMKIyAtIEluc3RhbGwgW2Ryb25lLWNsaV0oaHR0cDovL3JlYWRtZS5kcm9uZS5pby91c2FnZS9nZXR0aW5nLXN0YXJ0ZWQtY2xpLykKIyAtIENvcHkgeW91ciB0b2tlbiBmcm9tICBodHRwOi8vZG90dHktY2kuZXBmbC5jaC9hY2NvdW50IChDbGljayBTSE9XIFRPS0VOKQojIC0gKGV4cG9ydCBEUk9ORV9UT0tFTj15b3VyLXRva2VuOyBleHBvcnQgRFJPTkVfU0VSVkVSPWh0dHA6Ly9kb3R0eS1jaS5lcGZsLmNoOyBkcm9uZSBzaWduIGxhbXBlcGZsL2RvdHR5KQoKcGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXg0MDk2bSAtSi1YWDpSZXNlcnZlZENvZGVDYWNoZVNpemU9NTEybSAtSi1YWDpNYXhNZXRhc3BhY2VTaXplPTEwMjRtIC1EZG90dHkuZHJvbmUubWVtPTQwOTZtICIke1RFU1R9IgogICAgd2hlbjoKICAgICAgYnJhbmNoOgogICAgICAgIGV4Y2x1ZGU6IGdoLXBhZ2VzCgogIGRvY3VtZW50YXRpb246CiAgICBpbWFnZTogbGFtcGVwZmwvZG90dHk6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBjb21tYW5kczoKICAgICAgLSAuL3Byb2plY3Qvc2NyaXB0cy9nZW5Eb2NzICIke1RFU1R9IiAkQk9UX1BBU1MKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGdpdHRlcjoKICAgIGltYWdlOiBwbHVnaW5zL2dpdHRlcgogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgogIHNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIGNoYW5uZWw6IGRvdHR5CiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgogICAgICBzdGF0dXM6IGNoYW5nZWQKCm1hdHJpeDoKICBURVNUOgogICAgLSA7c2V0IHRlc3RPcHRpb25zIGluIExvY2FsUHJvamVjdCgiZG90dHktY29tcGlsZXIiKSArPSBUZXN0cy5Bcmd1bWVudChUZXN0RnJhbWV3b3Jrcy5KVW5pdCwgIi0tZXhjbHVkZS1jYXRlZ29yaWVzPWRvdHR5LnRvb2xzLmRvdGMuUGFyYWxsZWxUZXN0aW5nIikgO3Rlc3QgO2RvdHR5LWJpbi10ZXN0cy90ZXN0CiAgICAtIDtzZXQgdGVzdE9wdGlvbnMgaW4gTG9jYWxQcm9qZWN0KCJkb3R0eS1jb21waWxlci1ib290c3RyYXBwZWQiKSArPSBUZXN0cy5Bcmd1bWVudChUZXN0RnJhbWV3b3Jrcy5KVW5pdCwgIi0tZXhjbHVkZS1jYXRlZ29yaWVzPWRvdHR5LnRvb2xzLmRvdGMuUGFyYWxsZWxUZXN0aW5nIikgO3B1Ymxpc2hMb2NhbCA7ZG90dHktYm9vdHN0cmFwcGVkL3Rlc3QKICAgIC0gO3NldCB0ZXN0T3B0aW9ucyBpbiBMb2NhbFByb2plY3QoImRvdHR5LWNvbXBpbGVyIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDtwYXJ0ZXN0LW9ubHktbm8tYm9vdHN0cmFwIC0tc2hvdy1kaWZmIC0tdmVyYm9zZQogICAgLSA7c2V0IHRlc3RPcHRpb25zIGluIExvY2FsUHJvamVjdCgiZG90dHktY29tcGlsZXItYm9vdHN0cmFwcGVkIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDtwYXJ0ZXN0LW9ubHkgLS1zaG93LWRpZmYgLS12ZXJib3NlCiAgICAtIDtkb3R0eS1jb21waWxlci90ZXN0T25seSBkb3R0eS50b29scy5kb3RjLkNvbXBpbGF0aW9uVGVzdHMKICAgIC0gO3B1Ymxpc2hMb2NhbCA7ZG90dHktYm9vdHN0cmFwcGVkL3Rlc3RPbmx5IGRvdHR5LnRvb2xzLmRvdGMuQ29tcGlsYXRpb25UZXN0cwo.z4tT2XteNQ9O0oTt_l26-K2mhgBHKyWQf6k_IRdZvgQ \ No newline at end of file +eyJhbGciOiJIUzI1NiJ9.IyBBZnRlciB1cGRhdGluZyB0aGlzIGZpbGUsIHlvdSBuZWVkIHRvIHJlLXNpZ24gaXQ6CiMKIyAtIEluc3RhbGwgW2Ryb25lLWNsaV0oaHR0cDovL3JlYWRtZS5kcm9uZS5pby91c2FnZS9nZXR0aW5nLXN0YXJ0ZWQtY2xpLykKIyAtIENvcHkgeW91ciB0b2tlbiBmcm9tICBodHRwOi8vZG90dHktY2kuZXBmbC5jaC9hY2NvdW50IChDbGljayBTSE9XIFRPS0VOKQojIC0gKGV4cG9ydCBEUk9ORV9UT0tFTj15b3VyLXRva2VuOyBleHBvcnQgRFJPTkVfU0VSVkVSPWh0dHA6Ly9kb3R0eS1jaS5lcGZsLmNoOyBkcm9uZSBzaWduIGxhbXBlcGZsL2RvdHR5KQoKcGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXg0MDk2bSAtSi1YWDpSZXNlcnZlZENvZGVDYWNoZVNpemU9NTEybSAtSi1YWDpNYXhNZXRhc3BhY2VTaXplPTEwMjRtIC1EZG90dHkuZHJvbmUubWVtPTQwOTZtICIke1RFU1R9IgogICAgd2hlbjoKICAgICAgYnJhbmNoOgogICAgICAgIGV4Y2x1ZGU6IGdoLXBhZ2VzCgogIGRvY3VtZW50YXRpb246CiAgICBpbWFnZTogbGFtcGVwZmwvZG90dHk6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBjb21tYW5kczoKICAgICAgLSAuL3Byb2plY3Qvc2NyaXB0cy9nZW5Eb2NzICIke1RFU1R9IiAkQk9UX1BBU1MKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGdpdHRlcjoKICAgIGltYWdlOiBwbHVnaW5zL2dpdHRlcgogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgogIHNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIGNoYW5uZWw6IGRvdHR5CiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgogICAgICBzdGF0dXM6IGNoYW5nZWQKCm1hdHJpeDoKICBURVNUOgogICAgLSB0ZXN0CiAgICAtIDtwdWJsaXNoTG9jYWwgO2RvdHR5LWJvb3RzdHJhcHBlZC90ZXN0CiAgICAtIGxlZ2FjeVRlc3RzCiAgICAtIGRvdHR5LWJpbi10ZXN0cy90ZXN0Cg.7jaA1Gh5FpzKvXQsaf2_of5tUEMBcR_3Mzo0wL8pE3E \ No newline at end of file diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index 8e7337524..4bb09fd02 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -11,14 +11,11 @@ import java.io.{ File => JFile } import scala.reflect.io.Directory import scala.io.Source -/** Marker class to indicate sequential unit tests */ -class SequentialUnitTests - /** WARNING * ======= * These are legacy, do not add tests here, see `CompilationTests.scala` */ -@Category(Array(classOf[SequentialUnitTests])) +@Category(Array(classOf[java.lang.Exception])) class tests extends CompilerTest { // tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be diff --git a/project/Build.scala b/project/Build.scala index 391799ef7..04e75de4c 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -47,8 +47,8 @@ object Build { // Spawns a repl with the correct classpath lazy val repl = inputKey[Unit]("run the REPL with correct classpath") - // Run tests with filter - lazy val filterTest = inputKey[Unit]("runs integration test with the supplied filter") + // Run tests with filter through vulpix test suite + lazy val vulpix = inputKey[Unit]("runs integration test with the supplied filter") // Used to compile files similar to ./bin/dotc script lazy val dotc = @@ -134,7 +134,9 @@ object Build { settings( triggeredMessage in ThisBuild := Watched.clearWhenTriggered, - addCommandAlias("run", "dotty-compiler/run") + addCommandAlias("run", "dotty-compiler/run") ++ + addCommandAlias("test", "testOnly -- --exclude-categories=java.lang.Exception") ++ + addCommandAlias("legacyTests", "dotty-compiler/testOnly dotc.tests") ). settings(publishing) @@ -142,7 +144,8 @@ object Build { lazy val `dotty-bootstrapped` = project. aggregate(`dotty-library-bootstrapped`, `dotty-compiler-bootstrapped`). settings( - publishArtifact := false + publishArtifact := false, + addCommandAlias("test", "testOnly -- --exclude-categories=java.lang.Exception") ) lazy val `dotty-interfaces` = project.in(file("interfaces")). @@ -283,12 +286,13 @@ object Build { ) }.evaluated, - filterTest := Def.inputTaskDyn { + vulpix := Def.inputTaskDyn { val args: Seq[String] = spaceDelimited("").parsed - testOptions := Seq() - (testOnly in Test).toTask( - " dotty.tools.dotc.CompilationTests -- -Ddotty.partest.filter=" + args.head - ) + val cmd = " dotty.tools.dotc.CompilationTests" + { + if (args.nonEmpty) " -- -Ddotty.tests.filter=" + args.mkString(" ") + else "" + } + (testOnly in Test).toTask(cmd) }.evaluated, // Override run to be able to run compiled classfiles @@ -332,12 +336,6 @@ object Build { "--run-listener=dotty.tools.ContextEscapeDetector" ), - // Ignore old sequential unit tests when running `test` in sbt - testOptions in Test += Tests.Argument( - TestFrameworks.JUnit, - "--exclude-categories=dotc.SequentialUnitTests" - ), - /* Add the sources of scalajs-ir. * To guarantee that dotty can bootstrap without depending on a version * of scalajs-ir built with a different Scala compiler, we add its -- cgit v1.2.3 From 923533ea86b53b90e343e4fc0f88956996a2ed5b Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 16:12:43 +0200 Subject: Move vulpix to `dotty.tools.vulpix` --- compiler/test/dotc/comptest.scala | 2 +- .../test/dotty/tools/dotc/vulpix/ChildMain.scala | 83 -- .../dotty/tools/dotc/vulpix/ParallelTesting.scala | 1099 -------------------- .../tools/dotc/vulpix/RunnerOrchestration.scala | 153 --- .../test/dotty/tools/dotc/vulpix/Statuses.java | 25 - .../dotty/tools/dotc/vulpix/SummaryReport.java | 96 -- .../test/dotty/tools/dotc/vulpix/VulpixTests.scala | 63 -- compiler/test/dotty/tools/vulpix/ChildMain.scala | 82 ++ .../test/dotty/tools/vulpix/ParallelTesting.scala | 1099 ++++++++++++++++++++ .../dotty/tools/vulpix/RunnerOrchestration.scala | 153 +++ compiler/test/dotty/tools/vulpix/Statuses.java | 25 + .../test/dotty/tools/vulpix/SummaryReport.java | 96 ++ compiler/test/dotty/tools/vulpix/VulpixTests.scala | 61 ++ 13 files changed, 1517 insertions(+), 1520 deletions(-) delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/Statuses.java delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java delete mode 100644 compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala create mode 100644 compiler/test/dotty/tools/vulpix/ChildMain.scala create mode 100644 compiler/test/dotty/tools/vulpix/ParallelTesting.scala create mode 100644 compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala create mode 100644 compiler/test/dotty/tools/vulpix/Statuses.java create mode 100644 compiler/test/dotty/tools/vulpix/SummaryReport.java create mode 100644 compiler/test/dotty/tools/vulpix/VulpixTests.scala diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala index ab55e0271..8737ef165 100644 --- a/compiler/test/dotc/comptest.scala +++ b/compiler/test/dotc/comptest.scala @@ -1,6 +1,6 @@ package dotc -import dotty.tools.dotc.vulpix.ParallelTesting +import dotty.tools.vulpix.ParallelTesting import scala.concurrent.duration._ diff --git a/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala b/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala deleted file mode 100644 index fdd602379..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/ChildMain.scala +++ /dev/null @@ -1,83 +0,0 @@ -package dotty.tools.dotc -package vulpix - -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream, - ByteArrayOutputStream, PrintStream -} -import java.lang.reflect.InvocationTargetException - -import dotty.tools.dotc.vulpix.Statuses._ - -object ChildMain { - val realStdin = System.in - val realStderr = System.err - val realStdout = System.out - - private def runMain(dir: JFile): Status = { - def renderStackTrace(ex: Throwable): String = - ex.getStackTrace - .takeWhile(_.getMethodName != "invoke0") - .mkString(" ", "\n ", "") - - def resetOutDescriptors(): Unit = { - System.setOut(realStdout) - System.setErr(realStderr) - } - - import java.net.{ URL, URLClassLoader } - - val printStream = new ByteArrayOutputStream - - try { - // Do classloading magic and running here: - val ucl = new URLClassLoader(Array(dir.toURI.toURL)) - val cls = ucl.loadClass("Test") - val meth = cls.getMethod("main", classOf[Array[String]]) - - try { - val ps = new PrintStream(printStream) - System.setOut(ps) - System.setErr(ps) - Console.withOut(printStream) { - Console.withErr(printStream) { - // invoke main with "jvm" as arg - meth.invoke(null, Array("jvm")) - } - } - resetOutDescriptors() - } catch { - case t: Throwable => - resetOutDescriptors() - throw t - } - new Success(printStream.toString("utf-8")) - } - catch { - case ex: NoSuchMethodException => - val msg = s"test in '$dir' did not contain method: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: ClassNotFoundException => - val msg = s"test in '$dir' did not contain class: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: InvocationTargetException => - val msg = s"An exception ocurred when running main: ${ex.getCause}" - new Failure(msg, renderStackTrace(ex.getCause)) - } - } - - def main(args: Array[String]): Unit = { - val stdin = new ObjectInputStream(System.in); - val stdout = new ObjectOutputStream(System.out); - - while (true) { - val dir = stdin.readObject().asInstanceOf[JFile] - stdout.writeObject(runMain(dir)) - stdout.flush() - } - } -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala deleted file mode 100644 index 28c2ea22b..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/ParallelTesting.scala +++ /dev/null @@ -1,1099 +0,0 @@ -package dotty -package tools -package dotc -package vulpix - -import java.io.{ File => JFile } -import java.text.SimpleDateFormat -import java.util.HashMap -import java.nio.file.StandardCopyOption.REPLACE_EXISTING -import java.nio.file.{ Files, Path, Paths, NoSuchFileException } -import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } - -import scala.io.Source -import scala.util.control.NonFatal -import scala.util.Try -import scala.collection.mutable -import scala.util.matching.Regex -import scala.util.Random - -import core.Contexts._ -import reporting.{ Reporter, TestReporter } -import reporting.diagnostic.MessageContainer -import interfaces.Diagnostic.ERROR -import dotc.util.DiffUtil - -import vulpix.Statuses._ - -/** A parallel testing suite whose goal is to integrate nicely with JUnit - * - * This trait can be mixed in to offer parallel testing to compile runs. When - * using this, you should be running your JUnit tests **sequentially**, as the - * test suite itself runs with a high level of concurrency. - */ -trait ParallelTesting extends RunnerOrchestration { self => - - import ParallelTesting._ - import SummaryReport._ - - /** If the running environment supports an interactive terminal, each `Test` - * will be run with a progress bar and real time feedback - */ - def isInteractive: Boolean - - /** A regex which is used to filter which tests to run, if `None` will run - * all tests - */ - def testFilter: Option[Regex] - - /** A test source whose files or directory of files is to be compiled - * in a specific way defined by the `Test` - */ - private sealed trait TestSource { self => - def name: String - def outDir: JFile - def flags: Array[String] - - - def title: String = self match { - case self: JointCompilationSource => - if (self.files.length > 1) name - else self.files.head.getPath - - case self: SeparateCompilationSource => - self.dir.getPath - } - - /** Adds the flags specified in `newFlags0` if they do not already exist */ - def withFlags(newFlags0: String*) = { - val newFlags = newFlags0.toArray - if (!flags.containsSlice(newFlags)) self match { - case self: JointCompilationSource => - self.copy(flags = flags ++ newFlags) - case self: SeparateCompilationSource => - self.copy(flags = flags ++ newFlags) - } - else self - } - - /** Generate the instructions to redo the test from the command line */ - def buildInstructions(errors: Int, warnings: Int): String = { - val sb = new StringBuilder - val maxLen = 80 - var lineLen = 0 - - sb.append( - s"""| - |Test '$title' compiled with $errors error(s) and $warnings warning(s), - |the test can be reproduced by running:""".stripMargin - ) - sb.append("\n\n./bin/dotc ") - flags.foreach { arg => - if (lineLen > maxLen) { - sb.append(" \\\n ") - lineLen = 4 - } - sb.append(arg) - lineLen += arg.length - sb += ' ' - } - - self match { - case JointCompilationSource(_, files, _, _) => { - files.map(_.getAbsolutePath).foreach { path => - sb.append("\\\n ") - sb.append(path) - sb += ' ' - } - sb.toString + "\n\n" - } - case self: SeparateCompilationSource => { - val command = sb.toString - val fsb = new StringBuilder(command) - self.compilationGroups.foreach { files => - files.map(_.getPath).foreach { path => - fsb.append("\\\n ") - lineLen = 8 - fsb.append(path) - fsb += ' ' - } - fsb.append("\n\n") - fsb.append(command) - } - fsb.toString + "\n\n" - } - } - } - } - - /** A group of files that may all be compiled together, with the same flags - * and output directory - */ - private final case class JointCompilationSource( - name: String, - files: Array[JFile], - flags: Array[String], - outDir: JFile - ) extends TestSource { - def sourceFiles: Array[JFile] = files.filter(isSourceFile) - - override def toString() = outDir.toString - } - - /** A test source whose files will be compiled separately according to their - * suffix `_X` - */ - private final case class SeparateCompilationSource( - name: String, - dir: JFile, - flags: Array[String], - outDir: JFile - ) extends TestSource { - - /** Get the files grouped by `_X` as a list of groups, files missing this - * suffix will be put into the same group - * - * Filters out all none source files - */ - def compilationGroups: List[Array[JFile]] = - dir - .listFiles - .groupBy { file => - val name = file.getName - Try { - val potentialNumber = name - .substring(0, name.lastIndexOf('.')) - .reverse.takeWhile(_ != '_').reverse - - potentialNumber.toInt.toString - } - .toOption - .getOrElse("") - } - .toList.sortBy(_._1).map(_._2.filter(isSourceFile)) - } - - /** Each `Test` takes the `testSources` and performs the compilation and assertions - * according to the implementing class "neg", "run" or "pos". - */ - private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { - protected final val realStdout = System.out - protected final val realStderr = System.err - - /** Actual compilation run logic, the test behaviour is defined here */ - protected def compilationRunnable(testSource: TestSource): Runnable - - /** All testSources left after filtering out */ - private val filteredSources = - if (!testFilter.isDefined) testSources - else testSources.filter { - case JointCompilationSource(_, files, _, _) => - files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined) - case SeparateCompilationSource(_, dir, _, _) => - testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined - } - - /** Total amount of test sources being compiled by this test */ - val sourceCount = filteredSources.length - - private[this] var _errorCount = 0 - def errorCount: Int = _errorCount - - private[this] var _testSourcesCompiled = 0 - private def testSourcesCompiled: Int = _testSourcesCompiled - - /** Complete the current compilation with the amount of errors encountered */ - protected final def registerCompilation(errors: Int) = synchronized { - _testSourcesCompiled += 1 - _errorCount += errors - } - - private[this] var _failed = false - /** Fail the current test */ - protected[this] final def fail(): Unit = synchronized { _failed = true } - def didFail: Boolean = _failed - - protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { - val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) - addFailureInstruction(errorMsg) - failTestSource(testSource) - } - - /** Instructions on how to reproduce failed test source compilations */ - private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String] - protected final def addFailureInstruction(ins: String): Unit = - synchronized { reproduceInstructions.append(ins) } - - /** The test sources that failed according to the implementing subclass */ - private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] - protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized { - val extra = reason.map(" with reason: " + _).getOrElse("") - failedTestSources.append(testSource.name + " failed" + extra) - fail() - } - - /** Prints to `System.err` if we're not suppressing all output */ - protected def echo(msg: String): Unit = if (!suppressAllOutput) { - // pad right so that output is at least as large as progress bar line - val paddingRight = " " * math.max(0, 80 - msg.length) - realStderr.println(msg + paddingRight) - } - - /** A single `Runnable` that prints a progress bar for the curent `Test` */ - private def createProgressMonitor: Runnable = new Runnable { - def run(): Unit = { - val start = System.currentTimeMillis - var tCompiled = testSourcesCompiled - while (tCompiled < sourceCount) { - val timestamp = (System.currentTimeMillis - start) / 1000 - val progress = (tCompiled.toDouble / sourceCount * 40).toInt - - realStdout.print( - "[" + ("=" * (math.max(progress - 1, 0))) + - (if (progress > 0) ">" else "") + - (" " * (39 - progress)) + - s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" - ) - - Thread.sleep(100) - tCompiled = testSourcesCompiled - } - // println, otherwise no newline and cursor at start of line - realStdout.println( - s"[=======================================] compiled ($sourceCount/$sourceCount, " + - s"${(System.currentTimeMillis - start) / 1000}s) " - ) - } - } - - /** Wrapper function to make sure that the compiler itself did not crash - - * if it did, the test should automatically fail. - */ - protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = - try { - if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") - op - } catch { - case NonFatal(e) => { - // if an exception is thrown during compilation, the complete test - // run should fail - failTestSource(testSource) - e.printStackTrace() - registerCompilation(1) - throw e - } - } - - protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = { - - val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath) - - def flattenFiles(f: JFile): Array[JFile] = - if (f.isDirectory) f.listFiles.flatMap(flattenFiles) - else Array(f) - - val files: Array[JFile] = files0.flatMap(flattenFiles) - - def findJarFromRuntime(partialName: String) = { - val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) - urls.find(_.contains(partialName)).getOrElse { - throw new java.io.FileNotFoundException( - s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" - ) - } - } - - def addOutDir(xs: Array[String]): Array[String] = { - val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") - if (cpAndAfter.nonEmpty) { - val (cp :: cpArg :: rest) = cpAndAfter - (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray - } - else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray - } - - def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { - val scalaLib = findJarFromRuntime("scala-library-2.") - val fullArgs = Array( - "javac", - "-classpath", - s".:$scalaLib:${targetDir.getAbsolutePath}" - ) ++ flags.takeRight(2) ++ fs - - Runtime.getRuntime.exec(fullArgs).waitFor() == 0 - } else true - - val reporter = - TestReporter.reporter(realStdout, logLevel = - if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) - - val driver = - if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } - else new Driver { - def newCompiler(implicit ctx: Context) = new Compiler - - private def ntimes(n: Int)(op: Int => Reporter): Reporter = - (emptyReporter /: (1 to n)) ((_, i) => op(i)) - - override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) = - ntimes(times) { run => - val start = System.nanoTime() - val rep = super.doCompile(comp, files) - ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms") - rep - } - } - - val allArgs = addOutDir(flags) - driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) - - val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) - assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") - - reporter - } - - private[ParallelTesting] def executeTestSuite(): this.type = { - assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`") - - if (filteredSources.nonEmpty) { - val pool = threadLimit match { - case Some(i) => JExecutors.newWorkStealingPool(i) - case None => JExecutors.newWorkStealingPool() - } - - if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) - - filteredSources.foreach { target => - pool.submit(compilationRunnable(target)) - } - - pool.shutdown() - if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { - pool.shutdownNow() - System.setOut(realStdout) - System.setErr(realStderr) - throw new TimeoutException("Compiling targets timed out") - } - - if (didFail) { - reportFailed() - failedTestSources.toSet.foreach(addFailedTest) - reproduceInstructions.iterator.foreach(addReproduceInstruction) - } - else reportPassed() - } - else echo { - testFilter - .map(r => s"""No files matched regex "$r" in test""") - .getOrElse("No tests available under target - erroneous test?") - } - - this - } - } - - private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val reporter = compile(testSource.sourceFiles, flags, false, outDir) - registerCompilation(reporter.errorCount) - - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) - val errorCount = reporters.foldLeft(0) { (acc, reporter) => - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - acc + reporter.errorCount - } - - def warningCount = reporters.foldLeft(0)(_ + _.warningCount) - - registerCompilation(errorCount) - - if (errorCount > 0) - echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) - } - } - } - } - } - - private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - runMain(dir) match { - case success: Success => { - val outputLines = success.output.lines.toArray - val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray - val sourceTitle = testSource.title - - def linesMatch = - outputLines - .zip(checkLines) - .forall { case (x, y) => x == y } - - if (outputLines.length != checkLines.length || !linesMatch) { - // Print diff to files and summary: - val diff = outputLines.zip(checkLines).map { case (act, exp) => - DiffUtil.mkColoredLineDiff(exp, act) - }.mkString("\n") - - val msg = - s"""|Output from '$sourceTitle' did not match check file. - |Diff ('e' is expected, 'a' is actual): - |""".stripMargin + diff + "\n" - echo(msg) - addFailureInstruction(msg) - - // Print build instructions to file and summary: - val buildInstr = testSource.buildInstructions(0, warnings) - addFailureInstruction(buildInstr) - - // Fail target: - failTestSource(testSource) - } - } - - case failure: Failure => - echo(renderFailure(failure)) - failTestSource(testSource) - - case _: Timeout => - echo("failed because test " + testSource.title + " timed out") - failTestSource(testSource, Some("test timed out")) - } - } - - private def renderFailure(failure: Failure): String = - failure.message + "\n" + failure.stacktrace - - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val checkFile = files.flatMap { file => - if (file.isDirectory) Nil - else { - val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check" - val checkFile = new JFile(fname) - if (checkFile.exists) List(checkFile) - else Nil - } - }.headOption - val reporter = compile(testSource.sourceFiles, flags, false, outDir) - - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - registerCompilation(reporter.errorCount) - (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") - val (errorCount, warningCount) = - testSource - .compilationGroups - .map(compile(_, flags, false, outDir)) - .foldLeft((0,0)) { case ((errors, warnings), reporter) => - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - - (errors + reporter.errorCount, warnings + reporter.warningCount) - } - - if (errorCount > 0) fail() - - registerCompilation(errorCount) - (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) - } - } - - if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.outDir) match { - case status: Failure => - echo(renderFailure(status)) - failTestSource(testSource) - case _: Timeout => - echo("failed because test " + testSource.title + " timed out") - failTestSource(testSource, Some("test timed out")) - case _: Success => // success! - } - else if (errorCount > 0) { - echo(s"\nCompilation failed for: '$testSource'") - val buildInstr = testSource.buildInstructions(errorCount, warningCount) - addFailureInstruction(buildInstr) - failTestSource(testSource) - } - } - } - } - - private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) - extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - // In neg-tests we allow two types of error annotations, - // "nopos-error" which doesn't care about position and "error" which - // has to be annotated on the correct line number. - // - // We collect these in a map `"file:row" -> numberOfErrors`, for - // nopos errors we save them in `"file" -> numberOfNoPosErrors` - def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = { - val errorMap = new HashMap[String, Integer]() - var expectedErrors = 0 - files.filter(_.getName.endsWith(".scala")).foreach { file => - Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) => - val errors = line.sliding("// error".length).count(_.mkString == "// error") - if (errors > 0) - errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors) - - val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error") - if (noposErrors > 0) { - val nopos = errorMap.get("nopos") - val existing: Integer = if (nopos eq null) 0 else nopos - errorMap.put("nopos", noposErrors + existing) - } - - expectedErrors += noposErrors + errors - } - } - - (errorMap, expectedErrors) - } - - def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error => - val key = if (error.pos.exists) { - val fileName = error.pos.source.file.toString - s"$fileName:${error.pos.line}" - - } else "nopos" - - val errors = errorMap.get(key) - - if (errors ne null) { - if (errors == 1) errorMap.remove(key) - else errorMap.put(key, errors - 1) - true - } - else { - echo { - s"Error reported in ${error.pos.source}, but no annotation found" - } - false - } - } - - val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { - case testSource @ JointCompilationSource(_, files, flags, outDir) => { - val sourceFiles = testSource.sourceFiles - val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) - val reporter = compile(sourceFiles, flags, true, outDir) - val actualErrors = reporter.errorCount - - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) - } - - case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { - val compilationGroups = testSource.compilationGroups - val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) - val reporters = compilationGroups.map(compile(_, flags, true, outDir)) - val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) - val errors = reporters.iterator.flatMap(_.errors) - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) - } - } - - if (expectedErrors != actualErrors) { - echo { - s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" - } - failTestSource(testSource) - } - else if (hasMissingAnnotations()) { - echo { - s"\nErrors found on incorrect row numbers when compiling $testSource" - } - failTestSource(testSource) - } - else if (!errorMap.isEmpty) { - echo { - s"\nExpected error(s) have {=}: $errorMap" - } - failTestSource(testSource) - } - - registerCompilation(actualErrors) - } - } - } - - /** The `CompilationTest` is the main interface to `ParallelTesting`, it - * can be instantiated via one of the following methods: - * - * - `compileFile` - * - `compileDir` - * - `compileList` - * - `compileFilesInDir` - * - `compileShallowFilesInDir` - * - * Each compilation test can then be turned into either a "pos", "neg" or - * "run" test: - * - * ``` - * compileFile("../tests/pos/i1103.scala", opts).pos() - * ``` - * - * These tests can be customized before calling one of the execution - * methods, for instance: - * - * ``` - * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos() - * ``` - * - * Which would compile `i1103.scala` twice with the verbose flag as a "pos" - * test. - * - * pos tests - * ========= - * Pos tests verify that the compiler is able to compile the given - * `TestSource`s and that they generate no errors or exceptions during - * compilation - * - * neg tests - * ========= - * Neg tests are expected to generate a certain amount of errors - but not - * crash the compiler. In each `.scala` file, you specifiy the line on which - * the error will be generated, e.g: - * - * ``` - * val x: String = 1 // error - * ``` - * - * if a line generates multiple errors, you need to annotate it multiple - * times. For a line that generates two errors: - * - * ``` - * val y: String = { val y1: String = 1; 2 } // error // error - * ``` - * - * Certain errors have no position, if you need to check these annotate the - * file anywhere with `// nopos-error` - * - * run tests - * ========= - * Run tests are a superset of pos tests, they both verify compilation and - * that the compiler does not crash. In addition, run tests verify that the - * tests are able to run as expected. - * - * Run tests need to have the following form: - * - * ``` - * object Test { - * def main(args: Array[String]): Unit = () - * } - * ``` - * - * This is because the runner instantiates the `Test` class and calls the - * main method. - * - * Other definitions are allowed in the same file, but the file needs to at - * least have the `Test` object with a `main` method. - * - * To verify output you may use `.check` files. These files should share the - * name of the file or directory that they are testing. For instance: - * - * ```none - * . - * └── tests - * ├── i1513.scala - * └── i1513.check - * ``` - * - * If you are testing a directory under separate compilation, you would - * have: - * - * ```none - * . - * └── tests - * ├── myTestDir - * │ ├── T_1.scala - * │ ├── T_2.scala - * │ └── T_3.scala - * └── myTestDir.check - * ``` - * - * In the above example, `i1513.scala` and one of the files `T_X.scala` - * would contain a `Test` object with a main method. - * - * Composing tests - * =============== - * Since this is a parallel test suite, it is essential to be able to - * compose tests to take advantage of the concurrency. This is done using - * the `+` function. This function will make sure that tests being combined - * are compatible according to the `require`s in `+`. - */ - final class CompilationTest private ( - private[ParallelTesting] val targets: List[TestSource], - private[ParallelTesting] val times: Int, - private[ParallelTesting] val shouldDelete: Boolean, - private[ParallelTesting] val threadLimit: Option[Int], - private[ParallelTesting] val shouldFail: Boolean - ) { - import org.junit.Assert.fail - - private[ParallelTesting] def this(target: TestSource) = - this(List(target), 1, true, None, false) - - private[ParallelTesting] def this(targets: List[TestSource]) = - this(targets, 1, true, None, false) - - /** Compose test targets from `this` with `other` - * - * It does this, only if the two tests are compatible. Otherwise it throws - * an `IllegalArgumentException`. - * - * Grouping tests together like this allows us to take advantage of the - * concurrency offered by this test suite as each call to an executing - * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the - * maximum allowed level of concurrency. Doing this for only a few targets - * does not yield any real benefit over sequential compilation. - * - * As such, each `CompilationTest` should contain as many targets as - * possible. - */ - def +(other: CompilationTest) = { - require(other.times == times, "can't combine tests that are meant to be benchmark compiled") - require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output") - require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome") - new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail) - } - - /** Creates a "pos" test run, which makes sure that all tests pass - * compilation without generating errors and that they do not crash the - * compiler - */ - def checkCompile(): this.type = { - val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail(s"Expected no errors when compiling, but found: ${test.errorCount}") - } - else if (shouldFail && !test.didFail) { - fail("Pos test should have failed, but didn't") - } - - cleanup() - } - - /** Creates a "neg" test run, which makes sure that each test generates the - * correct amount of errors at the correct positions. It also makes sure - * that none of these tests crash the compiler - */ - def checkExpectedErrors(): this.type = { - val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail("Neg test shouldn't have failed, but did") - } - else if (shouldFail && !test.didFail) { - fail("Neg test should have failed, but did not") - } - - cleanup() - } - - /** Creates a "run" test run, which is a superset of "pos". In addition to - * making sure that all tests pass compilation and that they do not crash - * the compiler; it also makes sure that all tests can run with the - * expected output - */ - def checkRuns(): this.type = { - val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() - - if (!shouldFail && test.didFail) { - fail("Run test failed, but should not") - } - else if (shouldFail && !test.didFail) { - fail("Run test should have failed, but did not") - } - - cleanup() - } - - /** Deletes output directories and files */ - private def cleanup(): this.type = { - if (shouldDelete) delete() - this - } - - /** Copies `file` to `dir` - taking into account if `file` is a directory, - * and if so copying recursively - */ - private def copyToDir(dir: JFile, file: JFile): JFile = { - val target = Paths.get(dir.getAbsolutePath, file.getName) - Files.copy(file.toPath, target, REPLACE_EXISTING) - if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _)) - target.toFile - } - - /** Builds a new `CompilationTest` where we have copied the target files to - * the out directory. This is needed for tests that modify the original - * source, such as `-rewrite` tests - */ - def copyToTarget(): CompilationTest = new CompilationTest ( - targets.map { - case target @ JointCompilationSource(_, files, _, outDir) => - target.copy(files = files.map(copyToDir(outDir,_))) - case target @ SeparateCompilationSource(_, dir, _, outDir) => - target.copy(dir = copyToDir(outDir, dir)) - }, - times, shouldDelete, threadLimit, shouldFail - ) - - /** Builds a `CompilationTest` which performs the compilation `i` times on - * each target - */ - def times(i: Int): CompilationTest = - new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail) - - /** Builds a `Compilationtest` which passes the verbose flag and logs the - * classpath - */ - def verbose: CompilationTest = new CompilationTest( - targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")), - times, shouldDelete, threadLimit, shouldFail - ) - - /** Builds a `CompilationTest` which keeps the generated output files - * - * This is needed for tests like `tastyBootstrap` which relies on first - * compiling a certain part of the project and then compiling a second - * part which depends on the first - */ - def keepOutput: CompilationTest = - new CompilationTest(targets, times, false, threadLimit, shouldFail) - - /** Builds a `CompilationTest` with a limited level of concurrency with - * maximum `i` threads - */ - def limitThreads(i: Int): CompilationTest = - new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail) - - /** Builds a `CompilationTest` where the executed test is expected to fail - * - * This behaviour is mainly needed for the tests that test the test suite. - */ - def expectFailure: CompilationTest = - new CompilationTest(targets, times, shouldDelete, threadLimit, true) - - /** Delete all output files generated by this `CompilationTest` */ - def delete(): Unit = targets.foreach(t => delete(t.outDir)) - - private def delete(file: JFile): Unit = { - if (file.isDirectory) file.listFiles.foreach(delete) - try Files.delete(file.toPath) - catch { - case _: NoSuchFileException => // already deleted, everything's fine - } - } - } - - /** Create out directory for directory `d` */ - private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = { - val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}") - targetDir.mkdirs() - targetDir - } - - /** Create out directory for `file` */ - private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = { - val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.')) - val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir") - targetDir.mkdirs() - targetDir - } - - /** Make sure that directory string is as expected */ - private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = { - require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`") - require(outDir.last == '/', "please specify an `outDir` with a trailing slash") - } - - /** Separates directories from files and returns them as `(dirs, files)` */ - private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) = - sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) => - if (f.isDirectory) (f :: dirs, files) - else if (isSourceFile(f)) (dirs, f :: files) - else (dirs, files) - } - - /** Gets the name of the calling method via reflection. - * - * It does this in a way that needs to work both with the bootstrapped dotty - * and the non-bootstrapped version. Since the two compilers generate - * different bridges, we first need to filter out methods with the same name - * (bridges) - and then find the `@Test` method in our extending class - */ - private def getCallingMethod(): String = { - val seen = mutable.Set.empty[String] - Thread.currentThread.getStackTrace - .filter { elem => - if (seen.contains(elem.getMethodName)) false - else { seen += elem.getMethodName; true } - } - .find { elem => - val callingClass = Class.forName(elem.getClassName) - classOf[ParallelTesting].isAssignableFrom(callingClass) && - elem.getFileName != "ParallelTesting.scala" - } - .map(_.getMethodName) - .getOrElse { - throw new IllegalStateException("Unable to reflectively find calling method") - } - } - - /** Compiles a single file from the string path `f` using the supplied flags */ - def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val sourceFile = new JFile(f) - val parent = sourceFile.getParentFile - val outDir = - outDirectory + callingMethod + "/" + - sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/" - - require( - sourceFile.exists && !sourceFile.isDirectory && - (parent ne null) && parent.exists && parent.isDirectory, - s"Source file: $f, didn't exist" - ) - - val target = JointCompilationSource( - callingMethod, - Array(sourceFile), - flags, - createOutputDirsForFile(sourceFile, parent, outDir) - ) - new CompilationTest(target) - } - - /** Compiles a directory `f` using the supplied `flags`. This method does - * deep compilation, that is - it compiles all files and subdirectories - * contained within the directory `f`. - * - * By default, files are compiled in alphabetical order. An optional seed - * can be used for randomization. - */ - def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - def flatten(f: JFile): Array[JFile] = - if (f.isDirectory) f.listFiles.flatMap(flatten) - else Array(f) - - // Sort files either alphabetically or randomly using the provided seed: - val sortedFiles = flatten(sourceDir).sorted - val randomized = randomOrder match { - case None => sortedFiles - case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray - } - - // Directories in which to compile all containing files with `flags`: - val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/") - targetDir.mkdirs() - - val target = JointCompilationSource(callingMethod, randomized, flags, targetDir) - new CompilationTest(target) - } - - /** Compiles all `files` together as a single compilation run. It is given a - * `testName` since files can be in separate directories and or be otherwise - * dissociated - */ - def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" + testName + "/" - - // Directories in which to compile all containing files with `flags`: - val targetDir = new JFile(outDir) - targetDir.mkdirs() - assert(targetDir.exists, s"couldn't create target directory: $targetDir") - - val target = JointCompilationSource(callingMethod, files.map(new JFile(_)).toArray, flags, targetDir) - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(target) - } - - /** This function compiles the files and folders contained within directory - * `f` in a specific way. - * - * - Each file is compiled separately as a single compilation run - * - Each directory is compiled as a `SeparateCompilationTaret`, in this - * target all files are grouped according to the file suffix `_X` where `X` - * is a number. These groups are then ordered in ascending order based on - * the value of `X` and each group is compiled one after the other. - * - * For this function to work as expected, we use the same convention for - * directory layout as the old partest. That is: - * - * - Single files can have an associated check-file with the same name (but - * with file extension `.check`) - * - Directories can have an associated check-file, where the check file has - * the same name as the directory (with the file extension `.check`) - */ - def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - val (dirs, files) = compilationTargets(sourceDir) - - val targets = - files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++ - dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir))) - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(targets) - } - - /** This function behaves similar to `compileFilesInDir` but it ignores - * sub-directories and as such, does **not** perform separate compilation - * tests. - */ - def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { - val callingMethod = getCallingMethod - val outDir = outDirectory + callingMethod + "/" - val sourceDir = new JFile(f) - checkRequirements(f, sourceDir, outDir) - - val (_, files) = compilationTargets(sourceDir) - - val targets = files.map { file => - JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir)) - } - - // Create a CompilationTest and let the user decide whether to execute a pos or a neg test - new CompilationTest(targets) - } -} - -object ParallelTesting { - def isSourceFile(f: JFile): Boolean = { - val name = f.getName - name.endsWith(".scala") || name.endsWith(".java") - } -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala deleted file mode 100644 index 9dc808af7..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/RunnerOrchestration.scala +++ /dev/null @@ -1,153 +0,0 @@ -package dotty.tools.dotc -package vulpix - -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream -} -import java.util.concurrent.TimeoutException - -import scala.concurrent.duration.Duration -import scala.concurrent.{ Await, Future } -import scala.concurrent.ExecutionContext.Implicits.global -import scala.collection.mutable - -import vulpix.Statuses._ - -trait RunnerOrchestration { - - /** The maximum amount of active runners, which contain a child JVM */ - def numberOfSlaves: Int - - /** The maximum duration the child process is allowed to consume before - * getting destroyed - */ - def maxDuration: Duration - - /** Destroy and respawn process after each test */ - def safeMode: Boolean - - /** Running a `Test` class's main method from the specified `dir` */ - def runMain(dir: JFile): Status = monitor.runMain(dir) - - private[this] val monitor = new RunnerMonitor - - private class RunnerMonitor { - - def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) - - private class Runner(private var process: Process) { - private[this] var ois: ObjectInputStream = _ - private[this] var oos: ObjectOutputStream = _ - - /** Checks if `process` is still alive - * - * When `process.exitValue()` is called on an active process the caught - * exception is thrown. As such we can know if the subprocess exited or - * not. - * - * @note used for debug - */ - def isAlive: Boolean = - try { process.exitValue(); false } - catch { case _: IllegalThreadStateException => true } - - /** Destroys the underlying process and kills IO streams */ - def kill(): Unit = { - if (process ne null) process.destroy() - process = null - ois = null - oos = null - } - - /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ - def runMain(dir: JFile): Status = { - assert(process ne null, - "Runner was killed and then reused without setting a new process") - - // Makes the encapsulating RunnerMonitor spawn a new runner - def respawn(): Unit = { - process.destroy() - process = createProcess - ois = null - oos = null - } - - if (oos eq null) oos = new ObjectOutputStream(process.getOutputStream) - - // pass file to running process - oos.writeObject(dir) - oos.flush() - - // Create a future reading the object: - val readObject = Future { - if (ois eq null) ois = new ObjectInputStream(process.getInputStream) - ois.readObject().asInstanceOf[Status] - } - - // Await result for `maxDuration` and then timout and destroy the - // process: - val status = - try Await.result(readObject, maxDuration) - catch { case _: TimeoutException => new Timeout() } - - // Handle failure of the VM: - status match { - case _ if safeMode => respawn() - case _: Failure => respawn() - case _: Timeout => respawn() - case _ => () - } - status - } - } - - private def createProcess: Process = { - val sep = sys.props("file.separator") - val cp = sys.props("java.class.path") - val java = sys.props("java.home") + sep + "bin" + sep + "java" - new ProcessBuilder(java, "-cp", cp, "dotty.tools.dotc.vulpix.ChildMain")//classOf[ChildMain].getName) - .redirectErrorStream(true) - .redirectInput(ProcessBuilder.Redirect.PIPE) - .redirectOutput(ProcessBuilder.Redirect.PIPE) - .start() - } - - private[this] val allRunners = List.fill(numberOfSlaves)(new Runner(createProcess)) - - private[this] val freeRunners = mutable.Queue(allRunners: _*) - private[this] val busyRunners = mutable.Set.empty[Runner] - - private def getRunner(): Runner = synchronized { - while (freeRunners.isEmpty) wait() - - val runner = freeRunners.dequeue() - busyRunners += runner - - notify() - runner - } - - private def freeRunner(runner: Runner): Unit = synchronized { - freeRunners.enqueue(runner) - busyRunners -= runner - notify() - } - - private def withRunner[T](op: Runner => T): T = { - val runner = getRunner() - val result = op(runner) - freeRunner(runner) - result - } - - private def killAll(): Unit = allRunners.foreach(_.kill()) - - // On shutdown, we need to kill all runners: - sys.addShutdownHook(killAll()) - // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we - // need to clean up ourselves. - SummaryReport.addCleanup(killAll) - } -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/Statuses.java b/compiler/test/dotty/tools/dotc/vulpix/Statuses.java deleted file mode 100644 index bec687d01..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/Statuses.java +++ /dev/null @@ -1,25 +0,0 @@ -package dotty.tools.dotc.vulpix; - -import java.io.Serializable; - -/** The status of each call to `main` in the test applications */ -public class Statuses { - interface Status {} - - static class Success implements Status, Serializable { - public final String output; - public Success(String output) { this.output = output; } - } - - static class Failure implements Status, Serializable { - public final String message; - public final String stacktrace; - - public Failure(String message, String stacktrace) { - this.message = message; - this.stacktrace = stacktrace; - } - } - - static class Timeout implements Status, Serializable {} -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java b/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java deleted file mode 100644 index 4f2b6350b..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/SummaryReport.java +++ /dev/null @@ -1,96 +0,0 @@ -package dotty.tools.dotc.vulpix; - -import org.junit.BeforeClass; -import org.junit.AfterClass; -import java.util.ArrayDeque; -import java.util.function.Supplier; -import scala.Function0; -import scala.Unit; - -import dotty.tools.dotc.reporting.TestReporter; - -/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with - * this class - */ -public class SummaryReport { - public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); - - private static TestReporter rep = TestReporter.reporter(System.out, -1); - private static ArrayDeque failedTests = new ArrayDeque<>(); - private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); - private static Supplier cleanup; - private static int passed; - private static int failed; - - public final static void reportFailed() { - failed++; - } - - public final static void reportPassed() { - passed++; - } - - public final static void addFailedTest(String msg) { - failedTests.offer(msg); - } - - public final static void addReproduceInstruction(String msg) { - reproduceInstructions.offer(msg); - } - - public final static void addCleanup(Function0 func) { - // Wow, look at how neatly we - compose cleanup callbacks: - if (cleanup == null) { - cleanup = () -> { - func.apply(); - return null; - }; - } else { - Supplier oldCleanup = cleanup; - cleanup = () -> { - oldCleanup.get(); - func.apply(); - return null; - }; - } - } - - @BeforeClass public final static void setup() { - rep = TestReporter.reporter(System.out, -1); - failedTests = new ArrayDeque<>(); - reproduceInstructions = new ArrayDeque<>(); - } - - @AfterClass public final static void teardown() { - rep.echo( - "\n================================================================================" + - "\nTest Report" + - "\n================================================================================" + - "\n" + - passed + " passed, " + failed + " failed, " + (passed + failed) + " total" + - "\n" - ); - - failedTests - .stream() - .map(x -> " " + x) - .forEach(rep::echo); - - // If we're compiling locally, we don't need reproduce instructions - if (isInteractive) rep.flushToStdErr(); - - rep.echo(""); - - reproduceInstructions - .stream() - .forEach(rep::echo); - - // If we're on the CI, we want everything - if (!isInteractive) rep.flushToStdErr(); - - if (failed > 0) rep.flushToFile(); - - // Perform cleanup callback: - if (cleanup != null) cleanup.get(); - } -} diff --git a/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala deleted file mode 100644 index 1a1775995..000000000 --- a/compiler/test/dotty/tools/dotc/vulpix/VulpixTests.scala +++ /dev/null @@ -1,63 +0,0 @@ -package dotty -package tools -package dotc -package vulpix - -import org.junit.Assert._ -import org.junit.Test - -import scala.concurrent.duration._ -import scala.util.control.NonFatal - -class VulpixTests extends ParallelTesting { - import CompilationTests._ - - def maxDuration = 3.seconds - def numberOfSlaves = 5 - def safeMode = sys.env.get("SAFEMODE").isDefined - def isInteractive = !sys.env.contains("DRONE") - def testFilter = None - - @Test def missingFile: Unit = - try { - compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors() - fail("didn't fail properly") - } - catch { - case _: IllegalArgumentException => // pass! - case NonFatal(_) => fail("wrong exception thrown") - } - - @Test def pos1Error: Unit = - compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile() - - @Test def negMissingAnnot: Unit = - compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negAnnotWrongLine: Unit = - compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negTooManyAnnots: Unit = - compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def negNoPositionAnnot: Unit = - compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() - - @Test def runCompileFail: Unit = - compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runWrongOutput1: Unit = - compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runWrongOutput2: Unit = - compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runDiffOutput1: Unit = - compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runStackOverflow: Unit = - compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns() - - @Test def runOutRedirects: Unit = - compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() -} diff --git a/compiler/test/dotty/tools/vulpix/ChildMain.scala b/compiler/test/dotty/tools/vulpix/ChildMain.scala new file mode 100644 index 000000000..30059a9c5 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ChildMain.scala @@ -0,0 +1,82 @@ +package dotty.tools.vulpix + +import java.io.{ + File => JFile, + InputStream, ObjectInputStream, + OutputStream, ObjectOutputStream, + ByteArrayOutputStream, PrintStream +} +import java.lang.reflect.InvocationTargetException + +import dotty.tools.vulpix.Statuses._ + +object ChildMain { + val realStdin = System.in + val realStderr = System.err + val realStdout = System.out + + private def runMain(dir: JFile): Status = { + def renderStackTrace(ex: Throwable): String = + ex.getStackTrace + .takeWhile(_.getMethodName != "invoke0") + .mkString(" ", "\n ", "") + + def resetOutDescriptors(): Unit = { + System.setOut(realStdout) + System.setErr(realStderr) + } + + import java.net.{ URL, URLClassLoader } + + val printStream = new ByteArrayOutputStream + + try { + // Do classloading magic and running here: + val ucl = new URLClassLoader(Array(dir.toURI.toURL)) + val cls = ucl.loadClass("Test") + val meth = cls.getMethod("main", classOf[Array[String]]) + + try { + val ps = new PrintStream(printStream) + System.setOut(ps) + System.setErr(ps) + Console.withOut(printStream) { + Console.withErr(printStream) { + // invoke main with "jvm" as arg + meth.invoke(null, Array("jvm")) + } + } + resetOutDescriptors() + } catch { + case t: Throwable => + resetOutDescriptors() + throw t + } + new Success(printStream.toString("utf-8")) + } + catch { + case ex: NoSuchMethodException => + val msg = s"test in '$dir' did not contain method: ${ex.getMessage}" + new Failure(msg, renderStackTrace(ex.getCause)) + + case ex: ClassNotFoundException => + val msg = s"test in '$dir' did not contain class: ${ex.getMessage}" + new Failure(msg, renderStackTrace(ex.getCause)) + + case ex: InvocationTargetException => + val msg = s"An exception ocurred when running main: ${ex.getCause}" + new Failure(msg, renderStackTrace(ex.getCause)) + } + } + + def main(args: Array[String]): Unit = { + val stdin = new ObjectInputStream(System.in); + val stdout = new ObjectOutputStream(System.out); + + while (true) { + val dir = stdin.readObject().asInstanceOf[JFile] + stdout.writeObject(runMain(dir)) + stdout.flush() + } + } +} diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala new file mode 100644 index 000000000..9b9f0a2bb --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -0,0 +1,1099 @@ +package dotty +package tools +package vulpix + +import java.io.{ File => JFile } +import java.text.SimpleDateFormat +import java.util.HashMap +import java.nio.file.StandardCopyOption.REPLACE_EXISTING +import java.nio.file.{ Files, Path, Paths, NoSuchFileException } +import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException } + +import scala.io.Source +import scala.util.control.NonFatal +import scala.util.Try +import scala.collection.mutable +import scala.util.matching.Regex +import scala.util.Random + +import dotc.core.Contexts._ +import dotc.reporting.{ Reporter, TestReporter } +import dotc.reporting.diagnostic.MessageContainer +import dotc.interfaces.Diagnostic.ERROR +import dotc.util.DiffUtil +import dotc.{ Driver, Compiler } + +import vulpix.Statuses._ + +/** A parallel testing suite whose goal is to integrate nicely with JUnit + * + * This trait can be mixed in to offer parallel testing to compile runs. When + * using this, you should be running your JUnit tests **sequentially**, as the + * test suite itself runs with a high level of concurrency. + */ +trait ParallelTesting extends RunnerOrchestration { self => + + import ParallelTesting._ + import SummaryReport._ + + /** If the running environment supports an interactive terminal, each `Test` + * will be run with a progress bar and real time feedback + */ + def isInteractive: Boolean + + /** A regex which is used to filter which tests to run, if `None` will run + * all tests + */ + def testFilter: Option[Regex] + + /** A test source whose files or directory of files is to be compiled + * in a specific way defined by the `Test` + */ + private sealed trait TestSource { self => + def name: String + def outDir: JFile + def flags: Array[String] + + + def title: String = self match { + case self: JointCompilationSource => + if (self.files.length > 1) name + else self.files.head.getPath + + case self: SeparateCompilationSource => + self.dir.getPath + } + + /** Adds the flags specified in `newFlags0` if they do not already exist */ + def withFlags(newFlags0: String*) = { + val newFlags = newFlags0.toArray + if (!flags.containsSlice(newFlags)) self match { + case self: JointCompilationSource => + self.copy(flags = flags ++ newFlags) + case self: SeparateCompilationSource => + self.copy(flags = flags ++ newFlags) + } + else self + } + + /** Generate the instructions to redo the test from the command line */ + def buildInstructions(errors: Int, warnings: Int): String = { + val sb = new StringBuilder + val maxLen = 80 + var lineLen = 0 + + sb.append( + s"""| + |Test '$title' compiled with $errors error(s) and $warnings warning(s), + |the test can be reproduced by running:""".stripMargin + ) + sb.append("\n\n./bin/dotc ") + flags.foreach { arg => + if (lineLen > maxLen) { + sb.append(" \\\n ") + lineLen = 4 + } + sb.append(arg) + lineLen += arg.length + sb += ' ' + } + + self match { + case JointCompilationSource(_, files, _, _) => { + files.map(_.getAbsolutePath).foreach { path => + sb.append("\\\n ") + sb.append(path) + sb += ' ' + } + sb.toString + "\n\n" + } + case self: SeparateCompilationSource => { + val command = sb.toString + val fsb = new StringBuilder(command) + self.compilationGroups.foreach { files => + files.map(_.getPath).foreach { path => + fsb.append("\\\n ") + lineLen = 8 + fsb.append(path) + fsb += ' ' + } + fsb.append("\n\n") + fsb.append(command) + } + fsb.toString + "\n\n" + } + } + } + } + + /** A group of files that may all be compiled together, with the same flags + * and output directory + */ + private final case class JointCompilationSource( + name: String, + files: Array[JFile], + flags: Array[String], + outDir: JFile + ) extends TestSource { + def sourceFiles: Array[JFile] = files.filter(isSourceFile) + + override def toString() = outDir.toString + } + + /** A test source whose files will be compiled separately according to their + * suffix `_X` + */ + private final case class SeparateCompilationSource( + name: String, + dir: JFile, + flags: Array[String], + outDir: JFile + ) extends TestSource { + + /** Get the files grouped by `_X` as a list of groups, files missing this + * suffix will be put into the same group + * + * Filters out all none source files + */ + def compilationGroups: List[Array[JFile]] = + dir + .listFiles + .groupBy { file => + val name = file.getName + Try { + val potentialNumber = name + .substring(0, name.lastIndexOf('.')) + .reverse.takeWhile(_ != '_').reverse + + potentialNumber.toInt.toString + } + .toOption + .getOrElse("") + } + .toList.sortBy(_._1).map(_._2.filter(isSourceFile)) + } + + /** Each `Test` takes the `testSources` and performs the compilation and assertions + * according to the implementing class "neg", "run" or "pos". + */ + private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { + protected final val realStdout = System.out + protected final val realStderr = System.err + + /** Actual compilation run logic, the test behaviour is defined here */ + protected def compilationRunnable(testSource: TestSource): Runnable + + /** All testSources left after filtering out */ + private val filteredSources = + if (!testFilter.isDefined) testSources + else testSources.filter { + case JointCompilationSource(_, files, _, _) => + files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined) + case SeparateCompilationSource(_, dir, _, _) => + testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined + } + + /** Total amount of test sources being compiled by this test */ + val sourceCount = filteredSources.length + + private[this] var _errorCount = 0 + def errorCount: Int = _errorCount + + private[this] var _testSourcesCompiled = 0 + private def testSourcesCompiled: Int = _testSourcesCompiled + + /** Complete the current compilation with the amount of errors encountered */ + protected final def registerCompilation(errors: Int) = synchronized { + _testSourcesCompiled += 1 + _errorCount += errors + } + + private[this] var _failed = false + /** Fail the current test */ + protected[this] final def fail(): Unit = synchronized { _failed = true } + def didFail: Boolean = _failed + + protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { + val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) + addFailureInstruction(errorMsg) + failTestSource(testSource) + } + + /** Instructions on how to reproduce failed test source compilations */ + private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String] + protected final def addFailureInstruction(ins: String): Unit = + synchronized { reproduceInstructions.append(ins) } + + /** The test sources that failed according to the implementing subclass */ + private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] + protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized { + val extra = reason.map(" with reason: " + _).getOrElse("") + failedTestSources.append(testSource.name + " failed" + extra) + fail() + } + + /** Prints to `System.err` if we're not suppressing all output */ + protected def echo(msg: String): Unit = if (!suppressAllOutput) { + // pad right so that output is at least as large as progress bar line + val paddingRight = " " * math.max(0, 80 - msg.length) + realStderr.println(msg + paddingRight) + } + + /** A single `Runnable` that prints a progress bar for the curent `Test` */ + private def createProgressMonitor: Runnable = new Runnable { + def run(): Unit = { + val start = System.currentTimeMillis + var tCompiled = testSourcesCompiled + while (tCompiled < sourceCount) { + val timestamp = (System.currentTimeMillis - start) / 1000 + val progress = (tCompiled.toDouble / sourceCount * 40).toInt + + realStdout.print( + "[" + ("=" * (math.max(progress - 1, 0))) + + (if (progress > 0) ">" else "") + + (" " * (39 - progress)) + + s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" + ) + + Thread.sleep(100) + tCompiled = testSourcesCompiled + } + // println, otherwise no newline and cursor at start of line + realStdout.println( + s"[=======================================] compiled ($sourceCount/$sourceCount, " + + s"${(System.currentTimeMillis - start) / 1000}s) " + ) + } + } + + /** Wrapper function to make sure that the compiler itself did not crash - + * if it did, the test should automatically fail. + */ + protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = + try { + if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") + op + } catch { + case NonFatal(e) => { + // if an exception is thrown during compilation, the complete test + // run should fail + failTestSource(testSource) + e.printStackTrace() + registerCompilation(1) + throw e + } + } + + protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = { + + val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath) + + def flattenFiles(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flattenFiles) + else Array(f) + + val files: Array[JFile] = files0.flatMap(flattenFiles) + + def findJarFromRuntime(partialName: String) = { + val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) + urls.find(_.contains(partialName)).getOrElse { + throw new java.io.FileNotFoundException( + s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" + ) + } + } + + def addOutDir(xs: Array[String]): Array[String] = { + val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") + if (cpAndAfter.nonEmpty) { + val (cp :: cpArg :: rest) = cpAndAfter + (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray + } + else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray + } + + def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { + val scalaLib = findJarFromRuntime("scala-library-2.") + val fullArgs = Array( + "javac", + "-classpath", + s".:$scalaLib:${targetDir.getAbsolutePath}" + ) ++ flags.takeRight(2) ++ fs + + Runtime.getRuntime.exec(fullArgs).waitFor() == 0 + } else true + + val reporter = + TestReporter.reporter(realStdout, logLevel = + if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) + + val driver = + if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } + else new Driver { + def newCompiler(implicit ctx: Context) = new Compiler + + private def ntimes(n: Int)(op: Int => Reporter): Reporter = + (emptyReporter /: (1 to n)) ((_, i) => op(i)) + + override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) = + ntimes(times) { run => + val start = System.nanoTime() + val rep = super.doCompile(comp, files) + ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms") + rep + } + } + + val allArgs = addOutDir(flags) + driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) + + val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) + assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") + + reporter + } + + private[ParallelTesting] def executeTestSuite(): this.type = { + assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`") + + if (filteredSources.nonEmpty) { + val pool = threadLimit match { + case Some(i) => JExecutors.newWorkStealingPool(i) + case None => JExecutors.newWorkStealingPool() + } + + if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) + + filteredSources.foreach { target => + pool.submit(compilationRunnable(target)) + } + + pool.shutdown() + if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { + pool.shutdownNow() + System.setOut(realStdout) + System.setErr(realStderr) + throw new TimeoutException("Compiling targets timed out") + } + + if (didFail) { + reportFailed() + failedTestSources.toSet.foreach(addFailedTest) + reproduceInstructions.iterator.foreach(addReproduceInstruction) + } + else reportPassed() + } + else echo { + testFilter + .map(r => s"""No files matched regex "$r" in test""") + .getOrElse("No tests available under target - erroneous test?") + } + + this + } + } + + private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + registerCompilation(reporter.errorCount) + + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) + val errorCount = reporters.foldLeft(0) { (acc, reporter) => + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + acc + reporter.errorCount + } + + def warningCount = reporters.foldLeft(0)(_ + _.warningCount) + + registerCompilation(errorCount) + + if (errorCount > 0) + echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) + } + } + } + } + } + + private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { + runMain(dir) match { + case success: Success => { + val outputLines = success.output.lines.toArray + val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray + val sourceTitle = testSource.title + + def linesMatch = + outputLines + .zip(checkLines) + .forall { case (x, y) => x == y } + + if (outputLines.length != checkLines.length || !linesMatch) { + // Print diff to files and summary: + val diff = outputLines.zip(checkLines).map { case (act, exp) => + DiffUtil.mkColoredLineDiff(exp, act) + }.mkString("\n") + + val msg = + s"""|Output from '$sourceTitle' did not match check file. + |Diff ('e' is expected, 'a' is actual): + |""".stripMargin + diff + "\n" + echo(msg) + addFailureInstruction(msg) + + // Print build instructions to file and summary: + val buildInstr = testSource.buildInstructions(0, warnings) + addFailureInstruction(buildInstr) + + // Fail target: + failTestSource(testSource) + } + } + + case failure: Failure => + echo(renderFailure(failure)) + failTestSource(testSource) + + case _: Timeout => + echo("failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + } + } + + private def renderFailure(failure: Failure): String = + failure.message + "\n" + failure.stacktrace + + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val checkFile = files.flatMap { file => + if (file.isDirectory) Nil + else { + val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check" + val checkFile = new JFile(fname) + if (checkFile.exists) List(checkFile) + else Nil + } + }.headOption + val reporter = compile(testSource.sourceFiles, flags, false, outDir) + + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + registerCompilation(reporter.errorCount) + (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") + val (errorCount, warningCount) = + testSource + .compilationGroups + .map(compile(_, flags, false, outDir)) + .foldLeft((0,0)) { case ((errors, warnings), reporter) => + if (reporter.errorCount > 0) + echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + + (errors + reporter.errorCount, warnings + reporter.warningCount) + } + + if (errorCount > 0) fail() + + registerCompilation(errorCount) + (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) + } + } + + if (errorCount == 0 && hasCheckFile) verifier() + else if (errorCount == 0) runMain(testSource.outDir) match { + case status: Failure => + echo(renderFailure(status)) + failTestSource(testSource) + case _: Timeout => + echo("failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + case _: Success => // success! + } + else if (errorCount > 0) { + echo(s"\nCompilation failed for: '$testSource'") + val buildInstr = testSource.buildInstructions(errorCount, warningCount) + addFailureInstruction(buildInstr) + failTestSource(testSource) + } + } + } + } + + private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + extends Test(testSources, times, threadLimit, suppressAllOutput) { + protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { + def run(): Unit = tryCompile(testSource) { + // In neg-tests we allow two types of error annotations, + // "nopos-error" which doesn't care about position and "error" which + // has to be annotated on the correct line number. + // + // We collect these in a map `"file:row" -> numberOfErrors`, for + // nopos errors we save them in `"file" -> numberOfNoPosErrors` + def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = { + val errorMap = new HashMap[String, Integer]() + var expectedErrors = 0 + files.filter(_.getName.endsWith(".scala")).foreach { file => + Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) => + val errors = line.sliding("// error".length).count(_.mkString == "// error") + if (errors > 0) + errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors) + + val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error") + if (noposErrors > 0) { + val nopos = errorMap.get("nopos") + val existing: Integer = if (nopos eq null) 0 else nopos + errorMap.put("nopos", noposErrors + existing) + } + + expectedErrors += noposErrors + errors + } + } + + (errorMap, expectedErrors) + } + + def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error => + val key = if (error.pos.exists) { + val fileName = error.pos.source.file.toString + s"$fileName:${error.pos.line}" + + } else "nopos" + + val errors = errorMap.get(key) + + if (errors ne null) { + if (errors == 1) errorMap.remove(key) + else errorMap.put(key, errors - 1) + true + } + else { + echo { + s"Error reported in ${error.pos.source}, but no annotation found" + } + false + } + } + + val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { + case testSource @ JointCompilationSource(_, files, flags, outDir) => { + val sourceFiles = testSource.sourceFiles + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) + val reporter = compile(sourceFiles, flags, true, outDir) + val actualErrors = reporter.errorCount + + (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) + } + + case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { + val compilationGroups = testSource.compilationGroups + val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) + val reporters = compilationGroups.map(compile(_, flags, true, outDir)) + val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) + val errors = reporters.iterator.flatMap(_.errors) + (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) + } + } + + if (expectedErrors != actualErrors) { + echo { + s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" + } + failTestSource(testSource) + } + else if (hasMissingAnnotations()) { + echo { + s"\nErrors found on incorrect row numbers when compiling $testSource" + } + failTestSource(testSource) + } + else if (!errorMap.isEmpty) { + echo { + s"\nExpected error(s) have {=}: $errorMap" + } + failTestSource(testSource) + } + + registerCompilation(actualErrors) + } + } + } + + /** The `CompilationTest` is the main interface to `ParallelTesting`, it + * can be instantiated via one of the following methods: + * + * - `compileFile` + * - `compileDir` + * - `compileList` + * - `compileFilesInDir` + * - `compileShallowFilesInDir` + * + * Each compilation test can then be turned into either a "pos", "neg" or + * "run" test: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).pos() + * ``` + * + * These tests can be customized before calling one of the execution + * methods, for instance: + * + * ``` + * compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos() + * ``` + * + * Which would compile `i1103.scala` twice with the verbose flag as a "pos" + * test. + * + * pos tests + * ========= + * Pos tests verify that the compiler is able to compile the given + * `TestSource`s and that they generate no errors or exceptions during + * compilation + * + * neg tests + * ========= + * Neg tests are expected to generate a certain amount of errors - but not + * crash the compiler. In each `.scala` file, you specifiy the line on which + * the error will be generated, e.g: + * + * ``` + * val x: String = 1 // error + * ``` + * + * if a line generates multiple errors, you need to annotate it multiple + * times. For a line that generates two errors: + * + * ``` + * val y: String = { val y1: String = 1; 2 } // error // error + * ``` + * + * Certain errors have no position, if you need to check these annotate the + * file anywhere with `// nopos-error` + * + * run tests + * ========= + * Run tests are a superset of pos tests, they both verify compilation and + * that the compiler does not crash. In addition, run tests verify that the + * tests are able to run as expected. + * + * Run tests need to have the following form: + * + * ``` + * object Test { + * def main(args: Array[String]): Unit = () + * } + * ``` + * + * This is because the runner instantiates the `Test` class and calls the + * main method. + * + * Other definitions are allowed in the same file, but the file needs to at + * least have the `Test` object with a `main` method. + * + * To verify output you may use `.check` files. These files should share the + * name of the file or directory that they are testing. For instance: + * + * ```none + * . + * └── tests + * ├── i1513.scala + * └── i1513.check + * ``` + * + * If you are testing a directory under separate compilation, you would + * have: + * + * ```none + * . + * └── tests + * ├── myTestDir + * │ ├── T_1.scala + * │ ├── T_2.scala + * │ └── T_3.scala + * └── myTestDir.check + * ``` + * + * In the above example, `i1513.scala` and one of the files `T_X.scala` + * would contain a `Test` object with a main method. + * + * Composing tests + * =============== + * Since this is a parallel test suite, it is essential to be able to + * compose tests to take advantage of the concurrency. This is done using + * the `+` function. This function will make sure that tests being combined + * are compatible according to the `require`s in `+`. + */ + final class CompilationTest private ( + private[ParallelTesting] val targets: List[TestSource], + private[ParallelTesting] val times: Int, + private[ParallelTesting] val shouldDelete: Boolean, + private[ParallelTesting] val threadLimit: Option[Int], + private[ParallelTesting] val shouldFail: Boolean + ) { + import org.junit.Assert.fail + + private[ParallelTesting] def this(target: TestSource) = + this(List(target), 1, true, None, false) + + private[ParallelTesting] def this(targets: List[TestSource]) = + this(targets, 1, true, None, false) + + /** Compose test targets from `this` with `other` + * + * It does this, only if the two tests are compatible. Otherwise it throws + * an `IllegalArgumentException`. + * + * Grouping tests together like this allows us to take advantage of the + * concurrency offered by this test suite as each call to an executing + * method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the + * maximum allowed level of concurrency. Doing this for only a few targets + * does not yield any real benefit over sequential compilation. + * + * As such, each `CompilationTest` should contain as many targets as + * possible. + */ + def +(other: CompilationTest) = { + require(other.times == times, "can't combine tests that are meant to be benchmark compiled") + require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output") + require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome") + new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail) + } + + /** Creates a "pos" test run, which makes sure that all tests pass + * compilation without generating errors and that they do not crash the + * compiler + */ + def checkCompile(): this.type = { + val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail(s"Expected no errors when compiling, but found: ${test.errorCount}") + } + else if (shouldFail && !test.didFail) { + fail("Pos test should have failed, but didn't") + } + + cleanup() + } + + /** Creates a "neg" test run, which makes sure that each test generates the + * correct amount of errors at the correct positions. It also makes sure + * that none of these tests crash the compiler + */ + def checkExpectedErrors(): this.type = { + val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Neg test shouldn't have failed, but did") + } + else if (shouldFail && !test.didFail) { + fail("Neg test should have failed, but did not") + } + + cleanup() + } + + /** Creates a "run" test run, which is a superset of "pos". In addition to + * making sure that all tests pass compilation and that they do not crash + * the compiler; it also makes sure that all tests can run with the + * expected output + */ + def checkRuns(): this.type = { + val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() + + if (!shouldFail && test.didFail) { + fail("Run test failed, but should not") + } + else if (shouldFail && !test.didFail) { + fail("Run test should have failed, but did not") + } + + cleanup() + } + + /** Deletes output directories and files */ + private def cleanup(): this.type = { + if (shouldDelete) delete() + this + } + + /** Copies `file` to `dir` - taking into account if `file` is a directory, + * and if so copying recursively + */ + private def copyToDir(dir: JFile, file: JFile): JFile = { + val target = Paths.get(dir.getAbsolutePath, file.getName) + Files.copy(file.toPath, target, REPLACE_EXISTING) + if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _)) + target.toFile + } + + /** Builds a new `CompilationTest` where we have copied the target files to + * the out directory. This is needed for tests that modify the original + * source, such as `-rewrite` tests + */ + def copyToTarget(): CompilationTest = new CompilationTest ( + targets.map { + case target @ JointCompilationSource(_, files, _, outDir) => + target.copy(files = files.map(copyToDir(outDir,_))) + case target @ SeparateCompilationSource(_, dir, _, outDir) => + target.copy(dir = copyToDir(outDir, dir)) + }, + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which performs the compilation `i` times on + * each target + */ + def times(i: Int): CompilationTest = + new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail) + + /** Builds a `Compilationtest` which passes the verbose flag and logs the + * classpath + */ + def verbose: CompilationTest = new CompilationTest( + targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")), + times, shouldDelete, threadLimit, shouldFail + ) + + /** Builds a `CompilationTest` which keeps the generated output files + * + * This is needed for tests like `tastyBootstrap` which relies on first + * compiling a certain part of the project and then compiling a second + * part which depends on the first + */ + def keepOutput: CompilationTest = + new CompilationTest(targets, times, false, threadLimit, shouldFail) + + /** Builds a `CompilationTest` with a limited level of concurrency with + * maximum `i` threads + */ + def limitThreads(i: Int): CompilationTest = + new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail) + + /** Builds a `CompilationTest` where the executed test is expected to fail + * + * This behaviour is mainly needed for the tests that test the test suite. + */ + def expectFailure: CompilationTest = + new CompilationTest(targets, times, shouldDelete, threadLimit, true) + + /** Delete all output files generated by this `CompilationTest` */ + def delete(): Unit = targets.foreach(t => delete(t.outDir)) + + private def delete(file: JFile): Unit = { + if (file.isDirectory) file.listFiles.foreach(delete) + try Files.delete(file.toPath) + catch { + case _: NoSuchFileException => // already deleted, everything's fine + } + } + } + + /** Create out directory for directory `d` */ + private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = { + val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}") + targetDir.mkdirs() + targetDir + } + + /** Create out directory for `file` */ + private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = { + val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.')) + val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir") + targetDir.mkdirs() + targetDir + } + + /** Make sure that directory string is as expected */ + private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = { + require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`") + require(outDir.last == '/', "please specify an `outDir` with a trailing slash") + } + + /** Separates directories from files and returns them as `(dirs, files)` */ + private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) = + sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) => + if (f.isDirectory) (f :: dirs, files) + else if (isSourceFile(f)) (dirs, f :: files) + else (dirs, files) + } + + /** Gets the name of the calling method via reflection. + * + * It does this in a way that needs to work both with the bootstrapped dotty + * and the non-bootstrapped version. Since the two compilers generate + * different bridges, we first need to filter out methods with the same name + * (bridges) - and then find the `@Test` method in our extending class + */ + private def getCallingMethod(): String = { + val seen = mutable.Set.empty[String] + Thread.currentThread.getStackTrace + .filter { elem => + if (seen.contains(elem.getMethodName)) false + else { seen += elem.getMethodName; true } + } + .find { elem => + val callingClass = Class.forName(elem.getClassName) + classOf[ParallelTesting].isAssignableFrom(callingClass) && + elem.getFileName != "ParallelTesting.scala" + } + .map(_.getMethodName) + .getOrElse { + throw new IllegalStateException("Unable to reflectively find calling method") + } + } + + /** Compiles a single file from the string path `f` using the supplied flags */ + def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val sourceFile = new JFile(f) + val parent = sourceFile.getParentFile + val outDir = + outDirectory + callingMethod + "/" + + sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/" + + require( + sourceFile.exists && !sourceFile.isDirectory && + (parent ne null) && parent.exists && parent.isDirectory, + s"Source file: $f, didn't exist" + ) + + val target = JointCompilationSource( + callingMethod, + Array(sourceFile), + flags, + createOutputDirsForFile(sourceFile, parent, outDir) + ) + new CompilationTest(target) + } + + /** Compiles a directory `f` using the supplied `flags`. This method does + * deep compilation, that is - it compiles all files and subdirectories + * contained within the directory `f`. + * + * By default, files are compiled in alphabetical order. An optional seed + * can be used for randomization. + */ + def compileDir(f: String, flags: Array[String], randomOrder: Option[Int] = None)(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + def flatten(f: JFile): Array[JFile] = + if (f.isDirectory) f.listFiles.flatMap(flatten) + else Array(f) + + // Sort files either alphabetically or randomly using the provided seed: + val sortedFiles = flatten(sourceDir).sorted + val randomized = randomOrder match { + case None => sortedFiles + case Some(seed) => new Random(seed).shuffle(sortedFiles.toList).toArray + } + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/") + targetDir.mkdirs() + + val target = JointCompilationSource(callingMethod, randomized, flags, targetDir) + new CompilationTest(target) + } + + /** Compiles all `files` together as a single compilation run. It is given a + * `testName` since files can be in separate directories and or be otherwise + * dissociated + */ + def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + testName + "/" + + // Directories in which to compile all containing files with `flags`: + val targetDir = new JFile(outDir) + targetDir.mkdirs() + assert(targetDir.exists, s"couldn't create target directory: $targetDir") + + val target = JointCompilationSource(callingMethod, files.map(new JFile(_)).toArray, flags, targetDir) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(target) + } + + /** This function compiles the files and folders contained within directory + * `f` in a specific way. + * + * - Each file is compiled separately as a single compilation run + * - Each directory is compiled as a `SeparateCompilationTaret`, in this + * target all files are grouped according to the file suffix `_X` where `X` + * is a number. These groups are then ordered in ascending order based on + * the value of `X` and each group is compiled one after the other. + * + * For this function to work as expected, we use the same convention for + * directory layout as the old partest. That is: + * + * - Single files can have an associated check-file with the same name (but + * with file extension `.check`) + * - Directories can have an associated check-file, where the check file has + * the same name as the directory (with the file extension `.check`) + */ + def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (dirs, files) = compilationTargets(sourceDir) + + val targets = + files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++ + dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir))) + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } + + /** This function behaves similar to `compileFilesInDir` but it ignores + * sub-directories and as such, does **not** perform separate compilation + * tests. + */ + def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = { + val callingMethod = getCallingMethod + val outDir = outDirectory + callingMethod + "/" + val sourceDir = new JFile(f) + checkRequirements(f, sourceDir, outDir) + + val (_, files) = compilationTargets(sourceDir) + + val targets = files.map { file => + JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir)) + } + + // Create a CompilationTest and let the user decide whether to execute a pos or a neg test + new CompilationTest(targets) + } +} + +object ParallelTesting { + def isSourceFile(f: JFile): Boolean = { + val name = f.getName + name.endsWith(".scala") || name.endsWith(".java") + } +} diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala new file mode 100644 index 000000000..a75b1c564 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -0,0 +1,153 @@ +package dotty.tools +package vulpix + +import java.io.{ + File => JFile, + InputStream, ObjectInputStream, + OutputStream, ObjectOutputStream +} +import java.util.concurrent.TimeoutException + +import scala.concurrent.duration.Duration +import scala.concurrent.{ Await, Future } +import scala.concurrent.ExecutionContext.Implicits.global +import scala.collection.mutable + +import vulpix.Statuses._ + +trait RunnerOrchestration { + + /** The maximum amount of active runners, which contain a child JVM */ + def numberOfSlaves: Int + + /** The maximum duration the child process is allowed to consume before + * getting destroyed + */ + def maxDuration: Duration + + /** Destroy and respawn process after each test */ + def safeMode: Boolean + + /** Running a `Test` class's main method from the specified `dir` */ + def runMain(dir: JFile): Status = monitor.runMain(dir) + + private[this] val monitor = new RunnerMonitor + + private class RunnerMonitor { + + def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) + + private class Runner(private var process: Process) { + private[this] var ois: ObjectInputStream = _ + private[this] var oos: ObjectOutputStream = _ + + /** Checks if `process` is still alive + * + * When `process.exitValue()` is called on an active process the caught + * exception is thrown. As such we can know if the subprocess exited or + * not. + * + * @note used for debug + */ + def isAlive: Boolean = + try { process.exitValue(); false } + catch { case _: IllegalThreadStateException => true } + + /** Destroys the underlying process and kills IO streams */ + def kill(): Unit = { + if (process ne null) process.destroy() + process = null + ois = null + oos = null + } + + /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ + def runMain(dir: JFile): Status = { + assert(process ne null, + "Runner was killed and then reused without setting a new process") + + // Makes the encapsulating RunnerMonitor spawn a new runner + def respawn(): Unit = { + process.destroy() + process = createProcess + ois = null + oos = null + } + + if (oos eq null) oos = new ObjectOutputStream(process.getOutputStream) + + // pass file to running process + oos.writeObject(dir) + oos.flush() + + // Create a future reading the object: + val readObject = Future { + if (ois eq null) ois = new ObjectInputStream(process.getInputStream) + ois.readObject().asInstanceOf[Status] + } + + // Await result for `maxDuration` and then timout and destroy the + // process: + val status = + try Await.result(readObject, maxDuration) + catch { case _: TimeoutException => new Timeout() } + + // Handle failure of the VM: + status match { + case _ if safeMode => respawn() + case _: Failure => respawn() + case _: Timeout => respawn() + case _ => () + } + status + } + } + + private def createProcess: Process = { + val sep = sys.props("file.separator") + val cp = sys.props("java.class.path") + val java = sys.props("java.home") + sep + "bin" + sep + "java" + new ProcessBuilder(java, "-cp", cp, "dotty.tools.dotc.vulpix.ChildMain")//classOf[ChildMain].getName) + .redirectErrorStream(true) + .redirectInput(ProcessBuilder.Redirect.PIPE) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .start() + } + + private[this] val allRunners = List.fill(numberOfSlaves)(new Runner(createProcess)) + + private[this] val freeRunners = mutable.Queue(allRunners: _*) + private[this] val busyRunners = mutable.Set.empty[Runner] + + private def getRunner(): Runner = synchronized { + while (freeRunners.isEmpty) wait() + + val runner = freeRunners.dequeue() + busyRunners += runner + + notify() + runner + } + + private def freeRunner(runner: Runner): Unit = synchronized { + freeRunners.enqueue(runner) + busyRunners -= runner + notify() + } + + private def withRunner[T](op: Runner => T): T = { + val runner = getRunner() + val result = op(runner) + freeRunner(runner) + result + } + + private def killAll(): Unit = allRunners.foreach(_.kill()) + + // On shutdown, we need to kill all runners: + sys.addShutdownHook(killAll()) + // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we + // need to clean up ourselves. + SummaryReport.addCleanup(killAll) + } +} diff --git a/compiler/test/dotty/tools/vulpix/Statuses.java b/compiler/test/dotty/tools/vulpix/Statuses.java new file mode 100644 index 000000000..68add30eb --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/Statuses.java @@ -0,0 +1,25 @@ +package dotty.tools.vulpix; + +import java.io.Serializable; + +/** The status of each call to `main` in the test applications */ +public class Statuses { + interface Status {} + + static class Success implements Status, Serializable { + public final String output; + public Success(String output) { this.output = output; } + } + + static class Failure implements Status, Serializable { + public final String message; + public final String stacktrace; + + public Failure(String message, String stacktrace) { + this.message = message; + this.stacktrace = stacktrace; + } + } + + static class Timeout implements Status, Serializable {} +} diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.java b/compiler/test/dotty/tools/vulpix/SummaryReport.java new file mode 100644 index 000000000..61a708f26 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.java @@ -0,0 +1,96 @@ +package dotty.tools.vulpix; + +import org.junit.BeforeClass; +import org.junit.AfterClass; +import java.util.ArrayDeque; +import java.util.function.Supplier; +import scala.Function0; +import scala.Unit; + +import dotty.tools.dotc.reporting.TestReporter; + +/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with + * this class + */ +public class SummaryReport { + public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); + + private static TestReporter rep = TestReporter.reporter(System.out, -1); + private static ArrayDeque failedTests = new ArrayDeque<>(); + private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); + private static Supplier cleanup; + private static int passed; + private static int failed; + + public final static void reportFailed() { + failed++; + } + + public final static void reportPassed() { + passed++; + } + + public final static void addFailedTest(String msg) { + failedTests.offer(msg); + } + + public final static void addReproduceInstruction(String msg) { + reproduceInstructions.offer(msg); + } + + public final static void addCleanup(Function0 func) { + // Wow, look at how neatly we - compose cleanup callbacks: + if (cleanup == null) { + cleanup = () -> { + func.apply(); + return null; + }; + } else { + Supplier oldCleanup = cleanup; + cleanup = () -> { + oldCleanup.get(); + func.apply(); + return null; + }; + } + } + + @BeforeClass public final static void setup() { + rep = TestReporter.reporter(System.out, -1); + failedTests = new ArrayDeque<>(); + reproduceInstructions = new ArrayDeque<>(); + } + + @AfterClass public final static void teardown() { + rep.echo( + "\n================================================================================" + + "\nTest Report" + + "\n================================================================================" + + "\n" + + passed + " passed, " + failed + " failed, " + (passed + failed) + " total" + + "\n" + ); + + failedTests + .stream() + .map(x -> " " + x) + .forEach(rep::echo); + + // If we're compiling locally, we don't need reproduce instructions + if (isInteractive) rep.flushToStdErr(); + + rep.echo(""); + + reproduceInstructions + .stream() + .forEach(rep::echo); + + // If we're on the CI, we want everything + if (!isInteractive) rep.flushToStdErr(); + + if (failed > 0) rep.flushToFile(); + + // Perform cleanup callback: + if (cleanup != null) cleanup.get(); + } +} diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala new file mode 100644 index 000000000..bebcc7601 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -0,0 +1,61 @@ +package dotty.tools +package vulpix + +import org.junit.Assert._ +import org.junit.Test + +import scala.concurrent.duration._ +import scala.util.control.NonFatal + +class VulpixTests extends ParallelTesting { + import dotc.CompilationTests._ + + def maxDuration = 3.seconds + def numberOfSlaves = 5 + def safeMode = sys.env.get("SAFEMODE").isDefined + def isInteractive = !sys.env.contains("DRONE") + def testFilter = None + + @Test def missingFile: Unit = + try { + compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors() + fail("didn't fail properly") + } + catch { + case _: IllegalArgumentException => // pass! + case NonFatal(_) => fail("wrong exception thrown") + } + + @Test def pos1Error: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile() + + @Test def negMissingAnnot: Unit = + compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negAnnotWrongLine: Unit = + compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negTooManyAnnots: Unit = + compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def negNoPositionAnnot: Unit = + compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors() + + @Test def runCompileFail: Unit = + compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput1: Unit = + compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runWrongOutput2: Unit = + compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runDiffOutput1: Unit = + compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runStackOverflow: Unit = + compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns() + + @Test def runOutRedirects: Unit = + compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() +} -- cgit v1.2.3 From c1e787f7560807ca95e021d9cb7f1406c5953c3c Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 19:22:58 +0200 Subject: Make inter JVM communication be string based --- compiler/test/dotty/Jars.scala | 13 ++++ compiler/test/dotty/tools/vulpix/ChildJVMMain.java | 34 ++++++++ compiler/test/dotty/tools/vulpix/ChildMain.scala | 82 ------------------- .../test/dotty/tools/vulpix/ParallelTesting.scala | 49 +++++------- .../dotty/tools/vulpix/RunnerOrchestration.scala | 91 ++++++++++++++-------- compiler/test/dotty/tools/vulpix/Status.scala | 7 ++ compiler/test/dotty/tools/vulpix/Statuses.java | 25 ------ 7 files changed, 133 insertions(+), 168 deletions(-) create mode 100644 compiler/test/dotty/tools/vulpix/ChildJVMMain.java delete mode 100644 compiler/test/dotty/tools/vulpix/ChildMain.scala create mode 100644 compiler/test/dotty/tools/vulpix/Status.scala delete mode 100644 compiler/test/dotty/tools/vulpix/Statuses.java diff --git a/compiler/test/dotty/Jars.scala b/compiler/test/dotty/Jars.scala index f062f8b25..06df9c891 100644 --- a/compiler/test/dotty/Jars.scala +++ b/compiler/test/dotty/Jars.scala @@ -19,4 +19,17 @@ object Jars { val dottyTestDeps: List[String] = dottyLib :: dottyCompiler :: dottyInterfaces :: dottyExtras + + + def scalaLibraryFromRuntime: String = findJarFromRuntime("scala-library-2.") + + private def findJarFromRuntime(partialName: String) = { + val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) + urls.find(_.contains(partialName)).getOrElse { + throw new java.io.FileNotFoundException( + s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" + ) + } + } + } diff --git a/compiler/test/dotty/tools/vulpix/ChildJVMMain.java b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java new file mode 100644 index 000000000..90b795898 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java @@ -0,0 +1,34 @@ +package dotty.tools.vulpix; + +import java.io.File; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.lang.reflect.Method; + +public class ChildJVMMain { + static final String MessageEnd = "##THIS IS THE END FOR ME, GOODBYE##"; + + private static void runMain(String dir) throws Exception { + ArrayList cp = new ArrayList<>(); + for (String path : dir.split(":")) + cp.add(new File(path).toURI().toURL()); + + URLClassLoader ucl = new URLClassLoader(cp.toArray(new URL[cp.size()])); + Class cls = ucl.loadClass("Test"); + Method meth = cls.getMethod("main", String[].class); + Object[] args = new Object[]{ new String[]{ "jvm" } }; + meth.invoke(null, args); + } + + public static void main(String[] args) throws Exception { + BufferedReader stdin = new BufferedReader(new InputStreamReader(System.in)); + + while (true) { + runMain(stdin.readLine()); + System.out.println(MessageEnd); + } + } +} diff --git a/compiler/test/dotty/tools/vulpix/ChildMain.scala b/compiler/test/dotty/tools/vulpix/ChildMain.scala deleted file mode 100644 index 30059a9c5..000000000 --- a/compiler/test/dotty/tools/vulpix/ChildMain.scala +++ /dev/null @@ -1,82 +0,0 @@ -package dotty.tools.vulpix - -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream, - ByteArrayOutputStream, PrintStream -} -import java.lang.reflect.InvocationTargetException - -import dotty.tools.vulpix.Statuses._ - -object ChildMain { - val realStdin = System.in - val realStderr = System.err - val realStdout = System.out - - private def runMain(dir: JFile): Status = { - def renderStackTrace(ex: Throwable): String = - ex.getStackTrace - .takeWhile(_.getMethodName != "invoke0") - .mkString(" ", "\n ", "") - - def resetOutDescriptors(): Unit = { - System.setOut(realStdout) - System.setErr(realStderr) - } - - import java.net.{ URL, URLClassLoader } - - val printStream = new ByteArrayOutputStream - - try { - // Do classloading magic and running here: - val ucl = new URLClassLoader(Array(dir.toURI.toURL)) - val cls = ucl.loadClass("Test") - val meth = cls.getMethod("main", classOf[Array[String]]) - - try { - val ps = new PrintStream(printStream) - System.setOut(ps) - System.setErr(ps) - Console.withOut(printStream) { - Console.withErr(printStream) { - // invoke main with "jvm" as arg - meth.invoke(null, Array("jvm")) - } - } - resetOutDescriptors() - } catch { - case t: Throwable => - resetOutDescriptors() - throw t - } - new Success(printStream.toString("utf-8")) - } - catch { - case ex: NoSuchMethodException => - val msg = s"test in '$dir' did not contain method: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: ClassNotFoundException => - val msg = s"test in '$dir' did not contain class: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: InvocationTargetException => - val msg = s"An exception ocurred when running main: ${ex.getCause}" - new Failure(msg, renderStackTrace(ex.getCause)) - } - } - - def main(args: Array[String]): Unit = { - val stdin = new ObjectInputStream(System.in); - val stdout = new ObjectOutputStream(System.out); - - while (true) { - val dir = stdin.readObject().asInstanceOf[JFile] - stdout.writeObject(runMain(dir)) - stdout.flush() - } - } -} diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 9b9f0a2bb..d23ab0778 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -23,8 +23,6 @@ import dotc.interfaces.Diagnostic.ERROR import dotc.util.DiffUtil import dotc.{ Driver, Compiler } -import vulpix.Statuses._ - /** A parallel testing suite whose goal is to integrate nicely with JUnit * * This trait can be mixed in to offer parallel testing to compile runs. When @@ -54,6 +52,15 @@ trait ParallelTesting extends RunnerOrchestration { self => def outDir: JFile def flags: Array[String] + def classPath: String = { + val (beforeCp, cpAndAfter) = flags.toList.span(_ != "-classpath") + if (cpAndAfter.nonEmpty) { + val (_ :: cpArg :: _) = cpAndAfter + s"${outDir.getAbsolutePath}:" + cpArg + } + else outDir.getAbsolutePath + } + def title: String = self match { case self: JointCompilationSource => @@ -294,15 +301,6 @@ trait ParallelTesting extends RunnerOrchestration { self => val files: Array[JFile] = files0.flatMap(flattenFiles) - def findJarFromRuntime(partialName: String) = { - val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) - urls.find(_.contains(partialName)).getOrElse { - throw new java.io.FileNotFoundException( - s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" - ) - } - } - def addOutDir(xs: Array[String]): Array[String] = { val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") if (cpAndAfter.nonEmpty) { @@ -313,11 +311,10 @@ trait ParallelTesting extends RunnerOrchestration { self => } def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { - val scalaLib = findJarFromRuntime("scala-library-2.") val fullArgs = Array( "javac", "-classpath", - s".:$scalaLib:${targetDir.getAbsolutePath}" + s".:${Jars.scalaLibraryFromRuntime}:${targetDir.getAbsolutePath}" ) ++ flags.takeRight(2) ++ fs Runtime.getRuntime.exec(fullArgs).waitFor() == 0 @@ -430,9 +427,9 @@ trait ParallelTesting extends RunnerOrchestration { self => private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - runMain(dir) match { - case success: Success => { - val outputLines = success.output.lines.toArray + runMain(testSource.classPath) match { + case Success(output) => { + val outputLines = output.lines.toArray val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray val sourceTitle = testSource.title @@ -463,19 +460,16 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - case failure: Failure => - echo(renderFailure(failure)) + case Failure(output) => + echo(output) failTestSource(testSource) - case _: Timeout => + case Timeout => echo("failed because test " + testSource.title + " timed out") failTestSource(testSource, Some("test timed out")) } } - private def renderFailure(failure: Failure): String = - failure.message + "\n" + failure.stacktrace - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { def run(): Unit = tryCompile(testSource) { val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { @@ -519,17 +513,14 @@ trait ParallelTesting extends RunnerOrchestration { self => } if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.outDir) match { - case status: Failure => - echo(renderFailure(status)) + else if (errorCount == 0) runMain(testSource.classPath) match { + case Success(_) => // success! + case Failure(output) => failTestSource(testSource) - case _: Timeout => - echo("failed because test " + testSource.title + " timed out") + case Timeout => failTestSource(testSource, Some("test timed out")) - case _: Success => // success! } else if (errorCount > 0) { - echo(s"\nCompilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) addFailureInstruction(buildInstr) failTestSource(testSource) diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala index a75b1c564..22bebf714 100644 --- a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -1,11 +1,8 @@ -package dotty.tools +package dotty +package tools package vulpix -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream -} +import java.io.{ File => JFile, InputStreamReader, BufferedReader, PrintStream } import java.util.concurrent.TimeoutException import scala.concurrent.duration.Duration @@ -13,8 +10,25 @@ import scala.concurrent.{ Await, Future } import scala.concurrent.ExecutionContext.Implicits.global import scala.collection.mutable -import vulpix.Statuses._ - +/** Vulpix spawns JVM subprocesses (`numberOfSlaves`) in order to run tests + * without compromising the main JVM + * + * These need to be orchestrated in a safe manner with a simple protocol. This + * interface provides just that. + * + * The protocol is defined as: + * + * - master sends classpath to for which to run `Test#main` and waits for + * `maxDuration` + * - slave invokes the method and waits until completion + * - upon completion it sends back a `RunComplete` message + * - the master checks if the child is still alive + * - child is still alive, the output was valid + * - child is dead, the output is the failure message + * + * If this whole chain of events is not completed within `maxDuration`, the + * child process is destroyed and a new child is spawned. + */ trait RunnerOrchestration { /** The maximum amount of active runners, which contain a child JVM */ @@ -29,25 +43,24 @@ trait RunnerOrchestration { def safeMode: Boolean /** Running a `Test` class's main method from the specified `dir` */ - def runMain(dir: JFile): Status = monitor.runMain(dir) + def runMain(classPath: String): Status = monitor.runMain(classPath) private[this] val monitor = new RunnerMonitor + /** Look away now, sweet child of summer */ private class RunnerMonitor { - def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) + def runMain(classPath: String): Status = withRunner(_.runMain(classPath)) private class Runner(private var process: Process) { - private[this] var ois: ObjectInputStream = _ - private[this] var oos: ObjectOutputStream = _ + private[this] var childStdout: BufferedReader = _ + private[this] var childStdin: PrintStream = _ /** Checks if `process` is still alive * * When `process.exitValue()` is called on an active process the caught * exception is thrown. As such we can know if the subprocess exited or * not. - * - * @note used for debug */ def isAlive: Boolean = try { process.exitValue(); false } @@ -57,12 +70,12 @@ trait RunnerOrchestration { def kill(): Unit = { if (process ne null) process.destroy() process = null - ois = null - oos = null + childStdout = null + childStdin = null } /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ - def runMain(dir: JFile): Status = { + def runMain(classPath: String): Status = { assert(process ne null, "Runner was killed and then reused without setting a new process") @@ -70,33 +83,45 @@ trait RunnerOrchestration { def respawn(): Unit = { process.destroy() process = createProcess - ois = null - oos = null + childStdout = null + childStdin = null } - if (oos eq null) oos = new ObjectOutputStream(process.getOutputStream) + if (childStdin eq null) + childStdin = new PrintStream(process.getOutputStream, /* autoFlush = */ true) // pass file to running process - oos.writeObject(dir) - oos.flush() + childStdin.println(classPath) // Create a future reading the object: - val readObject = Future { - if (ois eq null) ois = new ObjectInputStream(process.getInputStream) - ois.readObject().asInstanceOf[Status] + val readOutput = Future { + val sb = new StringBuilder + + if (childStdout eq null) + childStdout = new BufferedReader(new InputStreamReader(process.getInputStream)) + + var childOutput = childStdout.readLine() + while (childOutput != ChildJVMMain.MessageEnd && childOutput != null) { + sb.append(childOutput) + sb += '\n' + childOutput = childStdout.readLine() + } + + if (process.isAlive && childOutput != null) Success(sb.toString) + else Failure(sb.toString) } // Await result for `maxDuration` and then timout and destroy the // process: val status = - try Await.result(readObject, maxDuration) - catch { case _: TimeoutException => new Timeout() } + try Await.result(readOutput, maxDuration) + catch { case _: TimeoutException => Timeout } // Handle failure of the VM: status match { - case _ if safeMode => respawn() + case _: Success if safeMode => respawn() case _: Failure => respawn() - case _: Timeout => respawn() + case Timeout => respawn() case _ => () } status @@ -105,9 +130,11 @@ trait RunnerOrchestration { private def createProcess: Process = { val sep = sys.props("file.separator") - val cp = sys.props("java.class.path") - val java = sys.props("java.home") + sep + "bin" + sep + "java" - new ProcessBuilder(java, "-cp", cp, "dotty.tools.dotc.vulpix.ChildMain")//classOf[ChildMain].getName) + val cp = + classOf[ChildJVMMain].getProtectionDomain.getCodeSource.getLocation.getFile + ":" + + Jars.scalaLibraryFromRuntime + val javaBin = sys.props("java.home") + sep + "bin" + sep + "java" + new ProcessBuilder(javaBin, "-cp", cp, "dotty.tools.vulpix.ChildJVMMain") .redirectErrorStream(true) .redirectInput(ProcessBuilder.Redirect.PIPE) .redirectOutput(ProcessBuilder.Redirect.PIPE) diff --git a/compiler/test/dotty/tools/vulpix/Status.scala b/compiler/test/dotty/tools/vulpix/Status.scala new file mode 100644 index 000000000..3de7aff2b --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/Status.scala @@ -0,0 +1,7 @@ +package dotty.tools +package vulpix + +sealed trait Status +final case class Success(output: String) extends Status +final case class Failure(output: String) extends Status +final case object Timeout extends Status diff --git a/compiler/test/dotty/tools/vulpix/Statuses.java b/compiler/test/dotty/tools/vulpix/Statuses.java deleted file mode 100644 index 68add30eb..000000000 --- a/compiler/test/dotty/tools/vulpix/Statuses.java +++ /dev/null @@ -1,25 +0,0 @@ -package dotty.tools.vulpix; - -import java.io.Serializable; - -/** The status of each call to `main` in the test applications */ -public class Statuses { - interface Status {} - - static class Success implements Status, Serializable { - public final String output; - public Success(String output) { this.output = output; } - } - - static class Failure implements Status, Serializable { - public final String message; - public final String stacktrace; - - public Failure(String message, String stacktrace) { - this.message = message; - this.stacktrace = stacktrace; - } - } - - static class Timeout implements Status, Serializable {} -} -- cgit v1.2.3 From 95732a3576feb477fda74d11907878b808c9bc66 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 19:23:27 +0200 Subject: Improve test failure output --- compiler/test/dotty/tools/vulpix/ParallelTesting.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index d23ab0778..82847e041 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -235,7 +235,7 @@ trait ParallelTesting extends RunnerOrchestration { self => private[this] val failedTestSources = mutable.ArrayBuffer.empty[String] protected final def failTestSource(testSource: TestSource, reason: Option[String] = None) = synchronized { val extra = reason.map(" with reason: " + _).getOrElse("") - failedTestSources.append(testSource.name + " failed" + extra) + failedTestSources.append(testSource.title + s" failed (in ${testSource.name})" + extra) fail() } @@ -516,11 +516,15 @@ trait ParallelTesting extends RunnerOrchestration { self => else if (errorCount == 0) runMain(testSource.classPath) match { case Success(_) => // success! case Failure(output) => + echo(s" failed when running '${testSource.title}'") + echo(output) failTestSource(testSource) case Timeout => + echo(" failed because test " + testSource.title + " timed out") failTestSource(testSource, Some("test timed out")) } else if (errorCount > 0) { + echo(s"\n Compilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) addFailureInstruction(buildInstr) failTestSource(testSource) -- cgit v1.2.3 From 0ee99771cf12c059f79ef0b9eb7b6e2bd09da942 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 19:20:44 +0200 Subject: Disable scalac internals tests --- .../scalac-dependent/WeakHashSetTest.scala | 174 +++++++++++++++++++++ tests/disabled/scalac-dependent/shortClass.scala | 26 +++ .../scalac-dependent/showraw_nosymbol.scala | 5 + .../scalac-dependent/sm-interpolator.scala | 41 +++++ tests/disabled/scalac-dependent/structural.scala | 25 +++ tests/disabled/scalac-dependent/t6732.scala | 12 ++ tests/run/WeakHashSetTest.scala | 174 --------------------- tests/run/shortClass.scala | 26 --- tests/run/showraw_nosymbol.scala | 5 - tests/run/sm-interpolator.scala | 41 ----- tests/run/structural.scala | 25 --- tests/run/t429.scala | 1 + tests/run/t6732.scala | 12 -- 13 files changed, 284 insertions(+), 283 deletions(-) create mode 100644 tests/disabled/scalac-dependent/WeakHashSetTest.scala create mode 100644 tests/disabled/scalac-dependent/shortClass.scala create mode 100644 tests/disabled/scalac-dependent/showraw_nosymbol.scala create mode 100644 tests/disabled/scalac-dependent/sm-interpolator.scala create mode 100644 tests/disabled/scalac-dependent/structural.scala create mode 100644 tests/disabled/scalac-dependent/t6732.scala delete mode 100644 tests/run/WeakHashSetTest.scala delete mode 100644 tests/run/shortClass.scala delete mode 100644 tests/run/showraw_nosymbol.scala delete mode 100644 tests/run/sm-interpolator.scala delete mode 100644 tests/run/structural.scala delete mode 100644 tests/run/t6732.scala diff --git a/tests/disabled/scalac-dependent/WeakHashSetTest.scala b/tests/disabled/scalac-dependent/WeakHashSetTest.scala new file mode 100644 index 000000000..8bcb95091 --- /dev/null +++ b/tests/disabled/scalac-dependent/WeakHashSetTest.scala @@ -0,0 +1,174 @@ +object Test { + def main(args: Array[String]): Unit = { + val test = scala.reflect.internal.util.WeakHashSetTest + test.checkEmpty + test.checkPlusEquals + test.checkPlusEqualsCollisions + test.checkRehashing + test.checkRehashCollisions + test.checkFindOrUpdate + test.checkMinusEquals + test.checkMinusEqualsCollisions + test.checkClear + test.checkIterator + test.checkIteratorCollisions + + // This test is commented out because it relies on gc behavior which isn't reliable enough in an automated environment + // test.checkRemoveUnreferencedObjects + } +} + +// put the main test object in the same package as WeakHashSet because +// it uses the package private "diagnostics" method +package scala.reflect.internal.util { + + object WeakHashSetTest { + // a class guaranteed to provide hash collisions + case class Collider(x : String) extends Comparable[Collider] with Serializable { + override def hashCode = 0 + def compareTo(y : Collider) = this.x compareTo y.x + } + + // basic emptiness check + def checkEmpty: Unit = { + val hs = new WeakHashSet[String]() + assert(hs.size == 0) + hs.diagnostics.fullyValidate + } + + // make sure += works + def checkPlusEquals: Unit = { + val hs = new WeakHashSet[String]() + val elements = List("hello", "goodbye") + elements foreach (hs += _) + assert(hs.size == 2) + assert(hs contains "hello") + assert(hs contains "goodbye") + hs.diagnostics.fullyValidate + } + + // make sure += works when there are collisions + def checkPlusEqualsCollisions: Unit = { + val hs = new WeakHashSet[Collider]() + val elements = List("hello", "goodbye") map Collider + elements foreach (hs += _) + assert(hs.size == 2) + assert(hs contains Collider("hello")) + assert(hs contains Collider("goodbye")) + hs.diagnostics.fullyValidate + } + + // add a large number of elements to force rehashing and then validate + def checkRehashing: Unit = { + val size = 200 + val hs = new WeakHashSet[String]() + val elements = (0 until size).toList map ("a" + _) + elements foreach (hs += _) + elements foreach {i => assert(hs contains i)} + hs.diagnostics.fullyValidate + } + + // make sure rehashing works properly when the set is rehashed + def checkRehashCollisions: Unit = { + val size = 200 + val hs = new WeakHashSet[Collider]() + val elements = (0 until size).toList map {x => Collider("a" + x)} + elements foreach (hs += _) + elements foreach {i => assert(hs contains i)} + hs.diagnostics.fullyValidate + } + + // test that unreferenced objects are removed + // not run in an automated environment because gc behavior can't be relied on + def checkRemoveUnreferencedObjects: Unit = { + val size = 200 + val hs = new WeakHashSet[Collider]() + val elements = (0 until size).toList map {x => Collider("a" + x)} + elements foreach (hs += _) + // don't throw the following into a retained collection so gc + // can remove them + for (i <- 0 until size) { + hs += Collider("b" + i) + } + System.gc() + Thread.sleep(1000) + assert(hs.size == 200) + elements foreach {i => assert(hs contains i)} + for (i <- 0 until size) { + assert(!(hs contains Collider("b" + i))) + } + hs.diagnostics.fullyValidate + } + + // make sure findOrUpdate returns the originally entered element + def checkFindOrUpdate: Unit = { + val size = 200 + val hs = new WeakHashSet[Collider]() + val elements = (0 until size).toList map {x => Collider("a" + x)} + elements foreach {x => assert(hs findEntryOrUpdate x eq x)} + for (i <- 0 until size) { + // when we do a lookup the result should be the same reference we + // original put in + assert(hs findEntryOrUpdate(Collider("a" + i)) eq elements(i)) + } + hs.diagnostics.fullyValidate + } + + // check -= functionality + def checkMinusEquals: Unit = { + val hs = new WeakHashSet[String]() + val elements = List("hello", "goodbye") + elements foreach (hs += _) + hs -= "goodbye" + assert(hs.size == 1) + assert(hs contains "hello") + assert(!(hs contains "goodbye")) + hs.diagnostics.fullyValidate + } + + // check -= when there are collisions + def checkMinusEqualsCollisions: Unit = { + val hs = new WeakHashSet[Collider] + val elements = List(Collider("hello"), Collider("goodbye")) + elements foreach (hs += _) + hs -= Collider("goodbye") + assert(hs.size == 1) + assert(hs contains Collider("hello")) + assert(!(hs contains Collider("goodbye"))) + hs -= Collider("hello") + assert(hs.size == 0) + assert(!(hs contains Collider("hello"))) + hs.diagnostics.fullyValidate + } + + // check that the clear method actually cleans everything + def checkClear: Unit = { + val size = 200 + val hs = new WeakHashSet[String]() + val elements = (0 until size).toList map ("a" + _) + elements foreach (hs += _) + hs.clear() + assert(hs.size == 0) + elements foreach {i => assert(!(hs contains i))} + hs.diagnostics.fullyValidate + } + + // check that the iterator covers all the contents + def checkIterator: Unit = { + val hs = new WeakHashSet[String]() + val elements = (0 until 20).toList map ("a" + _) + elements foreach (hs += _) + assert(elements.iterator.toList.sorted == elements.sorted) + hs.diagnostics.fullyValidate + } + + // check that the iterator covers all the contents even when there is a collision + def checkIteratorCollisions: Unit = { + val hs = new WeakHashSet[Collider] + val elements = (0 until 20).toList map {x => Collider("a" + x)} + elements foreach (hs += _) + assert(elements.iterator.toList.sorted == elements.sorted) + hs.diagnostics.fullyValidate + } + } +} diff --git a/tests/disabled/scalac-dependent/shortClass.scala b/tests/disabled/scalac-dependent/shortClass.scala new file mode 100644 index 000000000..c5c2043f4 --- /dev/null +++ b/tests/disabled/scalac-dependent/shortClass.scala @@ -0,0 +1,26 @@ +import scala.reflect.internal.util._ + +package bippity { + trait DingDongBippy + + package bop { + class Foo { + class Bar + object Bar + } + } +} + +object Test { + import bippity._ + import bop._ + + def printSanitized(x: String) = println(x.filterNot(_.isDigit)) + + def main(args: Array[String]): Unit = { + val f = new Foo + val instances = List(f, new f.Bar, f.Bar, new Foo with DingDongBippy, new f.Bar with DingDongBippy) + instances map (_.getClass.getName) foreach printSanitized + instances map shortClassOfInstance foreach printSanitized + } +} diff --git a/tests/disabled/scalac-dependent/showraw_nosymbol.scala b/tests/disabled/scalac-dependent/showraw_nosymbol.scala new file mode 100644 index 000000000..191647583 --- /dev/null +++ b/tests/disabled/scalac-dependent/showraw_nosymbol.scala @@ -0,0 +1,5 @@ +import scala.reflect.runtime.universe._ + +object Test extends dotty.runtime.LegacyApp { + println(showRaw(NoSymbol)) +} diff --git a/tests/disabled/scalac-dependent/sm-interpolator.scala b/tests/disabled/scalac-dependent/sm-interpolator.scala new file mode 100644 index 000000000..e4bec7afb --- /dev/null +++ b/tests/disabled/scalac-dependent/sm-interpolator.scala @@ -0,0 +1,41 @@ +object Test extends dotty.runtime.LegacyApp { + import scala.reflect.internal.util.StringContextStripMarginOps + def check(actual: Any, expected: Any) = if (actual != expected) sys.error(s"\nexpected:\n$expected\n\nactual:\n$actual") + + val bar = "|\n ||" + + check( + sm"""|ab + |de + |${bar} | ${1}""", + "ab\nde\n|\n || | 1") + + check( + sm"|", + "") + + check( + sm"${0}", + "0") + + check( + sm"${0}", + "0") + + check( + sm"""${0}|${1} + |""", + "0|1\n") + + check( + sm""" ||""", + "|") + + check( + sm""" ${" "} ||""", + " ||") + + check( + sm"\n", + raw"\n".stripMargin) +} diff --git a/tests/disabled/scalac-dependent/structural.scala b/tests/disabled/scalac-dependent/structural.scala new file mode 100644 index 000000000..0f18f4579 --- /dev/null +++ b/tests/disabled/scalac-dependent/structural.scala @@ -0,0 +1,25 @@ +case class Record(elems: (String, Any)*) extends Selectable { + def selectDynamic(name: String): Any = elems.find(_._1 == name).get._2 +} + +object Test { + import scala.reflect.Selectable.reflectiveSelectable + + def f(closeable: { def close(): Unit }) = + closeable.close() + + type RN = Record { val name: String; val age: Int } + + def g(r: RN) = r.name + + val rr: RN = Record("name" -> "Bob", "age" -> 42).asInstanceOf[RN] + + def main(args: Array[String]): Unit = { + f(new java.io.PrintStream("foo")) + assert(g(rr) == "Bob") + + val s: { def concat(s: String): String } = "abc" + assert(s.concat("def") == "abcdef") + } +} + diff --git a/tests/disabled/scalac-dependent/t6732.scala b/tests/disabled/scalac-dependent/t6732.scala new file mode 100644 index 000000000..ff0f0494d --- /dev/null +++ b/tests/disabled/scalac-dependent/t6732.scala @@ -0,0 +1,12 @@ +import scala.reflect.runtime.universe._ +import definitions._ + +object Test extends dotty.runtime.LegacyApp { + def test(sym: Symbol): Unit = { + println(s"${showRaw(sym, printKinds = true)}: ${sym.isModule}, ${sym.isModuleClass}, ${sym.isPackage}, ${sym.isPackageClass}") + } + test(ScalaPackage) + test(ScalaPackageClass) + test(ListModule) + test(ListModule.moduleClass) +} diff --git a/tests/run/WeakHashSetTest.scala b/tests/run/WeakHashSetTest.scala deleted file mode 100644 index 8bcb95091..000000000 --- a/tests/run/WeakHashSetTest.scala +++ /dev/null @@ -1,174 +0,0 @@ -object Test { - def main(args: Array[String]): Unit = { - val test = scala.reflect.internal.util.WeakHashSetTest - test.checkEmpty - test.checkPlusEquals - test.checkPlusEqualsCollisions - test.checkRehashing - test.checkRehashCollisions - test.checkFindOrUpdate - test.checkMinusEquals - test.checkMinusEqualsCollisions - test.checkClear - test.checkIterator - test.checkIteratorCollisions - - // This test is commented out because it relies on gc behavior which isn't reliable enough in an automated environment - // test.checkRemoveUnreferencedObjects - } -} - -// put the main test object in the same package as WeakHashSet because -// it uses the package private "diagnostics" method -package scala.reflect.internal.util { - - object WeakHashSetTest { - // a class guaranteed to provide hash collisions - case class Collider(x : String) extends Comparable[Collider] with Serializable { - override def hashCode = 0 - def compareTo(y : Collider) = this.x compareTo y.x - } - - // basic emptiness check - def checkEmpty: Unit = { - val hs = new WeakHashSet[String]() - assert(hs.size == 0) - hs.diagnostics.fullyValidate - } - - // make sure += works - def checkPlusEquals: Unit = { - val hs = new WeakHashSet[String]() - val elements = List("hello", "goodbye") - elements foreach (hs += _) - assert(hs.size == 2) - assert(hs contains "hello") - assert(hs contains "goodbye") - hs.diagnostics.fullyValidate - } - - // make sure += works when there are collisions - def checkPlusEqualsCollisions: Unit = { - val hs = new WeakHashSet[Collider]() - val elements = List("hello", "goodbye") map Collider - elements foreach (hs += _) - assert(hs.size == 2) - assert(hs contains Collider("hello")) - assert(hs contains Collider("goodbye")) - hs.diagnostics.fullyValidate - } - - // add a large number of elements to force rehashing and then validate - def checkRehashing: Unit = { - val size = 200 - val hs = new WeakHashSet[String]() - val elements = (0 until size).toList map ("a" + _) - elements foreach (hs += _) - elements foreach {i => assert(hs contains i)} - hs.diagnostics.fullyValidate - } - - // make sure rehashing works properly when the set is rehashed - def checkRehashCollisions: Unit = { - val size = 200 - val hs = new WeakHashSet[Collider]() - val elements = (0 until size).toList map {x => Collider("a" + x)} - elements foreach (hs += _) - elements foreach {i => assert(hs contains i)} - hs.diagnostics.fullyValidate - } - - // test that unreferenced objects are removed - // not run in an automated environment because gc behavior can't be relied on - def checkRemoveUnreferencedObjects: Unit = { - val size = 200 - val hs = new WeakHashSet[Collider]() - val elements = (0 until size).toList map {x => Collider("a" + x)} - elements foreach (hs += _) - // don't throw the following into a retained collection so gc - // can remove them - for (i <- 0 until size) { - hs += Collider("b" + i) - } - System.gc() - Thread.sleep(1000) - assert(hs.size == 200) - elements foreach {i => assert(hs contains i)} - for (i <- 0 until size) { - assert(!(hs contains Collider("b" + i))) - } - hs.diagnostics.fullyValidate - } - - // make sure findOrUpdate returns the originally entered element - def checkFindOrUpdate: Unit = { - val size = 200 - val hs = new WeakHashSet[Collider]() - val elements = (0 until size).toList map {x => Collider("a" + x)} - elements foreach {x => assert(hs findEntryOrUpdate x eq x)} - for (i <- 0 until size) { - // when we do a lookup the result should be the same reference we - // original put in - assert(hs findEntryOrUpdate(Collider("a" + i)) eq elements(i)) - } - hs.diagnostics.fullyValidate - } - - // check -= functionality - def checkMinusEquals: Unit = { - val hs = new WeakHashSet[String]() - val elements = List("hello", "goodbye") - elements foreach (hs += _) - hs -= "goodbye" - assert(hs.size == 1) - assert(hs contains "hello") - assert(!(hs contains "goodbye")) - hs.diagnostics.fullyValidate - } - - // check -= when there are collisions - def checkMinusEqualsCollisions: Unit = { - val hs = new WeakHashSet[Collider] - val elements = List(Collider("hello"), Collider("goodbye")) - elements foreach (hs += _) - hs -= Collider("goodbye") - assert(hs.size == 1) - assert(hs contains Collider("hello")) - assert(!(hs contains Collider("goodbye"))) - hs -= Collider("hello") - assert(hs.size == 0) - assert(!(hs contains Collider("hello"))) - hs.diagnostics.fullyValidate - } - - // check that the clear method actually cleans everything - def checkClear: Unit = { - val size = 200 - val hs = new WeakHashSet[String]() - val elements = (0 until size).toList map ("a" + _) - elements foreach (hs += _) - hs.clear() - assert(hs.size == 0) - elements foreach {i => assert(!(hs contains i))} - hs.diagnostics.fullyValidate - } - - // check that the iterator covers all the contents - def checkIterator: Unit = { - val hs = new WeakHashSet[String]() - val elements = (0 until 20).toList map ("a" + _) - elements foreach (hs += _) - assert(elements.iterator.toList.sorted == elements.sorted) - hs.diagnostics.fullyValidate - } - - // check that the iterator covers all the contents even when there is a collision - def checkIteratorCollisions: Unit = { - val hs = new WeakHashSet[Collider] - val elements = (0 until 20).toList map {x => Collider("a" + x)} - elements foreach (hs += _) - assert(elements.iterator.toList.sorted == elements.sorted) - hs.diagnostics.fullyValidate - } - } -} diff --git a/tests/run/shortClass.scala b/tests/run/shortClass.scala deleted file mode 100644 index c5c2043f4..000000000 --- a/tests/run/shortClass.scala +++ /dev/null @@ -1,26 +0,0 @@ -import scala.reflect.internal.util._ - -package bippity { - trait DingDongBippy - - package bop { - class Foo { - class Bar - object Bar - } - } -} - -object Test { - import bippity._ - import bop._ - - def printSanitized(x: String) = println(x.filterNot(_.isDigit)) - - def main(args: Array[String]): Unit = { - val f = new Foo - val instances = List(f, new f.Bar, f.Bar, new Foo with DingDongBippy, new f.Bar with DingDongBippy) - instances map (_.getClass.getName) foreach printSanitized - instances map shortClassOfInstance foreach printSanitized - } -} diff --git a/tests/run/showraw_nosymbol.scala b/tests/run/showraw_nosymbol.scala deleted file mode 100644 index 191647583..000000000 --- a/tests/run/showraw_nosymbol.scala +++ /dev/null @@ -1,5 +0,0 @@ -import scala.reflect.runtime.universe._ - -object Test extends dotty.runtime.LegacyApp { - println(showRaw(NoSymbol)) -} diff --git a/tests/run/sm-interpolator.scala b/tests/run/sm-interpolator.scala deleted file mode 100644 index e4bec7afb..000000000 --- a/tests/run/sm-interpolator.scala +++ /dev/null @@ -1,41 +0,0 @@ -object Test extends dotty.runtime.LegacyApp { - import scala.reflect.internal.util.StringContextStripMarginOps - def check(actual: Any, expected: Any) = if (actual != expected) sys.error(s"\nexpected:\n$expected\n\nactual:\n$actual") - - val bar = "|\n ||" - - check( - sm"""|ab - |de - |${bar} | ${1}""", - "ab\nde\n|\n || | 1") - - check( - sm"|", - "") - - check( - sm"${0}", - "0") - - check( - sm"${0}", - "0") - - check( - sm"""${0}|${1} - |""", - "0|1\n") - - check( - sm""" ||""", - "|") - - check( - sm""" ${" "} ||""", - " ||") - - check( - sm"\n", - raw"\n".stripMargin) -} diff --git a/tests/run/structural.scala b/tests/run/structural.scala deleted file mode 100644 index 0f18f4579..000000000 --- a/tests/run/structural.scala +++ /dev/null @@ -1,25 +0,0 @@ -case class Record(elems: (String, Any)*) extends Selectable { - def selectDynamic(name: String): Any = elems.find(_._1 == name).get._2 -} - -object Test { - import scala.reflect.Selectable.reflectiveSelectable - - def f(closeable: { def close(): Unit }) = - closeable.close() - - type RN = Record { val name: String; val age: Int } - - def g(r: RN) = r.name - - val rr: RN = Record("name" -> "Bob", "age" -> 42).asInstanceOf[RN] - - def main(args: Array[String]): Unit = { - f(new java.io.PrintStream("foo")) - assert(g(rr) == "Bob") - - val s: { def concat(s: String): String } = "abc" - assert(s.concat("def") == "abcdef") - } -} - diff --git a/tests/run/t429.scala b/tests/run/t429.scala index eeed4b080..411d199ee 100644 --- a/tests/run/t429.scala +++ b/tests/run/t429.scala @@ -11,5 +11,6 @@ object Test { } def main (args: Array[String]): Unit = { Console.print((new B).y); + println() } } diff --git a/tests/run/t6732.scala b/tests/run/t6732.scala deleted file mode 100644 index ff0f0494d..000000000 --- a/tests/run/t6732.scala +++ /dev/null @@ -1,12 +0,0 @@ -import scala.reflect.runtime.universe._ -import definitions._ - -object Test extends dotty.runtime.LegacyApp { - def test(sym: Symbol): Unit = { - println(s"${showRaw(sym, printKinds = true)}: ${sym.isModule}, ${sym.isModuleClass}, ${sym.isPackage}, ${sym.isPackageClass}") - } - test(ScalaPackage) - test(ScalaPackageClass) - test(ListModule) - test(ListModule.moduleClass) -} -- cgit v1.2.3 From 67065d070d869f5eca01ca7e698afac8ba784e0a Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 19:50:46 +0200 Subject: Add vulpix docstrings and change defines --- compiler/test/dotty/tools/dotc/CompilationTests.scala | 4 ++-- compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala | 13 +++++++++++-- compiler/test/dotty/tools/vulpix/VulpixTests.scala | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 4d5ab6963..a3f44c74f 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -16,9 +16,9 @@ class CompilationTests extends SummaryReport with ParallelTesting { def maxDuration = 180.seconds def numberOfSlaves = 5 - def safeMode = sys.env.get("SAFEMODE").isDefined + def safeMode = sys.env.get("dotty.tests.safemode").isDefined def isInteractive = SummaryReport.isInteractive - def testFilter = sys.props.get("dotty.partest.filter").map(r => new Regex(r)) + def testFilter = sys.props.get("dotty.tests.filter").map(r => new Regex(r)) // Positive tests ------------------------------------------------------------ diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala index 22bebf714..476012d1d 100644 --- a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -47,7 +47,14 @@ trait RunnerOrchestration { private[this] val monitor = new RunnerMonitor - /** Look away now, sweet child of summer */ + /** The runner monitor object keeps track of child JVM processes by keeping + * them in two structures - one for free, and one for busy children. + * + * When a user calls `runMain` the monitor makes takes a free JVM and blocks + * until the run is complete - or `maxDuration` has passed. It then performs + * cleanup by returning the used JVM to the free list, or respawning it if + * it died + */ private class RunnerMonitor { def runMain(classPath: String): Status = withRunner(_.runMain(classPath)) @@ -128,6 +135,9 @@ trait RunnerOrchestration { } } + /** Create a process which has the classpath of the `ChildJVMMain` and the + * scala library. + */ private def createProcess: Process = { val sep = sys.props("file.separator") val cp = @@ -142,7 +152,6 @@ trait RunnerOrchestration { } private[this] val allRunners = List.fill(numberOfSlaves)(new Runner(createProcess)) - private[this] val freeRunners = mutable.Queue(allRunners: _*) private[this] val busyRunners = mutable.Set.empty[Runner] diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala index bebcc7601..154008bd1 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -7,6 +7,7 @@ import org.junit.Test import scala.concurrent.duration._ import scala.util.control.NonFatal +/** Meta tests for the Vulpix test suite */ class VulpixTests extends ParallelTesting { import dotc.CompilationTests._ -- cgit v1.2.3 From 1fced2bb8d684cc56672e84b2e164716c92a21b9 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 6 Apr 2017 09:51:26 +0200 Subject: Only complete tests after run has been performed --- .../test/dotty/tools/vulpix/ParallelTesting.scala | 52 ++++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 82847e041..8cafd543b 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -206,12 +206,12 @@ trait ParallelTesting extends RunnerOrchestration { self => private[this] var _errorCount = 0 def errorCount: Int = _errorCount - private[this] var _testSourcesCompiled = 0 - private def testSourcesCompiled: Int = _testSourcesCompiled + private[this] var _testSourcesCompleted = 0 + private def testSourcesCompleted: Int = _testSourcesCompleted /** Complete the current compilation with the amount of errors encountered */ - protected final def registerCompilation(errors: Int) = synchronized { - _testSourcesCompiled += 1 + protected final def registerCompletion(errors: Int) = synchronized { + _testSourcesCompleted += 1 _errorCount += errors } @@ -250,7 +250,7 @@ trait ParallelTesting extends RunnerOrchestration { self => private def createProgressMonitor: Runnable = new Runnable { def run(): Unit = { val start = System.currentTimeMillis - var tCompiled = testSourcesCompiled + var tCompiled = testSourcesCompleted while (tCompiled < sourceCount) { val timestamp = (System.currentTimeMillis - start) / 1000 val progress = (tCompiled.toDouble / sourceCount * 40).toInt @@ -259,15 +259,15 @@ trait ParallelTesting extends RunnerOrchestration { self => "[" + ("=" * (math.max(progress - 1, 0))) + (if (progress > 0) ">" else "") + (" " * (39 - progress)) + - s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" + s"] completed ($tCompiled/$sourceCount, ${timestamp}s)\r" ) Thread.sleep(100) - tCompiled = testSourcesCompiled + tCompiled = testSourcesCompleted } // println, otherwise no newline and cursor at start of line realStdout.println( - s"[=======================================] compiled ($sourceCount/$sourceCount, " + + s"[=======================================] completed ($sourceCount/$sourceCount, " + s"${(System.currentTimeMillis - start) / 1000}s) " ) } @@ -286,7 +286,7 @@ trait ParallelTesting extends RunnerOrchestration { self => // run should fail failTestSource(testSource) e.printStackTrace() - registerCompilation(1) + registerCompletion(1) throw e } } @@ -351,7 +351,7 @@ trait ParallelTesting extends RunnerOrchestration { self => } private[ParallelTesting] def executeTestSuite(): this.type = { - assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`") + assert(_testSourcesCompleted == 0, "not allowed to re-use a `CompileRun`") if (filteredSources.nonEmpty) { val pool = threadLimit match { @@ -397,7 +397,7 @@ trait ParallelTesting extends RunnerOrchestration { self => testSource match { case testSource @ JointCompilationSource(_, files, flags, outDir) => { val reporter = compile(testSource.sourceFiles, flags, false, outDir) - registerCompilation(reporter.errorCount) + registerCompletion(reporter.errorCount) if (reporter.errorCount > 0) echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) @@ -414,7 +414,7 @@ trait ParallelTesting extends RunnerOrchestration { self => def warningCount = reporters.foldLeft(0)(_ + _.warningCount) - registerCompilation(errorCount) + registerCompletion(errorCount) if (errorCount > 0) echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) @@ -488,7 +488,6 @@ trait ParallelTesting extends RunnerOrchestration { self => if (reporter.errorCount > 0) echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - registerCompilation(reporter.errorCount) (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) } @@ -507,28 +506,33 @@ trait ParallelTesting extends RunnerOrchestration { self => if (errorCount > 0) fail() - registerCompilation(errorCount) (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) } } if (errorCount == 0 && hasCheckFile) verifier() else if (errorCount == 0) runMain(testSource.classPath) match { - case Success(_) => // success! - case Failure(output) => - echo(s" failed when running '${testSource.title}'") - echo(output) - failTestSource(testSource) - case Timeout => - echo(" failed because test " + testSource.title + " timed out") - failTestSource(testSource, Some("test timed out")) - } + case Success(_) => // success! + case Failure(output) => + echo(s" failed when running '${testSource.title}'") + echo(output) + failTestSource(testSource) + case Timeout => + echo(" failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + } else if (errorCount > 0) { echo(s"\n Compilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) addFailureInstruction(buildInstr) failTestSource(testSource) } + else { + realStdout.println("Got a super weird error that I haven't handled yet") + realStdout.println("errorCount: " + errorCount) + realStdout.println("test: " + testSource.title + " " + testSource.name) + } + registerCompletion(errorCount) } } } @@ -627,7 +631,7 @@ trait ParallelTesting extends RunnerOrchestration { self => failTestSource(testSource) } - registerCompilation(actualErrors) + registerCompletion(actualErrors) } } } -- cgit v1.2.3 From 07fab41be7d24e790cd37f625f3d7a10363c45ff Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 6 Apr 2017 13:46:29 +0200 Subject: Add `Properties` object for dotty testing props and env --- compiler/test/dotc/tests.scala | 6 +-- compiler/test/dotty/Jars.scala | 24 ++++++++---- compiler/test/dotty/Properties.scala | 44 ++++++++++++++++++++++ .../test/dotty/tools/dotc/CompilationTests.scala | 4 +- 4 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 compiler/test/dotty/Properties.scala diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index 4bb09fd02..c2c38d152 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -21,8 +21,6 @@ class tests extends CompilerTest { // tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be // executed as benchmarks. - def isRunByDrone: Boolean = sys.props.isDefinedAt("DRONE") - val defaultOutputDir = "../out/" val noCheckOptions = List( @@ -70,7 +68,7 @@ class tests extends CompilerTest { } implicit val defaultOptions: List[String] = noCheckOptions ++ { - if (isRunByDrone) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 + if (dotty.Properties.isRunByDrone) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") } ++ checkOptions ++ classPath @@ -229,7 +227,7 @@ class tests extends CompilerTest { |../scala-scala/src/library/scala/collection/parallel/mutable/ParSet.scala |../scala-scala/src/library/scala/collection/mutable/SetLike.scala""".stripMargin)(scala2mode ++ defaultOptions) - @Test def dotty = { + @Test def dottyBooted = { dottyBootedLib dottyDependsOnBootedLib } diff --git a/compiler/test/dotty/Jars.scala b/compiler/test/dotty/Jars.scala index 06df9c891..bc000fced 100644 --- a/compiler/test/dotty/Jars.scala +++ b/compiler/test/dotty/Jars.scala @@ -2,25 +2,34 @@ package dotty /** Jars used when compiling test, normally set from the sbt build */ object Jars { + /** Dotty library Jar */ val dottyLib: String = sys.env.get("DOTTY_LIB") - .getOrElse(sys.props("dotty.tests.classes.library")) + .getOrElse(Properties.dottyLib) + /** Dotty Compiler Jar */ val dottyCompiler: String = sys.env.get("DOTTY_COMPILER") - .getOrElse(sys.props("dotty.tests.classes.compiler")) + .getOrElse(Properties.dottyCompiler) + /** Dotty Interfaces Jar */ val dottyInterfaces: String = sys.env.get("DOTTY_INTERFACE") - .getOrElse(sys.props("dotty.tests.classes.interfaces")) + .getOrElse(Properties.dottyInterfaces) - val dottyExtras: List[String] = Option(sys.env.get("DOTTY_EXTRAS") - .getOrElse(sys.props("dotty.tests.extraclasspath"))) - .map(_.split(":").toList).getOrElse(Nil) + /** Dotty extras classpath from env or properties */ + val dottyExtras: List[String] = sys.env.get("DOTTY_EXTRAS") + .map(_.split(":").toList).getOrElse(Properties.dottyExtras) + /** Dotty REPL dependencies */ val dottyReplDeps: List[String] = dottyLib :: dottyExtras + /** Dotty test dependencies */ val dottyTestDeps: List[String] = dottyLib :: dottyCompiler :: dottyInterfaces :: dottyExtras - + /** Gets the scala 2.* library at runtime, note that doing this is unsafe + * unless you know that the library will be on the classpath of the running + * application. It is currently safe to call this function if the tests are + * run by sbt. + */ def scalaLibraryFromRuntime: String = findJarFromRuntime("scala-library-2.") private def findJarFromRuntime(partialName: String) = { @@ -31,5 +40,4 @@ object Jars { ) } } - } diff --git a/compiler/test/dotty/Properties.scala b/compiler/test/dotty/Properties.scala new file mode 100644 index 000000000..6106c75b9 --- /dev/null +++ b/compiler/test/dotty/Properties.scala @@ -0,0 +1,44 @@ +package dotty + +/** Runtime properties from defines or environmnent */ +object Properties { + + /** If property is unset or "TRUE" we consider it `true` */ + private[this] def propIsNullOrTrue(prop: String): Boolean = { + val prop = System.getProperty("dotty.tests.interactive") + prop == null || prop == "TRUE" + } + + /** Are we running on the Drone CI? */ + val isRunByDrone: Boolean = sys.env.isDefinedAt("DRONE") + + /** Tests should run interactive? */ + val testsInteractive: Boolean = propIsNullOrTrue("dotty.tests.interactive") + + /** Filter out tests not matching the regex supplied by "dotty.tests.filter" + * define + */ + val testsFilter: Option[String] = sys.props.get("dotty.tests.filter") + + /** Should Unit tests run in safe mode? + * + * For run tests this means that we respawn child JVM processes after each + * test, so that they are never reused. + */ + val testsSafeMode: Boolean = sys.props.isDefinedAt("dotty.tests.safemode") + + /** Dotty compiler path provided through define */ + def dottyCompiler: String = sys.props("dotty.tests.classes.compiler") + + /** Dotty classpath extras provided through define */ + def dottyExtras: List[String] = + Option(sys.props("dotty.tests.extraclasspath")) + .map(_.split(":").toList) + .getOrElse(Nil) + + /** Dotty interfaces path provided through define */ + def dottyInterfaces: String = sys.props("dotty.tests.classes.interfaces") + + /** Dotty library path provided through define */ + def dottyLib: String = sys.props("dotty.tests.classes.library") +} diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index a3f44c74f..ab7dda850 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -16,9 +16,9 @@ class CompilationTests extends SummaryReport with ParallelTesting { def maxDuration = 180.seconds def numberOfSlaves = 5 - def safeMode = sys.env.get("dotty.tests.safemode").isDefined + def safeMode = Properties.testsSafeMode def isInteractive = SummaryReport.isInteractive - def testFilter = sys.props.get("dotty.tests.filter").map(r => new Regex(r)) + def testFilter = Properties.testsFilter.map(r => new Regex(r)) // Positive tests ------------------------------------------------------------ -- cgit v1.2.3 From 8fe0d14c1d2bdc26088fc4a29388b961ae1552a2 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 6 Apr 2017 11:50:20 +0200 Subject: Fix interactive mode in eclipse --- compiler/test/dotty/tools/vulpix/SummaryReport.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.java b/compiler/test/dotty/tools/vulpix/SummaryReport.java index 61a708f26..1a7fc2a61 100644 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.java +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.java @@ -8,12 +8,14 @@ import scala.Function0; import scala.Unit; import dotty.tools.dotc.reporting.TestReporter; +import dotty.Properties; /** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with * this class */ public class SummaryReport { - public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); + public final static boolean isInteractive = + Properties.testsInteractive() && !Properties.isRunByDrone(); private static TestReporter rep = TestReporter.reporter(System.out, -1); private static ArrayDeque failedTests = new ArrayDeque<>(); -- cgit v1.2.3 From 55803b2657a473a1ebbebfd9ab7ba4c1b4e27d38 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Fri, 7 Apr 2017 13:47:07 +0200 Subject: Implement meta tests suggested by @DarkDimius Note that merging this as-is will not protect us against fork bombs. This is because the timeout of tests is currently 180 seconds. A forkbomb that is allowed to run for that long... --- compiler/test/dotty/tools/vulpix/VulpixTests.scala | 15 ++++++++ tests/partest-test/deadlock.scala | 44 ++++++++++++++++++++++ tests/partest-test/forkbomb.scala | 7 ++++ tests/partest-test/infinite.scala | 9 +++++ tests/partest-test/infiniteAlloc.scala | 9 +++++ tests/partest-test/infiniteTail.scala | 7 ++++ 6 files changed, 91 insertions(+) create mode 100644 tests/partest-test/deadlock.scala create mode 100644 tests/partest-test/forkbomb.scala create mode 100644 tests/partest-test/infinite.scala create mode 100644 tests/partest-test/infiniteAlloc.scala create mode 100644 tests/partest-test/infiniteTail.scala diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala index 154008bd1..2483bf6f0 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -59,4 +59,19 @@ class VulpixTests extends ParallelTesting { @Test def runOutRedirects: Unit = compileFile("../tests/partest-test/i2147.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiteNonRec: Unit = + compileFile("../tests/partest-test/infinite.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiteTailRec: Unit = + compileFile("../tests/partest-test/infiniteTail.scala", defaultOptions).expectFailure.checkRuns() + + @Test def infiniteAlloc: Unit = + compileFile("../tests/partest-test/infiniteAlloc.scala", defaultOptions).expectFailure.checkRuns() + + @Test def deadlock: Unit = + compileFile("../tests/partest-test/deadlock.scala", defaultOptions).expectFailure.checkRuns() + + @Test def forkbomb: Unit = + compileFile("../tests/partest-test/forkbomb.scala", defaultOptions).expectFailure.checkRuns() } diff --git a/tests/partest-test/deadlock.scala b/tests/partest-test/deadlock.scala new file mode 100644 index 000000000..df561aff3 --- /dev/null +++ b/tests/partest-test/deadlock.scala @@ -0,0 +1,44 @@ +object Test { + class Lock + val lock1 = new Lock + val lock2 = new Lock + + private[this] var took2: Boolean = false + def lock2Taken(): Unit = synchronized { + took2 = true + notify() + } + def tookLock2: Boolean = synchronized(took2) + + val thread1 = new Thread { + override def run(): Unit = synchronized { + lock1.synchronized { + while (!tookLock2) wait() + lock2.synchronized { + println("thread1 in lock2!") + } + } + println("thread1, done!") + } + } + + val thread2 = new Thread { + override def run(): Unit = synchronized { + lock2.synchronized { + lock2Taken() + lock1.synchronized { + println("thread2 in lock1!") + } + } + println("thread2, done!") + } + } + + def main(args: Array[String]): Unit = { + thread1.start() // takes lock1 then sleeps 1s - tries to take lock2 + thread2.start() // takes lock2 then sleeps 1s - tries to take lock1 + + thread1.join() // wait for threads to complete, can't because deadlock! + thread2.join() + } +} diff --git a/tests/partest-test/forkbomb.scala b/tests/partest-test/forkbomb.scala new file mode 100644 index 000000000..1d3cda172 --- /dev/null +++ b/tests/partest-test/forkbomb.scala @@ -0,0 +1,7 @@ +object Test { + def main(args: Array[String]): Unit = + while(true) + Runtime + .getRuntime() + .exec(Array("java", "-cp", System.getProperty("java.class.path"), "Test")); +} diff --git a/tests/partest-test/infinite.scala b/tests/partest-test/infinite.scala new file mode 100644 index 000000000..961382fea --- /dev/null +++ b/tests/partest-test/infinite.scala @@ -0,0 +1,9 @@ +object Test { + def main(args: Array[String]): Unit = { + var sum = 0 + while(true) { + sum += 1 + } + println(sum) + } +} diff --git a/tests/partest-test/infiniteAlloc.scala b/tests/partest-test/infiniteAlloc.scala new file mode 100644 index 000000000..89fa5d6ef --- /dev/null +++ b/tests/partest-test/infiniteAlloc.scala @@ -0,0 +1,9 @@ +import scala.collection.mutable +object Test { + val map = mutable.Map.empty[String, String] + + def main(args: Array[String]): Unit = while (true) { + val time = System.currentTimeMillis.toString + map += (time -> time) + } +} diff --git a/tests/partest-test/infiniteTail.scala b/tests/partest-test/infiniteTail.scala new file mode 100644 index 000000000..b3132cc19 --- /dev/null +++ b/tests/partest-test/infiniteTail.scala @@ -0,0 +1,7 @@ +object Test { + def foo: Int = bar + def bar: Int = foo + + def main(args: Array[String]): Unit = + println(foo) +} -- cgit v1.2.3 From d42a28d07683d95e6dffd27cdb9078ebeb599c15 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Fri, 7 Apr 2017 14:46:17 +0200 Subject: Add ability to only compile run tests --- compiler/test/dotty/Properties.scala | 5 +++ .../test/dotty/tools/vulpix/ParallelTesting.scala | 38 +++++++++++++++------- .../test/dotty/tools/vulpix/SummaryReport.java | 19 +++++++++-- compiler/test/dotty/tools/vulpix/VulpixTests.scala | 3 -- tests/partest-test/forkbomb.scala | 7 ---- 5 files changed, 49 insertions(+), 23 deletions(-) delete mode 100644 tests/partest-test/forkbomb.scala diff --git a/compiler/test/dotty/Properties.scala b/compiler/test/dotty/Properties.scala index 6106c75b9..70db82092 100644 --- a/compiler/test/dotty/Properties.scala +++ b/compiler/test/dotty/Properties.scala @@ -20,6 +20,11 @@ object Properties { */ val testsFilter: Option[String] = sys.props.get("dotty.tests.filter") + /** When set, the run tests are only compiled - not run, a warning will be + * issued + */ + val testsNoRun: Boolean = sys.props.get("dotty.tests.norun").isDefined + /** Should Unit tests run in safe mode? * * For run tests this means that we respawn child JVM processes after each diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 8cafd543b..e1babfb9c 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -426,8 +426,21 @@ trait ParallelTesting extends RunnerOrchestration { self => private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { + private[this] var didAddNoRunWarning = false + private[this] def addNoRunWarning() = if (!didAddNoRunWarning) { + didAddNoRunWarning = true + SummaryReport.addStartingMessage { + """|WARNING + |------- + |Run tests were only compiled, not run - this is due to `dotty.tests.norun` + |property being set + |""".stripMargin + } + } + private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - runMain(testSource.classPath) match { + if (Properties.testsNoRun) addNoRunWarning() + else runMain(testSource.classPath) match { case Success(output) => { val outputLines = output.lines.toArray val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray @@ -511,16 +524,19 @@ trait ParallelTesting extends RunnerOrchestration { self => } if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.classPath) match { - case Success(_) => // success! - case Failure(output) => - echo(s" failed when running '${testSource.title}'") - echo(output) - failTestSource(testSource) - case Timeout => - echo(" failed because test " + testSource.title + " timed out") - failTestSource(testSource, Some("test timed out")) - } + else if (errorCount == 0) { + if (Properties.testsNoRun) addNoRunWarning() + else runMain(testSource.classPath) match { + case Success(_) => // success! + case Failure(output) => + echo(s" failed when running '${testSource.title}'") + echo(output) + failTestSource(testSource) + case Timeout => + echo(" failed because test " + testSource.title + " timed out") + failTestSource(testSource, Some("test timed out")) + } + } else if (errorCount > 0) { echo(s"\n Compilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.java b/compiler/test/dotty/tools/vulpix/SummaryReport.java index 1a7fc2a61..23209eefc 100644 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.java +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.java @@ -10,8 +10,10 @@ import scala.Unit; import dotty.tools.dotc.reporting.TestReporter; import dotty.Properties; -/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with - * this class +/** This class adds summary reports to `ParallelTesting` + * + * It is written in Java because we currently cannot explicitly write static + * methods in Scala without SIP-25 (`@static` fields and methods in Scala) */ public class SummaryReport { public final static boolean isInteractive = @@ -20,6 +22,7 @@ public class SummaryReport { private static TestReporter rep = TestReporter.reporter(System.out, -1); private static ArrayDeque failedTests = new ArrayDeque<>(); private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); + private static ArrayDeque startingMessages = new ArrayDeque<>(); private static Supplier cleanup; private static int passed; private static int failed; @@ -40,6 +43,10 @@ public class SummaryReport { reproduceInstructions.offer(msg); } + public final static void addStartingMessage(String msg) { + startingMessages.offer(msg); + } + public final static void addCleanup(Function0 func) { // Wow, look at how neatly we - compose cleanup callbacks: if (cleanup == null) { @@ -61,6 +68,10 @@ public class SummaryReport { rep = TestReporter.reporter(System.out, -1); failedTests = new ArrayDeque<>(); reproduceInstructions = new ArrayDeque<>(); + startingMessages = new ArrayDeque<>(); + cleanup = null; + passed = 0; + failed = 0; } @AfterClass public final static void teardown() { @@ -73,6 +84,10 @@ public class SummaryReport { "\n" ); + startingMessages + .stream() + .forEach(rep::echo); + failedTests .stream() .map(x -> " " + x) diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala index 2483bf6f0..646e1bb29 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -71,7 +71,4 @@ class VulpixTests extends ParallelTesting { @Test def deadlock: Unit = compileFile("../tests/partest-test/deadlock.scala", defaultOptions).expectFailure.checkRuns() - - @Test def forkbomb: Unit = - compileFile("../tests/partest-test/forkbomb.scala", defaultOptions).expectFailure.checkRuns() } diff --git a/tests/partest-test/forkbomb.scala b/tests/partest-test/forkbomb.scala deleted file mode 100644 index 1d3cda172..000000000 --- a/tests/partest-test/forkbomb.scala +++ /dev/null @@ -1,7 +0,0 @@ -object Test { - def main(args: Array[String]): Unit = - while(true) - Runtime - .getRuntime() - .exec(Array("java", "-cp", System.getProperty("java.class.path"), "Test")); -} -- cgit v1.2.3 From f891b224228f7c4939d09ac1849ad562d1298640 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 10 Apr 2017 16:26:14 +0200 Subject: Make sure that everything is dumped to log files --- .../dotty/tools/dotc/reporting/TestReporter.scala | 22 +++- .../test/dotty/tools/vulpix/ParallelTesting.scala | 142 ++++++++++++++------- .../test/dotty/tools/vulpix/SummaryReport.java | 13 +- 3 files changed, 120 insertions(+), 57 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala index 5641240a7..efba2dc8f 100644 --- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala +++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala @@ -23,6 +23,10 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M final def errors: Iterator[MessageContainer] = _errorBuf.iterator protected final val _messageBuf = mutable.ArrayBuffer.empty[String] + final def messages: Iterator[String] = _messageBuf.iterator + + private[this] var _didCrash = false + final def compilerCrashed: Boolean = _didCrash final def flushToFile(): Unit = _messageBuf @@ -33,7 +37,6 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M final def flushToStdErr(): Unit = _messageBuf .iterator - .map(_.replaceAll("\u001b\\[.*?m", "")) .foreach(System.err.println) final def inlineInfo(pos: SourcePosition): String = @@ -44,9 +47,17 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M } else "" - def echo(msg: String) = + def log(msg: String) = _messageBuf.append(msg) + def logStackTrace(thrown: Throwable): Unit = { + _didCrash = true + val sw = new java.io.StringWriter + val pw = new java.io.PrintWriter(sw) + thrown.printStackTrace(pw) + log(sw.toString) + } + /** Prints the message with the given position indication. */ def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = { val msg = messageAndPos(m.contained, m.pos, diagnosticLevel(m)) @@ -73,15 +84,14 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M _errorBuf.append(m) printMessageAndPos(m, extra) } - case w: Warning => - printMessageAndPos(w, extra) - case _ => + case m => + printMessageAndPos(m, extra) } } } object TestReporter { - private[this] lazy val logWriter = { + lazy val logWriter = { val df = new SimpleDateFormat("yyyy-MM-dd-HH:mm") val timestamp = df.format(new Date) new JFile("../testlogs").mkdirs() diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index e1babfb9c..82a15c4a4 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -183,12 +183,39 @@ trait ParallelTesting extends RunnerOrchestration { self => /** Each `Test` takes the `testSources` and performs the compilation and assertions * according to the implementing class "neg", "run" or "pos". */ - private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { + private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { test => protected final val realStdout = System.out protected final val realStderr = System.err + /** A runnable that logs its contents in a buffer */ + trait LoggedRunnable extends Runnable { + import TestReporter.logWriter + + /** Instances of `LoggedRunnable` implement this method instead of the + * `run` method + */ + def checkTestSource(): Unit + + private[this] val logBuffer = mutable.ArrayBuffer.empty[String] + def log(msg: String): Unit = logBuffer.append(msg) + + def logReporterContents(reporter: TestReporter): Unit = + reporter.messages.foreach(log) + + def echo(msg: String): Unit = { + log(msg) + test.echo(msg) + } + + final def run(): Unit = { + checkTestSource() + logBuffer.iterator.foreach(logWriter.println) + logWriter.flush() + } + } + /** Actual compilation run logic, the test behaviour is defined here */ - protected def compilationRunnable(testSource: TestSource): Runnable + protected def encapsulatedCompilation(testSource: TestSource): LoggedRunnable /** All testSources left after filtering out */ private val filteredSources = @@ -220,7 +247,7 @@ trait ParallelTesting extends RunnerOrchestration { self => protected[this] final def fail(): Unit = synchronized { _failed = true } def didFail: Boolean = _failed - protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { + protected def logBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = { val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount) addFailureInstruction(errorMsg) failTestSource(testSource) @@ -278,7 +305,9 @@ trait ParallelTesting extends RunnerOrchestration { self => */ protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = try { - if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") + val testing = s"Testing ${testSource.title}" + TestReporter.logWriter.println(testing) + if (!isInteractive) realStdout.println(testing) op } catch { case NonFatal(e) => { @@ -342,10 +371,17 @@ trait ParallelTesting extends RunnerOrchestration { self => } val allArgs = addOutDir(flags) - driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) - val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) - assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") + // Compile with a try to catch any StackTrace generated by the compiler: + try { + driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter) + + val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath) + assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}") + } + catch { + case NonFatal(ex) => reporter.logStackTrace(ex) + } reporter } @@ -362,7 +398,7 @@ trait ParallelTesting extends RunnerOrchestration { self => if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor) filteredSources.foreach { target => - pool.submit(compilationRunnable(target)) + pool.submit(encapsulatedCompilation(target)) } pool.shutdown() @@ -392,22 +428,25 @@ trait ParallelTesting extends RunnerOrchestration { self => private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { testSource match { case testSource @ JointCompilationSource(_, files, flags, outDir) => { val reporter = compile(testSource.sourceFiles, flags, false, outDir) registerCompletion(reporter.errorCount) - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + if (reporter.compilerCrashed || reporter.errorCount > 0) { + logReporterContents(reporter) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } } case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) val errorCount = reporters.foldLeft(0) { (acc, reporter) => if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) acc + reporter.errorCount } @@ -416,8 +455,10 @@ trait ParallelTesting extends RunnerOrchestration { self => registerCompletion(errorCount) - if (errorCount > 0) - echoBuildInstructions(reporters.head, testSource, errorCount, warningCount) + if (compilerCrashed || errorCount > 0) { + reporters.foreach(logReporterContents) + logBuildInstructions(reporters.head, testSource, errorCount, warningCount) + } } } } @@ -483,9 +524,9 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { - val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { + val (compilerCrashed, errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { case testSource @ JointCompilationSource(_, files, flags, outDir) => { val checkFile = files.flatMap { file => if (file.isDirectory) Nil @@ -498,33 +539,37 @@ trait ParallelTesting extends RunnerOrchestration { self => }.headOption val reporter = compile(testSource.sourceFiles, flags, false, outDir) - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + if (reporter.compilerCrashed || reporter.errorCount > 0) { + logReporterContents(reporter) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + } - (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) + (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) } case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check") + val reporters = testSource.compilationGroups.map(compile(_, flags, false, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) val (errorCount, warningCount) = - testSource - .compilationGroups - .map(compile(_, flags, false, outDir)) - .foldLeft((0,0)) { case ((errors, warnings), reporter) => - if (reporter.errorCount > 0) - echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) + reporters.foldLeft((0,0)) { case ((errors, warnings), reporter) => + if (reporter.errorCount > 0) + logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) - (errors + reporter.errorCount, warnings + reporter.warningCount) - } + (errors + reporter.errorCount, warnings + reporter.warningCount) + } - if (errorCount > 0) fail() + if (errorCount > 0) { + reporters.foreach(logReporterContents) + logBuildInstructions(reporters.head, testSource, errorCount, warningCount) + } - (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) + (compilerCrashed, errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) } } - if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) { + if (!compilerCrashed && errorCount == 0 && hasCheckFile) verifier() + else if (!compilerCrashed && errorCount == 0) { if (Properties.testsNoRun) addNoRunWarning() else runMain(testSource.classPath) match { case Success(_) => // success! @@ -537,17 +582,12 @@ trait ParallelTesting extends RunnerOrchestration { self => failTestSource(testSource, Some("test timed out")) } } - else if (errorCount > 0) { + else { echo(s"\n Compilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) addFailureInstruction(buildInstr) failTestSource(testSource) } - else { - realStdout.println("Got a super weird error that I haven't handled yet") - realStdout.println("errorCount: " + errorCount) - realStdout.println("test: " + testSource.title + " " + testSource.name) - } registerCompletion(errorCount) } } @@ -555,8 +595,8 @@ trait ParallelTesting extends RunnerOrchestration { self => private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { - def run(): Unit = tryCompile(testSource) { + protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { + def checkTestSource(): Unit = tryCompile(testSource) { // In neg-tests we allow two types of error annotations, // "nopos-error" which doesn't care about position and "error" which // has to be annotated on the correct line number. @@ -608,27 +648,39 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { + val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match { case testSource @ JointCompilationSource(_, files, flags, outDir) => { val sourceFiles = testSource.sourceFiles val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles) val reporter = compile(sourceFiles, flags, true, outDir) val actualErrors = reporter.errorCount - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) + if (reporter.compilerCrashed || actualErrors > 0) + logReporterContents(reporter) + + (reporter.compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap) } case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { val compilationGroups = testSource.compilationGroups val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten) val reporters = compilationGroups.map(compile(_, flags, true, outDir)) + val compilerCrashed = reporters.exists(_.compilerCrashed) val actualErrors = reporters.foldLeft(0)(_ + _.errorCount) val errors = reporters.iterator.flatMap(_.errors) - (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) + + if (actualErrors > 0) + reporters.foreach(logReporterContents) + + (compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap) } } - if (expectedErrors != actualErrors) { + if (compilerCrashed) { + echo(s"Compiler crashed when compiling: ${testSource.title}") + failTestSource(testSource) + } + else if (expectedErrors != actualErrors) { echo { s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n" } diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.java b/compiler/test/dotty/tools/vulpix/SummaryReport.java index 23209eefc..b7aa423ff 100644 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.java +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.java @@ -2,6 +2,7 @@ package dotty.tools.vulpix; import org.junit.BeforeClass; import org.junit.AfterClass; +import java.util.Iterator; import java.util.ArrayDeque; import java.util.function.Supplier; import scala.Function0; @@ -75,7 +76,7 @@ public class SummaryReport { } @AfterClass public final static void teardown() { - rep.echo( + rep.log( "\n================================================================================" + "\nTest Report" + "\n================================================================================" + @@ -86,26 +87,26 @@ public class SummaryReport { startingMessages .stream() - .forEach(rep::echo); + .forEach(rep::log); failedTests .stream() .map(x -> " " + x) - .forEach(rep::echo); + .forEach(rep::log); // If we're compiling locally, we don't need reproduce instructions if (isInteractive) rep.flushToStdErr(); - rep.echo(""); + rep.log(""); reproduceInstructions .stream() - .forEach(rep::echo); + .forEach(rep::log); // If we're on the CI, we want everything if (!isInteractive) rep.flushToStdErr(); - if (failed > 0) rep.flushToFile(); + rep.flushToFile(); // Perform cleanup callback: if (cleanup != null) cleanup.get(); -- cgit v1.2.3 From bcdacee46f04f5bca4732bd487d3cc3c042e23db Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 11 Apr 2017 12:17:30 +0200 Subject: Remove need for java written summary reporter --- compiler/test/dotc/tests.scala | 4 +- .../test/dotty/tools/dotc/CompilationTests.scala | 70 ++----------- .../dotty/tools/dotc/reporting/TestReporter.scala | 21 ++-- .../dotc/transform/PatmatExhaustivityTest.scala | 3 +- .../test/dotty/tools/vulpix/ParallelTesting.scala | 20 ++-- .../dotty/tools/vulpix/RunnerOrchestration.scala | 21 ++-- .../test/dotty/tools/vulpix/SummaryReport.java | 114 --------------------- .../test/dotty/tools/vulpix/SummaryReport.scala | 103 +++++++++++++++++++ .../dotty/tools/vulpix/TestConfiguration.scala | 67 ++++++++++++ compiler/test/dotty/tools/vulpix/VulpixTests.scala | 4 +- 10 files changed, 222 insertions(+), 205 deletions(-) delete mode 100644 compiler/test/dotty/tools/vulpix/SummaryReport.java create mode 100644 compiler/test/dotty/tools/vulpix/SummaryReport.scala create mode 100644 compiler/test/dotty/tools/vulpix/TestConfiguration.scala diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index c2c38d152..efecc1df3 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -210,8 +210,8 @@ class tests extends CompilerTest { private val stdlibFiles: List[String] = StdLibSources.whitelisted @Test def compileStdLib = - if (!generatePartestFiles) - compileList("compileStdLib", stdlibFiles, "-migration" :: "-Yno-inline" :: scala2mode) + compileList("compileStdLib", stdlibFiles, "-migration" :: "-Yno-inline" :: scala2mode) + @Test def compileMixed = compileLine( """../tests/pos/B.scala |../scala-scala/src/library/scala/collection/immutable/Seq.scala diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index ab7dda850..023a87069 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -2,14 +2,15 @@ package dotty package tools package dotc -import org.junit.Test +import org.junit.{ Test, BeforeClass, AfterClass } import scala.util.matching.Regex import scala.concurrent.duration._ -import vulpix.{ ParallelTesting, SummaryReport } +import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration } -class CompilationTests extends SummaryReport with ParallelTesting { +class CompilationTests extends ParallelTesting { + import TestConfiguration._ import CompilationTests._ // Test suite configuration -------------------------------------------------- @@ -252,65 +253,6 @@ class CompilationTests extends SummaryReport with ParallelTesting { } object CompilationTests { - implicit val defaultOutputDir: String = "../out/" - - implicit class RichStringArray(val xs: Array[String]) extends AnyVal { - def and(args: String*): Array[String] = { - val argsArr: Array[String] = args.toArray - xs ++ argsArr - } - } - - val noCheckOptions = Array( - "-pagewidth", "120", - "-color:never" - ) - - val checkOptions = Array( - "-Yno-deep-subtypes", - "-Yno-double-bindings", - "-Yforce-sbt-phases" - ) - - val classPath = { - val paths = Jars.dottyTestDeps map { p => - val file = new java.io.File(p) - assert( - file.exists, - s"""|File "$p" couldn't be found. Run `packageAll` from build tool before - |testing. - | - |If running without sbt, test paths need to be setup environment variables: - | - | - DOTTY_LIBRARY - | - DOTTY_COMPILER - | - DOTTY_INTERFACES - | - DOTTY_EXTRAS - | - |Where these all contain locations, except extras which is a colon - |separated list of jars. - | - |When compiling with eclipse, you need the sbt-interfaces jar, put - |it in extras.""" - ) - file.getAbsolutePath - } mkString (":") - - Array("-classpath", paths) - } - - private val yCheckOptions = Array("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") - - val defaultOptions = noCheckOptions ++ checkOptions ++ yCheckOptions ++ classPath - val allowDeepSubtypes = defaultOptions diff Array("-Yno-deep-subtypes") - val allowDoubleBindings = defaultOptions diff Array("-Yno-double-bindings") - val picklingOptions = defaultOptions ++ Array( - "-Xprint-types", - "-Ytest-pickler", - "-Ystop-after:pickler", - "-Yprintpos" - ) - val scala2Mode = defaultOptions ++ Array("-language:Scala2") - val explicitUTF8 = defaultOptions ++ Array("-encoding", "UTF8") - val explicitUTF16 = defaultOptions ++ Array("-encoding", "UTF16") + implicit val summaryReport: SummaryReporting = new SummaryReport + @AfterClass def cleanup(): Unit = summaryReport.echoSummary() } diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala index efba2dc8f..8645882ca 100644 --- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala +++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala @@ -110,15 +110,22 @@ object TestReporter { val rep = new TestReporter(writer, writeToLog, WARNING) { /** Prints the message with the given position indication in a simplified manner */ override def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = { - val msg = s"${m.pos.line + 1}: " + m.contained.kind + extra - val extraInfo = inlineInfo(m.pos) + def report() = { + val msg = s"${m.pos.line + 1}: " + m.contained.kind + extra + val extraInfo = inlineInfo(m.pos) - writer.println(msg) - _messageBuf.append(msg) + writer.println(msg) + _messageBuf.append(msg) - if (extraInfo.nonEmpty) { - writer.println(extraInfo) - _messageBuf.append(extraInfo) + if (extraInfo.nonEmpty) { + writer.println(extraInfo) + _messageBuf.append(extraInfo) + } + } + m match { + case m: Error => report() + case m: Warning => report() + case _ => () } } } diff --git a/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala b/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala index eff86e6e7..1ec4a70a5 100644 --- a/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala +++ b/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala @@ -9,11 +9,12 @@ import scala.io.Source._ import scala.reflect.io.Directory import org.junit.Test import reporting.TestReporter +import vulpix.TestConfiguration class PatmatExhaustivityTest { val testsDir = "../tests/patmat" // stop-after: patmatexhaust-huge.scala crash compiler - val options = List("-color:never", "-Ystop-after:splitter", "-Ycheck-all-patmat") ++ CompilationTests.classPath + val options = List("-color:never", "-Ystop-after:splitter", "-Ycheck-all-patmat") ++ TestConfiguration.classPath private def compileFile(file: File) = { val stringBuffer = new StringWriter() diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 82a15c4a4..4c7328214 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -32,7 +32,6 @@ import dotc.{ Driver, Compiler } trait ParallelTesting extends RunnerOrchestration { self => import ParallelTesting._ - import SummaryReport._ /** If the running environment supports an interactive terminal, each `Test` * will be run with a progress bar and real time feedback @@ -183,7 +182,10 @@ trait ParallelTesting extends RunnerOrchestration { self => /** Each `Test` takes the `testSources` and performs the compilation and assertions * according to the implementing class "neg", "run" or "pos". */ - private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { test => + private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit val summaryReport: SummaryReporting) { test => + + import summaryReport._ + protected final val realStdout = System.out protected final val realStderr = System.err @@ -426,7 +428,7 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) extends Test(testSources, times, threadLimit, suppressAllOutput) { protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { def checkTestSource(): Unit = tryCompile(testSource) { @@ -465,12 +467,12 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) extends Test(testSources, times, threadLimit, suppressAllOutput) { private[this] var didAddNoRunWarning = false private[this] def addNoRunWarning() = if (!didAddNoRunWarning) { didAddNoRunWarning = true - SummaryReport.addStartingMessage { + summaryReport.addStartingMessage { """|WARNING |------- |Run tests were only compiled, not run - this is due to `dotty.tests.norun` @@ -593,7 +595,7 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) + private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting) extends Test(testSources, times, threadLimit, suppressAllOutput) { protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { def checkTestSource(): Unit = tryCompile(testSource) { @@ -849,7 +851,7 @@ trait ParallelTesting extends RunnerOrchestration { self => * compilation without generating errors and that they do not crash the * compiler */ - def checkCompile(): this.type = { + def checkCompile()(implicit summaryReport: SummaryReporting): this.type = { val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite() if (!shouldFail && test.didFail) { @@ -866,7 +868,7 @@ trait ParallelTesting extends RunnerOrchestration { self => * correct amount of errors at the correct positions. It also makes sure * that none of these tests crash the compiler */ - def checkExpectedErrors(): this.type = { + def checkExpectedErrors()(implicit summaryReport: SummaryReporting): this.type = { val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite() if (!shouldFail && test.didFail) { @@ -884,7 +886,7 @@ trait ParallelTesting extends RunnerOrchestration { self => * the compiler; it also makes sure that all tests can run with the * expected output */ - def checkRuns(): this.type = { + def checkRuns()(implicit summaryReport: SummaryReporting): this.type = { val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite() if (!shouldFail && test.didFail) { diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala index 476012d1d..ad068e9ef 100644 --- a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -43,7 +43,8 @@ trait RunnerOrchestration { def safeMode: Boolean /** Running a `Test` class's main method from the specified `dir` */ - def runMain(classPath: String): Status = monitor.runMain(classPath) + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = + monitor.runMain(classPath) private[this] val monitor = new RunnerMonitor @@ -57,7 +58,8 @@ trait RunnerOrchestration { */ private class RunnerMonitor { - def runMain(classPath: String): Status = withRunner(_.runMain(classPath)) + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = + withRunner(_.runMain(classPath)) private class Runner(private var process: Process) { private[this] var childStdout: BufferedReader = _ @@ -81,8 +83,16 @@ trait RunnerOrchestration { childStdin = null } + /** Did add hook to kill the child VMs? */ + private[this] var didAddCleanupCallback = false + /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ - def runMain(classPath: String): Status = { + def runMain(classPath: String)(implicit summaryReport: SummaryReporting): Status = { + if (!didAddCleanupCallback) { + // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we + // need to clean up ourselves. + summaryReport.addCleanup(killAll) + } assert(process ne null, "Runner was killed and then reused without setting a new process") @@ -127,9 +137,9 @@ trait RunnerOrchestration { // Handle failure of the VM: status match { case _: Success if safeMode => respawn() + case _: Success => // no need to respawn sub process case _: Failure => respawn() case Timeout => respawn() - case _ => () } status } @@ -182,8 +192,5 @@ trait RunnerOrchestration { // On shutdown, we need to kill all runners: sys.addShutdownHook(killAll()) - // If for some reason the test runner (i.e. sbt) doesn't kill the VM, we - // need to clean up ourselves. - SummaryReport.addCleanup(killAll) } } diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.java b/compiler/test/dotty/tools/vulpix/SummaryReport.java deleted file mode 100644 index b7aa423ff..000000000 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.java +++ /dev/null @@ -1,114 +0,0 @@ -package dotty.tools.vulpix; - -import org.junit.BeforeClass; -import org.junit.AfterClass; -import java.util.Iterator; -import java.util.ArrayDeque; -import java.util.function.Supplier; -import scala.Function0; -import scala.Unit; - -import dotty.tools.dotc.reporting.TestReporter; -import dotty.Properties; - -/** This class adds summary reports to `ParallelTesting` - * - * It is written in Java because we currently cannot explicitly write static - * methods in Scala without SIP-25 (`@static` fields and methods in Scala) - */ -public class SummaryReport { - public final static boolean isInteractive = - Properties.testsInteractive() && !Properties.isRunByDrone(); - - private static TestReporter rep = TestReporter.reporter(System.out, -1); - private static ArrayDeque failedTests = new ArrayDeque<>(); - private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); - private static ArrayDeque startingMessages = new ArrayDeque<>(); - private static Supplier cleanup; - private static int passed; - private static int failed; - - public final static void reportFailed() { - failed++; - } - - public final static void reportPassed() { - passed++; - } - - public final static void addFailedTest(String msg) { - failedTests.offer(msg); - } - - public final static void addReproduceInstruction(String msg) { - reproduceInstructions.offer(msg); - } - - public final static void addStartingMessage(String msg) { - startingMessages.offer(msg); - } - - public final static void addCleanup(Function0 func) { - // Wow, look at how neatly we - compose cleanup callbacks: - if (cleanup == null) { - cleanup = () -> { - func.apply(); - return null; - }; - } else { - Supplier oldCleanup = cleanup; - cleanup = () -> { - oldCleanup.get(); - func.apply(); - return null; - }; - } - } - - @BeforeClass public final static void setup() { - rep = TestReporter.reporter(System.out, -1); - failedTests = new ArrayDeque<>(); - reproduceInstructions = new ArrayDeque<>(); - startingMessages = new ArrayDeque<>(); - cleanup = null; - passed = 0; - failed = 0; - } - - @AfterClass public final static void teardown() { - rep.log( - "\n================================================================================" + - "\nTest Report" + - "\n================================================================================" + - "\n" + - passed + " passed, " + failed + " failed, " + (passed + failed) + " total" + - "\n" - ); - - startingMessages - .stream() - .forEach(rep::log); - - failedTests - .stream() - .map(x -> " " + x) - .forEach(rep::log); - - // If we're compiling locally, we don't need reproduce instructions - if (isInteractive) rep.flushToStdErr(); - - rep.log(""); - - reproduceInstructions - .stream() - .forEach(rep::log); - - // If we're on the CI, we want everything - if (!isInteractive) rep.flushToStdErr(); - - rep.flushToFile(); - - // Perform cleanup callback: - if (cleanup != null) cleanup.get(); - } -} diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.scala b/compiler/test/dotty/tools/vulpix/SummaryReport.scala new file mode 100644 index 000000000..53b0942ce --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.scala @@ -0,0 +1,103 @@ +package dotty +package tools +package vulpix + +import scala.collection.mutable +import dotc.reporting.TestReporter + +trait SummaryReporting { + def reportFailed(): Unit + def reportPassed(): Unit + def addFailedTest(msg: String): Unit + def addReproduceInstruction(instr: String): Unit + def addStartingMessage(msg: String): Unit + def addCleanup(f: () => Unit): Unit + def echoSummary(): Unit +} + +final class NoSummaryReport extends SummaryReporting { + def reportFailed(): Unit = () + def reportPassed(): Unit = () + def addFailedTest(msg: String): Unit = () + def addReproduceInstruction(instr: String): Unit = () + def addStartingMessage(msg: String): Unit = () + def addCleanup(f: () => Unit): Unit = () + def echoSummary(): Unit = () +} + +final class SummaryReport extends SummaryReporting { + + private val startingMessages = mutable.ArrayBuffer.empty[String] + private val failedTests = mutable.ArrayBuffer.empty[String] + private val reproduceInstructions = mutable.ArrayBuffer.empty[String] + private val cleanUps = mutable.ArrayBuffer.empty[() => Unit] + + private[this] var passed = 0 + private[this] var failed = 0 + + def reportFailed(): Unit = + failed += 1 + + def reportPassed(): Unit = + passed += 1 + + def addFailedTest(msg: String): Unit = + failedTests.append(msg) + + def addReproduceInstruction(instr: String): Unit = + reproduceInstructions.append(instr) + + def addStartingMessage(msg: String): Unit = + startingMessages.append(msg) + + def addCleanup(f: () => Unit): Unit = + cleanUps.append(f) + + /** Both echoes the summary to stdout and prints to file */ + def echoSummary(): Unit = { + import SummaryReport._ + + val rep = new StringBuilder + rep.append( + s"""| + |================================================================================ + |Test Report + |================================================================================ + | + |$passed passed, $failed failed, ${passed + failed} total + |""".stripMargin + ) + + startingMessages.foreach(rep.append) + + failedTests.map(x => " " + x).foreach(rep.append) + + // If we're compiling locally, we don't need instructions on how to + // reproduce failures + if (isInteractive) { + println(rep.toString) + if (failed > 0) println { + """| + |---------------------------------------------------------- + |Note: reproduction instructed have been dumped to log file + |----------------------------------------------------------""".stripMargin + } + } + + rep += '\n' + + reproduceInstructions.foreach(rep.append) + + // If we're on the CI, we want everything + if (!isInteractive) println(rep.toString) + + TestReporter.writeToLog(rep.toString) + + // Perform cleanup callback: + if (cleanUps.nonEmpty) cleanUps.foreach(_.apply()) + } +} + +object SummaryReport { + val isInteractive = Properties.testsInteractive && !Properties.isRunByDrone +} diff --git a/compiler/test/dotty/tools/vulpix/TestConfiguration.scala b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala new file mode 100644 index 000000000..dcf3fbaf0 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala @@ -0,0 +1,67 @@ +package dotty +package tools +package vulpix + +object TestConfiguration { + implicit val defaultOutputDir: String = "../out/" + + implicit class RichStringArray(val xs: Array[String]) extends AnyVal { + def and(args: String*): Array[String] = { + val argsArr: Array[String] = args.toArray + xs ++ argsArr + } + } + + val noCheckOptions = Array( + "-pagewidth", "120", + "-color:never" + ) + + val checkOptions = Array( + "-Yno-deep-subtypes", + "-Yno-double-bindings", + "-Yforce-sbt-phases" + ) + + val classPath = { + val paths = Jars.dottyTestDeps map { p => + val file = new java.io.File(p) + assert( + file.exists, + s"""|File "$p" couldn't be found. Run `packageAll` from build tool before + |testing. + | + |If running without sbt, test paths need to be setup environment variables: + | + | - DOTTY_LIBRARY + | - DOTTY_COMPILER + | - DOTTY_INTERFACES + | - DOTTY_EXTRAS + | + |Where these all contain locations, except extras which is a colon + |separated list of jars. + | + |When compiling with eclipse, you need the sbt-interfaces jar, put + |it in extras.""" + ) + file.getAbsolutePath + } mkString (":") + + Array("-classpath", paths) + } + + private val yCheckOptions = Array("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") + + val defaultOptions = noCheckOptions ++ checkOptions ++ yCheckOptions ++ classPath + val allowDeepSubtypes = defaultOptions diff Array("-Yno-deep-subtypes") + val allowDoubleBindings = defaultOptions diff Array("-Yno-double-bindings") + val picklingOptions = defaultOptions ++ Array( + "-Xprint-types", + "-Ytest-pickler", + "-Ystop-after:pickler", + "-Yprintpos" + ) + val scala2Mode = defaultOptions ++ Array("-language:Scala2") + val explicitUTF8 = defaultOptions ++ Array("-encoding", "UTF8") + val explicitUTF16 = defaultOptions ++ Array("-encoding", "UTF16") +} diff --git a/compiler/test/dotty/tools/vulpix/VulpixTests.scala b/compiler/test/dotty/tools/vulpix/VulpixTests.scala index 646e1bb29..f875e7c13 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixTests.scala @@ -9,7 +9,9 @@ import scala.util.control.NonFatal /** Meta tests for the Vulpix test suite */ class VulpixTests extends ParallelTesting { - import dotc.CompilationTests._ + import TestConfiguration._ + + implicit val _: SummaryReporting = new NoSummaryReport def maxDuration = 3.seconds def numberOfSlaves = 5 -- cgit v1.2.3 From 7c4227709666e5390afa1b5f43aeb3c32cc93016 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 12 Apr 2017 11:18:31 +0200 Subject: Make vulpix test filter work on substring instead of regexp --- .../test/dotty/tools/dotc/CompilationTests.scala | 2 +- .../test/dotty/tools/vulpix/ParallelTesting.scala | 31 +++++++++++----------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 023a87069..fa0c89f28 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -19,7 +19,7 @@ class CompilationTests extends ParallelTesting { def numberOfSlaves = 5 def safeMode = Properties.testsSafeMode def isInteractive = SummaryReport.isInteractive - def testFilter = Properties.testsFilter.map(r => new Regex(r)) + def testFilter = Properties.testsFilter // Positive tests ------------------------------------------------------------ diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 4c7328214..5957e3ec5 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -38,10 +38,11 @@ trait ParallelTesting extends RunnerOrchestration { self => */ def isInteractive: Boolean - /** A regex which is used to filter which tests to run, if `None` will run - * all tests + /** A string which is used to filter which tests to run, if `None` will run + * all tests. All absolute paths that contain the substring `testFilter` + * will be run */ - def testFilter: Option[Regex] + def testFilter: Option[String] /** A test source whose files or directory of files is to be compiled * in a specific way defined by the `Test` @@ -51,14 +52,14 @@ trait ParallelTesting extends RunnerOrchestration { self => def outDir: JFile def flags: Array[String] - def classPath: String = { - val (beforeCp, cpAndAfter) = flags.toList.span(_ != "-classpath") - if (cpAndAfter.nonEmpty) { - val (_ :: cpArg :: _) = cpAndAfter - s"${outDir.getAbsolutePath}:" + cpArg - } - else outDir.getAbsolutePath - } + def classPath: String = + outDir.getAbsolutePath + + flags + .dropWhile(_ != "-classpath") + .drop(1) + .headOption + .map(":" + _) + .getOrElse("") def title: String = self match { @@ -224,9 +225,9 @@ trait ParallelTesting extends RunnerOrchestration { self => if (!testFilter.isDefined) testSources else testSources.filter { case JointCompilationSource(_, files, _, _) => - files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined) + files.exists(file => file.getAbsolutePath.contains(testFilter.get)) case SeparateCompilationSource(_, dir, _, _) => - testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined + dir.getAbsolutePath.contains(testFilter.get) } /** Total amount of test sources being compiled by this test */ @@ -420,7 +421,7 @@ trait ParallelTesting extends RunnerOrchestration { self => } else echo { testFilter - .map(r => s"""No files matched regex "$r" in test""") + .map(r => s"""No files matched "$r" in test""") .getOrElse("No tests available under target - erroneous test?") } @@ -475,7 +476,7 @@ trait ParallelTesting extends RunnerOrchestration { self => summaryReport.addStartingMessage { """|WARNING |------- - |Run tests were only compiled, not run - this is due to `dotty.tests.norun` + |Run tests were only compiled, not run - this is due to the `dotty.tests.norun` |property being set |""".stripMargin } -- cgit v1.2.3 From ebd16700937d2e522f99ac5dd046217d889e8cb4 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 12 Apr 2017 11:19:02 +0200 Subject: Remove duplication with run test output verification --- .../test/dotty/tools/vulpix/ParallelTesting.scala | 26 ++++++---------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 5957e3ec5..f43462011 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -482,12 +482,13 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { + private def verifyOutput(checkFile: Option[JFile], dir: JFile, testSource: TestSource, warnings: Int) = { if (Properties.testsNoRun) addNoRunWarning() else runMain(testSource.classPath) match { + case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success! case Success(output) => { val outputLines = output.lines.toArray - val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray + val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines.toArray val sourceTitle = testSource.title def linesMatch = @@ -529,7 +530,7 @@ trait ParallelTesting extends RunnerOrchestration { self => protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable { def checkTestSource(): Unit = tryCompile(testSource) { - val (compilerCrashed, errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { + val (compilerCrashed, errorCount, warningCount, verifier: Function0[Unit]) = testSource match { case testSource @ JointCompilationSource(_, files, flags, outDir) => { val checkFile = files.flatMap { file => if (file.isDirectory) Nil @@ -547,7 +548,7 @@ trait ParallelTesting extends RunnerOrchestration { self => logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount) } - (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount)) + (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, () => verifyOutput(checkFile, outDir, testSource, reporter.warningCount)) } case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => { @@ -567,24 +568,11 @@ trait ParallelTesting extends RunnerOrchestration { self => logBuildInstructions(reporters.head, testSource, errorCount, warningCount) } - (compilerCrashed, errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount)) + (compilerCrashed, errorCount, warningCount, () => verifyOutput(Some(checkFile), outDir, testSource, warningCount)) } } - if (!compilerCrashed && errorCount == 0 && hasCheckFile) verifier() - else if (!compilerCrashed && errorCount == 0) { - if (Properties.testsNoRun) addNoRunWarning() - else runMain(testSource.classPath) match { - case Success(_) => // success! - case Failure(output) => - echo(s" failed when running '${testSource.title}'") - echo(output) - failTestSource(testSource) - case Timeout => - echo(" failed because test " + testSource.title + " timed out") - failTestSource(testSource, Some("test timed out")) - } - } + if (!compilerCrashed && errorCount == 0) verifier() else { echo(s"\n Compilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) -- cgit v1.2.3 From f6d519ab713cac49c6228f142cc4ccfc8880ef10 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 12 Apr 2017 11:20:36 +0200 Subject: Add docs to SummaryReport.scala --- .../test/dotty/tools/vulpix/SummaryReport.scala | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.scala b/compiler/test/dotty/tools/vulpix/SummaryReport.scala index 53b0942ce..8f3047f49 100644 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.scala +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.scala @@ -5,16 +5,38 @@ package vulpix import scala.collection.mutable import dotc.reporting.TestReporter +/** `SummaryReporting` can be used by unit tests by utilizing `@AfterClass` to + * call `echoSummary` + * + * This is used in vulpix by passing the companion object's `SummaryReporting` + * to each test, the `@AfterClass def` then calls the `SummaryReport`'s + * `echoSummary` method in order to dump the summary to both stdout and a log + * file + */ trait SummaryReporting { + /** Report a failed test */ def reportFailed(): Unit + + /** Report a test as passing */ def reportPassed(): Unit + + /** Add the name of the failed test */ def addFailedTest(msg: String): Unit + + /** Add instructions to reproduce the error */ def addReproduceInstruction(instr: String): Unit + + /** Add a message that will be issued in the beginning of the summary */ def addStartingMessage(msg: String): Unit + + /** Add a cleanup hook to be run upon completion */ def addCleanup(f: () => Unit): Unit + + /** Echo the summary report to the appropriate locations */ def echoSummary(): Unit } +/** A summary report that doesn't do anything */ final class NoSummaryReport extends SummaryReporting { def reportFailed(): Unit = () def reportPassed(): Unit = () @@ -25,6 +47,9 @@ final class NoSummaryReport extends SummaryReporting { def echoSummary(): Unit = () } +/** A summary report that logs to both stdout and the `TestReporter.logWriter` + * which outputs to a log file in `./testlogs/` + */ final class SummaryReport extends SummaryReporting { private val startingMessages = mutable.ArrayBuffer.empty[String] -- cgit v1.2.3