From 2f1a542034ddbc6cccf321e068e5161d32839f07 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 4 Apr 2017 23:13:41 +0200 Subject: Remove partest, keeping `dotc.tests` for now --- compiler/test/dotc/tests.scala | 17 +- compiler/test/dotty/partest/DPConfig.scala | 40 -- compiler/test/dotty/partest/DPConsoleRunner.scala | 411 --------------------- compiler/test/dotty/partest/DPDirectCompiler.scala | 36 -- .../test/dotty/tools/dotc/CompilationTests.scala | 3 +- compiler/test/dotty/tools/dotc/CompilerTest.scala | 231 ++---------- project/Build.scala | 92 +---- 7 files changed, 49 insertions(+), 781 deletions(-) delete mode 100644 compiler/test/dotty/partest/DPConfig.scala delete mode 100644 compiler/test/dotty/partest/DPConsoleRunner.scala delete mode 100644 compiler/test/dotty/partest/DPDirectCompiler.scala diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index af2c88e1a..8e7337524 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -3,6 +3,7 @@ package dotc import dotty.Jars import dotty.tools.dotc.CompilerTest import dotty.tools.StdLibSources +import org.junit.experimental.categories.Category import org.junit.{Before, Test} import org.junit.Assert._ @@ -10,10 +11,20 @@ import java.io.{ File => JFile } import scala.reflect.io.Directory import scala.io.Source -// tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be executed as benchmarks. +/** Marker class to indicate sequential unit tests */ +class SequentialUnitTests + +/** WARNING + * ======= + * These are legacy, do not add tests here, see `CompilationTests.scala` + */ +@Category(Array(classOf[SequentialUnitTests])) class tests extends CompilerTest { - def isRunByJenkins: Boolean = sys.props.isDefinedAt("dotty.jenkins.build") + // tests that match regex '(pos|dotc|run|java|compileStdLib)\.*' would be + // executed as benchmarks. + + def isRunByDrone: Boolean = sys.props.isDefinedAt("DRONE") val defaultOutputDir = "../out/" @@ -62,7 +73,7 @@ class tests extends CompilerTest { } implicit val defaultOptions: List[String] = noCheckOptions ++ { - if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 + if (isRunByDrone) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") } ++ checkOptions ++ classPath diff --git a/compiler/test/dotty/partest/DPConfig.scala b/compiler/test/dotty/partest/DPConfig.scala deleted file mode 100644 index 5c493f465..000000000 --- a/compiler/test/dotty/partest/DPConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -package dotty.partest - -import scala.collection.JavaConversions._ -import scala.reflect.io.Path -import java.io.File - -import scala.tools.partest.PartestDefaults - - -/** Dotty Partest runs all tests in the provided testDirs located under - * testRoot. There can be several directories with pos resp. neg tests, as - * long as the prefix is pos/neg. - * - * Each testDir can also have a __defaultFlags.flags file, which provides - * compiler flags and is used unless there's a specific flags file (e.g. for - * test pos/A.scala, if there's a pos/A.flags file those flags are used, - * otherwise pos/__defaultFlags.flags are used if the file exists). - */ -object DPConfig { - /** Options used for _running_ the run tests. - * Note that this is different from the options used when _compiling_ tests, - * those are determined by the sbt configuration. - */ - val runJVMOpts = s"-Xms64M -Xmx1024M ${PartestDefaults.javaOpts}" - - val testRoot = (Path("..") / Path("tests") / Path("partest-generated")).toString - val genLog = Path(testRoot) / Path("gen.log") - - lazy val testDirs = { - val root = new File(testRoot) - val dirs = if (!root.exists) Array.empty[String] else root.listFiles.filter(_.isDirectory).map(_.getName) - if (dirs.isEmpty) - throw new Exception("Partest did not detect any generated sources") - dirs - } - - // Tests finish faster when running in parallel, but console output is - // out of order and sometimes the compiler crashes - val runTestsInParallel = true -} diff --git a/compiler/test/dotty/partest/DPConsoleRunner.scala b/compiler/test/dotty/partest/DPConsoleRunner.scala deleted file mode 100644 index 3362d7a59..000000000 --- a/compiler/test/dotty/partest/DPConsoleRunner.scala +++ /dev/null @@ -1,411 +0,0 @@ -/* NOTE: Adapted from ScalaJSPartest.scala in - * https://github.com/scala-js/scala-js/ - * TODO make partest configurable */ - -package dotty.partest - -import dotty.tools.FatalError -import scala.reflect.io.AbstractFile -import scala.tools.partest._ -import scala.tools.partest.nest._ -import TestState.{ Pass, Fail, Crash, Uninitialized, Updated } -import ClassPath.{ join, split } -import FileManager.{ compareFiles, compareContents, joinPaths, withTempFile } -import scala.util.matching.Regex -import tools.nsc.io.{ File => NSCFile } -import java.io.{ File, PrintStream, FileOutputStream, PrintWriter, FileWriter } -import java.net.URLClassLoader - -/** Runs dotty partest from the Console, discovering test sources in - * DPConfig.testRoot that have been generated automatically by - * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are - * required by some run tests, add them to partestDeps in the sbt Build.scala. - */ -object DPConsoleRunner { - def main(args: Array[String]): Unit = { - // unfortunately sbt runTask passes args as single string - // extra jars for run tests are passed with -dottyJars ... - val jarFinder = """-dottyJars (\d*) (.*)""".r - val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined) - val (extraJars, moreArgs) = jarList match { - case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars *\".") - case jarFinder(nr, jarString) :: Nil => - val jars = jarString.split(" ").toList - val count = nr.toInt - if (jars.length < count) - sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr) - else (jars.take(count), jars.drop(count)) - case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list) - } - new DPConsoleRunner((otherArgs ::: moreArgs) mkString (" "), extraJars).runPartest - } -} - -// console runner has a suite runner which creates a test runner for each test -class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) { - override val suiteRunner = new DPSuiteRunner ( - testSourcePath = optSourcePath getOrElse DPConfig.testRoot, - fileManager = new DottyFileManager(extraJars), - updateCheck = optUpdateCheck, - failed = optFailed, - consoleArgs = args) - - override def run = {} - def runPartest = super.run -} - -class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) { - lazy val extraJarList = extraJars.map(NSCFile(_)) - override lazy val libraryUnderTest = Path(extraJars.find(_.contains("scala-library")).getOrElse("")) - override lazy val reflectUnderTest = Path(extraJars.find(_.contains("scala-reflect")).getOrElse("")) - override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse("")) -} - -class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending" - fileManager: DottyFileManager, - updateCheck: Boolean, - failed: Boolean, - consoleArgs: String, - javaCmdPath: String = PartestDefaults.javaCmd, - javacCmdPath: String = PartestDefaults.javacCmd, - scalacExtraArgs: Seq[String] = Seq.empty, - javaOpts: String = DPConfig.runJVMOpts) -extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs, javaOpts) { - - if (!DPConfig.runTestsInParallel) - sys.props("partest.threads") = "1" - - sys.props("partest.root") = "." - - // override to provide Dotty banner - override def banner: String = { - s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString} - |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main - |Generated test sources: ${PathSettings.srcDir}${File.separator} - |Test directories: ${DPConfig.testDirs.toList.mkString(", ")} - |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj - |Parallel: ${DPConfig.runTestsInParallel} - |Options: (use partest --help for usage information) ${consoleArgs} - """.stripMargin - } - - /** Some tests require a limitation of resources, tests which are compiled - * with one or more of the flags in this list will be run with - * `limitedThreads`. This is necessary because some test flags require a lot - * of memory when running the compiler and may exhaust the available memory - * when run in parallel with too many other tests. - * - * This number could be increased on the CI, but might fail locally if - * scaled too extreme - override with: - * - * ``` - * -Ddotty.tests.limitedThreads=X - * ``` - */ - def limitResourceFlags = List("-Ytest-pickler") - private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2") - - override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = { - val (limitResourceTests, parallelTests) = - kindFiles partition { kindFile => - val flags = kindFile.changeExtension("flags").fileContents - limitResourceFlags.exists(seqFlag => flags.contains(seqFlag)) - } - - val seqResults = - if (!limitResourceTests.isEmpty) { - val savedThreads = sys.props("partest.threads") - sys.props("partest.threads") = { - assert( - savedThreads == null || limitedThreads.toInt <= savedThreads.toInt, - """|Should not use more threads than the default, when the point - |is to limit the amount of resources""".stripMargin - ) - limitedThreads - } - - NestUI.echo(s"## we will run ${limitResourceTests.length} tests using ${PartestDefaults.numThreads} thread(s) in parallel") - val res = super.runTestsForFiles(limitResourceTests, kind) - - if (savedThreads != null) - sys.props("partest.threads") = savedThreads - else - sys.props.remove("partest.threads") - - res - } else Array[TestState]() - - val parResults = - if (!parallelTests.isEmpty) { - NestUI.echo(s"## we will run ${parallelTests.length} tests in parallel using ${PartestDefaults.numThreads} thread(s)") - super.runTestsForFiles(parallelTests, kind) - } else Array[TestState]() - - seqResults ++ parResults - } - - // override for DPTestRunner and redirecting compilation output to test.clog - override def runTest(testFile: File): TestState = { - val runner = new DPTestRunner(testFile, this) - - val state = - try { - runner.run match { - // Append compiler output to transcript if compilation failed, - // printed with --verbose option - case TestState.Fail(f, r@"compilation failed", transcript) => - TestState.Fail(f, r, transcript ++ runner.cLogFile.fileLines.dropWhile(_ == "")) - case res => res - } - } catch { - case t: Throwable => throw new RuntimeException(s"Error running $testFile", t) - } - reportTest(state) - runner.cleanup() - - onFinishTest(testFile, state) - } - - // override NestUI.reportTest because --show-diff doesn't work. The diff used - // seems to add each line to transcript separately, whereas NestUI assumes - // that the diff string was added as one entry in the transcript - def reportTest(state: TestState) = { - import NestUI._ - import NestUI.color._ - - if (isTerse && state.isOk) { - NestUI.reportTest(state) - } else { - echo(statusLine(state)) - if (!state.isOk && isDiffy) { - val differ = bold(red("% ")) + "diff " - state.transcript.dropWhile(s => !(s startsWith differ)) foreach (echo(_)) - // state.transcript find (_ startsWith differ) foreach (echo(_)) // original - } - } - } -} - -class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runner(testFile, suiteRunner) { - val cLogFile = SFile(logFile).changeExtension("clog") - - // override to provide DottyCompiler - override def newCompiler = new dotty.partest.DPDirectCompiler(this) - - // Adapted from nest.Runner#javac because: - // - Our classpath handling is different and we need to pass extraClassPath - // to java to get the scala-library which is required for some java tests - // - The compiler output should be redirected to cLogFile, like the output of - // dotty itself - override def javac(files: List[File]): TestState = { - import fileManager._ - import suiteRunner._ - import FileManager.joinPaths - // compile using command-line javac compiler - val args = Seq( - suiteRunner.javacCmdPath, // FIXME: Dotty deviation just writing "javacCmdPath" doesn't work - "-d", - outDir.getAbsolutePath, - "-classpath", - joinPaths(outDir :: extraClasspath ++ testClassPath) - ) ++ files.map(_.getAbsolutePath) - - pushTranscript(args mkString " ") - - val captured = StreamCapture(runCommand(args, cLogFile)) - if (captured.result) genPass() else { - cLogFile appendAll captured.stderr - cLogFile appendAll captured.stdout - genFail("java compilation failed") - } - } - - // Overriden in order to recursively get all sources that should be handed to - // the compiler. Otherwise only sources in the top dir is compiled - works - // because the compiler is on the classpath. - override def sources(file: File): List[File] = - if (file.isDirectory) - file.listFiles.toList.flatMap { f => - if (f.isDirectory) sources(f) - else if (f.isJavaOrScala) List(f) - else Nil - } - else List(file) - - // Enable me to "fix" the depth issue - remove once completed - //override def compilationRounds(file: File): List[CompileRound] = { - // val srcs = sources(file) match { - // case Nil => - // System.err.println { - // s"""|================================================================================ - // |Warning! You attempted to compile sources from: - // | $file - // |but partest was unable to find any sources - uncomment DPConsoleRunner#sources - // |================================================================================""".stripMargin - // } - // List(new File("./tests/pos/HelloWorld.scala")) // "just compile some crap" - Guillaume - // case xs => - // xs - // } - // (groupedFiles(srcs) map mixedCompileGroup).flatten - //} - - // FIXME: This is copy-pasted from nest.Runner where it is private - // Remove this once https://github.com/scala/scala-partest/pull/61 is merged - /** Runs command redirecting standard out and - * error out to output file. - */ - def runCommand(args: Seq[String], outFile: File): Boolean = { - import scala.sys.process.{ Process, ProcessLogger } - //(Process(args) #> outFile !) == 0 or (Process(args) ! pl) == 0 - val pl = ProcessLogger(outFile) - val nonzero = 17 // rounding down from 17.3 - def run: Int = { - val p = Process(args) run pl - try p.exitValue - catch { - case e: InterruptedException => - NestUI verbose s"Interrupted waiting for command to finish (${args mkString " "})" - p.destroy - nonzero - case t: Throwable => - NestUI verbose s"Exception waiting for command to finish: $t (${args mkString " "})" - p.destroy - throw t - } - finally pl.close() - } - (pl buffer run) == 0 - } - - // override to provide default dotty flags from file in directory - override def flagsForCompilation(sources: List[File]): List[String] = { - val specificFlags = super.flagsForCompilation(sources) - if (specificFlags.isEmpty) defaultFlags - else specificFlags - } - - val defaultFlags = { - val defaultFile = parentFile.listFiles.toList.find(_.getName == "__defaultFlags.flags") - defaultFile.map({ file => - SFile(file).safeSlurp.map({ content => words(content).filter(_.nonEmpty) }).getOrElse(Nil) - }).getOrElse(Nil) - } - - // override to add the check for nr of compilation errors if there's a - // target.nerr file - override def runNegTest() = runInContext { - sealed abstract class NegTestState - // Don't get confused, the neg test passes when compilation fails for at - // least one round (optionally checking the number of compiler errors and - // compiler console output) - case object CompFailed extends NegTestState - // the neg test fails when all rounds return either of these: - case class CompFailedButWrongNErr(expected: String, found: String) extends NegTestState - case object CompFailedButWrongDiff extends NegTestState - case object CompSucceeded extends NegTestState - - def nerrIsOk(reason: String) = { - val nerrFinder = """compilation failed with (\d+) errors""".r - reason match { - case nerrFinder(found) => - SFile(FileOps(testFile) changeExtension "nerr").safeSlurp match { - case Some(exp) if (exp != found) => CompFailedButWrongNErr(exp, found) - case _ => CompFailed - } - case _ => CompFailed - } - } - - // we keep the partest semantics where only one round needs to fail - // compilation, not all - val compFailingRounds = - compilationRounds(testFile) - .map { round => - val ok = round.isOk - setLastState(if (ok) genPass else genFail("compilation failed")) - (round.result, ok) - } - .filter { case (_, ok) => !ok } - - val failureStates = compFailingRounds.map({ case (result, _) => result match { - // or, OK, we'll let you crash the compiler with a FatalError if you supply a check file - case Crash(_, t, _) if !checkFile.canRead || !t.isInstanceOf[FatalError] => CompSucceeded - case Fail(_, reason, _) => if (diffIsOk) nerrIsOk(reason) else CompFailedButWrongDiff - case _ => if (diffIsOk) CompFailed else CompFailedButWrongDiff - }}) - - if (failureStates.exists({ case CompFailed => true; case _ => false })) { - true - } else { - val existsNerr = failureStates.exists({ - case CompFailedButWrongNErr(exp, found) => - nextTestActionFailing(s"wrong number of compilation errors, expected: $exp, found: $found") - true - case _ => - false - }) - - if (existsNerr) false - else { - val existsDiff = failureStates.exists({ - case CompFailedButWrongDiff => - nextTestActionFailing(s"output differs") - true - case _ => - false - }) - if (existsDiff) false - else nextTestActionFailing("expected compilation failure") - } - } - } - - // override to change check file updating to original file, not generated - override def diffIsOk: Boolean = { - // always normalize the log first - normalizeLog() - val diff = currentDiff - // if diff is not empty, is update needed? - val updating: Option[Boolean] = ( - if (diff == "") None - else Some(suiteRunner.updateCheck) - ) - pushTranscript(s"diff $logFile $checkFile") - nextTestAction(updating) { - case Some(true) => - val origCheck = SFile(checkFile.changeExtension("checksrc").fileLines(1)) - NestUI.echo("Updating original checkfile " + origCheck) - origCheck writeAll file2String(logFile) - genUpdated() - case Some(false) => - // Get a word-highlighted diff from git if we can find it - val bestDiff = if (updating.isEmpty) "" else { - if (checkFile.canRead) - gitDiff(logFile, checkFile) getOrElse { - s"diff $logFile $checkFile\n$diff" - } - else diff - } - pushTranscript(bestDiff) - genFail("output differs") - case None => genPass() // redundant default case - } getOrElse true - } - - // override to add dotty and scala jars to classpath - override def extraClasspath = - suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath - - - // FIXME: Dotty deviation: error if return type is omitted: - // overriding method cleanup in class Runner of type ()Unit; - // method cleanup of type => Boolean | Unit has incompatible type - - // override to keep class files if failed and delete clog if ok - override def cleanup: Unit = if (lastState.isOk) { - logFile.delete - cLogFile.delete - Directory(outDir).deleteRecursively - } -} diff --git a/compiler/test/dotty/partest/DPDirectCompiler.scala b/compiler/test/dotty/partest/DPDirectCompiler.scala deleted file mode 100644 index 410dac338..000000000 --- a/compiler/test/dotty/partest/DPDirectCompiler.scala +++ /dev/null @@ -1,36 +0,0 @@ -package dotty.partest - -import dotty.tools.dotc.reporting.ConsoleReporter -import scala.tools.partest.{ TestState, nest } -import java.io.{ File, PrintWriter, FileWriter } - - -/* NOTE: Adapted from partest.DirectCompiler */ -class DPDirectCompiler(runner: DPTestRunner) extends nest.DirectCompiler(runner) { - - override def compile(opts0: List[String], sources: List[File]): TestState = { - val clogFWriter = new FileWriter(runner.cLogFile.jfile, true) - val clogWriter = new PrintWriter(clogFWriter, true) - clogWriter.println("\ncompiling " + sources.mkString(" ") + "\noptions: " + opts0.mkString(" ")) - - try { - val processor = - if (opts0.exists(_.startsWith("#"))) dotty.tools.dotc.Bench else dotty.tools.dotc.Main - val clogger = new ConsoleReporter(writer = clogWriter) - val reporter = processor.process((sources.map(_.toString) ::: opts0).toArray, clogger) - if (!reporter.hasErrors) runner.genPass() - else { - clogWriter.println(reporter.summary) - runner.genFail(s"compilation failed with ${reporter.errorCount} errors") - } - } catch { - case t: Throwable => - t.printStackTrace - t.printStackTrace(clogWriter) - runner.genCrash(t) - } finally { - clogFWriter.close - clogWriter.close - } - } -} diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index fc7ab8a8d..4d5ab6963 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -3,13 +3,12 @@ package tools package dotc import org.junit.Test -import org.junit.experimental.categories.Category import scala.util.matching.Regex import scala.concurrent.duration._ + import vulpix.{ ParallelTesting, SummaryReport } -@Category(Array(classOf[ParallelTesting])) class CompilationTests extends SummaryReport with ParallelTesting { import CompilationTests._ diff --git a/compiler/test/dotty/tools/dotc/CompilerTest.scala b/compiler/test/dotty/tools/dotc/CompilerTest.scala index f35f9f919..c5234ccca 100644 --- a/compiler/test/dotty/tools/dotc/CompilerTest.scala +++ b/compiler/test/dotty/tools/dotc/CompilerTest.scala @@ -2,7 +2,6 @@ package dotty.tools.dotc import repl.TestREPL import core.Contexts._ -import dotty.partest.DPConfig import interfaces.Diagnostic.ERROR import reporting._ import diagnostic.MessageContainer @@ -11,33 +10,11 @@ import config.CompilerCommand import dotty.tools.io.PlainFile import scala.collection.mutable.ListBuffer import scala.reflect.io.{ Path, Directory, File => SFile, AbstractFile } -import scala.tools.partest.nest.{ FileManager, NestUI } import scala.annotation.tailrec import java.io.{ RandomAccessFile, File => JFile } -/** This class has two modes: it can directly run compiler tests, or it can - * generate the necessary file structure for partest in the directory - * DPConfig.testRoot. Both modes are regular JUnit tests. Which mode is used - * depends on the existence of the tests/locks/partest-ppid.lock file which is - * created by sbt to trigger partest generation. Sbt will then run partest on - * the generated sources. - * - * Through overriding the partestableXX methods, tests can always be run as - * JUnit compiler tests. Run tests cannot be run by JUnit, only by partest. - * - * A test can either be a file or a directory. Partest will generate a - * -.log file with output of failed tests. Partest reads compiler - * flags and the number of errors expected from a neg test from .flags - * and .nerr files (also generated). The test is in a parent directory - * that determines the kind of test: - * - pos: checks that compilation succeeds - * - neg: checks that compilation fails with the given number of errors - * - run: compilation succeeds, partest: test run generates the output in - * .check. Run tests always need to be: - * object Test { def main(args: Array[String]): Unit = ... } - * Classpath jars can be added to partestDeps in the sbt Build.scala. - */ +/** Legacy compiler tests that run single threaded */ abstract class CompilerTest { /** Override with output dir of test so it can be patched. Partest expects @@ -49,32 +26,9 @@ abstract class CompilerTest { def partestableDir(prefix: String, dirName: String, args: List[String]) = true def partestableList(testName: String, files: List[String], args: List[String]) = true - val generatePartestFiles = { - /* Because we fork in test, the JVM in which this JUnit test runs has a - * different pid from the one that started the partest. But the forked VM - * receives the pid of the parent as system property. If the lock file - * exists, the parent is requesting partest generation. This mechanism - * allows one sbt instance to run test (JUnit only) and another partest. - * We cannot run two instances of partest at the same time, because they're - * writing to the same directories. The sbt lock file generation prevents - * this. - */ - val pid = System.getProperty("partestParentID") - if (pid == null) - false - else - new JFile(".." + JFile.separator + "tests" + JFile.separator + "locks" + JFile.separator + s"partest-$pid.lock").exists - } - - // Delete generated files from previous run and create new log - val logFile = if (!generatePartestFiles) None else Some(CompilerTest.init) - /** Always run with JUnit. */ - def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = { - if (generatePartestFiles) - log("WARNING: compileLine will always run with JUnit, no partest files generated.") + def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = compileArgs(cmdLine.split("\n"), Nil) - } /** Compiles the given code file. * @@ -88,36 +42,22 @@ abstract class CompilerTest { (implicit defaultOptions: List[String]): Unit = { val filePath = s"$prefix$fileName$extension" val expErrors = expectedErrors(filePath) - if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions)) { - if (runTest) - log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension") - if (args.contains("-rewrite")) { - val file = new PlainFile(filePath) - val data = file.toByteArray - // compile with rewrite - compileArgs((filePath :: args).toArray, expErrors) - // compile again, check that file now compiles without -language:Scala2 - val plainArgs = args.filter(arg => arg != "-rewrite" && arg != "-language:Scala2") - compileFile(prefix, fileName, plainArgs, extension, runTest) - // restore original test file - val out = file.output - out.write(data) - out.close() - } - else compileArgs((filePath :: args).toArray, expErrors) - } else { - val kind = testKind(prefix, runTest) - log(s"generating partest files for test file: $prefix$fileName$extension of kind $kind") - - val sourceFile = new JFile(prefix + fileName + extension) - if (sourceFile.exists) { - val firstDest = SFile(DPConfig.testRoot + JFile.separator + kind + JFile.separator + fileName + extension) - val xerrors = expErrors.map(_.totalErrors).sum - computeDestAndCopyFiles(sourceFile, firstDest, kind, args ++ defaultOptions, xerrors.toString) - } else { - throw new java.io.FileNotFoundException(s"Unable to locate test file $prefix$fileName") - } + if (runTest) + log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension") + if (args.contains("-rewrite")) { + val file = new PlainFile(filePath) + val data = file.toByteArray + // compile with rewrite + compileArgs((filePath :: args).toArray, expErrors) + // compile again, check that file now compiles without -language:Scala2 + val plainArgs = args.filter(arg => arg != "-rewrite" && arg != "-language:Scala2") + compileFile(prefix, fileName, plainArgs, extension, runTest) + // restore original test file + val out = file.output + out.write(data) + out.close() } + else compileArgs((filePath :: args).toArray, expErrors) } def runFile(prefix: String, fileName: String, args: List[String] = Nil, extension: String = ".scala") (implicit defaultOptions: List[String]): Unit = { @@ -167,33 +107,11 @@ abstract class CompilerTest { val expErrors = expectedErrors(filePaths.toList) (filePaths, javaFilePaths, normArgs, expErrors) } - if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions)) { - if (runTest) - log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName") - val (filePaths, javaFilePaths, normArgs, expErrors) = computeFilePathsAndExpErrors - compileWithJavac(javaFilePaths, Array.empty) // javac needs to run first on dotty-library - compileArgs(javaFilePaths ++ filePaths ++ normArgs, expErrors) - } else { - val (sourceDir, flags, deep) = args match { - case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep") - case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow") - } - val kind = testKind(prefix, runTest) - log(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind") - - if (sourceDir.exists) { - val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName) - val xerrors = if (isNegTest(prefix)) { - val (_, _, _, expErrors) = computeFilePathsAndExpErrors - expErrors.map(_.totalErrors).sum - } else 0 - computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString) - if (deep == "deep") - Directory(sourceDir).deleteRecursively - } else { - throw new java.io.FileNotFoundException(s"Unable to locate test dir $prefix$dirName") - } - } + if (runTest) + log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName") + val (filePaths, javaFilePaths, normArgs, expErrors) = computeFilePathsAndExpErrors + compileWithJavac(javaFilePaths, Array.empty) // javac needs to run first on dotty-library + compileArgs(javaFilePaths ++ filePaths ++ normArgs, expErrors) } def runDir(prefix: String, dirName: String, args: List[String] = Nil) (implicit defaultOptions: List[String]): Unit = @@ -222,19 +140,8 @@ abstract class CompilerTest { /** Compiles the given list of code files. */ def compileList(testName: String, files: List[String], args: List[String] = Nil) (implicit defaultOptions: List[String]): Unit = { - if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions)) { - val expErrors = expectedErrors(files) - compileArgs((files ++ args).toArray, expErrors) - } else { - val destDir = Directory(DPConfig.testRoot + JFile.separator + testName) - files.foreach({ file => - val sourceFile = new JFile(file) - val destFile = destDir / (if (file.startsWith("../")) file.substring(3) else file) - recCopyFiles(sourceFile, destFile) - }) - compileDir(DPConfig.testRoot + JFile.separator, testName, args) - destDir.deleteRecursively - } + val expErrors = expectedErrors(files) + compileArgs((files ++ args).toArray, expErrors) } // ========== HELPERS ============= @@ -425,60 +332,6 @@ abstract class CompilerTest { } import Difference._ - /** The same source might be used for several partest test cases (e.g. with - * different flags). Detects existing versions and computes the path to be - * used for this version, e.g. testname_v1 for the first alternative. */ - private def computeDestAndCopyFiles(source: JFile, dest: Path, kind: String, oldFlags: List[String], nerr: String, - nr: Int = 0, oldOutput: String = defaultOutputDir): Unit = { - - val partestOutput = dest.jfile.getParentFile + JFile.separator + dest.stripExtension + "-" + kind + ".obj" - - val altOutput = - source.getParentFile.getAbsolutePath.map(x => if (x == JFile.separatorChar) '_' else x) - - val (beforeCp, remaining) = oldFlags - .map(f => if (f == oldOutput) partestOutput else f) - .span(_ != "-classpath") - val flags = beforeCp ++ List("-classpath", (partestOutput :: remaining.drop(1)).mkString(":")) - - val difference = getExisting(dest).isDifferent(source, flags, nerr) - difference match { - case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind) - case ExistsSame => // nothing else to do - case ExistsDifferent => - val nextDest = dest.parent / (dest match { - case d: Directory => - val newVersion = replaceVersion(d.name, nr).getOrElse(altOutput) - Directory(newVersion) - case f => - val newVersion = replaceVersion(f.stripExtension, nr).getOrElse(altOutput) - SFile(newVersion).addExtension(f.extension) - }) - computeDestAndCopyFiles(source, nextDest, kind, flags, nerr, nr + 1, partestOutput) - } - } - - /** Copies the test sources. Creates flags, nerr, check and output files. */ - private def copyFiles(sourceFile: Path, dest: Path, partestOutput: String, flags: List[String], nerr: String, kind: String) = { - recCopyFiles(sourceFile, dest) - - new JFile(partestOutput).mkdirs - - if (flags.nonEmpty) - dest.changeExtension("flags").createFile(true).writeAll(flags.mkString(" ")) - if (nerr != "0") - dest.changeExtension("nerr").createFile(true).writeAll(nerr) - sourceFile.changeExtension("check").ifFile({ check => - if (kind == "run") { - FileManager.copyFile(check.jfile, dest.changeExtension("check").jfile) - dest.changeExtension("checksrc").createFile(true).writeAll("check file generated from source:\n" + check.toString) - } else { - log(s"WARNING: ignoring $check for test kind $kind") - } - }) - - } - /** Recursively copy over source files and directories, excluding extensions * that aren't in extensionsToCopy. */ private def recCopyFiles(sourceFile: Path, dest: Path): Unit = { @@ -576,38 +429,6 @@ abstract class CompilerTest { } } - /** Creates a temporary directory and copies all (deep) files over, thus - * flattening the directory structure. */ - private def flattenDir(prefix: String, dirName: String): JFile = { - val destDir = Directory(DPConfig.testRoot + JFile.separator + "_temp") - Directory(prefix + dirName).deepFiles.foreach(source => recCopyFiles(source, destDir / source.name)) - destDir.jfile - } - - /** Write either to console (JUnit) or log file (partest). */ - private def log(msg: String) = logFile.map(_.appendAll(msg + "\n")).getOrElse(println(msg)) -} - -object CompilerTest extends App { - - /** Deletes generated partest sources from a previous run, recreates - * directory and returns the freshly created log file. */ - lazy val init: SFile = { - scala.reflect.io.Directory(DPConfig.testRoot).deleteRecursively - new JFile(DPConfig.testRoot).mkdirs - val log = DPConfig.genLog.createFile(true) - println(s"CompilerTest is generating tests for partest, log: $log") - log - } - -// val dotcDir = "/Users/odersky/workspace/dotty/src/dotty/" - -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "CompilationUnit") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Compiler") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Driver") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Main") -// new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Run") - -// new CompilerTest().compileDir(dotcDir + "tools/dotc") - // new CompilerTest().compileFile(dotcDir + "tools/dotc/", "Run") + /** Write either to console */ + private def log(msg: String) = println(msg) } diff --git a/project/Build.scala b/project/Build.scala index bb02416cc..391799ef7 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -134,27 +134,7 @@ object Build { settings( triggeredMessage in ThisBuild := Watched.clearWhenTriggered, - addCommandAlias("run", "dotty-compiler/run") ++ - addCommandAlias( - "partest", - ";publishLocal" + // Non-bootstrapped dotty needs to be published first - ";dotty-compiler-bootstrapped/lockPartestFile" + - ";dotty-compiler-bootstrapped/test:test" + - ";dotty-compiler-bootstrapped/runPartestRunner" - ) ++ - addCommandAlias( - "partest-only", - ";publishLocal" + // Non-bootstrapped dotty needs to be published first - ";dotty-compiler-bootstrapped/lockPartestFile" + - ";dotty-compiler-bootstrapped/test:test-only dotc.tests" + - ";dotty-compiler-bootstrapped/runPartestRunner" - ) ++ - addCommandAlias( - "partest-only-no-bootstrap", - ";dotty-compiler/lockPartestFile" + - ";dotty-compiler/test:test-only dotc.tests" + - ";dotty-compiler/runPartestRunner" - ) + addCommandAlias("run", "dotty-compiler/run") ). settings(publishing) @@ -287,43 +267,6 @@ object Build { "org.scala-lang" % "scala-reflect" % scalacVersion, "org.scala-lang" % "scala-library" % scalacVersion % "test"), - // start partest specific settings: - libraryDependencies += "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test", - testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }), - // this option is needed so that partest doesn't run - partestDeps := Seq( - scalaCompiler, - "org.scala-lang" % "scala-reflect" % scalacVersion, - "org.scala-lang" % "scala-library" % scalacVersion % "test" - ), - lockPartestFile := { - // When this file is present, running `test` generates the files for - // partest. Otherwise it just executes the tests directly. - val lockDir = partestLockFile.getParentFile - lockDir.mkdirs - // Cannot have concurrent partests as they write to the same directory. - if (lockDir.list.size > 0) - throw new RuntimeException("ERROR: sbt partest: another partest is already running, pid in lock file: " + lockDir.list.toList.mkString(" ")) - partestLockFile.createNewFile - partestLockFile.deleteOnExit - }, - runPartestRunner := Def.inputTaskDyn { - // Magic! This is both an input task and a dynamic task. Apparently - // command line arguments get passed to the last task in an aliased - // sequence (see partest alias below), so this works. - val args = Def.spaceDelimited("").parsed - val jars = List( - (packageBin in Compile).value.getAbsolutePath, - packageAll.value("dotty-library"), - packageAll.value("dotty-interfaces") - ) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome) - val dottyJars = - s"""-dottyJars ${jars.length + 2} dotty.jar dotty-lib.jar ${jars.mkString(" ")}""" - // Provide the jars required on the classpath of run tests - runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars + " " + args.mkString(" ")) - }.evaluated, - // end partest specific settings - // enable improved incremental compilation algorithm incOptions := incOptions.value.withNameHashing(true), @@ -389,6 +332,12 @@ object Build { "--run-listener=dotty.tools.ContextEscapeDetector" ), + // Ignore old sequential unit tests when running `test` in sbt + testOptions in Test += Tests.Argument( + TestFrameworks.JUnit, + "--exclude-categories=dotc.SequentialUnitTests" + ), + /* Add the sources of scalajs-ir. * To guarantee that dotty can bootstrap without depending on a version * of scalajs-ir built with a different Scala compiler, we add its @@ -471,35 +420,10 @@ object Build { "-Ddotty.tests.classes.compiler=" + pA("dotty-compiler") ) - ("-DpartestParentID=" + pid) :: jars ::: tuning ::: agentOptions ::: ci_build ::: path.toList + jars ::: tuning ::: agentOptions ::: ci_build ::: path.toList } ) - // Partest tasks - lazy val partestDeps = - SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies") - lazy val runPartestRunner = - InputKey[Unit]("runPartestRunner", "Runs partest") - lazy val lockPartestFile = - TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-.lock") - lazy val partestLockFile = - new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock") - - def pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0)) - - def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match { - case Some(home) => - modules.map({ module => - val file = Path(home) / Path("cache") / - Path(module.organization) / Path(module.name) / Path("jars") / - Path(module.name + "-" + module.revision + ".jar") - if (!file.isFile) throw new RuntimeException("ERROR: sbt getJarPaths: dependency jar not found: " + file) - else file.jfile.getAbsolutePath - }) - case None => throw new RuntimeException("ERROR: sbt getJarPaths: ivyHome not defined") - } - // end partest tasks - lazy val `dotty-compiler` = project.in(file("compiler")). dependsOn(`dotty-interfaces`). dependsOn(`dotty-library`). -- cgit v1.2.3