Merge pull request #2125 from dotty-staging/topic/kill-partest

Operation Kill Partest (part 1)

51 files changed, 1868 insertions, 236 deletions

diff --git a/.drone.yml b/.drone.yml
index 83309ef4e..75f7d89de 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -34,7 +34,9 @@ pipeline:
 
 matrix:
   TEST:
-    - ;test;dotty-bin-tests/test
-    - ;publishLocal;dotty-bootstrapped/test
-    - partest-only-no-bootstrap --show-diff --verbose
-    - partest-only --show-diff --verbose
+    - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;test ;dotty-bin-tests/test
+    - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;publishLocal ;dotty-bootstrapped/test
+    - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only-no-bootstrap --show-diff --verbose
+    - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only --show-diff --verbose
+    - ;dotty-compiler/testOnly dotty.tools.dotc.CompilationTests
+    - ;publishLocal ;dotty-bootstrapped/testOnly dotty.tools.dotc.CompilationTests
diff --git a/.drone.yml.sig b/.drone.yml.sig
index 1093928f0..7f5049f14 100644
--- a/.drone.yml.sig
+++ b/.drone.yml.sig
@@ -1 +1 @@
-eyJhbGciOiJIUzI1NiJ9.cGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXg0MDk2bSAtSi1YWDpSZXNlcnZlZENvZGVDYWNoZVNpemU9NTEybSAtSi1YWDpNYXhNZXRhc3BhY2VTaXplPTEwMjRtIC1EZG90dHkuZHJvbmUubWVtPTQwOTZtICIke1RFU1R9IgogICAgd2hlbjoKICAgICAgYnJhbmNoOgogICAgICAgIGV4Y2x1ZGU6IGdoLXBhZ2VzCgogIGRvY3VtZW50YXRpb246CiAgICBpbWFnZTogbGFtcGVwZmwvZG90dHk6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBjb21tYW5kczoKICAgICAgLSAuL3Byb2plY3Qvc2NyaXB0cy9nZW5Eb2NzICIke1RFU1R9IiAkQk9UX1BBU1MKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGdpdHRlcjoKICAgIGltYWdlOiBwbHVnaW5zL2dpdHRlcgogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgogIHNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIGNoYW5uZWw6IGRvdHR5CiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgogICAgICBzdGF0dXM6IGNoYW5nZWQKCm1hdHJpeDoKICBURVNUOgogICAgLSA7dGVzdDtkb3R0eS1iaW4tdGVzdHMvdGVzdAogICAgLSA7cHVibGlzaExvY2FsO2RvdHR5LWJvb3RzdHJhcHBlZC90ZXN0CiAgICAtIHBhcnRlc3Qtb25seS1uby1ib290c3RyYXAgLS1zaG93LWRpZmYgLS12ZXJib3NlCiAgICAtIHBhcnRlc3Qtb25seSAtLXNob3ctZGlmZiAtLXZlcmJvc2UK.VRqZiSgeE6OumPlEvs4TWfxIHNOEVjR_ZmyBmapxZ-U
\ No newline at end of file
+eyJhbGciOiJIUzI1NiJ9.cGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXg0MDk2bSAtSi1YWDpSZXNlcnZlZENvZGVDYWNoZVNpemU9NTEybSAtSi1YWDpNYXhNZXRhc3BhY2VTaXplPTEwMjRtIC1EZG90dHkuZHJvbmUubWVtPTQwOTZtICIke1RFU1R9IgogICAgd2hlbjoKICAgICAgYnJhbmNoOgogICAgICAgIGV4Y2x1ZGU6IGdoLXBhZ2VzCgogIGRvY3VtZW50YXRpb246CiAgICBpbWFnZTogbGFtcGVwZmwvZG90dHk6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBjb21tYW5kczoKICAgICAgLSAuL3Byb2plY3Qvc2NyaXB0cy9nZW5Eb2NzICIke1RFU1R9IiAkQk9UX1BBU1MKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGdpdHRlcjoKICAgIGltYWdlOiBwbHVnaW5zL2dpdHRlcgogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgogIHNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIGNoYW5uZWw6IGRvdHR5CiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgogICAgICBzdGF0dXM6IGNoYW5nZWQKCm1hdHJpeDoKICBURVNUOgogICAgLSA7c2V0IHRlc3RPcHRpb25zIGluIExvY2FsUHJvamVjdCgiZG90dHktY29tcGlsZXIiKSArPSBUZXN0cy5Bcmd1bWVudChUZXN0RnJhbWV3b3Jrcy5KVW5pdCwgIi0tZXhjbHVkZS1jYXRlZ29yaWVzPWRvdHR5LnRvb2xzLmRvdGMuUGFyYWxsZWxUZXN0aW5nIikgO3Rlc3QgO2RvdHR5LWJpbi10ZXN0cy90ZXN0CiAgICAtIDtzZXQgdGVzdE9wdGlvbnMgaW4gTG9jYWxQcm9qZWN0KCJkb3R0eS1jb21waWxlci1ib290c3RyYXBwZWQiKSArPSBUZXN0cy5Bcmd1bWVudChUZXN0RnJhbWV3b3Jrcy5KVW5pdCwgIi0tZXhjbHVkZS1jYXRlZ29yaWVzPWRvdHR5LnRvb2xzLmRvdGMuUGFyYWxsZWxUZXN0aW5nIikgO3B1Ymxpc2hMb2NhbCA7ZG90dHktYm9vdHN0cmFwcGVkL3Rlc3QKICAgIC0gO3NldCB0ZXN0T3B0aW9ucyBpbiBMb2NhbFByb2plY3QoImRvdHR5LWNvbXBpbGVyIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDtwYXJ0ZXN0LW9ubHktbm8tYm9vdHN0cmFwIC0tc2hvdy1kaWZmIC0tdmVyYm9zZQogICAgLSA7c2V0IHRlc3RPcHRpb25zIGluIExvY2FsUHJvamVjdCgiZG90dHktY29tcGlsZXItYm9vdHN0cmFwcGVkIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDtwYXJ0ZXN0LW9ubHkgLS1zaG93LWRpZmYgLS12ZXJib3NlCiAgICAtIDtkb3R0eS1jb21waWxlci90ZXN0T25seSBkb3R0eS50b29scy5kb3RjLkNvbXBpbGF0aW9uVGVzdHMKICAgIC0gO3B1Ymxpc2hMb2NhbCA7ZG90dHktYm9vdHN0cmFwcGVkL3Rlc3RPbmx5IGRvdHR5LnRvb2xzLmRvdGMuQ29tcGlsYXRpb25UZXN0cwo.b9x4iSh27OqWMUT8eR6uiK0OH_eERpnKIaMglF8hKYA
\ No newline at end of file
diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala
index efde897cd..0d3fb5821 100644
--- a/compiler/src/dotty/tools/dotc/Compiler.scala
+++ b/compiler/src/dotty/tools/dotc/Compiler.scala
@@ -129,7 +129,7 @@ class Compiler {
       .setMode(Mode.ImplicitsEnabled)
       .setTyperState(new MutableTyperState(ctx.typerState, ctx.typerState.reporter, isCommittable = true))
       .setFreshNames(new FreshNameCreator.Default)
-  ctx.initialize()(start) // re-initialize the base context with start
+    ctx.initialize()(start) // re-initialize the base context with start
     def addImport(ctx: Context, refFn: () => TermRef) =
       ctx.fresh.setImportInfo(ImportInfo.rootImport(refFn)(ctx))
     (start.setRunInfo(new RunInfo(start)) /: defn.RootImportFns)(addImport)
diff --git a/compiler/src/dotty/tools/dotc/reporting/ConsoleReporter.scala b/compiler/src/dotty/tools/dotc/reporting/ConsoleReporter.scala
index 95f468995..9942b9ab9 100644
--- a/compiler/src/dotty/tools/dotc/reporting/ConsoleReporter.scala
+++ b/compiler/src/dotty/tools/dotc/reporting/ConsoleReporter.scala
@@ -17,9 +17,6 @@ class ConsoleReporter(
 
   import MessageContainer._
 
-  /** maximal number of error messages to be printed */
-  protected def ErrorLimit = 100
-
   /** Prints the message. */
   def printMessage(msg: String): Unit = { writer.print(msg + "\n"); writer.flush() }
 
diff --git a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala
index b2c7abec9..77ea1884d 100644
--- a/compiler/src/dotty/tools/dotc/reporting/Reporter.scala
+++ b/compiler/src/dotty/tools/dotc/reporting/Reporter.scala
@@ -63,7 +63,7 @@ trait Reporting { this: Context =>
            |This can be achieved by adding the import clause 'import $fqname'
            |or by setting the compiler option -language:$feature.
            |See the Scala docs for value $fqname for a discussion
-           |why the feature $req be explicitly enabled."""
+           |why the feature $req be explicitly enabled.""".stripMargin
       }
     }
 
diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala
index 5ae1823e8..dce002c81 100644
--- a/compiler/test/dotc/comptest.scala
+++ b/compiler/test/dotc/comptest.scala
@@ -1,24 +1,29 @@
 package dotc
 
-import dotty.tools.dotc.CompilerTest
+import dotty.tools.dotc.ParallelTesting
 
-object comptest extends CompilerTest {
+object comptest extends ParallelTesting {
 
-  override val generatePartestFiles = false
-  val defaultOutputDir: String = ""
+  def isInteractive = true
+  def testFilter = None
+
+  implicit val defaultOutputDir: String = "."
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"
   val dotcDir = "./src/dotty/"
 
-  def main(args: Array[String]) =
-    compileList("comptest", List(
-      dotcDir + "tools/dotc/CompilationUnit.scala",
-      dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala"), List(
-      "#runs", "2",
-      "-Ylog:frontend",
-      "-Xprompt"))(Nil)
-
-//    compileDir(dotcDir + "tools/dotc/", "printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
+  def main(args: Array[String]): Unit =
+    compileList(
+      "comptest",
+      List(
+        dotcDir + "tools/dotc/CompilationUnit.scala",
+        dotcDir + "tools/dotc/core/Types.scala",
+        dotcDir + "tools/dotc/ast/Trees.scala"
+      ),
+      Array(
+        "-Ylog:frontend",
+        "-Xprompt"
+      )
+  )
 }
diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala
index 3ebf7f2c4..1c80767ee 100644
--- a/compiler/test/dotc/tests.scala
+++ b/compiler/test/dotc/tests.scala
@@ -85,7 +85,6 @@ class tests extends CompilerTest {
   val negDir        = testsDir + "neg/"
   val runDir        = testsDir + "run/"
   val newDir        = testsDir + "new/"
-  val replDir       = testsDir + "repl/"
   val javaDir       = testsDir + "pos-java-interop/"
 
   val sourceDir = "./src/"
@@ -173,7 +172,6 @@ class tests extends CompilerTest {
   @Test def pos_utf16 = compileFile(posSpecialDir, "utf16encoded", explicitUTF16)
 
   @Test def new_all = compileFiles(newDir, twice)
-  @Test def repl_all = replFiles(replDir)
 
   @Test def neg_all = compileFiles(negDir, verbose = true, compileSubDirs = false)
   @Test def neg_typedIdents() = compileDir(negDir, "typedIdents")
@@ -205,18 +203,6 @@ class tests extends CompilerTest {
 
   private val stdlibFiles: List[String] = StdLibSources.whitelisted
 
-  @Test def checkWBLists = {
-    val stdlibFilesBlackListed = StdLibSources.blacklisted
-
-    val duplicates = stdlibFilesBlackListed.groupBy(x => x).filter(_._2.size > 1).filter(_._2.size > 1)
-    val msg = duplicates.map(x => s"'${x._1}' appears ${x._2.size} times").mkString(s"Duplicate entries in ${StdLibSources.blacklistFile}:\n", "\n", "\n")
-    assertTrue(msg, duplicates.isEmpty)
-
-    val filesNotInStdLib = stdlibFilesBlackListed.toSet -- StdLibSources.all
-    val msg2 = filesNotInStdLib.map(x => s"'$x'").mkString(s"Entries in ${StdLibSources.blacklistFile} where not found:\n", "\n", "\n")
-    assertTrue(msg2, filesNotInStdLib.isEmpty)
-  }
-
   @Test def compileStdLib = compileList("compileStdLib", stdlibFiles, "-migration" :: "-Yno-inline" :: scala2mode)
   @Test def compileMixed = compileLine(
       """../tests/pos/B.scala
diff --git a/compiler/test/dotty/partest/DPConsoleRunner.scala b/compiler/test/dotty/partest/DPConsoleRunner.scala
index aa926efe2..3362d7a59 100644
--- a/compiler/test/dotty/partest/DPConsoleRunner.scala
+++ b/compiler/test/dotty/partest/DPConsoleRunner.scala
@@ -90,18 +90,18 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
   }
 
   /** Some tests require a limitation of resources, tests which are compiled
-    * with one or more of the flags in this list will be run with
-    * `limitedThreads`. This is necessary because some test flags require a lot
-    * of memory when running the compiler and may exhaust the available memory
-    * when run in parallel with too many other tests.
-    *
-    * This number could be increased on the CI, but might fail locally if
-    * scaled too extreme - override with:
-    *
-    * ```
-    * -Ddotty.tests.limitedThreads=X
-    * ```
-    */
+   *  with one or more of the flags in this list will be run with
+   *  `limitedThreads`. This is necessary because some test flags require a lot
+   *  of memory when running the compiler and may exhaust the available memory
+   *  when run in parallel with too many other tests.
+   *
+   *  This number could be increased on the CI, but might fail locally if
+   *  scaled too extreme - override with:
+   *
+   *  ```
+   *  -Ddotty.tests.limitedThreads=X
+   *  ```
+   */
   def limitResourceFlags = List("-Ytest-pickler")
   private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2")
 
diff --git a/compiler/test/dotty/tools/StdLibSources.scala b/compiler/test/dotty/tools/StdLibSources.scala
index e3da36f22..0c1121058 100644
--- a/compiler/test/dotty/tools/StdLibSources.scala
+++ b/compiler/test/dotty/tools/StdLibSources.scala
@@ -1,8 +1,9 @@
 package dotty.tools
 
 import java.io.File
-
 import scala.io.Source
+import org.junit.Test
+import org.junit.Assert._
 
 object StdLibSources {
 
@@ -57,3 +58,17 @@ object StdLibSources {
     .toList
 
 }
+
+class StdLibSources {
+  @Test def checkWBLists = {
+    val stdlibFilesBlackListed = StdLibSources.blacklisted
+
+    val duplicates = stdlibFilesBlackListed.groupBy(x => x).filter(_._2.size > 1).filter(_._2.size > 1)
+    val msg = duplicates.map(x => s"'${x._1}' appears ${x._2.size} times").mkString(s"Duplicate entries in ${StdLibSources.blacklistFile}:\n", "\n", "\n")
+    assertTrue(msg, duplicates.isEmpty)
+
+    val filesNotInStdLib = stdlibFilesBlackListed.toSet -- StdLibSources.all
+    val msg2 = filesNotInStdLib.map(x => s"'$x'").mkString(s"Entries in ${StdLibSources.blacklistFile} where not found:\n", "\n", "\n")
+    assertTrue(msg2, filesNotInStdLib.isEmpty)
+  }
+}
diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala
new file mode 100644
index 000000000..788e30aa3
--- /dev/null
+++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala
@@ -0,0 +1,312 @@
+package dotty
+package tools
+package dotc
+
+import org.junit.Test
+import java.io.{ File => JFile }
+import org.junit.experimental.categories.Category
+
+import scala.util.matching.Regex
+
+@Category(Array(classOf[ParallelTesting]))
+class CompilationTests extends ParallelSummaryReport with ParallelTesting {
+  import CompilationTests._
+
+  def isInteractive: Boolean = !sys.env.contains("DRONE")
+
+  def testFilter: Option[Regex] = sys.props.get("dotty.partest.filter").map(r => new Regex(r))
+
+  // Positive tests ------------------------------------------------------------
+
+  @Test def compilePos: Unit = {
+    compileList("compileStdLib", StdLibSources.whitelisted, scala2Mode.and("-migration", "-Yno-inline")) +
+    compileFilesInDir("../tests/pos", defaultOptions)
+  }.checkCompile()
+
+  @Test def compilePosScala2: Unit =
+    compileFilesInDir("../tests/pos-scala2", scala2Mode).checkCompile()
+
+  @Test def compilePosMixedFlags: Unit = {
+    compileFile("../tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) +
+    compileFile("../tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() +
+    compileFile("../tests/pos-special/t8146a.scala", allowDeepSubtypes) +
+    compileFile("../tests/pos-special/utf8encoded.scala", explicitUTF8) +
+    compileFile("../tests/pos-special/utf16encoded.scala", explicitUTF16) +
+    compileList(
+      "compileMixed",
+      List(
+        "../tests/pos/B.scala",
+        "../scala-scala/src/library/scala/collection/immutable/Seq.scala",
+        "../scala-scala/src/library/scala/collection/parallel/ParSeq.scala",
+        "../scala-scala/src/library/scala/package.scala",
+        "../scala-scala/src/library/scala/collection/GenSeqLike.scala",
+        "../scala-scala/src/library/scala/collection/SeqLike.scala",
+        "../scala-scala/src/library/scala/collection/generic/GenSeqFactory.scala"
+      ),
+      defaultOptions
+    ) +
+    compileFilesInDir("../tests/pos-special/spec-t5545", defaultOptions) +
+    compileFile("../scala-scala/src/library/scala/collection/immutable/IndexedSeq.scala", defaultOptions) +
+    compileFile("../scala-scala/src/library/scala/collection/parallel/mutable/ParSetLike.scala", defaultOptions) +
+    compileList(
+      "parSetSubset",
+      List(
+       "../scala-scala/src/library/scala/collection/parallel/mutable/ParSetLike.scala",
+       "../scala-scala/src/library/scala/collection/parallel/mutable/ParSet.scala",
+       "../scala-scala/src/library/scala/collection/mutable/SetLike.scala"
+      ),
+      scala2Mode
+    )
+  }.checkCompile()
+
+  @Test def compileCoreNoCheck: Unit =
+    compileDir("../compiler/src/dotty/tools/dotc/core", noCheckOptions ++ classPath).checkCompile()
+
+  @Test def compileDotcInternals: Unit = {
+    compileDir("../compiler/src/dotty/tools/dotc/ast", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/config", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/core", allowDeepSubtypes) +
+    compileDir("../compiler/src/dotty/tools/dotc/transform", allowDeepSubtypes) +
+    compileDir("../compiler/src/dotty/tools/dotc/parsing", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/printing", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/reporting", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/typer", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/util", defaultOptions) +
+    compileDir("../compiler/src/dotty/tools/io", defaultOptions)
+  }.checkCompile()
+
+  @Test def posTwice: Unit = {
+    compileFile("../tests/pos/Labels.scala", defaultOptions) +
+    compileFilesInDir("../tests/pos-java-interop", defaultOptions) +
+    compileFile("../tests/pos/t2168.scala", defaultOptions) +
+    compileFile("../tests/pos/erasure.scala", defaultOptions) +
+    compileFile("../tests/pos/Coder.scala", defaultOptions) +
+    compileFile("../tests/pos/blockescapes.scala", defaultOptions) +
+    compileFile("../tests/pos/collections.scala", defaultOptions) +
+    compileFile("../tests/pos/functions1.scala", defaultOptions) +
+    compileFile("../tests/pos/implicits1.scala", defaultOptions) +
+    compileFile("../tests/pos/inferred.scala", defaultOptions) +
+    compileFile("../tests/pos/Patterns.scala", defaultOptions) +
+    compileFile("../tests/pos/selftypes.scala", defaultOptions) +
+    compileFile("../tests/pos/varargs.scala", defaultOptions) +
+    compileFile("../tests/pos/vararg-pattern.scala", defaultOptions) +
+    compileFile("../tests/pos/opassign.scala", defaultOptions) +
+    compileFile("../tests/pos/typedapply.scala", defaultOptions) +
+    compileFile("../tests/pos/nameddefaults.scala", defaultOptions) +
+    compileFile("../tests/pos/desugar.scala", defaultOptions) +
+    compileFile("../tests/pos/sigs.scala", defaultOptions) +
+    compileFile("../tests/pos/typers.scala", defaultOptions) +
+    compileDir("../tests/pos/typedIdents", defaultOptions) +
+    compileFile("../tests/pos/assignments.scala", defaultOptions) +
+    compileFile("../tests/pos/packageobject.scala", defaultOptions) +
+    compileFile("../tests/pos/overloaded.scala", defaultOptions) +
+    compileFile("../tests/pos/overrides.scala", defaultOptions) +
+    compileDir("../tests/pos/java-override", defaultOptions) +
+    compileFile("../tests/pos/templateParents.scala", defaultOptions) +
+    compileFile("../tests/pos/overloadedAccess.scala", defaultOptions) +
+    compileFile("../tests/pos/approximateUnion.scala", defaultOptions) +
+    compileFilesInDir("../tests/pos/tailcall", defaultOptions) +
+    compileShallowFilesInDir("../tests/pos/pos_valueclasses", defaultOptions) +
+    compileFile("../tests/pos/subtyping.scala", defaultOptions) +
+    compileFile("../tests/pos/i0239.scala", defaultOptions) +
+    compileFile("../tests/pos/anonClassSubtyping.scala", defaultOptions) +
+    compileFile("../tests/pos/extmethods.scala", defaultOptions) +
+    compileFile("../tests/pos/companions.scala", defaultOptions) +
+    compileList(
+      "testNonCyclic",
+      List(
+        "../compiler/src/dotty/tools/dotc/CompilationUnit.scala",
+        "../compiler/src/dotty/tools/dotc/core/Types.scala",
+        "../compiler/src/dotty/tools/dotc/ast/Trees.scala"
+      ),
+      defaultOptions.and("-Xprompt")
+    ) +
+    compileList(
+      "testIssue34",
+      List(
+        "../compiler/src/dotty/tools/dotc/config/Properties.scala",
+        "../compiler/src/dotty/tools/dotc/config/PathResolver.scala"
+      ),
+      defaultOptions.and("-Xprompt")
+    )
+  }.times(2).checkCompile()
+
+  // New tests -----------------------------------------------------------------
+
+  @Test def compileNew: Unit =
+    compileFilesInDir("../tests/new", defaultOptions).checkCompile()
+
+  // Negative tests ------------------------------------------------------------
+
+  @Test def compileNeg: Unit =
+    compileShallowFilesInDir("../tests/neg", defaultOptions).checkExpectedErrors()
+
+  @Test def compileNegCustomFlags: Unit = {
+    compileFile("../tests/neg/customArgs/typers.scala", allowDoubleBindings) +
+    compileFile("../tests/neg/customArgs/overrideClass.scala", scala2Mode) +
+    compileFile("../tests/neg/customArgs/autoTuplingTest.scala", defaultOptions.and("-language:noAutoTupling")) +
+    compileFile("../tests/neg/customArgs/i1050.scala", defaultOptions.and("-strict")) +
+    compileFile("../tests/neg/customArgs/i1240.scala", allowDoubleBindings) +
+    compileFile("../tests/neg/customArgs/i2002.scala", allowDoubleBindings) +
+    compileFile("../tests/neg/customArgs/nopredef.scala", defaultOptions.and("-Yno-predef")) +
+    compileFile("../tests/neg/customArgs/noimports.scala", defaultOptions.and("-Yno-imports")) +
+    compileFile("../tests/neg/customArgs/noimports2.scala", defaultOptions.and("-Yno-imports")) +
+    compileFile("../tests/neg/tailcall/t1672b.scala", defaultOptions) +
+    compileFile("../tests/neg/tailcall/t3275.scala", defaultOptions) +
+    compileFile("../tests/neg/tailcall/t6574.scala", defaultOptions) +
+    compileFile("../tests/neg/tailcall/tailrec.scala", defaultOptions) +
+    compileFile("../tests/neg/tailcall/tailrec-2.scala", defaultOptions) +
+    compileFile("../tests/neg/tailcall/tailrec-3.scala", defaultOptions) +
+    compileDir("../tests/neg/typedIdents", defaultOptions)
+  }.checkExpectedErrors()
+
+  // Run tests -----------------------------------------------------------------
+
+  @Test def runAll: Unit =
+    compileFilesInDir("../tests/run", defaultOptions).checkRuns()
+
+  // Pickling Tests ------------------------------------------------------------
+  //
+  // Pickling tests are very memory intensive and as such need to be run with a
+  // lower level of concurrency as to not kill their running VMs
+
+  @Test def testPickling1: Unit = {
+    compileFilesInDir("../tests/new", picklingOptions) +
+    compileFilesInDir("../tests/pickling", picklingOptions) +
+    compileDir("../library/src/dotty/runtime", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/backend/jvm", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/ast", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/core", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/config", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/parsing", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/printing", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/repl", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/rewrite", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/transform", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/typer", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/util", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/io", picklingOptions) +
+    compileFile("../tests/pos/pickleinf.scala", picklingOptions)
+  }.limitThreads(4).checkCompile()
+
+  @Test def testPickling2: Unit = {
+    compileDir("../compiler/src/dotty/tools/dotc/core/classfile", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/core/tasty", picklingOptions) +
+    compileDir("../compiler/src/dotty/tools/dotc/core/unpickleScala2", picklingOptions)
+  }.limitThreads(4).checkCompile()
+
+  @Test def testPickling3: Unit = {
+    compileDir("../compiler/src/dotty/tools", picklingOptions)
+  }.limitThreads(4).checkCompile()
+
+  @Test def testPickling4: Unit = {
+    compileDir("../compiler/src/dotty/tools/dotc", picklingOptions)
+  }.limitThreads(4).checkCompile()
+
+  /** The purpose of this test is two-fold, being able to compile dotty
+   *  bootstrapped, and making sure that TASTY can link against a compiled
+   *  version of Dotty
+   */
+  @Test def tastyBootstrap: Unit = {
+    val opt = Array(
+      "-classpath",
+      // compile with bootstrapped library on cp:
+      defaultOutputDir + "lib$1/src/:" +
+      // as well as bootstrapped compiler:
+      defaultOutputDir + "dotty1$1/dotty/:" +
+      Jars.dottyInterfaces
+    )
+
+    def lib =
+      compileDir("../library/src",
+        allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))
+
+    def dotty1 =
+      compileDir("../compiler/src/dotty", opt)
+
+    def dotty2 =
+      compileShallowFilesInDir("../compiler/src/dotty", opt)
+
+    {
+      lib.keepOutput :: dotty1.keepOutput :: {
+        dotty2 +
+        compileShallowFilesInDir("../compiler/src/dotty/tools", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/ast", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/config", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/parsing", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/printing", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/repl", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/reporting", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/rewrite", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/transform", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/typer", opt) +
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt)
+      } :: Nil
+    }.map(_.checkCompile()).foreach(_.delete())
+  }
+}
+
+object CompilationTests {
+  implicit val defaultOutputDir: String = "../out/"
+
+  implicit class RichStringArray(val xs: Array[String]) extends AnyVal {
+    def and(args: String*): Array[String] = {
+      val argsArr: Array[String] = args.toArray
+      xs ++ argsArr
+    }
+  }
+
+  val noCheckOptions = Array(
+    "-pagewidth", "120",
+    "-color:never"
+  )
+
+  val checkOptions = Array(
+    "-Yno-deep-subtypes",
+    "-Yno-double-bindings",
+    "-Yforce-sbt-phases"
+  )
+
+  val classPath = {
+    val paths = Jars.dottyTestDeps map { p =>
+      val file = new JFile(p)
+      assert(
+        file.exists,
+        s"""|File "$p" couldn't be found. Run `packageAll` from build tool before
+            |testing.
+            |
+            |If running without sbt, test paths need to be setup environment variables:
+            |
+            | - DOTTY_LIBRARY
+            | - DOTTY_COMPILER
+            | - DOTTY_INTERFACES
+            | - DOTTY_EXTRAS
+            |
+            |Where these all contain locations, except extras which is a colon
+            |separated list of jars.
+            |
+            |When compiling with eclipse, you need the sbt-interfaces jar, put
+            |it in extras."""
+      )
+      file.getAbsolutePath
+    } mkString (":")
+
+    Array("-classpath", paths)
+  }
+
+  private val yCheckOptions = Array("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef")
+
+  val defaultOptions = noCheckOptions ++ checkOptions ++ yCheckOptions ++ classPath
+  val allowDeepSubtypes = defaultOptions diff Array("-Yno-deep-subtypes")
+  val allowDoubleBindings = defaultOptions diff Array("-Yno-double-bindings")
+  val picklingOptions = defaultOptions ++ Array(
+    "-Xprint-types",
+    "-Ytest-pickler",
+    "-Ystop-after:pickler",
+    "-Yprintpos"
+  )
+  val scala2Mode = defaultOptions ++ Array("-language:Scala2")
+  val explicitUTF8 = defaultOptions ++ Array("-encoding", "UTF8")
+  val explicitUTF16 = defaultOptions ++ Array("-encoding", "UTF16")
+}
diff --git a/compiler/test/dotty/tools/dotc/CompilerTest.scala b/compiler/test/dotty/tools/dotc/CompilerTest.scala
index db12994f4..f35f9f919 100644
--- a/compiler/test/dotty/tools/dotc/CompilerTest.scala
+++ b/compiler/test/dotty/tools/dotc/CompilerTest.scala
@@ -237,23 +237,6 @@ abstract class CompilerTest {
     }
   }
 
-  def replFile(prefix: String, fileName: String): Unit = {
-    val path = s"$prefix$fileName"
-    val f = new PlainFile(path)
-    val repl = new TestREPL(new String(f.toCharArray))
-    repl.process(Array[String]())
-    repl.check()
-  }
-
-  def replFiles(path: String): Unit = {
-    val dir = Directory(path)
-    val fileNames = dir.files.toArray.map(_.jfile.getName).filter(_ endsWith ".check")
-    for (name <- fileNames) {
-      log(s"testing $path$name")
-      replFile(path, name)
-    }
-  }
-
   // ========== HELPERS =============
 
   private def expectedErrors(filePaths: List[String]): List[ErrorsInFile] = if (filePaths.exists(isNegTest(_))) filePaths.map(getErrors(_)) else Nil
diff --git a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java
new file mode 100644
index 000000000..9214e7d25
--- /dev/null
+++ b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java
@@ -0,0 +1,67 @@
+package dotty.tools.dotc;
+
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import java.util.ArrayDeque;
+
+import dotty.tools.dotc.reporting.TestReporter;
+import dotty.tools.dotc.reporting.TestReporter$;
+
+/** Note that while `ParallelTesting` runs in parallel, JUnit tests cannot with
+ *  this class
+ */
+public class ParallelSummaryReport {
+    private static TestReporter rep = TestReporter.reporter(-1);
+    private static ArrayDeque<String> failedTests = new ArrayDeque<>();
+    private static ArrayDeque<String> reproduceInstructions = new ArrayDeque<>();
+    private static int passed;
+    private static int failed;
+
+    public final static void reportFailed() {
+        failed++;
+    }
+
+    public final static void reportPassed() {
+        passed++;
+    }
+
+    public final static void addFailedTest(String msg) {
+        failedTests.offer(msg);
+    }
+
+    public final static void addReproduceInstruction(String msg) {
+        reproduceInstructions.offer(msg);
+    }
+
+    @BeforeClass public final static void setup() {
+        rep = TestReporter.reporter(-1);
+        failedTests = new ArrayDeque<>();
+        reproduceInstructions = new ArrayDeque<>();
+    }
+
+    @AfterClass public final static void teardown() {
+        rep.echo(
+            "\n================================================================================" +
+            "\nTest Report" +
+            "\n================================================================================" +
+            "\n" +
+            passed + " passed, " + failed + " failed, " + (passed + failed) + " total" +
+            "\n"
+        );
+
+        failedTests
+            .stream()
+            .map(x -> "    " + x)
+            .forEach(rep::echo);
+
+        rep.flushToStdErr();
+
+        rep.echo("");
+
+        reproduceInstructions
+            .stream()
+            .forEach(rep::echo);
+
+        if (failed > 0) rep.flushToFile();
+    }
+}
diff --git a/compiler/test/dotty/tools/dotc/ParallelTestTests.scala b/compiler/test/dotty/tools/dotc/ParallelTestTests.scala
new file mode 100644
index 000000000..9964be036
--- /dev/null
+++ b/compiler/test/dotty/tools/dotc/ParallelTestTests.scala
@@ -0,0 +1,55 @@
+package dotty
+package tools
+package dotc
+
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.util.control.NonFatal
+
+class ParallelTestTests extends ParallelTesting {
+  import CompilationTests._
+
+  def isInteractive = !sys.env.contains("DRONE")
+  def testFilter = None
+
+  @Test def missingFile: Unit =
+    try {
+      compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors()
+      fail("didn't fail properly")
+    }
+    catch {
+      case _: IllegalArgumentException => // pass!
+      case NonFatal(_) => fail("wrong exception thrown")
+    }
+
+  @Test def pos1Error: Unit =
+    compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkCompile()
+
+  @Test def negMissingAnnot: Unit =
+    compileFile("../tests/partest-test/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negAnnotWrongLine: Unit =
+    compileFile("../tests/partest-test/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negTooManyAnnots: Unit =
+    compileFile("../tests/partest-test/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negNoPositionAnnot: Unit =
+    compileFile("../tests/partest-test/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def runCompileFail: Unit =
+    compileFile("../tests/partest-test/posFail1Error.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runWrongOutput1: Unit =
+    compileFile("../tests/partest-test/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runWrongOutput2: Unit =
+    compileFile("../tests/partest-test/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runDiffOutput1: Unit =
+    compileFile("../tests/partest-test/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runStackOverflow: Unit =
+    compileFile("../tests/partest-test/stackOverflow.scala", defaultOptions).expectFailure.checkRuns()
+}
diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala
new file mode 100644
index 000000000..30679de9e
--- /dev/null
+++ b/compiler/test/dotty/tools/dotc/ParallelTesting.scala
@@ -0,0 +1,1062 @@
+package dotty
+package tools
+package dotc
+
+import java.io.{ File => JFile }
+import java.text.SimpleDateFormat
+import java.util.HashMap
+import java.lang.reflect.InvocationTargetException
+import java.nio.file.StandardCopyOption.REPLACE_EXISTING
+import java.nio.file.{ Files, Path, Paths, NoSuchFileException }
+import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException }
+
+import scala.io.Source
+import scala.util.control.NonFatal
+import scala.util.Try
+import scala.collection.mutable
+import scala.util.matching.Regex
+
+import core.Contexts._
+import reporting.{ Reporter, TestReporter }
+import reporting.diagnostic.MessageContainer
+import interfaces.Diagnostic.ERROR
+import dotc.util.DiffUtil
+
+/** A parallel testing suite whose goal is to integrate nicely with JUnit
+ *
+ *  This trait can be mixed in to offer parallel testing to compile runs. When
+ *  using this, you should be running your JUnit tests **sequentially**, as the
+ *  test suite itself runs with a high level of concurrency.
+ */
+trait ParallelTesting {
+
+  import ParallelTesting._
+  import ParallelSummaryReport._
+
+  /** If the running environment supports an interactive terminal, each `Test`
+   *  will be run with a progress bar and real time feedback
+   */
+  def isInteractive: Boolean
+
+  /** A regex which is used to filter which tests to run, if `None` will run
+   *  all tests
+   */
+  def testFilter: Option[Regex]
+
+  /** A test source whose files or directory of files is to be compiled
+   *  in a specific way defined by the `Test`
+   */
+  private sealed trait TestSource { self =>
+    def name: String
+    def outDir: JFile
+    def flags: Array[String]
+
+    /** Adds the flags specified in `newFlags0` if they do not already exist */
+    def withFlags(newFlags0: String*) = {
+      val newFlags = newFlags0.toArray
+      if (!flags.containsSlice(newFlags)) self match {
+        case self: JointCompilationSource =>
+          self.copy(flags = flags ++ newFlags)
+        case self: SeparateCompilationSource =>
+          self.copy(flags = flags ++ newFlags)
+      }
+      else self
+    }
+
+    /** Generate the instructions to redo the test from the command line */
+    def buildInstructions(errors: Int, warnings: Int): String = {
+      val sb = new StringBuilder
+      val maxLen = 80
+      var lineLen = 0
+
+      sb.append(s"\n\nTest compiled with $errors error(s) and $warnings warning(s), the test can be reproduced by running:")
+      sb.append("\n\n./bin/dotc ")
+      flags.foreach { arg =>
+        if (lineLen > maxLen) {
+          sb.append(" \\\n        ")
+          lineLen = 4
+        }
+        sb.append(arg)
+        lineLen += arg.length
+        sb += ' '
+      }
+
+      self match {
+        case JointCompilationSource(_, files, _, _) => {
+          files.map(_.getAbsolutePath).foreach { path =>
+            sb.append("\\\n        ")
+            sb.append(path)
+            sb += ' '
+          }
+          sb.toString + "\n\n"
+        }
+        case self: SeparateCompilationSource => {
+          val command = sb.toString
+          val fsb = new StringBuilder(command)
+          self.compilationGroups.foreach { files =>
+            files.map(_.getPath).foreach { path =>
+              fsb.append("\\\n        ")
+              lineLen = 8
+              fsb.append(path)
+              fsb += ' '
+            }
+            fsb.append("\n\n")
+            fsb.append(command)
+          }
+          fsb.toString + "\n\n"
+        }
+      }
+    }
+  }
+
+  /** A group of files that may all be compiled together, with the same flags
+   *  and output directory
+   */
+  private final case class JointCompilationSource(
+    name: String,
+    files: Array[JFile],
+    flags: Array[String],
+    outDir: JFile
+  ) extends TestSource {
+    def sourceFiles: Array[JFile] = files.filter(isSourceFile)
+
+    override def toString() = outDir.toString
+  }
+
+  /** A test source whose files will be compiled separately according to their
+   *  suffix `_X`
+   */
+  private final case class SeparateCompilationSource(
+    name: String,
+    dir: JFile,
+    flags: Array[String],
+    outDir: JFile
+  ) extends TestSource {
+
+    /** Get the files grouped by `_X` as a list of groups, files missing this
+     *  suffix will be put into the same group
+     *
+     *  Filters out all none source files
+     */
+    def compilationGroups: List[Array[JFile]] =
+      dir
+      .listFiles
+      .groupBy { file =>
+        val name = file.getName
+        Try {
+          val potentialNumber = name
+            .substring(0, name.lastIndexOf('.'))
+            .reverse.takeWhile(_ != '_').reverse
+
+          potentialNumber.toInt.toString
+        }
+        .toOption
+        .getOrElse("")
+      }
+      .toList.sortBy(_._1).map(_._2.filter(isSourceFile))
+  }
+
+  /** Each `Test` takes the `testSources` and performs the compilation and assertions
+   *  according to the implementing class "neg", "run" or "pos".
+   */
+  private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) {
+
+    /** Actual compilation run logic, the test behaviour is defined here */
+    protected def compilationRunnable(testSource: TestSource): Runnable
+
+    /** All testSources left after filtering out */
+    private val filteredSources =
+      if (!testFilter.isDefined) testSources
+      else testSources.filter {
+        case JointCompilationSource(_, files, _, _) =>
+          files.exists(file => testFilter.get.findFirstIn(file.getAbsolutePath).isDefined)
+        case SeparateCompilationSource(_, dir, _, _) =>
+          testFilter.get.findFirstIn(dir.getAbsolutePath).isDefined
+      }
+
+    /** Total amount of test sources being compiled by this test */
+    val sourceCount = filteredSources.length
+
+    private[this] var _errorCount =  0
+    def errorCount: Int = synchronized { _errorCount }
+
+    private[this] var _testSourcesCompiled = 0
+    private def testSourcesCompiled : Int = synchronized { _testSourcesCompiled }
+
+    /** Complete the current compilation with the amount of errors encountered */
+    protected final def registerCompilation(errors: Int) = synchronized {
+      _testSourcesCompiled += 1
+      _errorCount += errors
+    }
+
+    private[this] var _failed = false
+    /** Fail the current test */
+    protected[this] final def fail(): Unit = synchronized { _failed = true }
+    def didFail: Boolean = _failed
+
+    protected def echoBuildInstructions(reporter: TestReporter, testSource: TestSource, err: Int, war: Int) = {
+      val errorMsg = testSource.buildInstructions(reporter.errorCount, reporter.warningCount)
+      addFailureInstruction(errorMsg)
+      failTestSource(testSource)
+    }
+
+    /** Instructions on how to reproduce failed test source compilations */
+    private[this] val reproduceInstructions = mutable.ArrayBuffer.empty[String]
+    protected final def addFailureInstruction(ins: String): Unit =
+      synchronized { reproduceInstructions.append(ins) }
+
+    /** The test sources that failed according to the implementing subclass */
+    private[this] val failedTestSources = mutable.ArrayBuffer.empty[String]
+    protected final def failTestSource(testSource: TestSource) = synchronized {
+      failedTestSources.append(testSource.name + " failed")
+      fail()
+    }
+
+    /** Prints to `System.err` if we're not suppressing all output */
+    protected def echo(msg: String): Unit =
+      if (!suppressAllOutput) System.err.println(msg)
+
+    /** A single `Runnable` that prints a progress bar for the curent `Test` */
+    private def createProgressMonitor: Runnable = new Runnable {
+      def run(): Unit = {
+        val start = System.currentTimeMillis
+        var tCompiled = testSourcesCompiled
+        while (tCompiled < sourceCount) {
+          val timestamp = (System.currentTimeMillis - start) / 1000
+          val progress = (tCompiled.toDouble / sourceCount * 40).toInt
+          print(
+            "[" + ("=" * (math.max(progress - 1, 0))) +
+            (if (progress > 0) ">" else "") +
+            (" " * (39 - progress)) +
+            s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r"
+          )
+          Thread.sleep(100)
+          tCompiled = testSourcesCompiled
+        }
+        // println, otherwise no newline and cursor at start of line
+        println(
+          s"[=======================================] compiled ($sourceCount/$sourceCount, " +
+          s"${(System.currentTimeMillis - start) / 1000}s)  "
+        )
+      }
+    }
+
+    /** Wrapper function to make sure that the compiler itself did not crash -
+     *  if it did, the test should automatically fail.
+     */
+    protected def tryCompile(testSource: TestSource)(op: => Unit): Unit =
+      try op catch {
+        case NonFatal(e) => {
+          // if an exception is thrown during compilation, the complete test
+          // run should fail
+          failTestSource(testSource)
+          e.printStackTrace()
+          registerCompilation(1)
+          throw e
+        }
+      }
+
+    protected def compile(files0: Array[JFile], flags0: Array[String], suppressErrors: Boolean, targetDir: JFile): TestReporter = {
+
+      val flags = flags0 ++ Array("-d", targetDir.getAbsolutePath)
+
+      def flattenFiles(f: JFile): Array[JFile] =
+        if (f.isDirectory) f.listFiles.flatMap(flattenFiles)
+        else Array(f)
+
+      val files: Array[JFile] = files0.flatMap(flattenFiles)
+
+      def findJarFromRuntime(partialName: String) = {
+        val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString)
+        urls.find(_.contains(partialName)).getOrElse {
+          throw new java.io.FileNotFoundException(
+            s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}"""
+          )
+        }
+      }
+
+      def addOutDir(xs: Array[String]): Array[String] = {
+        val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath")
+        if (cpAndAfter.nonEmpty) {
+          val (cp :: cpArg :: rest) = cpAndAfter
+          (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray
+        }
+        else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray
+      }
+
+      def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) {
+        val scalaLib = findJarFromRuntime("scala-library-2.")
+        val fullArgs = Array(
+          "javac",
+          "-classpath",
+          s".:$scalaLib:${targetDir.getAbsolutePath}"
+        ) ++ flags.takeRight(2) ++ fs
+
+        Runtime.getRuntime.exec(fullArgs).waitFor() == 0
+      } else true
+
+      val reporter = TestReporter.parallelReporter(this, logLevel =
+        if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR)
+      val driver =
+        if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler }
+        else new Driver {
+          def newCompiler(implicit ctx: Context) = new Compiler
+
+          private def ntimes(n: Int)(op: Int => Reporter): Reporter =
+            (emptyReporter /: (1 to n)) ((_, i) => op(i))
+
+          override def doCompile(comp: Compiler, files: List[String])(implicit ctx: Context) =
+            ntimes(times) { run =>
+              val start = System.nanoTime()
+              val rep = super.doCompile(comp, files)
+              ctx.echo(s"\ntime run $run: ${(System.nanoTime - start) / 1000000}ms")
+              rep
+            }
+        }
+
+      val allArgs = addOutDir(flags)
+      driver.process(allArgs ++ files.map(_.getAbsolutePath), reporter = reporter)
+
+      val javaFiles = files.filter(_.getName.endsWith(".java")).map(_.getAbsolutePath)
+      assert(compileWithJavac(javaFiles), s"java compilation failed for ${javaFiles.mkString(", ")}")
+
+      reporter
+    }
+
+    private[ParallelTesting] def executeTestSuite(): this.type = {
+      assert(_testSourcesCompiled == 0, "not allowed to re-use a `CompileRun`")
+
+      if (filteredSources.nonEmpty) {
+        val pool = threadLimit match {
+          case Some(i) => JExecutors.newWorkStealingPool(i)
+          case None => JExecutors.newWorkStealingPool()
+        }
+
+        if (isInteractive && !suppressAllOutput) pool.submit(createProgressMonitor)
+
+        filteredSources.foreach { target =>
+          pool.submit(compilationRunnable(target))
+        }
+
+        pool.shutdown()
+        if (!pool.awaitTermination(10, TimeUnit.MINUTES))
+          throw new TimeoutException("Compiling targets timed out")
+
+        if (didFail) {
+          reportFailed()
+          failedTestSources.toSet.foreach(addFailedTest)
+          reproduceInstructions.iterator.foreach(addReproduceInstruction)
+        }
+        else reportPassed()
+      }
+      else echo {
+        testFilter
+          .map(r => s"""No files matched regex "$r" in test""")
+          .getOrElse("No tests available under target - erroneous test?")
+      }
+
+      this
+    }
+  }
+
+  private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)
+  extends Test(testSources, times, threadLimit, suppressAllOutput) {
+    protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable {
+      def run(): Unit = tryCompile(testSource) {
+        testSource match {
+          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+            val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+            registerCompilation(reporter.errorCount)
+
+            if (reporter.errorCount > 0)
+              echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+          }
+
+          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+            val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir))
+            val errorCount = reporters.foldLeft(0) { (acc, reporter) =>
+              if (reporter.errorCount > 0)
+                echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+
+              acc + reporter.errorCount
+            }
+
+            registerCompilation(errorCount)
+
+            if (errorCount > 0) failTestSource(testSource)
+          }
+        }
+
+      }
+    }
+  }
+
+  private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)
+  extends Test(testSources, times, threadLimit, suppressAllOutput) {
+    private def runMain(dir: JFile, testSource: TestSource): Array[String] = {
+      def renderStackTrace(ex: Throwable): String =
+        ex.getStackTrace
+          .takeWhile(_.getMethodName != "invoke0")
+          .mkString("    ", "\n    ", "")
+
+      import java.io.ByteArrayOutputStream
+      import java.net.{ URL, URLClassLoader }
+
+      val printStream = new ByteArrayOutputStream
+      try {
+        // Do classloading magic and running here:
+        val ucl = new URLClassLoader(Array(dir.toURI.toURL))
+        val cls = ucl.loadClass("Test")
+        val meth = cls.getMethod("main", classOf[Array[String]])
+
+        Console.withOut(printStream) {
+          meth.invoke(null, Array("jvm")) // partest passes at least "jvm" as an arg
+        }
+      }
+      catch {
+        case ex: NoSuchMethodException =>
+          echo(s"test in '$dir' did not contain method: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}")
+          failTestSource(testSource)
+
+        case ex: ClassNotFoundException =>
+          echo(s"test in '$dir' did not contain class: ${ex.getMessage}\n${renderStackTrace(ex.getCause)}")
+          failTestSource(testSource)
+
+        case ex: InvocationTargetException =>
+          echo(s"An exception ocurred when running main: ${ex.getCause}\n${renderStackTrace(ex.getCause)}")
+          failTestSource(testSource)
+      }
+      printStream.toString("utf-8").lines.toArray
+    }
+
+    private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = {
+      val outputLines = runMain(dir, testSource)
+      val checkLines = Source.fromFile(checkFile).getLines.toArray
+
+      def linesMatch =
+        outputLines
+        .zip(checkLines)
+        .forall { case (x, y) => x == y }
+
+      if (outputLines.length != checkLines.length || !linesMatch) {
+        // Print diff to files and summary:
+        val diff = outputLines.zip(checkLines).map { case (act, exp) =>
+          DiffUtil.mkColoredCodeDiff(exp, act, true)
+        }.mkString("\n")
+        val msg = s"\nOutput from run test '$checkFile' did not match expected, output:\n$diff\n"
+        echo(msg)
+        addFailureInstruction(msg)
+
+        // Print build instructions to file and summary:
+        val buildInstr = testSource.buildInstructions(0, warnings)
+        addFailureInstruction(buildInstr)
+
+        // Fail target:
+        failTestSource(testSource)
+      }
+    }
+
+    protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable {
+      def run(): Unit = tryCompile(testSource) {
+        val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match {
+          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+            val checkFile = files.flatMap { file =>
+              if (file.isDirectory) Nil
+              else {
+                val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check"
+                val checkFile = new JFile(fname)
+                if (checkFile.exists) List(checkFile)
+                else Nil
+              }
+            }.headOption
+            val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+
+            if (reporter.errorCount > 0)
+              echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+
+            registerCompilation(reporter.errorCount)
+            (reporter.errorCount, reporter.warningCount, checkFile.isDefined, () => verifyOutput(checkFile.get, outDir, testSource, reporter.warningCount))
+          }
+
+          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+            val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check")
+            val (errorCount, warningCount) =
+              testSource
+                .compilationGroups
+                .map(compile(_, flags, false, outDir))
+                .foldLeft((0,0)) { case ((errors, warnings), reporter) =>
+                  if (reporter.errorCount > 0)
+                    echoBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
+
+                  (errors + reporter.errorCount, warnings + reporter.warningCount)
+                }
+
+            if (errorCount > 0) fail()
+
+            registerCompilation(errorCount)
+            (errorCount, warningCount, checkFile.exists, () => verifyOutput(checkFile, outDir, testSource, warningCount))
+          }
+        }
+
+        if (errorCount == 0 && hasCheckFile) verifier()
+        else if (errorCount == 0) runMain(testSource.outDir, testSource)
+        else if (errorCount > 0) {
+          echo(s"\nCompilation failed for: '$testSource'")
+          val buildInstr = testSource.buildInstructions(errorCount, warningCount)
+          addFailureInstruction(buildInstr)
+          failTestSource(testSource)
+        }
+      }
+    }
+  }
+
+  private final class NegTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)
+  extends Test(testSources, times, threadLimit, suppressAllOutput) {
+    protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable {
+      def run(): Unit = tryCompile(testSource) {
+        // In neg-tests we allow two types of error annotations,
+        // "nopos-error" which doesn't care about position and "error" which
+        // has to be annotated on the correct line number.
+        //
+        // We collect these in a map `"file:row" -> numberOfErrors`, for
+        // nopos errors we save them in `"file" -> numberOfNoPosErrors`
+        def getErrorMapAndExpectedCount(files: Array[JFile]): (HashMap[String, Integer], Int) = {
+          val errorMap = new HashMap[String, Integer]()
+          var expectedErrors = 0
+          files.filter(_.getName.endsWith(".scala")).foreach { file =>
+            Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) =>
+              val errors = line.sliding("// error".length).count(_.mkString == "// error")
+              if (errors > 0)
+                errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors)
+
+              val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error")
+              if (noposErrors > 0) {
+                val nopos = errorMap.get("nopos")
+                val existing: Integer = if (nopos eq null) 0 else nopos
+                errorMap.put("nopos", noposErrors + existing)
+              }
+
+              expectedErrors += noposErrors + errors
+            }
+          }
+
+          (errorMap, expectedErrors)
+        }
+
+        def getMissingExpectedErrors(errorMap: HashMap[String, Integer], reporterErrors: Iterator[MessageContainer]) = !reporterErrors.forall { error =>
+          val key = if (error.pos.exists) {
+            val fileName = error.pos.source.file.toString
+            s"$fileName:${error.pos.line}"
+
+          } else "nopos"
+
+          val errors = errorMap.get(key)
+
+          if (errors ne null) {
+            if (errors == 1) errorMap.remove(key)
+            else errorMap.put(key, errors - 1)
+            true
+          }
+          else {
+            echo {
+              s"Error reported in ${error.pos.source}, but no annotation found"
+            }
+            false
+          }
+        }
+
+        val (expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
+          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+            val sourceFiles = testSource.sourceFiles
+            val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles)
+            val reporter = compile(sourceFiles, flags, true, outDir)
+            val actualErrors = reporter.errorCount
+
+            (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap)
+          }
+
+          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+            val compilationGroups = testSource.compilationGroups
+            val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(compilationGroups.toArray.flatten)
+            val reporters = compilationGroups.map(compile(_, flags, true, outDir))
+            val actualErrors = reporters.foldLeft(0)(_ + _.errorCount)
+            val errors = reporters.iterator.flatMap(_.errors)
+            (expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap)
+          }
+        }
+
+        if (expectedErrors != actualErrors) {
+          echo {
+            s"\nWrong number of errors encountered when compiling $testSource, expected: $expectedErrors, actual: $actualErrors\n"
+          }
+          failTestSource(testSource)
+        }
+        else if (hasMissingAnnotations()) {
+          echo {
+            s"\nErrors found on incorrect row numbers when compiling $testSource"
+          }
+          failTestSource(testSource)
+        }
+        else if (!errorMap.isEmpty) {
+          echo {
+            s"\nExpected error(s) have {<error position>=<unreported error>}: $errorMap"
+          }
+          failTestSource(testSource)
+        }
+
+        registerCompilation(actualErrors)
+      }
+    }
+  }
+
+  /** The `CompilationTest` is the main interface to `ParallelTesting`, it
+   *  can be instantiated via one of the following methods:
+   *
+   *  - `compileFile`
+   *  - `compileDir`
+   *  - `compileList`
+   *  - `compileFilesInDir`
+   *  - `compileShallowFilesInDir`
+   *
+   *  Each compilation test can then be turned into either a "pos", "neg" or
+   *  "run" test:
+   *
+   *  ```
+   *  compileFile("../tests/pos/i1103.scala", opts).pos()
+   *  ```
+   *
+   *  These tests can be customized before calling one of the execution
+   *  methods, for instance:
+   *
+   *  ```
+   *  compileFile("../tests/pos/i1103.scala", opts).times(2).verbose.pos()
+   *  ```
+   *
+   *  Which would compile `i1103.scala` twice with the verbose flag as a "pos"
+   *  test.
+   *
+   *  pos tests
+   *  =========
+   *  Pos tests verify that the compiler is able to compile the given
+   *  `TestSource`s and that they generate no errors or exceptions during
+   *  compilation
+   *
+   *  neg tests
+   *  =========
+   *  Neg tests are expected to generate a certain amount of errors - but not
+   *  crash the compiler. In each `.scala` file, you specifiy the line on which
+   *  the error will be generated, e.g:
+   *
+   *  ```
+   *  val x: String = 1 // error
+   *  ```
+   *
+   *  if a line generates multiple errors, you need to annotate it multiple
+   *  times. For a line that generates two errors:
+   *
+   *  ```
+   *  val y: String = { val y1: String = 1; 2 } // error // error
+   *  ```
+   *
+   *  Certain errors have no position, if you need to check these annotate the
+   *  file anywhere with `// nopos-error`
+   *
+   *  run tests
+   *  =========
+   *  Run tests are a superset of pos tests, they both verify compilation and
+   *  that the compiler does not crash. In addition, run tests verify that the
+   *  tests are able to run as expected.
+   *
+   *  Run tests need to have the following form:
+   *
+   *  ```
+   *  object Test {
+   *    def main(args: Array[String]): Unit = ()
+   *  }
+   *  ```
+   *
+   *  This is because the runner instantiates the `Test` class and calls the
+   *  main method.
+   *
+   *  Other definitions are allowed in the same file, but the file needs to at
+   *  least have the `Test` object with a `main` method.
+   *
+   *  To verify output you may use `.check` files. These files should share the
+   *  name of the file or directory that they are testing. For instance:
+   *
+   *  ```none
+   *  .
+   *  └── tests
+   *      ├── i1513.scala
+   *      └── i1513.check
+   *  ```
+   *
+   *  If you are testing a directory under separate compilation, you would
+   *  have:
+   *
+   *  ```none
+   *  .
+   *  └── tests
+   *      ├── myTestDir
+   *      │   ├── T_1.scala
+   *      │   ├── T_2.scala
+   *      │   └── T_3.scala
+   *      └── myTestDir.check
+   *  ```
+   *
+   *  In the above example, `i1513.scala` and one of the files `T_X.scala`
+   *  would contain a `Test` object with a main method.
+   *
+   *  Composing tests
+   *  ===============
+   *  Since this is a parallel test suite, it is essential to be able to
+   *  compose tests to take advantage of the concurrency. This is done using
+   *  the `+` function. This function will make sure that tests being combined
+   *  are compatible according to the `require`s in `+`.
+   */
+  final class CompilationTest private (
+    private[ParallelTesting] val targets: List[TestSource],
+    private[ParallelTesting] val times: Int,
+    private[ParallelTesting] val shouldDelete: Boolean,
+    private[ParallelTesting] val threadLimit: Option[Int],
+    private[ParallelTesting] val shouldFail: Boolean
+  ) {
+    import org.junit.Assert.fail
+
+    private[ParallelTesting] def this(target: TestSource) =
+      this(List(target), 1, true, None, false)
+
+    private[ParallelTesting] def this(targets: List[TestSource]) =
+      this(targets, 1, true, None, false)
+
+    /** Compose test targets from `this` with `other`
+     *
+     *  It does this, only if the two tests are compatible. Otherwise it throws
+     *  an `IllegalArgumentException`.
+     *
+     *  Grouping tests together like this allows us to take advantage of the
+     *  concurrency offered by this test suite as each call to an executing
+     *  method (`pos()` / `checkExpectedErrors()`/ `run()`) will spin up a thread pool with the
+     *  maximum allowed level of concurrency. Doing this for only a few targets
+     *  does not yield any real benefit over sequential compilation.
+     *
+     *  As such, each `CompilationTest` should contain as many targets as
+     *  possible.
+     */
+    def +(other: CompilationTest) = {
+      require(other.times == times, "can't combine tests that are meant to be benchmark compiled")
+      require(other.shouldDelete == shouldDelete, "can't combine tests that differ on deleting output")
+      require(other.shouldFail == shouldFail, "can't combine tests that have different expectations on outcome")
+      new CompilationTest(targets ++ other.targets, times, shouldDelete, threadLimit, shouldFail)
+    }
+
+    /** Creates a "pos" test run, which makes sure that all tests pass
+     *  compilation without generating errors and that they do not crash the
+     *  compiler
+     */
+    def checkCompile(): this.type = {
+      val test = new PosTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+      if (!shouldFail && test.didFail) {
+        fail(s"Expected no errors when compiling, but found: ${test.errorCount}")
+      }
+      else if (shouldFail && !test.didFail) {
+        fail("Pos test should have failed, but didn't")
+      }
+
+      cleanup()
+    }
+
+    /** Creates a "neg" test run, which makes sure that each test generates the
+     *  correct amount of errors at the correct positions. It also makes sure
+     *  that none of these tests crash the compiler
+     */
+    def checkExpectedErrors(): this.type = {
+      val test = new NegTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+      if (!shouldFail && test.didFail) {
+        fail("Neg test shouldn't have failed, but did")
+      }
+      else if (shouldFail && !test.didFail) {
+        fail("Neg test should have failed, but did not")
+      }
+
+      cleanup()
+    }
+
+    /** Creates a "run" test run, which is a superset of "pos". In addition to
+     *  making sure that all tests pass compilation and that they do not crash
+     *  the compiler; it also makes sure that all tests can run with the
+     *  expected output
+     */
+    def checkRuns(): this.type = {
+      val test = new RunTest(targets, times, threadLimit, shouldFail).executeTestSuite()
+
+      if (!shouldFail && test.didFail) {
+        fail("Run test failed, but should not")
+      }
+      else if (shouldFail && !test.didFail) {
+        fail("Run test should have failed, but did not")
+      }
+
+      cleanup()
+    }
+
+    /** Deletes output directories and files */
+    private def cleanup(): this.type = {
+      if (shouldDelete) delete()
+      this
+    }
+
+    /** Copies `file` to `dir` - taking into account if `file` is a directory,
+     *  and if so copying recursively
+     */
+    private def copyToDir(dir: JFile, file: JFile): JFile = {
+      val target = Paths.get(dir.getAbsolutePath, file.getName)
+      Files.copy(file.toPath, target, REPLACE_EXISTING)
+      if (file.isDirectory) file.listFiles.map(copyToDir(target.toFile, _))
+      target.toFile
+    }
+
+    /** Builds a new `CompilationTest` where we have copied the target files to
+     *  the out directory. This is needed for tests that modify the original
+     *  source, such as `-rewrite` tests
+     */
+    def copyToTarget(): CompilationTest = new CompilationTest (
+      targets.map {
+        case target @ JointCompilationSource(_, files, _, outDir) =>
+          target.copy(files = files.map(copyToDir(outDir,_)))
+        case target @ SeparateCompilationSource(_, dir, _, outDir) =>
+          target.copy(dir = copyToDir(outDir, dir))
+      },
+      times, shouldDelete, threadLimit, shouldFail
+    )
+
+    /** Builds a `CompilationTest` which performs the compilation `i` times on
+     *  each target
+     */
+    def times(i: Int): CompilationTest =
+      new CompilationTest(targets, i, shouldDelete, threadLimit, shouldFail)
+
+    /** Builds a `Compilationtest` which passes the verbose flag and logs the
+     *  classpath
+     */
+    def verbose: CompilationTest = new CompilationTest(
+      targets.map(t => t.withFlags("-verbose", "-Ylog-classpath")),
+      times, shouldDelete, threadLimit, shouldFail
+    )
+
+    /** Builds a `CompilationTest` which keeps the generated output files
+     *
+     *  This is needed for tests like `tastyBootstrap` which relies on first
+     *  compiling a certain part of the project and then compiling a second
+     *  part which depends on the first
+     */
+    def keepOutput: CompilationTest =
+      new CompilationTest(targets, times, false, threadLimit, shouldFail)
+
+    /** Builds a `CompilationTest` with a limited level of concurrency with
+     *  maximum `i` threads
+     */
+    def limitThreads(i: Int): CompilationTest =
+      new CompilationTest(targets, times, shouldDelete, Some(i), shouldFail)
+
+    /** Builds a `CompilationTest` where the executed test is expected to fail
+     *
+     *  This behaviour is mainly needed for the tests that test the test suite.
+     */
+    def expectFailure: CompilationTest =
+      new CompilationTest(targets, times, shouldDelete, threadLimit, true)
+
+    /** Delete all output files generated by this `CompilationTest` */
+    def delete(): Unit = targets.foreach(t => delete(t.outDir))
+
+    private def delete(file: JFile): Unit = {
+      if (file.isDirectory) file.listFiles.foreach(delete)
+      try Files.delete(file.toPath)
+      catch {
+        case _: NoSuchFileException => // already deleted, everything's fine
+      }
+    }
+  }
+
+  /** Create out directory for directory `d` */
+  private def createOutputDirsForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = {
+    val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}")
+    targetDir.mkdirs()
+    targetDir
+  }
+
+  /** Create out directory for `file` */
+  private def createOutputDirsForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = {
+    val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.'))
+    val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir")
+    targetDir.mkdirs()
+    targetDir
+  }
+
+  /** Make sure that directory string is as expected */
+  private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = {
+    require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`")
+    require(outDir.last == '/', "please specify an `outDir` with a trailing slash")
+  }
+
+  /** Separates directories from files and returns them as `(dirs, files)` */
+  private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) =
+    sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) =>
+      if (f.isDirectory) (f :: dirs, files)
+      else if (f.getName.endsWith(".check")) (dirs, files)
+      else if (f.getName.endsWith(".flags")) (dirs, files)
+      else (dirs, f :: files)
+    }
+
+  /** Gets the name of the calling method via reflection.
+   *
+   *  It does this in a way that needs to work both with the bootstrapped dotty
+   *  and the non-bootstrapped version. Since the two compilers generate
+   *  different bridges, we first need to filter out methods with the same name
+   *  (bridges) - and then find the `@Test` method in our extending class
+   */
+  private def getCallingMethod(): String = {
+    val seen = mutable.Set.empty[String]
+    Thread.currentThread.getStackTrace
+      .filter { elem =>
+        if (seen.contains(elem.getMethodName)) false
+        else { seen += elem.getMethodName; true }
+      }
+      .find { elem =>
+        val callingClass = Class.forName(elem.getClassName)
+        classOf[ParallelTesting].isAssignableFrom(callingClass) &&
+        elem.getFileName != "ParallelTesting.scala"
+      }
+      .map(_.getMethodName)
+      .getOrElse {
+        throw new IllegalStateException("Unable to reflectively find calling method")
+      }
+  }
+
+  /** Compiles a single file from the string path `f` using the supplied flags */
+  def compileFile(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    val callingMethod = getCallingMethod
+    val sourceFile = new JFile(f)
+    val parent = sourceFile.getParentFile
+    val outDir =
+      outDirectory + callingMethod + "/" +
+      sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/"
+
+    require(
+      sourceFile.exists && !sourceFile.isDirectory &&
+      (parent ne null) && parent.exists && parent.isDirectory,
+      s"Source file: $f, didn't exist"
+    )
+
+    val target = JointCompilationSource(
+      callingMethod,
+      Array(sourceFile),
+      flags,
+      createOutputDirsForFile(sourceFile, parent, outDir)
+    )
+    new CompilationTest(target)
+  }
+
+  /** Compiles a directory `f` using the supplied `flags`. This method does
+   *  deep compilation, that is - it compiles all files and subdirectories
+   *  contained within the directory `f`.
+   */
+  def compileDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    val callingMethod = getCallingMethod
+    val outDir = outDirectory + callingMethod + "/"
+    val sourceDir = new JFile(f)
+    checkRequirements(f, sourceDir, outDir)
+
+    def flatten(f: JFile): Array[JFile] =
+      if (f.isDirectory) f.listFiles.flatMap(flatten)
+      else Array(f)
+
+    // Directories in which to compile all containing files with `flags`:
+    val targetDir = new JFile(outDir + "/" + sourceDir.getName + "/")
+    targetDir.mkdirs()
+
+    val target = JointCompilationSource(callingMethod, flatten(sourceDir), flags, targetDir)
+    new CompilationTest(target)
+  }
+
+  /** Compiles all `files` together as a single compilation run. It is given a
+   *  `testName` since files can be in separate directories and or be otherwise
+   *  dissociated
+   */
+  def compileList(testName: String, files: List[String], flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    val callingMethod = getCallingMethod
+    val outDir = outDirectory + callingMethod + "/" + testName + "/"
+
+    // Directories in which to compile all containing files with `flags`:
+    val targetDir = new JFile(outDir)
+    targetDir.mkdirs()
+    assert(targetDir.exists, s"couldn't create target directory: $targetDir")
+
+    val target = JointCompilationSource(callingMethod, files.map(new JFile(_)).toArray, flags, targetDir)
+
+    // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+    new CompilationTest(target)
+  }
+
+  /** This function compiles the files and folders contained within directory
+   *  `f` in a specific way.
+   *
+   *  - Each file is compiled separately as a single compilation run
+   *  - Each directory is compiled as a `SeparateCompilationTaret`, in this
+   *    target all files are grouped according to the file suffix `_X` where `X`
+   *    is a number. These groups are then ordered in ascending order based on
+   *    the value of `X` and each group is compiled one after the other.
+   *
+   *  For this function to work as expected, we use the same convention for
+   *  directory layout as the old partest. That is:
+   *
+   *  - Single files can have an associated check-file with the same name (but
+   *    with file extension `.check`)
+   *  - Directories can have an associated check-file, where the check file has
+   *    the same name as the directory (with the file extension `.check`)
+   */
+  def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    val callingMethod = getCallingMethod
+    val outDir = outDirectory + callingMethod + "/"
+    val sourceDir = new JFile(f)
+    checkRequirements(f, sourceDir, outDir)
+
+    val (dirs, files) = compilationTargets(sourceDir)
+
+    val targets =
+      files.map(f => JointCompilationSource(callingMethod, Array(f), flags, createOutputDirsForFile(f, sourceDir, outDir))) ++
+      dirs.map(dir => SeparateCompilationSource(callingMethod, dir, flags, createOutputDirsForDir(dir, sourceDir, outDir)))
+
+    // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+    new CompilationTest(targets)
+  }
+
+  /** This function behaves similar to `compileFilesInDir` but it ignores
+   *  sub-directories and as such, does **not** perform separate compilation
+   *  tests.
+   */
+  def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    val callingMethod = getCallingMethod
+    val outDir = outDirectory + callingMethod + "/"
+    val sourceDir = new JFile(f)
+    checkRequirements(f, sourceDir, outDir)
+
+    val (_, files) = compilationTargets(sourceDir)
+
+    val targets = files.map { file =>
+      JointCompilationSource(callingMethod, Array(file), flags, createOutputDirsForFile(file, sourceDir, outDir))
+    }
+
+    // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+    new CompilationTest(targets)
+  }
+}
+
+object ParallelTesting {
+  def isSourceFile(f: JFile): Boolean = {
+    val name = f.getName
+    name.endsWith(".scala") || name.endsWith(".java")
+  }
+}
diff --git a/compiler/test/dotty/tools/dotc/repl/TestREPL.scala b/compiler/test/dotty/tools/dotc/repl/TestREPL.scala
index a38abcbab..131a88ab1 100644
--- a/compiler/test/dotty/tools/dotc/repl/TestREPL.scala
+++ b/compiler/test/dotty/tools/dotc/repl/TestREPL.scala
@@ -5,6 +5,8 @@ package repl
 import core.Contexts.Context
 import collection.mutable
 import java.io.StringWriter
+import dotty.tools.io.{ PlainFile, Directory }
+import org.junit.Test
 
 /** A subclass of REPL used for testing.
  *  It takes a transcript of a REPL session in `script`. The transcript
@@ -62,3 +64,24 @@ class TestREPL(script: String) extends REPL {
     }
   }
 }
+
+class REPLTests {
+  def replFile(prefix: String, fileName: String): Unit = {
+    val path = s"$prefix$fileName"
+    val f = new PlainFile(path)
+    val repl = new TestREPL(new String(f.toCharArray))
+    repl.process(Array[String]())
+    repl.check()
+  }
+
+  def replFiles(path: String): Unit = {
+    val dir = Directory(path)
+    val fileNames = dir.files.toArray.map(_.jfile.getName).filter(_ endsWith ".check")
+    for (name <- fileNames) {
+      println(s"testing $path$name")
+      replFile(path, name)
+    }
+  }
+
+  @Test def replAll = replFiles("../tests/repl/")
+}
diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala
index 169908c4f..521cf9576 100644
--- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala
+++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala
@@ -2,35 +2,57 @@ package dotty.tools
 package dotc
 package reporting
 
+import java.io.{ PrintWriter, File => JFile, FileOutputStream }
+import java.text.SimpleDateFormat
+import java.util.Date
+
 import scala.collection.mutable
+
 import util.SourcePosition
 import core.Contexts._
 import Reporter._
-import java.io.PrintWriter
 import diagnostic.{ Message, MessageContainer, NoExplanation }
 import diagnostic.messages._
+import interfaces.Diagnostic.{ ERROR, WARNING, INFO }
 
-class TestReporter(writer: PrintWriter) extends Reporter
-with UniqueMessagePositions with HideNonSensicalMessages {
-
+class TestReporter protected (outWriter: PrintWriter, filePrintln: String => Unit, logLevel: Int)
+extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with MessageRendering {
   import MessageContainer._
 
-  /** maximal number of error messages to be printed */
-  protected def ErrorLimit = 100
+  protected final val _errorBuf = mutable.ArrayBuffer.empty[MessageContainer]
+  final def errors: Iterator[MessageContainer] = _errorBuf.iterator
+
+  protected final val _messageBuf = mutable.ArrayBuffer.empty[String]
 
-  def printPos(pos: SourcePosition): Unit =
+  final def flushToFile(): Unit =
+    _messageBuf.iterator.foreach(filePrintln)
+
+  final def flushToStdErr(): Unit =
+    _messageBuf.iterator.foreach(System.err.println)
+
+  final def inlineInfo(pos: SourcePosition): String =
     if (pos.exists) {
-      if (pos.outer.exists) {
-        writer.println(s"\ninlined at ${pos.outer}:\n")
-        printPos(pos.outer)
-      }
+      if (pos.outer.exists)
+        s"\ninlined at ${pos.outer}:\n" + inlineInfo(pos.outer)
+      else ""
     }
+    else ""
+
+  def echo(msg: String) =
+    _messageBuf.append(msg)
 
   /** Prints the message with the given position indication. */
-  def printMessageAndPos(msg: String, pos: SourcePosition)(implicit ctx: Context): Unit = {
-    val posStr = s"${pos.line + 1}: "
-    writer.println(posStr + msg)
-    printPos(pos)
+  def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = {
+    val msg = messageAndPos(m.contained, m.pos, diagnosticLevel(m))
+    val extraInfo = inlineInfo(m.pos)
+
+    if (m.level >= logLevel) {
+      outWriter.println(msg)
+      if (extraInfo.nonEmpty) outWriter.println(extraInfo)
+    }
+
+    _messageBuf.append(msg)
+    if (extraInfo.nonEmpty) _messageBuf.append(extraInfo)
   }
 
   override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = {
@@ -41,11 +63,61 @@ with UniqueMessagePositions with HideNonSensicalMessages {
     }
 
     m match {
-      case m: Error =>
-        printMessageAndPos(m.contained.kind + extra, m.pos)
+      case m: Error => {
+        _errorBuf.append(m)
+        printMessageAndPos(m, extra)
+      }
       case w: Warning =>
-        printMessageAndPos(w.contained.kind + extra, w.pos)
+        printMessageAndPos(w, extra)
       case _ =>
     }
   }
 }
+
+object TestReporter {
+  private[this] val logWriter = {
+    val df = new SimpleDateFormat("yyyy-MM-dd-HH:mm")
+    val timestamp = df.format(new Date)
+    new PrintWriter(new FileOutputStream(new JFile(s"../tests-$timestamp.log"), true))
+  }
+
+  def writeToLog(str: String) = {
+    logWriter.println(str)
+    logWriter.flush()
+  }
+
+  def parallelReporter(lock: AnyRef, logLevel: Int): TestReporter = new TestReporter(
+    new PrintWriter(Console.err, true),
+    str => lock.synchronized {
+      logWriter.println(str)
+      logWriter.flush()
+    },
+    logLevel
+  )
+
+  def reporter(logLevel: Int): TestReporter = new TestReporter(
+    new PrintWriter(Console.err, true),
+    writeToLog,
+    logLevel
+  )
+
+  def simplifiedReporter(writer: PrintWriter): TestReporter = new TestReporter(
+    writer,
+    writeToLog,
+    WARNING
+  ) {
+    /** Prints the message with the given position indication in a simplified manner */
+    override def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = {
+      val msg = s"${m.pos.line + 1}: " + m.contained.kind + extra
+      val extraInfo = inlineInfo(m.pos)
+
+      writer.println(msg)
+      _messageBuf.append(msg)
+
+      if (extraInfo.nonEmpty) {
+        writer.println(extraInfo)
+        _messageBuf.append(extraInfo)
+      }
+    }
+  }
+}
diff --git a/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala b/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala
index d9a5d8a38..eff86e6e7 100644
--- a/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala
+++ b/compiler/test/dotty/tools/dotc/transform/PatmatExhaustivityTest.scala
@@ -1,21 +1,23 @@
-package test.transform
+package dotty
+package tools
+package dotc
+package transform
 
 import java.io._
 
 import scala.io.Source._
 import scala.reflect.io.Directory
 import org.junit.Test
-import dotty.tools.dotc.Main
-import dotty.tools.dotc.reporting.TestReporter
+import reporting.TestReporter
 
 class PatmatExhaustivityTest {
   val testsDir = "../tests/patmat"
   // stop-after: patmatexhaust-huge.scala crash compiler
-  val options = List("-color:never", "-Ystop-after:splitter", "-Ycheck-all-patmat") ++ (new dotc.tests).classPath
+  val options = List("-color:never", "-Ystop-after:splitter", "-Ycheck-all-patmat") ++ CompilationTests.classPath
 
   private def compileFile(file: File) = {
     val stringBuffer = new StringWriter()
-    val reporter = new TestReporter(new PrintWriter(stringBuffer))
+    val reporter = TestReporter.simplifiedReporter(new PrintWriter(stringBuffer))
 
     try {
       Main.process((file.getPath::options).toArray, reporter, null)
@@ -38,7 +40,7 @@ class PatmatExhaustivityTest {
   /** A single test with multiple files grouped in a folder */
   private def compileDir(file: File) = {
     val stringBuffer = new StringWriter()
-    val reporter = new TestReporter(new PrintWriter(stringBuffer))
+    val reporter = TestReporter.simplifiedReporter(new PrintWriter(stringBuffer))
 
     val files = Directory(file.getPath).list.toList
       .filter(f => f.extension == "scala" || f.extension == "java" )
diff --git a/doc-tool/test/DottyDocTest.scala b/doc-tool/test/DottyDocTest.scala
index 4202dca73..9e7b70c8f 100644
--- a/doc-tool/test/DottyDocTest.scala
+++ b/doc-tool/test/DottyDocTest.scala
@@ -9,8 +9,13 @@ import dotc.typer.FrontEnd
 import dottydoc.core.{ DocASTPhase, ContextDottydoc }
 import model.Package
 import dotty.tools.dottydoc.util.syntax._
+import dotc.reporting.{ StoreReporter, MessageRendering }
+import dotc.interfaces.Diagnostic.ERROR
+import org.junit.Assert.fail
 
-trait DottyDocTest {
+import java.io.{ BufferedWriter, OutputStreamWriter }
+
+trait DottyDocTest extends MessageRendering {
   dotty.tools.dotc.parsing.Scanners // initialize keywords
 
   implicit val ctx: FreshContext = {
@@ -26,6 +31,7 @@ trait DottyDocTest {
       ctx.settings.classpath,
       dotty.Jars.dottyLib
     )
+    ctx.setReporter(new StoreReporter(ctx.reporter))
     base.initialize()(ctx)
     ctx
   }
@@ -36,17 +42,47 @@ trait DottyDocTest {
         def phaseName = "assertionPhase"
         override def run(implicit ctx: Context): Unit =
           assertion(ctx.docbase.packages)
+          if (ctx.reporter.hasErrors) {
+            System.err.println("reporter had errors:")
+            ctx.reporter.removeBufferedMessages.foreach { msg =>
+              System.err.println {
+                messageAndPos(msg.contained, msg.pos, diagnosticLevel(msg))
+              }
+            }
+          }
       }) :: Nil
 
     override def phases =
       super.phases ++ assertionPhase
   }
 
+  private def callingMethod: String =
+    Thread.currentThread.getStackTrace.find {
+      _.getMethodName match {
+        case "checkSource" | "callingMethod" | "getStackTrace" | "currentThread" =>
+          false
+        case _ =>
+          true
+      }
+    }
+    .map(_.getMethodName)
+    .getOrElse {
+      throw new IllegalStateException("couldn't get calling method via reflection")
+    }
+
+  private def sourceFileFromString(name: String, contents: String): SourceFile = {
+    val virtualFile = new scala.reflect.io.VirtualFile(name)
+    val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8"))
+    writer.write(contents)
+    writer.close()
+    new SourceFile(virtualFile, scala.io.Codec.UTF8)
+  }
+
   def checkSource(source: String)(assertion: Map[String, Package] => Unit): Unit = {
     val c = compilerWithChecker(assertion)
     c.rootContext(ctx)
     val run = c.newRun
-    run.compile(source)
+    run.compileSources(sourceFileFromString(callingMethod, source) :: Nil)
   }
 
   def checkFiles(sources: List[String])(assertion: Map[String, Package] => Unit): Unit = {
diff --git a/doc-tool/test/WhitelistedStdLib.scala b/doc-tool/test/WhitelistedStdLib.scala
index 9092d1ded..d59457605 100644
--- a/doc-tool/test/WhitelistedStdLib.scala
+++ b/doc-tool/test/WhitelistedStdLib.scala
@@ -6,18 +6,16 @@ import org.junit.Assert._
 
 class TestWhitelistedCollections extends DottyDocTest {
 
-  @Test def arrayHasDocumentation =
+  @Test def arrayAndImmutableHasDocumentation =
     checkFiles(TestWhitelistedCollections.files) { packages =>
       val array =
         packages("scala")
         .children.find(_.path.mkString(".") == "scala.Array")
         .get
 
-      assert(array.comment.get.body.length > 0)
-    }
+      assert(array.comment.get.body.length > 0,
+        "scala.Array didn't have any documentation")
 
-  @Test def traitImmutableHasDocumentation =
-    checkFiles(TestWhitelistedCollections.files) { packages =>
       val imm =
         packages("scala")
         .children.find(_.path.mkString(".") == "scala.Immutable")
diff --git a/out/.keep b/out/.keep
deleted file mode 100644
index e69de29bb..000000000
--- a/out/.keep
+++ /dev/null
diff --git a/project/Build.scala b/project/Build.scala
index ce560d8e4..bb02416cc 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -47,6 +47,9 @@ object Build {
   // Spawns a repl with the correct classpath
   lazy val repl = inputKey[Unit]("run the REPL with correct classpath")
 
+  // Run tests with filter
+  lazy val filterTest = inputKey[Unit]("runs integration test with the supplied filter")
+
   // Used to compile files similar to ./bin/dotc script
   lazy val dotc =
     inputKey[Unit]("run the compiler using the correct classpath, or the user supplied classpath")
@@ -276,23 +279,57 @@ object Build {
       com.typesafe.sbteclipse.plugin.EclipsePlugin.EclipseKeys.withSource := true,
 
       // get libraries onboard
-      partestDeps := Seq(scalaCompiler,
-                         "org.scala-lang" % "scala-reflect" % scalacVersion,
-                         "org.scala-lang" % "scala-library" % scalacVersion % "test"),
-      libraryDependencies ++= partestDeps.value,
-      libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1",
-                                  "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
-                                  "com.novocode" % "junit-interface" % "0.11" % "test"),
-
       resolvers += Resolver.typesafeIvyRepo("releases"), // For org.scala-sbt:interface
-      libraryDependencies += "org.scala-sbt" % "interface" % sbtVersion.value,
+      libraryDependencies ++= Seq(scalaCompiler,
+                                  "org.scala-sbt" % "interface" % sbtVersion.value,
+                                  "org.scala-lang.modules" %% "scala-xml" % "1.0.1",
+                                  "com.novocode" % "junit-interface" % "0.11" % "test",
+                                  "org.scala-lang" % "scala-reflect" % scalacVersion,
+                                  "org.scala-lang" % "scala-library" % scalacVersion % "test"),
+
+      // start partest specific settings:
+      libraryDependencies += "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
+      testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }),
+      // this option is needed so that partest doesn't run
+      partestDeps := Seq(
+        scalaCompiler,
+        "org.scala-lang" % "scala-reflect" % scalacVersion,
+        "org.scala-lang" % "scala-library" % scalacVersion % "test"
+      ),
+      lockPartestFile := {
+        // When this file is present, running `test` generates the files for
+        // partest. Otherwise it just executes the tests directly.
+        val lockDir = partestLockFile.getParentFile
+        lockDir.mkdirs
+        // Cannot have concurrent partests as they write to the same directory.
+        if (lockDir.list.size > 0)
+          throw new RuntimeException("ERROR: sbt partest: another partest is already running, pid in lock file: " + lockDir.list.toList.mkString(" "))
+        partestLockFile.createNewFile
+        partestLockFile.deleteOnExit
+      },
+      runPartestRunner := Def.inputTaskDyn {
+        // Magic! This is both an input task and a dynamic task. Apparently
+        // command line arguments get passed to the last task in an aliased
+        // sequence (see partest alias below), so this works.
+        val args = Def.spaceDelimited("<arg>").parsed
+        val jars = List(
+          (packageBin in Compile).value.getAbsolutePath,
+          packageAll.value("dotty-library"),
+          packageAll.value("dotty-interfaces")
+        ) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome)
+        val dottyJars  =
+          s"""-dottyJars ${jars.length + 2} dotty.jar dotty-lib.jar ${jars.mkString(" ")}"""
+        // Provide the jars required on the classpath of run tests
+        runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars + " " + args.mkString(" "))
+      }.evaluated,
+      // end partest specific settings
 
       // enable improved incremental compilation algorithm
       incOptions := incOptions.value.withNameHashing(true),
 
       // For convenience, change the baseDirectory when running the compiler
       baseDirectory in (Compile, run) := baseDirectory.value / "..",
-      // .. but not when running partest
+      // .. but not when running test
       baseDirectory in (Test, run) := baseDirectory.value,
 
       repl := Def.inputTaskDyn {
@@ -303,6 +340,14 @@ object Build {
         )
       }.evaluated,
 
+      filterTest := Def.inputTaskDyn {
+        val args: Seq[String] = spaceDelimited("<arg>").parsed
+        testOptions := Seq()
+        (testOnly in Test).toTask(
+          " dotty.tools.dotc.CompilationTests -- -Ddotty.partest.filter=" + args.head
+        )
+      }.evaluated,
+
       // Override run to be able to run compiled classfiles
       dotr := {
         val args: Seq[String] = spaceDelimited("<arg>").parsed
@@ -343,34 +388,6 @@ object Build {
         TestFrameworks.JUnit, "-a", "-v",
         "--run-listener=dotty.tools.ContextEscapeDetector"
       ),
-      testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }),
-
-      lockPartestFile := {
-        // When this file is present, running `test` generates the files for
-        // partest. Otherwise it just executes the tests directly.
-        val lockDir = partestLockFile.getParentFile
-        lockDir.mkdirs
-        // Cannot have concurrent partests as they write to the same directory.
-        if (lockDir.list.size > 0)
-          throw new RuntimeException("ERROR: sbt partest: another partest is already running, pid in lock file: " + lockDir.list.toList.mkString(" "))
-        partestLockFile.createNewFile
-        partestLockFile.deleteOnExit
-      },
-      runPartestRunner := Def.inputTaskDyn {
-        // Magic! This is both an input task and a dynamic task. Apparently
-        // command line arguments get passed to the last task in an aliased
-        // sequence (see partest alias below), so this works.
-        val args = Def.spaceDelimited("<arg>").parsed
-        val jars = List(
-          (packageBin in Compile).value.getAbsolutePath,
-          packageAll.value("dotty-library"),
-          packageAll.value("dotty-interfaces")
-        ) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome)
-        val dottyJars  =
-          s"""-dottyJars ${jars.length + 2} dotty.jar dotty-lib.jar ${jars.mkString(" ")}"""
-        // Provide the jars required on the classpath of run tests
-        runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars + " " + args.mkString(" "))
-      }.evaluated,
 
       /* Add the sources of scalajs-ir.
        * To guarantee that dotty can bootstrap without depending on a version
@@ -458,6 +475,31 @@ object Build {
       }
   )
 
+  // Partest tasks
+  lazy val partestDeps =
+    SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies")
+  lazy val runPartestRunner =
+    InputKey[Unit]("runPartestRunner", "Runs partest")
+  lazy val lockPartestFile =
+    TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-<pid>.lock")
+  lazy val partestLockFile =
+    new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock")
+
+  def pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0))
+
+  def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match {
+    case Some(home) =>
+      modules.map({ module =>
+        val file = Path(home) / Path("cache") /
+          Path(module.organization) / Path(module.name) / Path("jars") /
+          Path(module.name + "-" + module.revision + ".jar")
+        if (!file.isFile) throw new RuntimeException("ERROR: sbt getJarPaths: dependency jar not found: " + file)
+        else file.jfile.getAbsolutePath
+      })
+    case None => throw new RuntimeException("ERROR: sbt getJarPaths: ivyHome not defined")
+  }
+  // end partest tasks
+
   lazy val `dotty-compiler` = project.in(file("compiler")).
     dependsOn(`dotty-interfaces`).
     dependsOn(`dotty-library`).
@@ -793,26 +835,6 @@ object DottyInjectedPlugin extends AutoPlugin {
      )
    )
 
-  // Partest tasks
-  lazy val lockPartestFile = TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-<pid>.lock")
-  lazy val partestLockFile = new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock")
-  def pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0))
-
-  lazy val runPartestRunner = InputKey[Unit]("runPartestRunner", "Runs partest")
-
-  lazy val partestDeps = SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies")
-  def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match {
-    case Some(home) =>
-      modules.map({ module =>
-        val file = Path(home) / Path("cache") /
-          Path(module.organization) / Path(module.name) / Path("jars") /
-          Path(module.name + "-" + module.revision + ".jar")
-        if (!file.isFile) throw new RuntimeException("ERROR: sbt getJarPaths: dependency jar not found: " + file)
-        else file.jfile.getAbsolutePath
-      })
-    case None => throw new RuntimeException("ERROR: sbt getJarPaths: ivyHome not defined")
-  }
-
   // Compile with dotty
   lazy val compileWithDottySettings = {
     inConfig(Compile)(inTask(compile)(Defaults.runnerTask) ++ Seq(
diff --git a/tests/neg/i941.scala b/tests/neg/i941.scala
index 2643c2546..fc29cc27d 100644
--- a/tests/neg/i941.scala
+++ b/tests/neg/i941.scala
@@ -1,7 +1,7 @@
 object Test {
 
   def bar(tl: => String) = {
-    val x = tl _ //error
+    val x = tl _ // error
     val y = x _ // error
     val s: String = x() // error
   }
diff --git a/tests/neg/instantiateAbstract.scala b/tests/neg/instantiateAbstract.scala
index 10eeac64d..a2ff38ef4 100644
--- a/tests/neg/instantiateAbstract.scala
+++ b/tests/neg/instantiateAbstract.scala
@@ -15,7 +15,7 @@ object Test {
 
   @scala.annotation.Annotation type T = String // error
   @scala.annotation.Annotation val x = 1 // error
-  @scala.annotation.Annotation def f = 1 //error
+  @scala.annotation.Annotation def f = 1 // error
 
   (1: @scala.annotation.Annotation) // error
 
diff --git a/tests/neg/varargsInMethodsT1625/allParamsAreVarArgs.scala b/tests/neg/t1625.scala
index aabb1ecea..aabb1ecea 100644
--- a/tests/neg/varargsInMethodsT1625/allParamsAreVarArgs.scala
+++ b/tests/neg/t1625.scala
diff --git a/tests/neg/t1625b.scala b/tests/neg/t1625b.scala
new file mode 100644
index 000000000..f2c544f42
--- /dev/null
+++ b/tests/neg/t1625b.scala
@@ -0,0 +1,3 @@
+object T5 {
+  case class Abc(x: String*, c: String*) // error // error: varargs parameter must come last AND found: String* required: String
+}
diff --git a/tests/neg/varargsInMethodsT1625/classConstructorVarArgs.scala b/tests/neg/t1625c.scala
index 74595cb7d..74595cb7d 100644
--- a/tests/neg/varargsInMethodsT1625/classConstructorVarArgs.scala
+++ b/tests/neg/t1625c.scala
diff --git a/tests/neg/varargsInMethodsT1625/curriedNegExample.scala b/tests/neg/t1625d.scala
index 616ea0539..616ea0539 100644
--- a/tests/neg/varargsInMethodsT1625/curriedNegExample.scala
+++ b/tests/neg/t1625d.scala
diff --git a/tests/neg/varargsInMethodsT1625/firstParamIsVarArgs.scala b/tests/neg/t1625e.scala
index eceb7e696..eceb7e696 100644
--- a/tests/neg/varargsInMethodsT1625/firstParamIsVarArgs.scala
+++ b/tests/neg/t1625e.scala
diff --git a/tests/neg/varargsInMethodsT1625/caseClassConstructorVarArgs.scala b/tests/neg/varargsInMethodsT1625/caseClassConstructorVarArgs.scala
deleted file mode 100644
index 8f8a4fcf6..000000000
--- a/tests/neg/varargsInMethodsT1625/caseClassConstructorVarArgs.scala
+++ /dev/null
@@ -1,3 +0,0 @@
-object T5 {
-  case class Abc(x: String*, c: String*) // error //error: varargs parameter must come last AND found: String* required: String
-}
\ No newline at end of file
diff --git a/tests/partest-test/negAnnotWrongLine.scala b/tests/partest-test/negAnnotWrongLine.scala
new file mode 100644
index 000000000..06ba0a2c3
--- /dev/null
+++ b/tests/partest-test/negAnnotWrongLine.scala
@@ -0,0 +1,3 @@
+object Foo { // error
+  def bar: Int = "LOL"
+}
diff --git a/tests/partest-test/negMissingAnnot.scala b/tests/partest-test/negMissingAnnot.scala
new file mode 100644
index 000000000..acffb37a3
--- /dev/null
+++ b/tests/partest-test/negMissingAnnot.scala
@@ -0,0 +1 @@
+class Foo extends Bar
diff --git a/tests/partest-test/negNoPositionAnnots.scala b/tests/partest-test/negNoPositionAnnots.scala
new file mode 100644
index 000000000..7f3ce7e34
--- /dev/null
+++ b/tests/partest-test/negNoPositionAnnots.scala
@@ -0,0 +1,5 @@
+object Foo {
+  def bar: Int = "LOL"
+
+  // nopos-error
+}
diff --git a/tests/partest-test/negTooManyAnnots.scala b/tests/partest-test/negTooManyAnnots.scala
new file mode 100644
index 000000000..5dbd2fd75
--- /dev/null
+++ b/tests/partest-test/negTooManyAnnots.scala
@@ -0,0 +1,3 @@
+object Test {
+  def foo: Int = "LOL" // error // error
+}
diff --git a/tests/partest-test/posFail1Error.scala b/tests/partest-test/posFail1Error.scala
new file mode 100644
index 000000000..c8b565498
--- /dev/null
+++ b/tests/partest-test/posFail1Error.scala
@@ -0,0 +1,3 @@
+object Test extends Bar {
+  def main(args: Array[String]): Unit = ()
+}
diff --git a/tests/partest-test/runDiffOutput1.check b/tests/partest-test/runDiffOutput1.check
new file mode 100644
index 000000000..6234030de
--- /dev/null
+++ b/tests/partest-test/runDiffOutput1.check
@@ -0,0 +1,5 @@
+1
+2
+4
+4
+5
diff --git a/tests/partest-test/runDiffOutput1.scala b/tests/partest-test/runDiffOutput1.scala
new file mode 100644
index 000000000..32cf6f5b6
--- /dev/null
+++ b/tests/partest-test/runDiffOutput1.scala
@@ -0,0 +1,9 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    println(1)
+    println(2)
+    println(3)
+    println(4)
+    println(5)
+  }
+}
diff --git a/tests/partest-test/runWrongOutput1.check b/tests/partest-test/runWrongOutput1.check
new file mode 100644
index 000000000..b414108e8
--- /dev/null
+++ b/tests/partest-test/runWrongOutput1.check
@@ -0,0 +1,6 @@
+1
+2
+3
+4
+5
+6
diff --git a/tests/partest-test/runWrongOutput1.scala b/tests/partest-test/runWrongOutput1.scala
new file mode 100644
index 000000000..32cf6f5b6
--- /dev/null
+++ b/tests/partest-test/runWrongOutput1.scala
@@ -0,0 +1,9 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    println(1)
+    println(2)
+    println(3)
+    println(4)
+    println(5)
+  }
+}
diff --git a/tests/partest-test/runWrongOutput2.check b/tests/partest-test/runWrongOutput2.check
new file mode 100644
index 000000000..01e79c32a
--- /dev/null
+++ b/tests/partest-test/runWrongOutput2.check
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/tests/partest-test/runWrongOutput2.scala b/tests/partest-test/runWrongOutput2.scala
new file mode 100644
index 000000000..32cf6f5b6
--- /dev/null
+++ b/tests/partest-test/runWrongOutput2.scala
@@ -0,0 +1,9 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    println(1)
+    println(2)
+    println(3)
+    println(4)
+    println(5)
+  }
+}
diff --git a/tests/partest-test/stackOverflow.scala b/tests/partest-test/stackOverflow.scala
new file mode 100644
index 000000000..b3132cc19
--- /dev/null
+++ b/tests/partest-test/stackOverflow.scala
@@ -0,0 +1,7 @@
+object Test {
+  def foo: Int = bar
+  def bar: Int = foo
+
+  def main(args: Array[String]): Unit =
+    println(foo)
+}
diff --git a/tests/pos/i851.java b/tests/pos/i851.java
index 39e0bf350..cfd86ccc4 100644
--- a/tests/pos/i851.java
+++ b/tests/pos/i851.java
@@ -5,4 +5,4 @@ interface J<T> extends I<T> {
     };
 }
 
-class C<T> extends J<T> {}
\ No newline at end of file
+class C<T> implements J<T> {}
diff --git a/tests/pos/varargsInMethodsT1625/curriedPosExample.scala b/tests/pos/t1625.scala
index c614f2dee..c614f2dee 100644
--- a/tests/pos/varargsInMethodsT1625/curriedPosExample.scala
+++ b/tests/pos/t1625.scala
diff --git a/tests/pos/varargsInMethodsT1625/onlyOneVarArgs.scala b/tests/pos/t1625b.scala
index 834960b4a..834960b4a 100644
--- a/tests/pos/varargsInMethodsT1625/onlyOneVarArgs.scala
+++ b/tests/pos/t1625b.scala
diff --git a/tests/run/getclass.check b/tests/run/getclass.check
index 9d88762f4..ea73f6127 100644
--- a/tests/run/getclass.check
+++ b/tests/run/getclass.check
@@ -22,5 +22,5 @@ class [D
 class [Lscala.collection.immutable.List;
 
 Functions:
-class Test$$$Lambda$1
-class Test$$$Lambda$2
+class Test$$$Lambda$
+class Test$$$Lambda$
diff --git a/tests/run/getclass.scala b/tests/run/getclass.scala
index 7a13a61a2..b74e1b202 100644
--- a/tests/run/getclass.scala
+++ b/tests/run/getclass.scala
@@ -35,8 +35,8 @@ object Test {
 
     println("\nFunctions:")
     // FunctionN.getClass.toString has form of "class Test$$$Lambda$N/1349414238",
-    // but number (1349414238) depends on environment
-    println(f1.getClass.toString.takeWhile(_ != '/'))
-    println(f2.getClass.toString.takeWhile(_ != '/'))
+    // but "N/1349414238" depends on environment
+    println(f1.getClass.toString.take("class Test$$$Lambda$".length))
+    println(f2.getClass.toString.take("class Test$$$Lambda$".length))
   }
 }
diff --git a/tests/run/origins.check b/tests/run/origins.check
deleted file mode 100644
index 54d58296b..000000000
--- a/tests/run/origins.check
+++ /dev/null
@@ -1,6 +0,0 @@
-
->> Origins tag 'boop' logged 65 calls from 3 distinguished sources.
-
-     50   Test$.$anonfun$f3$1(origins.scala:16)
-     10   Test$.$anonfun$f2$1(origins.scala:15)
-      5   Test$.$anonfun$f1$1(origins.scala:14)
diff --git a/tests/run/origins.flags b/tests/run/origins.flags
deleted file mode 100644
index 690753d80..000000000
--- a/tests/run/origins.flags
+++ /dev/null
@@ -1 +0,0 @@
--no-specialization -Ydelambdafy:inline
\ No newline at end of file
diff --git a/tests/run/origins.scala b/tests/run/origins.scala
deleted file mode 100644
index 6529351d3..000000000
--- a/tests/run/origins.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-import scala.reflect.internal.util.Origins
-
-package goxbox {
-  object Socks {
-    val origins = Origins("boop")
-
-    def boop(x: Int): Int = origins { 5 }
-  }
-}
-
-object Test {
-  import goxbox.Socks.boop
-
-  def f1() = 1 to 5 map boop
-  def f2() = 1 to 10 map boop
-  def f3() = 1 to 50 map boop
-
-  def main(args: Array[String]): Unit = {
-    f1() ; f2() ; f3()
-  }
-}
diff --git a/tests/run/shutdownhooks.check b/tests/run/shutdownhooks.check
deleted file mode 100644
index 29956956e..000000000
--- a/tests/run/shutdownhooks.check
+++ /dev/null
@@ -1,3 +0,0 @@
-Fooblitzky!
-main#shutdown.
-Test#shutdown.
diff --git a/tests/run/shutdownhooks.scala b/tests/run/shutdownhooks.scala
deleted file mode 100644
index 5f512a391..000000000
--- a/tests/run/shutdownhooks.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-object Test {
-  scala.sys.addShutdownHook {
-    Thread.sleep(1000)
-    println("Test#shutdown.")
-  }
-
-  def daemon() = {
-    val t = new Thread {
-      override def run(): Unit = {
-        Thread.sleep(10000)
-        println("Hallelujah!") // should not see this
-      }
-    }
-    t.setDaemon(true)
-    t.start()
-    t
-  }
-
-  def nonDaemon() = {
-    val t = new Thread {
-      override def run(): Unit = {
-        Thread.sleep(100)
-        println("Fooblitzky!")
-      }
-    }
-    t.start()
-    t
-  }
-
-  def main(args: Array[String]): Unit = {
-    daemon()
-    nonDaemon()
-    scala.sys.addShutdownHook {
-      println("main#shutdown.")
-    }
-  }
-}