Partest for Dotty with pos tests and neg tests with error count

8 files changed, 602 insertions, 82 deletions

diff --git a/.gitignore b/.gitignore
index cbaf7ae60..4316f6008 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+*.DS_Store
 *.class
 *.log
 *~
@@ -24,3 +25,7 @@ classes/
 .idea
 .idea_modules
 /.worksheet/
+
+# Partest
+tests/partest-generated/
+tests/runPartest.flag
diff --git a/project/Build.scala b/project/Build.scala
index edb730815..ef9d1375a 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -32,10 +32,11 @@ object DottyBuild extends Build {
     libraryDependencies ++= Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value,
                                 "org.scala-lang.modules" %% "scala-xml" % "1.0.1",
                                 "me.d-d" % "scala-compiler" % "2.11.5-20150416-144435-09c4a520e1",
+                                "org.scala-lang.modules" %% "scala-partest" % "1.0.5" % "test",
                                 "jline" % "jline" % "2.12"),
 
     // get junit onboard
-    libraryDependencies += "com.novocode" % "junit-interface" % "0.11-RC1" % "test",
+    libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
 
     // scalac options
     scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:_"),
@@ -46,7 +47,13 @@ object DottyBuild extends Build {
     incOptions := incOptions.value.withNameHashing(true),
 
     // enable verbose exception messages for JUnit
-    testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+    testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+    // when this file is present, running test generates the files for partest
+    // otherwise it just executes the tests directly
+    createPartestFile := { new java.io.File("./tests", "runPartest.flag").createNewFile },
+    runPartestRunner <<= runTask(Test, "dotty.partest.DPConsoleRunner", "") dependsOn (test in Test),
+    deletePartestFile := { new java.io.File("./tests", "runPartest.flag").delete },
+
     // Adjust classpath for running dotty
     mainClass in (Compile, run) := Some("dotty.tools.dotc.Main"),
     fork in run := true,
@@ -79,7 +86,7 @@ object DottyBuild extends Build {
 
       tuning ::: agentOptions ::: travis_build ::: fullpath
     }
-  )
+  ) ++ addCommandAlias("partest", ";createPartestFile;runPartestRunner;deletePartestFile")
 
   lazy val dotty = Project(id = "dotty", base = file("."), settings = defaults)
 
@@ -92,7 +99,7 @@ object DottyBuild extends Build {
 
 
     libraryDependencies ++= Seq("com.storm-enroute" %% "scalameter" % "0.6" % Test,
-      "com.novocode" % "junit-interface" % "0.11-RC1"),
+      "com.novocode" % "junit-interface" % "0.11"),
     testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"),
 
     // scalac options
@@ -129,4 +136,9 @@ object DottyBuild extends Build {
 
   lazy val benchmarks = Project(id = "dotty-bench", settings = benchmarkSettings,
     base = file("bench")) dependsOn(dotty % "compile->test")
+
+  lazy val createPartestFile = TaskKey[Unit]("createPartestFile", "Creates the tests/runPartest.flag file")
+  lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partests")
+  lazy val deletePartestFile = TaskKey[Unit]("deletePartestFile", "Deletes the tests/runPartest.flag file")
+
 }
diff --git a/test/dotc/comptest.scala b/test/dotc/comptest.scala
index 63c400304..5b6646f39 100644
--- a/test/dotc/comptest.scala
+++ b/test/dotc/comptest.scala
@@ -4,18 +4,21 @@ import test._
 
 object comptest extends CompilerTest {
 
+  override val generatePartestFiles = false
+  val defaultOutputDir: String = ""
+
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"
   val dotcDir = "./src/dotty/"
 
   def main(args: Array[String]) =
-    compileArgs(Array(
+    compileList("comptest", List(
       dotcDir + "tools/dotc/CompilationUnit.scala",
       dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
+      dotcDir + "tools/dotc/ast/Trees.scala"), List(
       "#runs", "2",
       "-Ylog:frontend",
       "-Xprompt"))(Nil)
 
-//    compileDir(dotcDir + "tools/dotc/printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
+//    compileDir(dotcDir + "tools/dotc/", "printing", List("-Xprompt", "-Ylog:frontend", "#runs", "2", "-uniqid"))
 }
diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala
index a015b9efe..ebd4673ef 100644
--- a/test/dotc/tests.scala
+++ b/test/dotc/tests.scala
@@ -1,7 +1,10 @@
 package dotc
 
-import org.junit.Test
 import test._
+import dotty.partest._
+import org.junit.Test
+import org.junit.experimental.categories._
+
 
 class tests extends CompilerTest {
 
@@ -13,12 +16,13 @@ class tests extends CompilerTest {
 //        "-Yshow-suppressed-errors",
         "-pagewidth", "160")
 
+  val defaultOutputDir = "./out/"
+
   implicit val defaultOptions = noCheckOptions ++ List(
       "-Yno-deep-subtypes", "-Yno-double-bindings",
       "-Ycheck:tailrec,resolveSuper,mixin,restoreScopes",
-      "-d", "./out/"
+      "-d", defaultOutputDir
   )
-
   val doEmitBytecode = List("-Ystop-before:terminal")
   val failedbyName = List("-Ystop-before:collectEntryPoints") // #288
   val testPickling = List("-Xprint-types", "-Ytest-pickler", "-Ystop-after:pickler")
@@ -30,19 +34,23 @@ class tests extends CompilerTest {
   val allowDeepSubtypes = defaultOptions diff List("-Yno-deep-subtypes")
   val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings")
 
-  val posDir = "./tests/pos/"
-  val posSpecialDir = "./tests/pos-special/"
-  val negDir = "./tests/neg/"
-  val newDir = "./tests/new/"
-  val dotcDir = "./src/dotty/"
-  val picklingDir = "./tests/pickling"
+  val testsDir      = "./tests/"
+  val posDir        = testsDir + "pos/"  
+  val posSpecialDir = testsDir + "pos-special/"
+  val negDir        = testsDir + "neg/"
+  val newDir        = testsDir + "new/"
 
-  @Test def pickle_pickleOK = compileDir(picklingDir, testPickling)
-  @Test def pickle_pickling = compileDir(dotcDir + "tools/dotc/core/pickling/", testPickling)
-  @Test def pickle_ast = compileDir(dotcDir + "tools/dotc/ast/", testPickling)
+  val dottyDir = "./src/dotty/"
+  val toolsDir = dottyDir + "tools/"
+  val dotcDir  = toolsDir + "dotc/"
+  val coreDir  = dotcDir + "core/"
 
-  //@Test def pickle_core = compileDir(dotcDir + "tools/dotc/core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  @Test def pickle_pickleOK = compileDir(testsDir, "pickling", testPickling)
+  @Test def pickle_pickling = compileDir(coreDir, "pickling", testPickling)
+  @Test def pickle_ast = compileDir(dotcDir, "ast", testPickling)
 
+  //@Test def pickle_core = compileDir(dotcDir, "core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps
+  
   @Test def pos_t2168_pat = compileFile(posDir, "t2168", twice)
   @Test def pos_erasure = compileFile(posDir, "erasure", twice)
   @Test def pos_Coder() = compileFile(posDir, "Coder", twice)
@@ -66,12 +74,12 @@ class tests extends CompilerTest {
   @Test def pos_packageobject() = compileFile(posDir, "packageobject", twice)
   @Test def pos_overloaded() = compileFile(posDir, "overloaded", twice)
   @Test def pos_overrides() = compileFile(posDir, "overrides", twice)
-  @Test def pos_javaOverride() = compileDir(posDir + "java-override", twice)
+  @Test def pos_javaOverride() = compileDir(posDir, "java-override", twice)
   @Test def pos_templateParents() = compileFile(posDir, "templateParents", twice)
   @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice)
   @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice)
-  @Test def pos_tailcall = compileDir(posDir + "tailcall/", twice)
-  @Test def pos_nullarify = compileFile(posDir, "nullarify", "-Ycheck:nullarify" :: Nil)
+  @Test def pos_tailcall = compileDir(posDir, "tailcall", twice)
+  @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil)
   @Test def pos_subtyping = compileFile(posDir, "subtyping", twice)
   @Test def pos_t2613 = compileFile(posSpecialDir, "t2613")(allowDeepSubtypes)
   @Test def pos_packageObj = compileFile(posDir, "i0239", twice)
@@ -92,7 +100,7 @@ class tests extends CompilerTest {
   @Test def neg_privates() = compileFile(negDir, "privates", xerrors = 2)
   @Test def neg_rootImports = compileFile(negDir, "rootImplicits", xerrors = 2)
   @Test def neg_templateParents() = compileFile(negDir, "templateParents", xerrors = 3)
-  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", "-language:noAutoTupling" :: Nil, xerrors = 4)
+  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, xerrors = 4)
   @Test def neg_autoTupling2 = compileFile(negDir, "autoTuplingTest", xerrors = 4)
   @Test def neg_companions = compileFile(negDir, "companions", xerrors = 1)
   @Test def neg_over = compileFile(negDir, "over", xerrors = 3)
@@ -102,14 +110,18 @@ class tests extends CompilerTest {
   @Test def neg_i50_volatile = compileFile(negDir, "i50-volatile", xerrors = 6)
   @Test def neg_t0273_doubledefs = compileFile(negDir, "t0273", xerrors = 1)
   @Test def neg_zoo = compileFile(negDir, "zoo", xerrors = 12)
-  @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
-  @Test def neg_tailcall_t1672b = compileFile(negDir, "tailcall/t1672b", xerrors = 6)
-  @Test def neg_tailcall_t3275 = compileFile(negDir, "tailcall/t3275", xerrors = 1)
-  @Test def neg_tailcall_t6574 = compileFile(negDir, "tailcall/t6574", xerrors = 2)
-  @Test def neg_tailcall = compileFile(negDir, "tailcall/tailrec", xerrors = 7)
-  @Test def neg_tailcall2 = compileFile(negDir, "tailcall/tailrec-2", xerrors = 2)
-  @Test def neg_tailcall3 = compileFile(negDir, "tailcall/tailrec-3", xerrors = 2)
-  @Test def nef_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
+  // TODO: this test file doesn't exist (anymore?), remove?
+  // @Test def neg_t1192_legalPrefix = compileFile(negDir, "t1192", xerrors = 1)
+
+  val negTailcallDir = negDir + "tailcall/"
+  @Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", xerrors = 6)
+  @Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", xerrors = 1)
+  @Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", xerrors = 2)
+  @Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", xerrors = 7)
+  @Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", xerrors = 2)
+  @Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", xerrors = 2)
+  
+  @Test def neg_t1279a = compileFile(negDir, "t1279a", xerrors = 1)
   @Test def neg_t1843_variances = compileFile(negDir, "t1843-variances", xerrors = 1)
   @Test def neg_t2660_ambi = compileFile(negDir, "t2660", xerrors = 2)
   @Test def neg_t2994 = compileFile(negDir, "t2994", xerrors = 2)
@@ -127,16 +139,16 @@ class tests extends CompilerTest {
   @Test def neg_moduleSubtyping = compileFile(negDir, "moduleSubtyping", xerrors = 4)
   @Test def neg_escapingRefs = compileFile(negDir, "escapingRefs", xerrors = 2)
 
-  @Test def dotc = compileDir(dotcDir + "tools/dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
-  @Test def dotc_ast = compileDir(dotcDir + "tools/dotc/ast", failedOther ++ twice)
+  @Test def dotc = compileDir(toolsDir, "dotc", failedOther)(allowDeepSubtypes ++ twice) // see dotc_core
+  @Test def dotc_ast = compileDir(dotcDir, "ast", failedOther ++ twice)
     //similar to dotc_core_pickling but for another anon class. Still during firstTransform
-  @Test def dotc_config = compileDir(dotcDir + "tools/dotc/config")
-  @Test def dotc_core = compileDir(dotcDir + "tools/dotc/core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_config = compileDir(dotcDir, "config")
+  @Test def dotc_core = compileDir(dotcDir, "core", failedOther)("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster
     // error: error while loading ConstraintHandling$$anon$1$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/core/ConstraintHandling$$anon$1.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_core_pickling = compileDir(dotcDir + "tools/dotc/core/pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
+  @Test def dotc_core_pickling = compileDir(coreDir, "pickling", failedOther)(allowDeepSubtypes)// twice omitted to make tests run faster
     // exception caught when loading class ClassfileParser$$anon$1: dotty.tools.dotc.core.Denotations$NotDefinedHere:
     // demanding denotation of module class ClassfileParser$$anon$1$ at phase frontend(1) outside defined interval:
     // defined periods are Period(31..36, run = 2) Period(3..24, run = 2) Period(25..26, run = 2)
@@ -144,45 +156,43 @@ class tests extends CompilerTest {
     // inside FirstTransform    at dotty.tools.dotc.transform.FirstTransform.transform(FirstTransform.scala:33)
     // weird.
 
-  @Test def dotc_transform = compileDir(dotcDir + "tools/dotc/transform")// twice omitted to make tests run faster
+  @Test def dotc_transform = compileDir(dotcDir, "transform")// twice omitted to make tests run faster
 
-  @Test def dotc_parsing = compileDir(dotcDir + "tools/dotc/parsing")// twice omitted to make tests run faster
+  @Test def dotc_parsing = compileDir(dotcDir, "parsing") // twice omitted to make tests run faster
 
-  @Test def dotc_printing = compileDir(dotcDir + "tools/dotc/printing") // twice omitted to make tests run faster
+  @Test def dotc_printing = compileDir(dotcDir, "printing") // twice omitted to make tests run faster
 
-  @Test def dotc_reporting = compileDir(dotcDir + "tools/dotc/reporting") // twice omitted to make tests run faster
+  @Test def dotc_reporting = compileDir(dotcDir, "reporting") // twice omitted to make tests run faster
 
-  @Test def dotc_typer = compileDir(dotcDir + "tools/dotc/typer", failedOther) // twice omitted to make tests run faster
+  @Test def dotc_typer = compileDir(dotcDir, "typer", failedOther)// twice omitted to make tests run faster
     // error: error while loading Checking$$anon$2$,
     // class file 'target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar(dotty/tools/dotc/typer/Checking$$anon$2.class)'
     // has location not matching its contents: contains class $anon
 
-  @Test def dotc_util = compileDir(dotcDir + "tools/dotc/util", failedOther ++ twice)
+  @Test def dotc_util = compileDir(dotcDir, "util", failedOther ++ twice)
     // java.lang.ClassCastException: dotty.tools.dotc.core.Types$NoType$ cannot be cast to dotty.tools.dotc.core.Types$ClassInfo
     // at dotty.tools.dotc.core.SymDenotations$ClassDenotation.classInfo(SymDenotations.scala:1026)
     // at dotty.tools.dotc.transform.ExtensionMethods.transform(ExtensionMethods.scala:38)
 
-  @Test def tools_io = compileDir(dotcDir + "tools/io", failedOther ++ twice) // inner class has symbol <none>
+  @Test def tools_io = compileDir(toolsDir, "io", failedOther ++ twice) // inner class has symbol <none>
 
   @Test def helloWorld = compileFile(posDir, "HelloWorld", twice)
   @Test def labels = compileFile(posDir, "Labels", twice)
-  //@Test def tools = compileDir(dotcDir + "tools", "-deep" :: Nil)(allowDeepSubtypes)
+  //@Test def tools = compileDir(dottyDir, "tools", "-deep" :: Nil)(allowDeepSubtypes)
 
-  @Test def testNonCyclic = compileArgs(Array(
-      dotcDir + "tools/dotc/CompilationUnit.scala",
-      dotcDir + "tools/dotc/core/Types.scala",
-      dotcDir + "tools/dotc/ast/Trees.scala",
-      "-Xprompt"
-      ) ++ staleSymbolError ++ twice)
+  @Test def testNonCyclic = compileList("testNonCyclic", List(
+      dotcDir + "CompilationUnit.scala",
+      coreDir + "Types.scala",
+      dotcDir + "ast/Trees.scala"
+    ), List("-Xprompt") ++ staleSymbolError ++ twice)
 
-  @Test def testIssue_34 = compileArgs(Array(
-      dotcDir + "tools/dotc/config/Properties.scala",
-      dotcDir + "tools/dotc/config/PathResolver.scala",
-      //"-Ylog:frontend",
-      "-Xprompt") ++ staleSymbolError ++ twice)
+  @Test def testIssue_34 = compileList("testIssue_34", List(
+      dotcDir + "config/Properties.scala",
+      dotcDir + "config/PathResolver.scala"
+    ), List(/* "-Ylog:frontend", */ "-Xprompt") ++ staleSymbolError ++ twice)
 
   val javaDir = "./tests/pos/java-interop/"
   @Test def java_all = compileFiles(javaDir, twice)
 
-  //@Test def dotc_compilercommand = compileFile(dotcDir + "tools/dotc/config/", "CompilerCommand")
+  //@Test def dotc_compilercommand = compileFile(dotcDir + "config/", "CompilerCommand")
 }
diff --git a/test/dotty/partest/DPConfig.scala b/test/dotty/partest/DPConfig.scala
new file mode 100644
index 000000000..c744a9562
--- /dev/null
+++ b/test/dotty/partest/DPConfig.scala
@@ -0,0 +1,31 @@
+package dotty.partest
+
+import java.io.File
+import scala.collection.JavaConversions._
+
+
+/** Dotty Partest runs all tests in the provided testDirs located under
+  * testRoot. There can be several directories with pos resp. neg tests, as
+  * long as the prefix is pos/neg.
+  *
+  * Each testDir can also have a __defaultFlags.flags file, which provides
+  * compiler flags and is used unless there's a specific flags file (e.g. for
+  * test pos/A.scala, if there's a pos/A.flags file those flags are used,
+  * otherwise pos/__defaultFlags.flags are used if the file exists).
+  */
+object DPConfig {
+  val testRoot = "./tests/partest-generated"
+  lazy val testDirs = {
+    val root = new File(testRoot)
+    val dirs = if (!root.exists) Array.empty[String] else root.listFiles.filter(_.isDirectory).map(_.getName)
+    if (dirs.length > 0)
+      println(s"Partest found generated source directories in $testRoot: " + dirs.mkString(", "))
+    else
+      throw new Exception("Partest did not detect any generated sources")
+    dirs
+  }
+
+  // Tests finish faster when running in parallel, but console output is
+  // out of order and sometimes the compiler crashes
+  val runTestsInParallel = false
+}
diff --git a/test/dotty/partest/DPConsoleRunner.scala b/test/dotty/partest/DPConsoleRunner.scala
new file mode 100644
index 000000000..d21b5e983
--- /dev/null
+++ b/test/dotty/partest/DPConsoleRunner.scala
@@ -0,0 +1,190 @@
+/* NOTE: Adapted from ScalaJSPartest.scala in
+ * https://github.com/scala-js/scala-js/
+ * TODO make partest configurable */
+
+package dotty.partest
+
+import scala.tools.partest._
+import scala.tools.partest.nest._
+import java.io.File
+import java.net.URLClassLoader
+
+/** Runs dotty partest from the Console, discovering test sources in
+  * DPConfig.testRoot that have been generated automatically by
+  * DPPrepJUnitRunner. Use `sbt test` to run.
+  */
+object DPConsoleRunner {
+  def main(args: Array[String]): Unit = {
+    new DPConsoleRunner(args mkString (" ")).runPartest
+  }
+}
+
+// console runner has a suite runner which creates a test runner for each test
+class DPConsoleRunner(args: String) extends ConsoleRunner(args) {
+  override val suiteRunner = new DPSuiteRunner (
+    testSourcePath = optSourcePath getOrElse DPConfig.testRoot,
+    fileManager = null, // new FileManager(ClassPath split PathResolver.Environment.javaUserClassPath map (Path(_))), // the script sets up our classpath for us via ant
+    updateCheck = optUpdateCheck,
+    failed = optFailed)
+
+  override def run = {}
+  def runPartest = super.run
+}
+
+class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending"
+  fileManager: FileManager,
+  updateCheck: Boolean,
+  failed: Boolean,
+  javaCmdPath: String = PartestDefaults.javaCmd,
+  javacCmdPath: String = PartestDefaults.javacCmd,
+  scalacExtraArgs: Seq[String] = Seq.empty) 
+extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs) {
+
+  if (!DPConfig.runTestsInParallel)
+    sys.props("partest.threads") = "1"
+
+  sys.props("partest.root") = "."
+
+  // override to provide Dotty banner
+  override def banner: String = {
+    s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString}
+        |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main
+        |Test root: ${PathSettings.srcDir}${File.separator}
+        |Test directories: ${DPConfig.testDirs.toList.mkString(", ")}
+        |Parallel: ${DPConfig.runTestsInParallel}
+    """.stripMargin
+  }
+
+  // override to provide DPTestRunner
+  override def runTest(testFile: File): TestState = {
+    val runner = new DPTestRunner(testFile, this)
+
+    // when option "--failed" is provided execute test only if log
+    // is present (which means it failed before)
+    val state =
+      if (failed && !runner.logFile.canRead)
+        runner.genPass()
+      else {
+        val (state, _) =
+          try timed(runner.run())
+          catch {
+            case t: Throwable => throw new RuntimeException(s"Error running $testFile", t)
+          }
+        NestUI.reportTest(state)
+        runner.cleanup()
+        state
+      }
+    onFinishTest(testFile, state)
+  }
+
+  // override val fileManager = new DottyFileManager(testClassLoader)
+  // sbt package generates a dotty compiler jar, currently
+  // ".../git/dotty/target/scala-2.11/dotty_2.11-0.1-SNAPSHOT.jar"
+  // but it doesn't seem to be used anywhere
+}
+
+class DPTestRunner(testFile: File, suiteRunner: SuiteRunner) extends nest.Runner(testFile, suiteRunner) {
+  // override to provide DottyCompiler
+  override def newCompiler = new dotty.partest.DPDirectCompiler(this)
+
+  // override to provide default dotty flags from file in directory
+  override def flagsForCompilation(sources: List[File]): List[String] = {
+    val specificFlags = super.flagsForCompilation(sources)
+    if (specificFlags.isEmpty) defaultFlags
+    else specificFlags
+  }
+  val defaultFlags = {
+    val defaultFile = parentFile.listFiles.toList.find(_.getName == "__defaultFlags.flags")
+    defaultFile.map({ file =>
+      SFile(file).safeSlurp.map({ content => words(content).filter(_.nonEmpty) }).getOrElse(Nil)
+    }).getOrElse(Nil)
+  }
+
+  // override to add the check for nr of compilation errors if there's a
+  // target.nerr file
+  override def runNegTest() = runInContext {
+    import TestState.{ Crash, Fail }
+    import scala.reflect.internal.FatalError
+
+    sealed abstract class NegTestState
+    // Don't get confused, the neg test passes when compilation fails for at
+    // least one round (optionally checking the number of compiler errors and
+    // compiler console output)
+    case class CompFailed() extends NegTestState
+    // the neg test fails when all rounds return either of these:
+    case class CompFailedButWrongNErr(expected: String, found: String) extends NegTestState
+    case class CompFailedButWrongDiff() extends NegTestState
+    case class CompSucceeded() extends NegTestState
+
+    def nerrIsOk(reason: String) = {
+      import scala.util.matching.Regex
+      val nerrFinder = """compilation failed with (\d+) errors""".r
+      reason match {
+        case nerrFinder(found) => 
+          SFile(FileOps(testFile) changeExtension "nerr").safeSlurp match {
+            case Some(exp) if (exp != found) => CompFailedButWrongNErr(exp, found)
+            case _ => CompFailed
+          }
+        case _ => CompFailed
+      }
+    }
+  
+    // we keep the partest semantics where only one round needs to fail
+    // compilation, not all
+    val compFailingRounds = compilationRounds(testFile).map({round => 
+      val ok = round.isOk
+      setLastState(if (ok) genPass else genFail("compilation failed"))
+      (round.result, ok)
+    }).filter({ case (_, ok) => !ok })
+
+    val failureStates = compFailingRounds.map({ case (result, _) => result match {
+      // or, OK, we'll let you crash the compiler with a FatalError if you supply a check file
+      case Crash(_, t, _) if !checkFile.canRead || !t.isInstanceOf[FatalError] => CompSucceeded
+      case Fail(_, reason, _) => if (diffIsOk) nerrIsOk(reason) else CompFailedButWrongDiff
+      case _ => if (diffIsOk) CompFailed else CompFailedButWrongDiff
+    }})
+
+    if (failureStates.exists({ case CompFailed => true; case _ => false })) {
+      true
+    } else {
+      val existsNerr = failureStates.exists({ 
+        case CompFailedButWrongNErr(exp, found) => nextTestActionFailing(s"wrong number of compilation errors, expected: $exp, found: $found"); true
+        case _ => false
+      })
+      if (existsNerr) {
+        false 
+      } else {
+        val existsDiff = failureStates.exists({ 
+          case CompFailedButWrongDiff() => nextTestActionFailing(s"output differs"); true
+          case _ => false
+        })
+        if (existsDiff) {
+          false
+        } else {
+          nextTestActionFailing("expected compilation failure")
+        }
+      }
+    }
+  }
+
+  // override because Dotty currently doesn't handle separate compilation well,
+  // so we ignore groups (tests suffixed with _1 and _2)
+  override def groupedFiles(sources: List[File]): List[List[File]] = {
+    val grouped = sources groupBy (_.group)
+    val flatGroup = List(grouped.keys.toList.sorted.map({ k => grouped(k) sortBy (_.getName) }).flatten)
+    try { // try/catch because of bug in partest
+      if (flatGroup != super.groupedFiles(sources))
+        NestUI.echoWarning("Warning: Overriding compilation groups for tests: " + sources)
+    } catch {
+      case e: java.lang.UnsupportedOperationException => NestUI.echoWarning("Warning: Overriding compilation groups for tests: " + sources)
+    }
+    flatGroup
+  }
+
+  // override to avoid separate compilation of scala and java sources
+  override def mixedCompileGroup(allFiles: List[File]): List[CompileRound] = List(OnlyDotty(allFiles))
+  case class OnlyDotty(fs: List[File]) extends CompileRound {
+    def description = s"dotc $fsString"
+    lazy val result = { pushTranscript(description) ; attemptCompile(fs) }
+  }
+}
diff --git a/test/dotty/partest/DPDirectCompiler.scala b/test/dotty/partest/DPDirectCompiler.scala
new file mode 100644
index 000000000..86e766505
--- /dev/null
+++ b/test/dotty/partest/DPDirectCompiler.scala
@@ -0,0 +1,34 @@
+package dotty.partest
+
+import scala.tools.partest.{ TestState, nest }
+import java.io.File
+
+
+/* NOTE: Adapted from partest.DirectCompiler and DottyTest */
+class DPDirectCompiler(runner: nest.Runner) extends nest.DirectCompiler(runner) {
+
+  override def compile(opts0: List[String], sources: List[File]): TestState = {
+    println("\ncompiling " + sources.mkString(" ") + "\noptions: " + opts0.mkString(" ")) 
+
+    implicit var ctx: dotty.tools.dotc.core.Contexts.Context = {
+      val base = new dotty.tools.dotc.core.Contexts.ContextBase
+      import base.settings._
+      val ctx = base.initialCtx.fresh.setSetting(printtypes, true)
+        .setSetting(pageWidth, 90).setSetting(log, List("<some"))
+      base.definitions.init(ctx)
+      ctx
+    }
+    
+    try {
+      val processor = if (opts0.exists(_.startsWith("#"))) dotty.tools.dotc.Bench else dotty.tools.dotc.Main
+      val reporter = processor.process((sources.map(_.toString) ::: opts0).toArray, ctx)
+      if (!reporter.hasErrors) runner.genPass()
+      else {
+        reporter.printSummary(ctx)
+        runner.genFail(s"compilation failed with ${reporter.errorCount} errors")
+      }
+    } catch { 
+      case t: Throwable => runner.genCrash(t)
+    }
+  }
+}
diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala
index ccee7467f..23bdfd829 100644
--- a/test/test/CompilerTest.scala
+++ b/test/test/CompilerTest.scala
@@ -1,52 +1,287 @@
 package test
 
-import scala.reflect.io._
-import org.junit.Test
-import scala.collection.mutable.ListBuffer
+import dotty.partest.DPConfig
 import dotty.tools.dotc.{Main, Bench, Driver}
 import dotty.tools.dotc.reporting.Reporter
+import scala.collection.mutable.ListBuffer
+import scala.reflect.io.{ Path, Directory, File => SFile }
+import scala.tools.partest.nest.FileManager
+import java.io.{ File => JFile }
+import org.junit.Test
+
+
+/** This class has two modes: it can directly run compiler tests, or it can
+  * generate the necessary file structure for partest in the directory
+  * DPConfig.testRoot. Both modes are regular JUnit tests. Which mode is used
+  * depends on the existence of the tests/runPartest.flag file which is created
+  * by sbt to trigger partest generation. Sbt can then run partest on the
+  * generated sources.
+  *
+  * Through overriding the partestableXX methods, tests can always be run as
+  * JUnit compiler tests.
+  *
+  * A test can either be a file or a directory. The test is in a parent
+  * directory that determines the kind of test:
+  * - pos: checks that compilation succeeds
+  * - neg: checks that compilation fails with the given number of errors
+  * (- run: compilation succeeds and running generates the given output.)
+  * For partest, compiler flags and the number of errors expected from a neg
+  * test are read from test.flags and test.nerr files (also generated).
+  */
+abstract class CompilerTest extends DottyTest {
+
+  /** Override with output dir of test so it can be patched. Partest expects
+    * classes to be in partest-generated/[kind]/[testname]-[kind].obj/ */
+  val defaultOutputDir: String
+
+  /** Override to filter out tests that should not be run by partest. */
+  def partestableFile(prefix: String, fileName: String, extension: String, args: List[String], xerrors: Int) = true
+  def partestableDir(prefix: String, dirName: String, args: List[String], xerrors: Int) = true
+  def partestableList(testName: String, files: List[String], args: List[String], xerrors: Int) = true
+
+  val generatePartestFiles = new JFile("tests", "runPartest.flag").exists
+
+  // Delete generated files from previous run
+  if (generatePartestFiles)
+    CompilerTest.init
+
+  /** Always run with JUnit. */
+  def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit =
+    compileArgs(cmdLine.split("\n"), xerrors)
+
+  /** Compiles the given code file.
+    *
+    * @param prefix    the parent directory (including separator at the end)
+    * @param fileName  the filename, by default without extension
+    * @param args      arguments to the compiler
+    * @param xerrors   if > 0, this test is a neg test with the expected number
+    *                  of compiler errors. Otherwise, this is a pos test.
+    * @param extension the file extension, .scala by default
+    * @param defaultOptions more arguments to the compiler
+    */
+  def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0, extension: String = ".scala")
+      (implicit defaultOptions: List[String]): Unit = {
+    if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions, xerrors)) {
+      compileArgs((s"$prefix$fileName$extension" :: args).toArray, xerrors)
+    } else {
+      val kind = testKind(xerrors)
+      println(s"generating partest files for test file: $prefix$fileName$extension of kind $kind")
+
+      val sourceFile = new JFile(prefix + fileName + extension)
+      if (sourceFile.exists) {
+        val firstDest = SFile(DPConfig.testRoot + JFile.separator + kind + JFile.separator + fileName + extension)
+        computeDestAndCopyFiles(sourceFile, firstDest, kind, args ++ defaultOptions, xerrors.toString)
+      } else {
+        throw new java.io.FileNotFoundException(s"Unable to locate test file $prefix$fileName")
+      }
+    }
+  }
+
+  /** Compiles the code files in the given directory together. If args starts
+    * with "-deep", all files in subdirectories (and so on) are included. */
+  def compileDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
+    if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions, xerrors)) {
+      val dir = Directory(prefix + dirName)
+      val (files, normArgs) = args match {
+        case "-deep" :: args1 => (dir.deepFiles, args1)
+        case _ => (dir.files, args)
+      }
+      val fileNames = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
+      compileArgs(fileNames ++ normArgs, xerrors)
+    } else {
+      val (sourceDir, flags, deep) = args match {
+        case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep")
+        case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow")
+      }
+      val kind = testKind(xerrors)
+      println(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind")
+
+      if (sourceDir.exists) {
+        val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName)
+        computeDestAndCopyFiles(sourceDir, firstDest, kind, args ++ defaultOptions, xerrors.toString)
+        if (deep == "deep") sourceDir.delete
+      } else {
+        throw new java.io.FileNotFoundException(s"Unable to locate test dir $prefix$dirName")
+      }
+    }
+  }
+
+  /** Compiles each source in the directory path separately by calling
+    * compileFile resp. compileDir. */
+  def compileFiles(path: String, args: List[String] = Nil, verbose: Boolean = true)
+      (implicit defaultOptions: List[String]): Unit = {
+    val dir = Directory(path)
+    val fileNames = dir.files.toArray.map(_.jfile.getName).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
+    for (name <- fileNames) {
+      if (verbose) println(s"testing $path$name")
+      compileFile(path, name, args, 0, "")
+    }
+    for (subdir <- dir.dirs) {
+      if (verbose) println(s"testing $subdir")
+      compileDir(path, subdir.jfile.getName, args, 0)
+    }
+  }
 
-class CompilerTest extends DottyTest {
+  /** Compiles the given list of code files. */
+  def compileList(testName: String, files: List[String], args: List[String] = Nil, xerrors: Int = 0)
+      (implicit defaultOptions: List[String]): Unit = {
+    if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions, xerrors)) {
+      compileArgs((files ++ args).toArray, xerrors)
+    } else {
+      val destDir = Directory(DPConfig.testRoot + JFile.separator + testName)
+      files.foreach({ file => 
+        val jfile = new JFile(file)
+        recCopyFiles(jfile, destDir / jfile.getName)
+      })
+      compileDir(DPConfig.testRoot + JFile.separator, testName, args, xerrors)
+    }
+  }
 
-  def compileArgs(args: Array[String], xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
+  // ========== HELPERS =============
+
+  private def compileArgs(args: Array[String], xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
     val allArgs = args ++ defaultOptions
     val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
     val nerrors = processor.process(allArgs, ctx).errorCount
     assert(nerrors == xerrors, s"Wrong # of errors. Expected: $xerrors, found: $nerrors")
   }
 
-  def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = compileArgs(cmdLine.split("\n"), xerrors)
+  // In particular, don't copy flags from scalac tests
+  private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")
 
-  def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit =
-    compileArgs((s"$prefix$fileName.scala" :: args).toArray, xerrors)
+  /** Determines what kind of test to run. */
+  private def testKind(xerrors: Int) = if (xerrors > 0) "neg" else "pos"
 
-  def compileDir(path: String, args: List[String] = Nil, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit =
-    compileDir(Directory(path), args, xerrors)
+  /** The three possibilities: no generated sources exist yet, the same sources
+    * exist already, different sources exist. */
+  object Difference extends Enumeration {
+    type Difference = Value
+    val NotExists, ExistsSame, ExistsDifferent = Value
+  }
+  import Difference._
 
-  def compileDir(dir: Directory, args: List[String], xerrors: Int)(implicit defaultOptions: List[String]): Unit = {
-    val (files, normArgs) = args match {
-      case "-deep" :: args1 => (dir.deepFiles, args1)
-      case _ => (dir.files, args)
+  /** The same source might be used for several partest test cases (e.g. with
+    * different flags). Detects existing versions and computes the path to be
+    * used for this version, e.g. testname_v1 for the first alternative. */
+  private def computeDestAndCopyFiles(source: JFile, dest: Path, kind: String, oldFlags: List[String], nerr: String,
+      nr: Int = 0, oldOutput: String = defaultOutputDir): Unit = {
+
+    val partestOutput = dest.jfile.getParentFile + JFile.separator + dest.stripExtension + "-" + kind + ".obj"
+    val flags = oldFlags.map(f => if (f == oldOutput) partestOutput else f)
+
+    getExisting(dest).isDifferent(source, flags, nerr) match {
+      case NotExists => copyFiles(source, dest, partestOutput, flags, nerr)
+      case ExistsSame => // nothing else to do
+      case ExistsDifferent =>
+        val nextDest = dest.parent / (dest match {
+          case f: SFile => SFile(replaceVersion(f.stripExtension, nr)).addExtension(f.extension)
+          case d: Directory => Directory(replaceVersion(d.name, nr))
+        })
+        computeDestAndCopyFiles(source, nextDest, kind, flags, nerr, nr + 1, partestOutput)
     }
-    val fileNames = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
-    compileArgs(fileNames ++ normArgs, xerrors)
   }
+  
+  /** Copies the test sources and creates flags, nerr and output files. */
+  private def copyFiles(sourceFile: Path, dest: Path, partestOutput: String, flags: List[String], nerr: String) = {
+    recCopyFiles(sourceFile, dest)
 
-  def compileFiles(path: String, args: List[String] = Nil)(implicit defaultOptions: List[String]): Unit = {
-    val dir = Directory(path)
-    val fileNames = dir.files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
-    for (name <- fileNames) {
-      println(s"testing $name")
-      compileArgs((name :: args).toArray, 0)
+    new JFile(partestOutput).mkdirs
+
+    if (flags.nonEmpty)
+      dest.changeExtension("flags").createFile(true).writeAll(flags.mkString(" "))
+    if (nerr != "0")
+      dest.changeExtension("nerr").createFile(true).writeAll(nerr)
+  }
+
+  /** Recursively copy over files and directories, excluding extensions that
+    * aren't in extensionsToCopy. */
+  private def recCopyFiles(sourceFile: Path, dest: Path): Unit = {
+    processFileDir(sourceFile, { sf =>
+      if (extensionsToCopy.contains(sf.extension)) {
+        dest.parent.jfile.mkdirs
+        FileManager.copyFile(sourceFile.jfile, dest.jfile)
+      } else {
+        println(s"warning: ignoring $sf")
+      }
+    }, { sdir => 
+      dest.jfile.mkdirs
+      sdir.list.foreach(path => recCopyFiles(path, dest / path.name))
+    }, Some("DPCompilerTest.recCopyFiles: sourceFile not found: " + sourceFile))
+  }
+
+  /** Reads the existing files for the given test source if any. */
+  private def getExisting(dest: Path): ExistingFiles = {
+    val content: Option[Option[String]] = processFileDir(dest, f => f.safeSlurp, d => Some(""))
+    if (content.isDefined && content.get.isDefined) {
+      val flags = (dest changeExtension "flags").toFile.safeSlurp
+      val nerr = (dest changeExtension "nerr").toFile.safeSlurp
+      ExistingFiles(content.get, flags, nerr)
+    } else ExistingFiles()
+  }
+
+  /** Encapsulates existing generated test files. */
+  case class ExistingFiles(genSrc: Option[String] = None, flags: Option[String] = None, nerr: Option[String] = None) {
+    def isDifferent(sourceFile: JFile, otherFlags: List[String], otherNerr: String): Difference = {
+      if (!genSrc.isDefined) {
+        NotExists
+      } else {
+        val source = processFileDir(sourceFile, { f => f.safeSlurp }, { d => Some("") }, 
+            Some("DPCompilerTest sourceFile doesn't exist: " + sourceFile)).get
+        if (source == genSrc) {
+          nerr match {
+            case Some(n) if (n != otherNerr) => ExistsDifferent
+            case None if (otherNerr != "0") => ExistsDifferent
+            case _ if (flags.map(_ == otherFlags.mkString(" ")).getOrElse(otherFlags.isEmpty)) => ExistsSame
+            case _ => ExistsDifferent
+          }
+        } else ExistsDifferent
+      }
     }
-    for (subdir <- dir.dirs) {
-      println(s"testing $subdir")
-      compileDir(subdir, args, 0)
+  }
+
+  import scala.util.matching.Regex
+  val nrFinder = """(.*_v)(\d+)""".r
+  /** Changes the version number suffix in the name (without extension). */
+  private def replaceVersion(name: String, nr: Int): String = {
+    val nrString = nr.toString
+    name match {
+      case nrFinder(prefix, `nrString`) => prefix + (nr + 1)
+      case _ => 
+        assert(nr == 0, "DPCompilerTest couldn't create new version of files, match error")
+        name + "_v1"
     }
   }
+
+  /** Returns None if the given path doesn't exist, otherwise returns Some of
+    * applying either processFile or processDir, depending on what the path
+    * refers to in the file system. If failMsgOnNone is defined, this function
+    * asserts that the file exists using the provided message. */
+  private def processFileDir[T](input: Path, processFile: SFile => T, processDir: Directory => T, failMsgOnNone: Option[String] = None): Option[T] = {
+    val res = input.ifFile(f => processFile(f)).orElse(input.ifDirectory(d => processDir(d)))
+    (failMsgOnNone, res) match {
+      case (Some(msg), None) => assert(false, msg); None
+      case _ => res
+    }
+  }
+
+  /** Creates a temporary directory and copies all (deep) files over, thus
+    * flattening the directory structure. */
+  private def flattenDir(prefix: String, dirName: String): JFile = {
+    val destDir = Directory(DPConfig.testRoot + JFile.separator + "_temp")
+    Directory(prefix + dirName).deepFiles.foreach(source => recCopyFiles(source, destDir / source.name))
+    destDir.jfile
+  }
+  
 }
+
 object CompilerTest extends App {
 
+  /** Delete generated partest sources from a previous run. */
+  lazy val init = {
+    scala.reflect.io.Directory(DPConfig.testRoot).deleteRecursively
+    new java.io.File(DPConfig.testRoot).mkdirs
+  }
+
 //  val dotcDir = "/Users/odersky/workspace/dotty/src/dotty/"
 
 //  new CompilerTest().compileFile(dotcDir + "tools/dotc/", "CompilationUnit")