Add support for error annotations in neg tests

2 files changed, 188 insertions, 42 deletions

diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala
new file mode 100644
index 000000000..22f7e6d91
--- /dev/null
+++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala
@@ -0,0 +1,61 @@
+package dotty
+package tools
+package dotc
+
+import org.junit.Test
+import java.io.{ File => JFile }
+
+class CompilationTests extends ParallelTesting {
+  import CompilationTests.{ defaultOutputDir, defaultOptions }
+
+  @Test def compilePos =
+    compileFilesInDir("../tests/pos", defaultOptions).pos
+
+  @Test def compileNeg =
+    compileShallowFilesInDir("../tests/neg", defaultOptions).neg
+}
+
+object CompilationTests {
+  implicit val defaultOutputDir: String = "../out/"
+
+  private val noCheckOptions = Array(
+    "-pagewidth", "80"
+  )
+
+  private val checkOptions = Array(
+    "-Yno-deep-subtypes",
+    "-Yno-double-bindings",
+    "-Yforce-sbt-phases"
+  )
+
+  private val classPath = {
+    val paths = Jars.dottyTestDeps map { p =>
+      val file = new JFile(p)
+      assert(
+        file.exists,
+        s"""|File "$p" couldn't be found. Run `packageAll` from build tool before
+            |testing.
+            |
+            |If running without sbt, test paths need to be setup environment variables:
+            |
+            | - DOTTY_LIBRARY
+            | - DOTTY_COMPILER
+            | - DOTTY_INTERFACES
+            | - DOTTY_EXTRAS
+            |
+            |Where these all contain locations, except extras which is a colon
+            |separated list of jars.
+            |
+            |When compiling with eclipse, you need the sbt-interfaces jar, put
+            |it in extras."""
+      )
+      file.getAbsolutePath
+    } mkString (":")
+
+    Array("-classpath", paths)
+  }
+
+  private val yCheckOptions = Array("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef")
+
+  val defaultOptions = noCheckOptions ++ checkOptions ++ yCheckOptions ++ classPath
+}
diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala
index 8c4750f9f..f1363e49f 100644
--- a/compiler/test/dotty/tools/dotc/ParallelTesting.scala
+++ b/compiler/test/dotty/tools/dotc/ParallelTesting.scala
@@ -13,6 +13,7 @@ import java.nio.file.StandardCopyOption.REPLACE_EXISTING
 import java.nio.file.{ Files, Path, Paths }
 import java.util.concurrent.{ Executors => JExecutors, TimeUnit }
 import scala.util.control.NonFatal
+import java.util.HashMap
 
 trait ParallelTesting {
 
@@ -49,8 +50,8 @@ trait ParallelTesting {
         }
         // println, otherwise no newline and cursor at start of line
         println(
-          s"Compiled tests in $fromDir " +
-          s"[========================================] $totalTargets/$totalTargets, " +
+          s"Compiled tests in $fromDir  " +
+          s"[=======================================] $totalTargets/$totalTargets, " +
           s"${(System.currentTimeMillis - start) / 1000}s, errors: $errors  "
         )
       }
@@ -79,8 +80,8 @@ trait ParallelTesting {
       def run(): Unit =
         try {
           val sourceFiles = dir.listFiles.filter(f => f.getName.endsWith(".scala") || f.getName.endsWith(".java"))
-          val errors = compile(sourceFiles, flags ++ Array("-d", dir.getAbsolutePath), false)
-          completeCompilation(errors.length)
+          val reporter = compile(sourceFiles, flags ++ Array("-d", dir.getAbsolutePath), false)
+          completeCompilation(reporter.errorCount)
         }
         catch {
           case NonFatal(e) => {
@@ -104,12 +105,30 @@ trait ParallelTesting {
         try {
           val sourceFiles = dir.listFiles.filter(f => f.getName.endsWith(".scala") || f.getName.endsWith(".java"))
 
-          val expectedErrors = dir.listFiles.filter(_.getName.endsWith(".scala")).foldLeft(0) { (acc, file) =>
-            acc + Source.fromFile(file).sliding("// error".length).count(_.mkString == "// error")
+          // In neg-tests we allow two types of error annotations,
+          // "nopos-error" which doesn't care about position and "error" which
+          // has to be annotated on the correct line number.
+          //
+          // We collect these in a map `"file:row" -> numberOfErrors`, for
+          // nopos errors we save them in `"file" -> numberOfNoPosErrors`
+          val errorMap = new HashMap[String, Integer]()
+          var expectedErrors = 0
+          dir.listFiles.filter(_.getName.endsWith(".scala")).foreach { file =>
+            Source.fromFile(file).getLines.zipWithIndex.foreach { case (line, lineNbr) =>
+              val errors = line.sliding("// error".length).count(_.mkString == "// error")
+              if (errors > 0)
+                errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors)
+
+              val noposErrors = line.sliding("// nopos-error".length).count(_.mkString == "// nopos-error")
+              if (noposErrors > 0)
+                errorMap.put(file.getAbsolutePath, noposErrors)
+
+              expectedErrors += noposErrors + errors
+            }
           }
 
-          val errors = compile(sourceFiles, flags ++ Array("-d", dir.getAbsolutePath), true)
-          val actualErrors = errors.length
+          val reporter = compile(sourceFiles, flags ++ Array("-d", dir.getAbsolutePath), true)
+          val actualErrors = reporter.errorCount
 
           if (expectedErrors != actualErrors) {
             System.err.println {
@@ -117,6 +136,45 @@ trait ParallelTesting {
             }
             fail()
           }
+          else if (
+            // Here we check that there is a correpsonding error reported for
+            // each annotation
+            !reporter.errors.forall { error =>
+              val fileName = error.pos.source.file.toString
+              val fileAndRow = s"$fileName:${error.pos.line}"
+
+              val rowErrors = errorMap.get(fileAndRow)
+              lazy val noposErrors = errorMap.get(fileName)
+
+              if (rowErrors ne null) {
+                if (rowErrors == 1) errorMap.remove(fileAndRow)
+                else errorMap.put(fileAndRow, rowErrors - 1)
+                true
+              }
+              else if (noposErrors ne null) {
+                if (noposErrors == 1) errorMap.remove(fileName)
+                else errorMap.put(fileName, noposErrors - 1)
+                true
+              }
+              else {
+                System.err.println {
+                  s"Error reported in ${error.pos}, but no annotation found"
+                }
+                false
+              }
+            }
+          ) {
+            System.err.println {
+              s"\nErrors found on incorrect row numbers when compiling $dir"
+            }
+            fail()
+          }
+          else if (!errorMap.isEmpty) {
+            System.err.println {
+              s"\nError annotation(s) have {<error position>=<unreported error>}: $errorMap"
+            }
+            fail()
+          }
 
           completeCompilation(actualErrors)
         }
@@ -148,7 +206,7 @@ trait ParallelTesting {
     }
   }
 
-  private def compile(files: Array[JFile], flags: Array[String], suppressErrors: Boolean): List[MessageContainer] = {
+  private def compile(files: Array[JFile], flags: Array[String], suppressErrors: Boolean): DaftReporter = {
 
     def findJarFromRuntime(partialName: String) = {
       val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString)
@@ -174,7 +232,7 @@ trait ParallelTesting {
 
     val reporter = new DaftReporter(suppress = suppressErrors)
     driver.process(flags ++ files.map(_.getAbsolutePath), reporter = reporter)
-    reporter.errors
+    reporter
   }
 
 
@@ -190,48 +248,75 @@ trait ParallelTesting {
     )
   }
 
-  def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
-    // each calling method gets its own unique output directory, in which we
-    // place the dir being compiled:
-    val callingMethod = Thread.currentThread.getStackTrace.apply(3).getMethodName
-    val outDir = outDirectory + callingMethod + "/"
+  private def toCompilerDirFromDir(d: JFile, sourceDir: JFile, outDir: String): JFile = {
+    val targetDir = new JFile(outDir + s"${sourceDir.getName}/${d.getName}")
+    // create if not exists
+    targetDir.mkdirs()
+    d.listFiles.foreach(copyToDir(targetDir, _))
+    targetDir
+  }
+
+  private def toCompilerDirFromFile(file: JFile, sourceDir: JFile, outDir: String): JFile = {
+    val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.'))
+    val targetDir = new JFile(outDir + s"${sourceDir.getName}/$uniqueSubdir")
+    // create if not exists
+    targetDir.mkdirs()
+    // copy file to dir:
+    copyToDir(targetDir, file)
+    targetDir
+  }
+
+  private def copyToDir(dir: JFile, file: JFile): Unit = {
+    val target = Paths.get(dir.getAbsolutePath, file.getName)
+    Files.copy(file.toPath, target, REPLACE_EXISTING).toFile
+  }
 
-    val dir = new JFile(f)
+  private def requirements(f: String, sourceDir: JFile, outDir: String): Unit = {
     require(f.contains("/tests"), "only allowed to run integration tests from `tests` dir using this method")
-    require(dir.isDirectory && dir.exists, "passed non-directory to `compileFilesInDir`")
+    require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`")
     require(outDir.last == '/', "please specify an `outDir` with a trailing slash")
+  }
 
-    def toCompilerDirFromDir(d: JFile): JFile = {
-      val targetDir = new JFile(outDir + s"${dir.getName}/${d.getName}")
-      // create if not exists
-      targetDir.mkdirs()
-      d.listFiles.foreach(copyToDir(targetDir, _))
-      targetDir
-    }
-    def toCompilerDirFromFile(file: JFile): JFile = {
-      val uniqueSubdir = file.getName.substring(0, file.getName.lastIndexOf('.'))
-      val targetDir = new JFile(outDir + s"${dir.getName}/$uniqueSubdir")
-      // create if not exists
-      targetDir.mkdirs()
-      // copy file to dir:
-      copyToDir(targetDir, file)
-      targetDir
-    }
-    def copyToDir(dir: JFile, file: JFile): Unit = {
-      val target = Paths.get(dir.getAbsolutePath, file.getName)
-      Files.copy(file.toPath, target, REPLACE_EXISTING).toFile
+  private def compilationTargets(sourceDir: JFile): (List[JFile], List[JFile]) =
+    sourceDir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) =>
+      if (f.isDirectory) (f :: dirs, files)
+      else (dirs, f :: files)
     }
 
-    val (dirs, files) =
-      dir.listFiles.foldLeft((List.empty[JFile], List.empty[JFile])) { case ((dirs, files), f) =>
-        if (f.isDirectory) (f :: dirs, files)
-        else (dirs, f :: files)
-      }
+  def compileFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    // each calling method gets its own unique output directory, in which we
+    // place the dir being compiled:
+    val callingMethod = Thread.currentThread.getStackTrace.apply(3).getMethodName
+    val outDir = outDirectory + callingMethod + "/"
+    val sourceDir = new JFile(f)
+    requirements(f, sourceDir, outDir)
+
+    val (dirs, files) = compilationTargets(sourceDir)
 
     // Directories in which to compile all containing files with `flags`:
-    val dirsToCompile = files.map(toCompilerDirFromFile) ++ dirs.map(toCompilerDirFromDir)
+    val dirsToCompile =
+      files.map(toCompilerDirFromFile(_, sourceDir, outDir)) ++
+      dirs.map(toCompilerDirFromDir(_, sourceDir, outDir))
 
     // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
     new CompilationTest(dirsToCompile, f, flags)
   }
+
+  def compileShallowFilesInDir(f: String, flags: Array[String])(implicit outDirectory: String): CompilationTest = {
+    // each calling method gets its own unique output directory, in which we
+    // place the dir being compiled:
+    val callingMethod = Thread.currentThread.getStackTrace.apply(3).getMethodName
+    val outDir = outDirectory + callingMethod + "/"
+    val sourceDir = new JFile(f)
+    requirements(f, sourceDir, outDir)
+
+    val (_, files) = compilationTargets(sourceDir)
+
+    // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+    new CompilationTest(
+      files.map(toCompilerDirFromFile(_, sourceDir, outDir)),
+      f,
+      flags
+    )
+  }
 }