Neg tests: remove xerror parameter from tests (compute based on // error)

2 files changed, 81 insertions, 121 deletions

diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala
index 0bb27ab14..7cc6bafbd 100644
--- a/test/dotc/tests.scala
+++ b/test/dotc/tests.scala
@@ -108,87 +108,22 @@ class tests extends CompilerTest {
 
   @Test def new_all = compileFiles(newDir, twice)
 
-  @Test def neg_abstractOverride() = compileFile(negDir, "abstract-override", xerrors = 2)
-  @Test def neg_blockescapes() = compileFile(negDir, "blockescapesNeg", xerrors = 1)
-  @Test def neg_bounds() = compileFile(negDir, "bounds", xerrors = 2)
-  @Test def neg_functionArity() = compileFile(negDir, "function-arity", xerrors = 7)
-  @Test def neg_typedapply() = compileFile(negDir, "typedapply", xerrors = 3)
-  @Test def neg_typedIdents() = compileDir(negDir, "typedIdents", xerrors = 2)
-  @Test def neg_assignments() = compileFile(negDir, "assignments", xerrors = 3)
-  @Test def neg_typers() = compileFile(negDir, "typers", xerrors = 14)(allowDoubleBindings)
-  @Test def neg_privates() = compileFile(negDir, "privates", xerrors = 2)
-  @Test def neg_rootImports = compileFile(negDir, "rootImplicits", xerrors = 2)
-  @Test def neg_templateParents() = compileFile(negDir, "templateParents", xerrors = 3)
-  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, xerrors = 3)
-  @Test def neg_autoTupling2 = compileFile(negDir, "autoTuplingTest", xerrors = 3)
-  @Test def neg_companions = compileFile(negDir, "companions", xerrors = 1)
-  @Test def namedParams = compileFile(negDir, "named-params", xerrors = 14)
-  @Test def neg_over = compileFile(negDir, "over", xerrors = 3)
-  @Test def neg_overrides = compileFile(negDir, "overrides", xerrors = 14)
-  @Test def neg_overrideClass = compileFile(negDir, "overrideClass", List("-language:Scala2"), xerrors = 1)
-  @Test def neg_i39 = compileFile(negDir, "i39", xerrors = 2)
-  @Test def neg_i50_volatile = compileFile(negDir, "i50-volatile", xerrors = 3)
-  @Test def neg_zoo = compileFile(negDir, "zoo", xerrors = 12)
+  @Test def neg_all = compileFiles(negDir, verbose = true, negTest = true, compileSubDirs = false)
+  @Test def neg_typedIdents() = compileDir(negDir, "typedIdents", negTest = true)
+
+  val negCustomArgs = negDir + "customArgs/"
+  @Test def neg_typers() = compileFile(negCustomArgs, "typers", negTest = true)(allowDoubleBindings)
+  @Test def neg_overrideClass = compileFile(negCustomArgs, "overrideClass", List("-language:Scala2"), negTest = true)
+  @Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, negTest = true)
+  @Test def neg_i1050 = compileFile(negCustomArgs, "i1050", List("-strict"), negTest = true)
 
   val negTailcallDir = negDir + "tailcall/"
-  @Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", xerrors = 5)
-  @Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", xerrors = 1)
-  @Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", xerrors = 2)
-  @Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", xerrors = 7)
-  @Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", xerrors = 2)
-  @Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", xerrors = 2)
-
-  @Test def neg_t1843_variances = compileFile(negDir, "t1843-variances", xerrors = 1)
-  @Test def neg_t2660_ambi = compileFile(negDir, "t2660", xerrors = 2)
-  @Test def neg_t2994 = compileFile(negDir, "t2994", xerrors = 2)
-  @Test def neg_subtyping = compileFile(negDir, "subtyping", xerrors = 5)
-  @Test def neg_variances = compileFile(negDir, "variances", xerrors = 2)
-  @Test def neg_variancesConstr = compileFile(negDir, "variances-constr", xerrors = 2)
-  @Test def neg_i871_missingReturnType = compileFile(negDir, "i871", xerrors = 2)
-  @Test def neg_badAuxConstr = compileFile(negDir, "badAuxConstr", xerrors = 2)
-  @Test def neg_typetest = compileFile(negDir, "typetest", xerrors = 1)
-  @Test def neg_t1569_failedAvoid = compileFile(negDir, "t1569-failedAvoid", xerrors = 1)
-  @Test def neg_clashes = compileFile(negDir, "clashes", xerrors = 2)
-  @Test def neg_cycles = compileFile(negDir, "cycles", xerrors = 7)
-  @Test def neg_boundspropagation = compileFile(negDir, "boundspropagation", xerrors = 5)
-  @Test def neg_refinedSubtyping = compileFile(negDir, "refinedSubtyping", xerrors = 2)
-  @Test def neg_hklower = compileFile(negDir, "hklower", xerrors = 4)
-  @Test def neg_Iter2 = compileFile(negDir, "Iter2", xerrors = 2)
-  @Test def neg_i0091_infpaths = compileFile(negDir, "i0091-infpaths", xerrors = 3)
-  @Test def neg_i0248_inherit_refined = compileFile(negDir, "i0248-inherit-refined", xerrors = 4)
-  @Test def neg_i0281 = compileFile(negDir, "i0281-null-primitive-conforms", xerrors = 3)
-  @Test def neg_i324 = compileFile(negDir, "i324", xerrors = 2)
-  @Test def neg_i583 = compileFile(negDir, "i0583-skolemize", xerrors = 2)
-  @Test def neg_i941 = compileFile(negDir, "i941", xerrors = 3)
-  @Test def neg_finalSealed = compileFile(negDir, "final-sealed", xerrors = 2)
-  @Test def neg_i645 = compileFile(negDir, "amp", xerrors = 4)
-  @Test def neg_i705 = compileFile(negDir, "i705-inner-value-class", xerrors = 7)
-  @Test def neg_i803 = compileFile(negDir, "i803", xerrors = 2)
-  @Test def neg_i866 = compileFile(negDir, "i866", xerrors = 2)
-  @Test def neg_i974 = compileFile(negDir, "i974", xerrors = 2)
-  @Test def neg_i997 = compileFile(negDir, "i997", xerrors = 15)
-  @Test def neg_i997a = compileFile(negDir, "i997a", xerrors = 2)
-  @Test def neg_i1050 = compileFile(negDir, "i1050", List("-strict"), xerrors = 11)
-  @Test def neg_i1050a = compileFile(negDir, "i1050a", xerrors = 2)
-  @Test def neg_i1050c = compileFile(negDir, "i1050c", xerrors = 8)
-  @Test def neg_moduleSubtyping = compileFile(negDir, "moduleSubtyping", xerrors = 4)
-  @Test def neg_escapingRefs = compileFile(negDir, "escapingRefs", xerrors = 2)
-  @Test def neg_instantiateAbstract = compileFile(negDir, "instantiateAbstract", xerrors = 8)
-  @Test def neg_partialApplications = compileFile(negDir, "partialApplications", xerrors = 3)
-  @Test def neg_selfInheritance = compileFile(negDir, "selfInheritance", xerrors = 6)
-  @Test def neg_selfreq = compileFile(negDir, "selfreq", xerrors = 2)
-  @Test def neg_singletons = compileFile(negDir, "singletons", xerrors = 8)
-  @Test def neg_ski = compileFile(negDir, "ski", xerrors = 12)
-  @Test def neg_traitParamsTyper = compileFile(negDir, "traitParamsTyper", xerrors = 5)
-  @Test def neg_traitParamsMixin = compileFile(negDir, "traitParamsMixin", xerrors = 2)
-  @Test def neg_firstError = compileFile(negDir, "firstError", xerrors = 3)
-  @Test def neg_implicitLowerBound = compileFile(negDir, "implicit-lower-bound", xerrors = 1)
-  @Test def neg_validate = compileFile(negDir, "validate", xerrors = 18)
-  @Test def neg_validateParsing = compileFile(negDir, "validate-parsing", xerrors = 7)
-  @Test def neg_validateRefchecks = compileFile(negDir, "validate-refchecks", xerrors = 2)
-  @Test def neg_skolemize = compileFile(negDir, "skolemize", xerrors = 2)
-  @Test def neg_nested_bounds = compileFile(negDir, "nested_bounds", xerrors = 1)
-  @Test def neg_valueClasses = compileFile(negDir, "valueClasses", xerrors = 2)
+  @Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", negTest = true)
+  @Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", negTest = true)
+  @Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", negTest = true)
+  @Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", negTest = true)
+  @Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", negTest = true)
+  @Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", negTest = true)
 
   @Test def run_all = runFiles(runDir)
 
diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala
index b8cd88f2d..367c36d59 100644
--- a/test/test/CompilerTest.scala
+++ b/test/test/CompilerTest.scala
@@ -43,9 +43,9 @@ abstract class CompilerTest {
   val defaultOutputDir: String
 
   /** Override to filter out tests that should not be run by partest. */
-  def partestableFile(prefix: String, fileName: String, extension: String, args: List[String], xerrors: Int) = true
-  def partestableDir(prefix: String, dirName: String, args: List[String], xerrors: Int) = true
-  def partestableList(testName: String, files: List[String], args: List[String], xerrors: Int) = true
+  def partestableFile(prefix: String, fileName: String, extension: String, args: List[String]) = true
+  def partestableDir(prefix: String, dirName: String, args: List[String]) = true
+  def partestableList(testName: String, files: List[String], args: List[String]) = true
 
   val generatePartestFiles = {
     /* Because we fork in test, the JVM in which this JUnit test runs has a
@@ -68,10 +68,10 @@ abstract class CompilerTest {
   val logFile = if (!generatePartestFiles) None else Some(CompilerTest.init)
 
   /** Always run with JUnit. */
-  def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
+  def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = {
     if (generatePartestFiles)
       log("WARNING: compileLine will always run with JUnit, no partest files generated.")
-    compileArgs(cmdLine.split("\n"), xerrors)
+    compileArgs(cmdLine.split("\n"), Nil, negTest = false)
   }
 
   /** Compiles the given code file.
@@ -79,59 +79,72 @@ abstract class CompilerTest {
     * @param prefix    the parent directory (including separator at the end)
     * @param fileName  the filename, by default without extension
     * @param args      arguments to the compiler
-    * @param xerrors   if > 0, this test is a neg test with the expected number
+    * @param negTest   if negTest is true, this test is a neg test with the expected number
     *                  of compiler errors. Otherwise, this is a pos test.
     * @param extension the file extension, .scala by default
     * @param defaultOptions more arguments to the compiler
     */
-  def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0,
+  def compileFile(prefix: String, fileName: String, args: List[String] = Nil, negTest: Boolean = false,
       extension: String = ".scala", runTest: Boolean = false)
       (implicit defaultOptions: List[String]): Unit = {
-    if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions, xerrors)) {
+    val filePath = s"$prefix$fileName$extension"
+    val expErrors = expectedErrors(filePath, negTest)
+    if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions)) {
       if (runTest)
         log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension")
-      compileArgs((s"$prefix$fileName$extension" :: args).toArray, xerrors)
+      compileArgs((s"$filePath" :: args).toArray, expErrors, negTest)
     } else {
-      val kind = testKind(prefix, xerrors, runTest)
+      val kind = testKind(prefix, negTest, runTest)
       log(s"generating partest files for test file: $prefix$fileName$extension of kind $kind")
 
       val sourceFile = new JFile(prefix + fileName + extension)
       if (sourceFile.exists) {
         val firstDest = SFile(DPConfig.testRoot + JFile.separator + kind + JFile.separator + fileName + extension)
+        val xerrors = if (negTest) expErrors.map(_.totalErrors).sum else 0
         computeDestAndCopyFiles(sourceFile, firstDest, kind, args ++ defaultOptions, xerrors.toString)
       } else {
         throw new java.io.FileNotFoundException(s"Unable to locate test file $prefix$fileName")
       }
     }
   }
-  def runFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0,
-      extension: String = ".scala")(implicit defaultOptions: List[String]): Unit =
-    compileFile(prefix, fileName, args, xerrors, extension, true)
+  def runFile(prefix: String, fileName: String, args: List[String] = Nil, negTest: Boolean = false,
+      extension: String = ".scala")(implicit defaultOptions: List[String]): Unit = {
+    compileFile(prefix, fileName, args, negTest, extension, true)
+  }
 
   /** Compiles the code files in the given directory together. If args starts
     * with "-deep", all files in subdirectories (and so on) are included. */
-  def compileDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0, runTest: Boolean = false)
+  def compileDir(prefix: String, dirName: String, args: List[String] = Nil, negTest: Boolean = false, runTest: Boolean = false)
       (implicit defaultOptions: List[String]): Unit = {
-    if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions, xerrors)) {
-      if (runTest)
-        log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName")
+    val computeFilePathsAndExpErrors = { () =>
       val dir = Directory(prefix + dirName)
       val (files, normArgs) = args match {
         case "-deep" :: args1 => (dir.deepFiles, args1)
         case _ => (dir.files, args)
       }
-      val fileNames = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
-      compileArgs(fileNames ++ normArgs, xerrors)
+      val filePaths = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
+      val expErrors = expectedErrors(filePaths.toList, negTest)
+      (filePaths, normArgs, expErrors)
+    }
+    if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions)) {
+      if (runTest)
+        log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName")
+      val (filePaths, normArgs, expErrors) = computeFilePathsAndExpErrors()
+      compileArgs(filePaths ++ normArgs, expErrors, negTest)
     } else {
       val (sourceDir, flags, deep) = args match {
         case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep")
         case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow")
       }
-      val kind = testKind(prefix, xerrors, runTest)
+      val kind = testKind(prefix, negTest, runTest)
       log(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind")
 
       if (sourceDir.exists) {
         val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName)
+        val xerrors = if (negTest) {
+          val (_, _, expErrors) = computeFilePathsAndExpErrors()
+          expErrors.map(_.totalErrors).sum
+        } else 0
         computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString)
         if (deep == "deep")
           Directory(sourceDir).deleteRecursively
@@ -140,54 +153,69 @@ abstract class CompilerTest {
       }
     }
   }
-  def runDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0)
+  def runDir(prefix: String, dirName: String, args: List[String] = Nil, negTest: Boolean = false)
       (implicit defaultOptions: List[String]): Unit =
-    compileDir(prefix, dirName, args, xerrors, true)
+    compileDir(prefix, dirName, args, negTest, true)
 
   /** Compiles each source in the directory path separately by calling
     * compileFile resp. compileDir. */
-  def compileFiles(path: String, args: List[String] = Nil, verbose: Boolean = true, runTest: Boolean = false)
-      (implicit defaultOptions: List[String]): Unit = {
+  def compileFiles(path: String, args: List[String] = Nil, verbose: Boolean = true, runTest: Boolean = false,
+                   negTest: Boolean = false, compileSubDirs: Boolean = true)(implicit defaultOptions: List[String]): Unit = {
     val dir = Directory(path)
     val fileNames = dir.files.toArray.map(_.jfile.getName).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
     for (name <- fileNames) {
       if (verbose) log(s"testing $path$name")
-      compileFile(path, name, args, 0, "", runTest)
-    }
-    for (subdir <- dir.dirs) {
-      if (verbose) log(s"testing $subdir")
-      compileDir(path, subdir.jfile.getName, args, 0, runTest)
+      compileFile(path, name, args, negTest, "", runTest)
     }
+    if (compileSubDirs)
+      for (subdir <- dir.dirs) {
+        if (verbose) log(s"testing $subdir")
+        compileDir(path, subdir.jfile.getName, args, negTest, runTest)
+      }
   }
   def runFiles(path: String, args: List[String] = Nil, verbose: Boolean = true)
       (implicit defaultOptions: List[String]): Unit =
     compileFiles(path, args, verbose, true)
 
   /** Compiles the given list of code files. */
-  def compileList(testName: String, files: List[String], args: List[String] = Nil, xerrors: Int = 0)
+  def compileList(testName: String, files: List[String], args: List[String] = Nil, negTest: Boolean = false)
       (implicit defaultOptions: List[String]): Unit = {
-    if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions, xerrors)) {
-      compileArgs((files ++ args).toArray, xerrors)
+    if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions)) {
+      val expErrors = expectedErrors(files, negTest)
+      compileArgs((files ++ args).toArray, expErrors, negTest)
     } else {
       val destDir = Directory(DPConfig.testRoot + JFile.separator + testName)
       files.foreach({ file =>
         val jfile = new JFile(file)
         recCopyFiles(jfile, destDir / jfile.getName)
       })
-      compileDir(DPConfig.testRoot + JFile.separator, testName, args, xerrors)
+      compileDir(DPConfig.testRoot + JFile.separator, testName, args, negTest)
       destDir.deleteRecursively
     }
   }
 
   // ========== HELPERS =============
 
-  private def compileArgs(args: Array[String], xerrors: Int = 0)
+  private def expectedErrors(filePaths: List[String], negTest: Boolean): List[ErrorsInFile] = {
+    if (negTest) {
+      filePaths.map(getErrors(_))
+    } else Nil
+  }
+
+  private def expectedErrors(filePath: String, negTest: Boolean): List[ErrorsInFile] = expectedErrors(List(filePath), negTest)
+
+  private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile], negTest: Boolean)
       (implicit defaultOptions: List[String]): Unit = {
     val allArgs = args ++ defaultOptions
     val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
     val reporter = processor.process(allArgs)
 
     val nerrors = reporter.errorCount
+    val xerrors = (expectedErrorsPerFile map {_.totalErrors}).sum
+    if (negTest)
+      assert((negTest && xerrors > 0), s"No errors found in neg test")
+    else
+      assert((!negTest && xerrors == 0), s"There shouldn't be expected errors in non-neg test")
     assert(nerrors == xerrors, s"Wrong # of errors. Expected: $xerrors, found: $nerrors")
 
     // NEG TEST
@@ -196,11 +224,6 @@ abstract class CompilerTest {
       // reporter didn't record as many errors as its errorCount says
       assert(errorLines.length == nerrors, s"Not enough errors recorded.")
 
-      val allFiles = (allArgs filter {
-          arg => !arg.startsWith("-") && (arg.endsWith(".scala") || arg.endsWith(".java"))
-      }).toList
-      val expectedErrorsPerFile = allFiles.map(getErrors(_))
-
       // Some compiler errors have an associated source position. Each error
       // needs to correspond to a "// error" marker on that line in the source
       // file and vice versa.
@@ -233,7 +256,9 @@ abstract class CompilerTest {
   /** Captures the number of nopos-errors in the given file and the number of
     * errors with a position, represented as a tuple of source line and number
     * of errors on that line. */
-  case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)])
+  case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)]) {
+    def totalErrors = noposErrorNr + posErrorLinesToNr.map(_._2).sum
+  }
 
   /** Extracts the errors expected for the given neg test file. */
   def getErrors(fileName: String): ErrorsInFile = {
@@ -319,9 +344,9 @@ abstract class CompilerTest {
   private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")
 
   /** Determines what kind of test to run. */
-  private def testKind(prefixDir: String, xerrors: Int, runTest: Boolean) = {
+  private def testKind(prefixDir: String, negTest: Boolean, runTest: Boolean) = {
     if (runTest) "run"
-    else if (xerrors > 0) "neg"
+    else if (negTest) "neg"
     else if (prefixDir.endsWith("run" + JFile.separator)) {
       log("WARNING: test is being run as pos test despite being in a run directory. " +
         "Use runFile/runDir instead of compileFile/compileDir to do a run test")