aboutsummaryrefslogtreecommitdiff
path: root/compiler/test/dotty/partest/DPConsoleRunner.scala
blob: 3362d7a59343e5071ed2b3d430a9690cd63ab603 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
/* NOTE: Adapted from ScalaJSPartest.scala in
 * https://github.com/scala-js/scala-js/
 * TODO make partest configurable */

package dotty.partest

import dotty.tools.FatalError
import scala.reflect.io.AbstractFile
import scala.tools.partest._
import scala.tools.partest.nest._
import TestState.{ Pass, Fail, Crash, Uninitialized, Updated }
import ClassPath.{ join, split }
import FileManager.{ compareFiles, compareContents, joinPaths, withTempFile }
import scala.util.matching.Regex
import tools.nsc.io.{ File => NSCFile }
import java.io.{ File, PrintStream, FileOutputStream, PrintWriter, FileWriter }
import java.net.URLClassLoader

/** Runs dotty partest from the Console, discovering test sources in
  * DPConfig.testRoot that have been generated automatically by
  * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are
  * required by some run tests, add them to partestDeps in the sbt Build.scala.
  */
object DPConsoleRunner {
  def main(args: Array[String]): Unit = {
    // unfortunately sbt runTask passes args as single string
    // extra jars for run tests are passed with -dottyJars <count> <jar1> <jar2> ...
    val jarFinder = """-dottyJars (\d*) (.*)""".r
    val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined)
    val (extraJars, moreArgs) = jarList match {
      case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars <jarCount> <jars>*\".")
      case jarFinder(nr, jarString) :: Nil =>
        val jars = jarString.split(" ").toList
        val count = nr.toInt
        if (jars.length < count)
          sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr)
        else (jars.take(count), jars.drop(count))
      case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list)
    }
    new DPConsoleRunner((otherArgs ::: moreArgs) mkString (" "), extraJars).runPartest
  }
}

// console runner has a suite runner which creates a test runner for each test
class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) {
  override val suiteRunner = new DPSuiteRunner (
    testSourcePath = optSourcePath getOrElse DPConfig.testRoot,
    fileManager = new DottyFileManager(extraJars),
    updateCheck = optUpdateCheck,
    failed = optFailed,
    consoleArgs = args)

  override def run = {}
  def runPartest = super.run
}

class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) {
  lazy val extraJarList = extraJars.map(NSCFile(_))
  override lazy val libraryUnderTest  = Path(extraJars.find(_.contains("scala-library")).getOrElse(""))
  override lazy val reflectUnderTest  = Path(extraJars.find(_.contains("scala-reflect")).getOrElse(""))
  override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse(""))
}

class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending"
  fileManager: DottyFileManager,
  updateCheck: Boolean,
  failed: Boolean,
  consoleArgs: String,
  javaCmdPath: String = PartestDefaults.javaCmd,
  javacCmdPath: String = PartestDefaults.javacCmd,
  scalacExtraArgs: Seq[String] = Seq.empty,
  javaOpts: String = DPConfig.runJVMOpts)
extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs, javaOpts) {

  if (!DPConfig.runTestsInParallel)
    sys.props("partest.threads") = "1"

  sys.props("partest.root") = "."

  // override to provide Dotty banner
  override def banner: String = {
    s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString}
        |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main
        |Generated test sources: ${PathSettings.srcDir}${File.separator}
        |Test directories: ${DPConfig.testDirs.toList.mkString(", ")}
        |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj
        |Parallel: ${DPConfig.runTestsInParallel}
        |Options: (use partest --help for usage information) ${consoleArgs}
    """.stripMargin
  }

  /** Some tests require a limitation of resources, tests which are compiled
   *  with one or more of the flags in this list will be run with
   *  `limitedThreads`. This is necessary because some test flags require a lot
   *  of memory when running the compiler and may exhaust the available memory
   *  when run in parallel with too many other tests.
   *
   *  This number could be increased on the CI, but might fail locally if
   *  scaled too extreme - override with:
   *
   *  ```
   *  -Ddotty.tests.limitedThreads=X
   *  ```
   */
  def limitResourceFlags = List("-Ytest-pickler")
  private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2")

  override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = {
    val (limitResourceTests, parallelTests) =
      kindFiles partition { kindFile =>
        val flags = kindFile.changeExtension("flags").fileContents
        limitResourceFlags.exists(seqFlag => flags.contains(seqFlag))
      }

    val seqResults =
      if (!limitResourceTests.isEmpty) {
        val savedThreads = sys.props("partest.threads")
        sys.props("partest.threads") = {
          assert(
            savedThreads == null || limitedThreads.toInt <= savedThreads.toInt,
            """|Should not use more threads than the default, when the point
               |is to limit the amount of resources""".stripMargin
          )
          limitedThreads
        }

        NestUI.echo(s"## we will run ${limitResourceTests.length} tests using ${PartestDefaults.numThreads} thread(s) in parallel")
        val res = super.runTestsForFiles(limitResourceTests, kind)

        if (savedThreads != null)
          sys.props("partest.threads") = savedThreads
        else
          sys.props.remove("partest.threads")

        res
      } else Array[TestState]()

    val parResults =
      if (!parallelTests.isEmpty) {
        NestUI.echo(s"## we will run ${parallelTests.length} tests in parallel using ${PartestDefaults.numThreads} thread(s)")
        super.runTestsForFiles(parallelTests, kind)
      } else Array[TestState]()

    seqResults ++ parResults
  }

  // override for DPTestRunner and redirecting compilation output to test.clog
  override def runTest(testFile: File): TestState = {
    val runner = new DPTestRunner(testFile, this)

    val state =
      try {
        runner.run match {
          // Append compiler output to transcript if compilation failed,
          // printed with --verbose option
          case TestState.Fail(f, r@"compilation failed", transcript) =>
            TestState.Fail(f, r, transcript ++ runner.cLogFile.fileLines.dropWhile(_ == ""))
          case res => res
        }
      } catch {
        case t: Throwable => throw new RuntimeException(s"Error running $testFile", t)
      }
    reportTest(state)
    runner.cleanup()

    onFinishTest(testFile, state)
  }

  // override NestUI.reportTest because --show-diff doesn't work. The diff used
  // seems to add each line to transcript separately, whereas NestUI assumes
  // that the diff string was added as one entry in the transcript
  def reportTest(state: TestState) = {
    import NestUI._
    import NestUI.color._

    if (isTerse && state.isOk) {
      NestUI.reportTest(state)
    } else {
      echo(statusLine(state))
      if (!state.isOk && isDiffy) {
        val differ = bold(red("% ")) + "diff "
        state.transcript.dropWhile(s => !(s startsWith differ)) foreach (echo(_))
        // state.transcript find (_ startsWith differ) foreach (echo(_)) // original
      }
    }
  }
}

class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runner(testFile, suiteRunner) {
  val cLogFile = SFile(logFile).changeExtension("clog")

  // override to provide DottyCompiler
  override def newCompiler = new dotty.partest.DPDirectCompiler(this)

  // Adapted from nest.Runner#javac because:
  // - Our classpath handling is different and we need to pass extraClassPath
  //   to java to get the scala-library which is required for some java tests
  // - The compiler output should be redirected to cLogFile, like the output of
  //   dotty itself
  override def javac(files: List[File]): TestState = {
    import fileManager._
    import suiteRunner._
    import FileManager.joinPaths
    // compile using command-line javac compiler
    val args = Seq(
      suiteRunner.javacCmdPath, // FIXME: Dotty deviation just writing "javacCmdPath" doesn't work
      "-d",
      outDir.getAbsolutePath,
      "-classpath",
      joinPaths(outDir :: extraClasspath ++ testClassPath)
    ) ++ files.map(_.getAbsolutePath)

    pushTranscript(args mkString " ")

    val captured = StreamCapture(runCommand(args, cLogFile))
    if (captured.result) genPass() else {
      cLogFile appendAll captured.stderr
      cLogFile appendAll captured.stdout
      genFail("java compilation failed")
    }
  }

  // Overriden in order to recursively get all sources that should be handed to
  // the compiler. Otherwise only sources in the top dir is compiled - works
  // because the compiler is on the classpath.
  override def sources(file: File): List[File] =
    if (file.isDirectory)
      file.listFiles.toList.flatMap { f =>
        if (f.isDirectory) sources(f)
        else if (f.isJavaOrScala) List(f)
        else Nil
      }
    else List(file)

  // Enable me to "fix" the depth issue - remove once completed
  //override def compilationRounds(file: File): List[CompileRound] = {
  //  val srcs = sources(file) match {
  //    case Nil =>
  //      System.err.println {
  //        s"""|================================================================================
  //            |Warning! You attempted to compile sources from:
  //            |  $file
  //            |but partest was unable to find any sources - uncomment DPConsoleRunner#sources
  //            |================================================================================""".stripMargin
  //      }
  //      List(new File("./tests/pos/HelloWorld.scala")) // "just compile some crap" - Guillaume
  //    case xs =>
  //    xs
  //  }
  //  (groupedFiles(srcs) map mixedCompileGroup).flatten
  //}

  // FIXME: This is copy-pasted from nest.Runner where it is private
  // Remove this once https://github.com/scala/scala-partest/pull/61 is merged
  /** Runs command redirecting standard out and
   *  error out to output file.
   */
  def runCommand(args: Seq[String], outFile: File): Boolean = {
    import scala.sys.process.{ Process, ProcessLogger }
    //(Process(args) #> outFile !) == 0 or (Process(args) ! pl) == 0
    val pl = ProcessLogger(outFile)
    val nonzero = 17     // rounding down from 17.3
    def run: Int = {
      val p = Process(args) run pl
      try p.exitValue
      catch {
        case e: InterruptedException =>
          NestUI verbose s"Interrupted waiting for command to finish (${args mkString " "})"
          p.destroy
          nonzero
        case t: Throwable =>
          NestUI verbose s"Exception waiting for command to finish: $t (${args mkString " "})"
          p.destroy
          throw t
      }
      finally pl.close()
    }
    (pl buffer run) == 0
  }

  // override to provide default dotty flags from file in directory
  override def flagsForCompilation(sources: List[File]): List[String] = {
    val specificFlags = super.flagsForCompilation(sources)
    if (specificFlags.isEmpty) defaultFlags
    else specificFlags
  }

  val defaultFlags = {
    val defaultFile = parentFile.listFiles.toList.find(_.getName == "__defaultFlags.flags")
    defaultFile.map({ file =>
      SFile(file).safeSlurp.map({ content => words(content).filter(_.nonEmpty) }).getOrElse(Nil)
    }).getOrElse(Nil)
  }

  // override to add the check for nr of compilation errors if there's a
  // target.nerr file
  override def runNegTest() = runInContext {
    sealed abstract class NegTestState
    // Don't get confused, the neg test passes when compilation fails for at
    // least one round (optionally checking the number of compiler errors and
    // compiler console output)
    case object CompFailed extends NegTestState
    // the neg test fails when all rounds return either of these:
    case class CompFailedButWrongNErr(expected: String, found: String) extends NegTestState
    case object CompFailedButWrongDiff extends NegTestState
    case object CompSucceeded extends NegTestState

    def nerrIsOk(reason: String) = {
      val nerrFinder = """compilation failed with (\d+) errors""".r
      reason match {
        case nerrFinder(found) =>
          SFile(FileOps(testFile) changeExtension "nerr").safeSlurp match {
            case Some(exp) if (exp != found) => CompFailedButWrongNErr(exp, found)
            case _ => CompFailed
          }
        case _ => CompFailed
      }
    }

    // we keep the partest semantics where only one round needs to fail
    // compilation, not all
    val compFailingRounds =
      compilationRounds(testFile)
      .map { round =>
        val ok = round.isOk
        setLastState(if (ok) genPass else genFail("compilation failed"))
        (round.result, ok)
      }
      .filter { case (_, ok) => !ok }

    val failureStates = compFailingRounds.map({ case (result, _) => result match {
      // or, OK, we'll let you crash the compiler with a FatalError if you supply a check file
      case Crash(_, t, _) if !checkFile.canRead || !t.isInstanceOf[FatalError] => CompSucceeded
      case Fail(_, reason, _) => if (diffIsOk) nerrIsOk(reason) else CompFailedButWrongDiff
      case _ => if (diffIsOk) CompFailed else CompFailedButWrongDiff
    }})

    if (failureStates.exists({ case CompFailed => true; case _ => false })) {
      true
    } else {
      val existsNerr = failureStates.exists({
        case CompFailedButWrongNErr(exp, found) =>
          nextTestActionFailing(s"wrong number of compilation errors, expected: $exp, found: $found")
          true
        case _ =>
          false
      })

      if (existsNerr) false
      else {
        val existsDiff = failureStates.exists({
          case CompFailedButWrongDiff =>
            nextTestActionFailing(s"output differs")
            true
          case _ =>
            false
        })
        if (existsDiff) false
        else nextTestActionFailing("expected compilation failure")
      }
    }
  }

  // override to change check file updating to original file, not generated
  override def diffIsOk: Boolean = {
    // always normalize the log first
    normalizeLog()
    val diff = currentDiff
    // if diff is not empty, is update needed?
    val updating: Option[Boolean] = (
      if (diff == "") None
      else Some(suiteRunner.updateCheck)
    )
    pushTranscript(s"diff $logFile $checkFile")
    nextTestAction(updating) {
      case Some(true) =>
        val origCheck = SFile(checkFile.changeExtension("checksrc").fileLines(1))
        NestUI.echo("Updating original checkfile " + origCheck)
        origCheck writeAll file2String(logFile)
        genUpdated()
      case Some(false) =>
        // Get a word-highlighted diff from git if we can find it
        val bestDiff = if (updating.isEmpty) "" else {
          if (checkFile.canRead)
            gitDiff(logFile, checkFile) getOrElse {
              s"diff $logFile $checkFile\n$diff"
            }
          else diff
        }
        pushTranscript(bestDiff)
        genFail("output differs")
      case None => genPass()  // redundant default case
    } getOrElse true
  }

  // override to add dotty and scala jars to classpath
  override def extraClasspath =
    suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath


  // FIXME: Dotty deviation: error if return type is omitted:
  //   overriding method cleanup in class Runner of type ()Unit;
  //    method cleanup of type => Boolean | Unit has incompatible type

  // override to keep class files if failed and delete clog if ok
  override def cleanup: Unit = if (lastState.isOk) {
    logFile.delete
    cLogFile.delete
    Directory(outDir).deleteRecursively
  }
}