Added support for res, script and shootout tests.

8 files changed, 381 insertions, 137 deletions

diff --git a/src/partest/scala/tools/partest/nest/AntRunner.scala b/src/partest/scala/tools/partest/nest/AntRunner.scala
index 0285e5a17f..621cc0dd44 100644
--- a/src/partest/scala/tools/partest/nest/AntRunner.scala
+++ b/src/partest/scala/tools/partest/nest/AntRunner.scala
@@ -23,6 +23,7 @@ class AntRunner extends DirectRunner {
     var JAVACMD: String = "java"
 
     var CLASSPATH: String = _
+    var EXT_CLASSPATH: String = _
     var LATEST_LIB: String = _
 
   }
diff --git a/src/partest/scala/tools/partest/nest/ConsoleFileManager.scala b/src/partest/scala/tools/partest/nest/ConsoleFileManager.scala
index a0052d92ff..3fc38ea09f 100644
--- a/src/partest/scala/tools/partest/nest/ConsoleFileManager.scala
+++ b/src/partest/scala/tools/partest/nest/ConsoleFileManager.scala
@@ -40,13 +40,14 @@ else
     testroot.getAbsolutePath
   }
 
-  CLASSPATH += File.pathSeparator + {
+  var EXT_CLASSPATH = {
     val libs = new File(TESTROOT, "files/lib")
     // add all jars in libs to EXT_CLASSPATH
     (libs.listFiles(new FilenameFilter {
       def accept(dir: File, name: String) = name endsWith ".jar"
     }) map {file => file.getCanonicalFile.getAbsolutePath}).mkString(""+File.pathSeparator)
   }
+
   def findLatest() {
     def prefixFile(relPath: String): File =
       (new File(PREFIX, relPath)).getCanonicalFile
@@ -82,8 +83,6 @@ else
     else
       error("Scala binaries could not be found")
 
-    println("latestPartestFile="+latestPartestFile)
-
     BIN_DIR = latestFile.getAbsolutePath
     LATEST_LIB = latestLibFile.getAbsolutePath
     LATEST_COMP = latestCompFile.getAbsolutePath
diff --git a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
index 294d62b8a1..e16f81e114 100644
--- a/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
+++ b/src/partest/scala/tools/partest/nest/ConsoleRunner.scala
@@ -26,6 +26,9 @@ class ConsoleRunner extends DirectRunner {
   private var jvmCheck = false
   private var jvm5Check = false
   private var resCheck = false
+  private var shootoutCheck = false
+  private var scriptCheck = false
+
   private var runAll = false
 
   private var testFiles: List[File] = List()
@@ -37,13 +40,15 @@ class ConsoleRunner extends DirectRunner {
 
   def denotesTestSet(arg: String) =
     arg match {
-      case "--pos"  => true
-      case "--neg"  => true
-      case "--run"  => true
-      case "--jvm"  => true
-      case "--jvm5" => true
-      case "--res"  => true
-      case _        => false
+      case "--pos"      => true
+      case "--neg"      => true
+      case "--run"      => true
+      case "--jvm"      => true
+      case "--jvm5"     => true
+      case "--res"      => true
+      case "--shootout" => true
+      case "--script"   => true
+      case _            => false
     }
 
   def main(argstr: String) {
@@ -57,15 +62,20 @@ class ConsoleRunner extends DirectRunner {
     if (args.length == 0)
       NestUI.usage()
     else {
-      if (!args.exists(denotesTestSet(_))) runAll = true
+      if (!args.exists(denotesTestSet(_)) && !args.exists(_.endsWith(".scala"))) runAll = true
       for (arg <- args) {
         arg match {
+          case "--all"          => runAll = true
+
           case "--pos"          => posCheck = true
           case "--neg"          => negCheck = true
           case "--run"          => runCheck = true
           case "--jvm"          => jvmCheck = true
           case "--jvm5"         => jvm5Check = true
           case "--res"          => resCheck = true
+          case "--shootout"     => shootoutCheck = true
+          case "--script"       => scriptCheck = true
+
           case "--verbose"      => NestUI._verbose = true
           case "--show-diff"    => fileManager.showDiff = true
           case "--show-log"     => fileManager.showLog = true
@@ -185,14 +195,18 @@ class ConsoleRunner extends DirectRunner {
       runCheck = true
       jvmCheck = true
       jvm5Check = true
-      //resCheck = true
+      resCheck = true
+      shootoutCheck = true
+      scriptCheck = true
     }
     val results = List(runTests("pos", posCheck, "Testing compiler (on files whose compilation should succeed)"),
                        runTests("neg", negCheck, "Testing compiler (on files whose compilation should fail)"),
                        runTests("run", runCheck, "Testing JVM backend"),
                        runTests("jvm", jvmCheck, "Testing JVM backend"),
                        runTests("jvm5", jvm5Check, "Testing JVM backend"),
-                       runTests("res", resCheck, "Testing resident compiler"))
+                       runTests("res", resCheck, "Testing resident compiler"),
+                       runTests("shootout", shootoutCheck, "Testing shootout tests"),
+                       runTests("script", scriptCheck, "Testing script tests"))
     results reduceLeft { (p: (Int, Int), q: (Int, Int)) =>
       (p._1+q._1, p._2+q._2) }
   }
diff --git a/src/partest/scala/tools/partest/nest/DirectRunner.scala b/src/partest/scala/tools/partest/nest/DirectRunner.scala
index d82b82b072..4e5124ca71 100644
--- a/src/partest/scala/tools/partest/nest/DirectRunner.scala
+++ b/src/partest/scala/tools/partest/nest/DirectRunner.scala
@@ -17,7 +17,7 @@ trait DirectRunner {
 
   def fileManager: FileManager
 
-  protected val numActors = Integer.parseInt(System.getProperty("scalatest.actors", "8"))
+  private val numActors = Integer.parseInt(System.getProperty("scalatest.actors", "8"))
 
   def runTestsForFiles(kindFiles: List[File], kind: String): (Int, Int) = {
     val len = kindFiles.length
@@ -46,11 +46,11 @@ trait DirectRunner {
       }
     }
     logsToDelete.foreach { log =>
-      NestUI.verbose("deleting "+log+"\n")
+      NestUI.verbose("deleting "+log)
       fileManager.deleteRecursive(log)
     }
     outdirsToDelete.foreach { outdir =>
-      NestUI.verbose("deleting "+outdir+"\n")
+      NestUI.verbose("deleting "+outdir)
       fileManager.deleteRecursive(outdir)
     }
 
diff --git a/src/partest/scala/tools/partest/nest/FileManager.scala b/src/partest/scala/tools/partest/nest/FileManager.scala
index f83882d078..8a814ea54b 100644
--- a/src/partest/scala/tools/partest/nest/FileManager.scala
+++ b/src/partest/scala/tools/partest/nest/FileManager.scala
@@ -47,6 +47,7 @@ trait FileManager {
   var JAVACMD: String
 
   var CLASSPATH: String
+  var EXT_CLASSPATH: String
   var LATEST_LIB: String
 
   var showDiff = false
diff --git a/src/partest/scala/tools/partest/nest/StreamAppender.scala b/src/partest/scala/tools/partest/nest/StreamAppender.scala
index 768d312398..ffaa2dee53 100644
--- a/src/partest/scala/tools/partest/nest/StreamAppender.scala
+++ b/src/partest/scala/tools/partest/nest/StreamAppender.scala
@@ -1,25 +1,35 @@
 /* NEST (New Scala Test)
+ * Copyright 2007-2008 LAMP/EPFL
  * @author Philipp Haller
  */
 
+// $Id$
+
 package scala.tools.partest.nest
 
 import java.io.{Writer, PrintWriter, Reader, BufferedReader,
-                IOException, InputStream, StringWriter, InputStreamReader}
+                IOException, InputStream, StringWriter, InputStreamReader,
+                OutputStreamWriter, StringReader, OutputStream}
 
 object StreamAppender {
-  def appenderToWriter(reader: BufferedReader, writer: Writer) = {
+  def apply(reader: BufferedReader, writer: Writer) = {
     val pwriter = new PrintWriter(writer, true)
     new StreamAppender(reader, pwriter)
   }
 
+  def apply(reader: Reader, writer: Writer) = {
+    val bufReader = new BufferedReader(reader)
+    val pwriter = new PrintWriter(writer, true)
+    new StreamAppender(bufReader, pwriter)
+  }
+
   def appendToString(in1: InputStream, in2: InputStream): String = {
     val swriter1 = new StringWriter
     val swriter2 = new StringWriter
     val reader1 = new BufferedReader(new InputStreamReader(in1))
     val reader2 = new BufferedReader(new InputStreamReader(in2))
-    val app1 = appenderToWriter(reader1, swriter1)
-    val app2 = appenderToWriter(reader2, swriter2)
+    val app1 = StreamAppender(reader1, swriter1)
+    val app2 = StreamAppender(reader2, swriter2)
 
     val async = new Thread(app2)
     async.start()
@@ -27,6 +37,40 @@ object StreamAppender {
     async.join()
     swriter1.toString + swriter2.toString
   }
+
+  def inParallel(t1: Runnable, t2: Runnable, t3: Runnable) {
+    val thr1 = new Thread(t1)
+    val thr2 = new Thread(t2)
+    thr1.start()
+    thr2.start()
+    t3.run()
+    thr1.join()
+    thr2.join()
+  }
+
+  def inParallel(t1: Runnable, t2: Runnable) {
+    val thr = new Thread(t2)
+    thr.start()
+    t1.run()
+    thr.join()
+  }
+
+  def concat(in: InputStream, err: InputStream, out: OutputStream) = new Runnable {
+    override def run() {
+      val outWriter = new PrintWriter(new OutputStreamWriter(out), true)
+      val inApp = new StreamAppender(new BufferedReader(new InputStreamReader(in)),
+                                     outWriter)
+      val errStringWriter = new StringWriter
+      val errApp = StreamAppender(new BufferedReader(new InputStreamReader(err)),
+                                  errStringWriter)
+      inParallel(inApp, errApp)
+
+      // append error string to out
+      val errStrApp = new StreamAppender(new BufferedReader(new StringReader(errStringWriter.toString)),
+                                         outWriter)
+      errStrApp.run()
+    }
+  }
 }
 
 class StreamAppender(reader: BufferedReader, writer: PrintWriter) extends Runnable {
diff --git a/src/partest/scala/tools/partest/nest/TestFile.scala b/src/partest/scala/tools/partest/nest/TestFile.scala
index 7edfa4dabc..94812e6c1f 100644
--- a/src/partest/scala/tools/partest/nest/TestFile.scala
+++ b/src/partest/scala/tools/partest/nest/TestFile.scala
@@ -3,7 +3,7 @@
  * @author Philipp Haller
  */
 
-// $Id: $
+// $Id$
 
 package scala.tools.partest.nest
 
@@ -77,21 +77,21 @@ case class RunTestFile(override val file: File, override val fileManager: FileMa
 }
 
 case class JvmTestFile(override val file: File, override val fileManager: FileManager) extends TestFile("jvm", file, fileManager) {
-  import fileManager.CLASSPATH
+  import fileManager.{CLASSPATH, EXT_CLASSPATH}
 
   override def defineSettings(settings: Settings) {
     baseSettings(settings)
-    settings.classpath.value = CLASSPATH
+    settings.classpath.value = CLASSPATH+File.pathSeparatorChar+EXT_CLASSPATH
     //println("settings.classpath.value="+settings.classpath.value)
   }
 }
 
 case class Jvm5TestFile(override val file: File, override val fileManager: FileManager) extends TestFile("jvm5", file, fileManager) {
-  import fileManager.CLASSPATH
+  import fileManager.{CLASSPATH, EXT_CLASSPATH}
 
   override def defineSettings(settings: Settings) {
     baseSettings(settings)
-    settings.classpath.value = CLASSPATH
+    settings.classpath.value = CLASSPATH+File.pathSeparatorChar+EXT_CLASSPATH
     settings.target.value = "jvm-1.5"
     //println("settings.classpath.value="+settings.classpath.value)
   }
@@ -99,9 +99,11 @@ case class Jvm5TestFile(override val file: File, override val fileManager: FileM
 
 case class ShootoutTestFile(override val file: File, override val fileManager: FileManager) extends TestFile("shootout", file, fileManager) {
   import fileManager.CLASSPATH
+
   override def defineSettings(settings: Settings) {
     baseSettings(settings)
     settings.classpath.value = CLASSPATH
-    //println("CLASSPATH="+settings.classpath.value)
+    //println("settings.classpath.value="+settings.classpath.value)
+    settings.outdir.value = file.getParent
   }
 }
diff --git a/src/partest/scala/tools/partest/nest/Worker.scala b/src/partest/scala/tools/partest/nest/Worker.scala
index 1faa21f0eb..84f8212084 100644
--- a/src/partest/scala/tools/partest/nest/Worker.scala
+++ b/src/partest/scala/tools/partest/nest/Worker.scala
@@ -24,6 +24,17 @@ class LogFile(parent: File, child: String) extends File(parent, child) {
 
 class Worker(val fileManager: FileManager) extends Actor {
   import fileManager._
+  import scala.tools.nsc.{Settings, CompilerCommand, Global}
+  import scala.tools.nsc.reporters.ConsoleReporter
+  import scala.tools.nsc.util.FakePos
+
+  var reporter: ConsoleReporter = _
+
+  def error(msg: String) {
+    reporter.error(FakePos("scalac"),
+                   msg + "\n  scalac -help  gives more information")
+  }
+
   def act() {
     react {
       case RunTests(kind, files) =>
@@ -111,14 +122,15 @@ class Worker(val fileManager: FileManager) extends Actor {
   def execTest(outDir: File, logFile: File) {
     val cmd =
       JAVACMD+
-      " -classpath "+outDir+File.pathSeparatorChar+CLASSPATH+
+      " -classpath "+outDir+File.pathSeparatorChar+CLASSPATH+File.pathSeparatorChar+EXT_CLASSPATH+
       " -Djava.library.path="+logFile.getParentFile.getAbsolutePath+
       " -Dscalatest.output="+outDir.getAbsolutePath+
       " -Dscalatest.lib="+LATEST_LIB+
+      " -Dscalatest.cwd="+outDir.getParent+
       " -Djavacmd="+JAVACMD+
       " scala.tools.nsc.MainGenericRunner"+
       " Test jvm"
-    //NestUI.verbose(cmd)
+    NestUI.verbose(cmd)
 
     val proc = Runtime.getRuntime.exec(cmd)
     val in = proc.getInputStream
@@ -132,9 +144,6 @@ class Worker(val fileManager: FileManager) extends Actor {
     async.start()
     inApp.run()
     async.join()
-
-    //proc.waitFor()
-
     writer.close()
 
     /*val execution = Runtime.getRuntime.exec(cmd)
@@ -204,7 +213,6 @@ class Worker(val fileManager: FileManager) extends Actor {
         printInfoStart(file, wr)
 
         try { // *catch-all*
-
           if (!compileMgr.shouldCompile(file, kind)) {
             NestUI.verbose("compilation of "+file+" failed\n")
             success = false
@@ -225,10 +233,6 @@ class Worker(val fileManager: FileManager) extends Actor {
               NestUI.verbose("output differs from log file\n")
               success = false
             }
-
-            // delete log file only if test was successful
-            if (success)
-              logFile.toDelete = true
           } // successful compile
         } catch { // *catch-all*
           case e: Exception =>
@@ -236,6 +240,10 @@ class Worker(val fileManager: FileManager) extends Actor {
             success = false
         }
 
+        // delete log file only if test was successful
+        if (success)
+          logFile.toDelete = true
+
         if (!success) {
           errors += 1
           NestUI.verbose("incremented errors: "+errors)
@@ -294,14 +302,16 @@ class Worker(val fileManager: FileManager) extends Actor {
               val dir = file.getParentFile
               val outDir = createOutputDir(dir, fileBase, kind)
 
-              if (!compileMgr.shouldCompile(file, kind)) {
+              if (!compileMgr.shouldCompile(file, kind))
                 succeeded = false
-                errors += 1
-              }
             } catch {
               case e: Exception =>
                 succeeded = false
-                errors += 1
+            }
+
+            if (!succeeded) {
+              errors += 1
+              NestUI.verbose("incremented errors: "+errors)
             }
           }
         }
@@ -333,46 +343,38 @@ class Worker(val fileManager: FileManager) extends Actor {
             try {
               if (!compileMgr.shouldFailCompile(file, kind, logFile)) {
                 succeeded = false
-                errors += 1
               } else { // compare log file to check file
                 val fileBase: String = basename(file.getName)
                 val dir = file.getParentFile
                 val outDir = createOutputDir(dir, fileBase, kind)
 
-                NestUI.verbose("comparing output with check file...")
                 diff = compareOutput(dir, fileBase, kind, logFile)
                 if (!diff.equals("")) {
                   NestUI.verbose("output differs from log file\n")
                   succeeded = false
-                  errors += 1
                 }
-
-                // delete log file only if test was successful
-                if (succeeded)
-                  logFile.toDelete = true
               }
             } catch {
               case e: Exception =>
                 NestUI.verbose("caught "+e)
                 succeeded = false
-                errors += 1
             }
 
+            // delete log file only if test was successful
+            if (succeeded)
+              logFile.toDelete = true
+
+            if (!succeeded) {
+              errors += 1
+              NestUI.verbose("incremented errors: "+errors)
+            }
             printInfoEnd(succeeded, wr)
             wr.flush()
             swr.flush()
             NestUI.normal(swr.toString)
 
             if (!succeeded && fileManager.showDiff) NestUI.normal(diff)
-            if (!succeeded && fileManager.showLog) {
-              // output log file
-              val logReader = new BufferedReader(new FileReader(logFile))
-              val logWriter = new PrintWriter(new StringWriter, true)
-              val logAppender = new StreamAppender(logReader, logWriter)
-              logAppender.run()
-              val log = logWriter.toString
-              NestUI.normal(log)
-            }
+            if (!succeeded && fileManager.showLog) showLog(logFile)
           }
         }
         NestUI.verbose("finished testing "+kind+" with "+errors+" errors")
@@ -390,95 +392,276 @@ class Worker(val fileManager: FileManager) extends Actor {
       }
       case "res" => {
         for (file <- files) {
-          val swr = new StringWriter
-          val wr = new PrintWriter(swr)
-          succeeded = true; diff = ""; log = ""
-          printInfoStart(file, wr)
-
-          val fileBase: String = basename(file.getName)
-          NestUI.verbose(this+" running test "+fileBase)
-          val dir = file.getParentFile
-          val outDir = createOutputDir(dir, fileBase, kind)
-          if (!outDir.exists) outDir.mkdir()
-          val logFile = createLogFile(dir, fileBase, kind)
-          val resFile = new File(dir, fileBase + ".res")
-
-          // run scalac in resident mode:
-          // $SCALAC -d "$os_dstbase".obj -Xresident -sourcepath . "$@"
-          val cmd =
-            JAVACMD+
-            " -classpath "+outDir+File.pathSeparator+CLASSPATH+
-            " -Djavacmd="+JAVACMD+
-            " scala.tools.nsc.Main"+
-            " -d "+outDir.getCanonicalFile.getAbsolutePath+
-            " -Xresident"+
-            " -sourcepath "+logFile.getParentFile.getCanonicalFile.getAbsolutePath
-          NestUI.verbose(cmd)
-
-          val proc = Runtime.getRuntime.exec(cmd, null, dir)
-
-          val in = proc.getInputStream
-          val err = proc.getErrorStream
-          val out = proc.getOutputStream
-
-          val writer = new PrintWriter(new FileWriter(logFile), true)
-          val reader = new BufferedReader(new FileReader(resFile))
-          val errWriter = new PrintWriter(new StringWriter, true)
-
-          val inApp = new StreamAppender(new BufferedReader(new InputStreamReader(in)),
-                                         writer)
-          val errApp = new StreamAppender(new BufferedReader(new InputStreamReader(err)),
-                                          errWriter)
-          val outApp = new StreamAppender(reader, new PrintWriter(new OutputStreamWriter(out), true))
-
-          val inThr = new Thread(inApp)
-          val errThr = new Thread(errApp)
-          inThr.start()
-          errThr.start()
-          outApp.run()
-
-          val exitCode = proc.waitFor()
-          NestUI.verbose("finished with exit code "+exitCode)
-
-          inThr.join()
-          errThr.join()
-
-          //writer.close()
-          //reader.close()
-
-          // compare log file with check file
-          diff = compareOutput(dir, fileBase, kind, logFile)
-          if (!diff.equals("")) {
-            NestUI.verbose("output differs from log file\n")
-            succeeded = false
-            errors += 1
-          }
+          // when option "--failed" is provided
+          // execute test only if log file is present
+          // (which means it failed before)
 
-          if (!succeeded && fileManager.showDiff) NestUI.normal(diff)
-          if (!succeeded && fileManager.showLog) {
-            // output log file
-            val logReader = new BufferedReader(new FileReader(logFile))
-            val logWriter = new PrintWriter(new StringWriter, true)
-            val logAppender = new StreamAppender(logReader, logWriter)
-            logAppender.run()
-            //TODO: close logReader
-            val log = logWriter.toString
-            NestUI.normal(log)
+          //val (logFileOut, logFileErr) = createLogFiles(file, kind)
+          val logFile = createLogFile(file, kind)
+          if (!fileManager.failed || (logFile.exists && logFile.canRead)) {
+            val swr = new StringWriter
+            val wr = new PrintWriter(swr)
+            succeeded = true; diff = ""; log = ""
+            printInfoStart(file, wr)
+
+            val fileBase: String = basename(file.getName)
+            NestUI.verbose(this+" running test "+fileBase)
+            val dir = file.getParentFile
+            val outDir = createOutputDir(dir, fileBase, kind)
+            if (!outDir.exists) outDir.mkdir()
+            val resFile = new File(dir, fileBase + ".res")
+            NestUI.verbose("outDir:  "+outDir)
+            NestUI.verbose("logFile: "+logFile)
+            //NestUI.verbose("logFileErr: "+logFileErr)
+            NestUI.verbose("resFile: "+resFile)
+
+            // run compiler in resident mode
+            // $SCALAC -d "$os_dstbase".obj -Xresident -sourcepath . "$@"
+
+            val sourcedir  = logFile.getParentFile.getCanonicalFile
+            val sourcepath = sourcedir.getAbsolutePath+"/"
+            NestUI.verbose("sourcepath: "+sourcepath)
+
+            val argString =
+              "-d "+outDir.getCanonicalFile.getAbsolutePath+
+              " -Xresident"+
+              " -sourcepath "+sourcepath
+            val argList = List.fromString(argString, ' ')
+
+            // configure input/output files
+            val logOut    = new FileOutputStream(logFile)
+            val logWriter = new PrintStream(logOut)
+            val resReader = new BufferedReader(new FileReader(resFile))
+            val logConsoleWriter = new PrintWriter(new OutputStreamWriter(logOut))
+
+            // create compiler
+            val settings = new Settings(error)
+            settings.sourcepath.value = sourcepath
+            reporter = new ConsoleReporter(settings, Console.in, logConsoleWriter)
+            val command = new CompilerCommand(argList, settings, error, false)
+            object compiler extends Global(command.settings, reporter)
+
+            // simulate resident compiler loop
+            val prompt = "\nnsc> "
+
+            val resCompile = (line: String) => {
+              NestUI.verbose("compiling "+line)
+              val cmdArgs = List.fromString(line, ' ')
+              val sett = new Settings(error)
+              sett.sourcepath.value = sourcepath
+              val command = new CompilerCommand(cmdArgs, sett, error, true)
+              (new compiler.Run) compile command.files
+            }
+
+            def loop(action: (String) => Unit) {
+              logWriter.print(prompt)
+              val line = resReader.readLine()
+              if ((line ne null) && line.length() > 0) {
+                action(line)
+                loop(action)
+              }
+            }
+            val oldStdOut = System.out
+            val oldStdErr = System.err
+            System.setOut(logWriter)
+            System.setErr(logWriter)
+            loop(resCompile)
+            resReader.close()
+            logWriter.flush()
+            logWriter.close()
+
+            System.setOut(oldStdOut)
+            System.setErr(oldStdErr)
+
+            diff = compareOutput(dir, fileBase, kind, logFile)
+            if (!diff.equals("")) {
+              NestUI.verbose("output differs from log file\n")
+              succeeded = false
+            }
+
+            // delete log file only if test was successful
+            if (succeeded)
+              logFile.toDelete = true
+
+            if (!succeeded) {
+              errors += 1
+              NestUI.verbose("incremented errors: "+errors)
+            }
+            printInfoEnd(succeeded, wr)
+            wr.flush()
+            swr.flush()
+            NestUI.normal(swr.toString)
+
+            if (!succeeded && fileManager.showDiff) NestUI.normal(diff)
+            if (!succeeded && fileManager.showLog) showLog(logFile)
           }
+        }
+        NestUI.verbose("finished testing "+kind+" with "+errors+" errors")
+        NestUI.verbose("created "+compileMgr.numSeparateCompilers+" separate compilers")
+        (files.length-errors, errors)
+      }
+      case "shootout" => {
+        for (file <- files) {
+          // when option "--failed" is provided
+          // execute test only if log file is present
+          // (which means it failed before)
+          val logFile = createLogFile(file, kind)
+          if (!fileManager.failed || (logFile.exists && logFile.canRead)) {
+            val swr = new StringWriter
+            val wr = new PrintWriter(swr)
+            succeeded = true; diff = ""; log = ""
+            printInfoStart(file, wr)
 
-          // delete log file only if test was successful
-          if (succeeded)
-            logFile.toDelete = true
+            val fileBase: String = basename(file.getName)
+            NestUI.verbose(this+" running test "+fileBase)
+            val dir = file.getParentFile
+            val outDir = createOutputDir(dir, fileBase, kind)
+            if (!outDir.exists) outDir.mkdir()
+
+            // 2. define file {outDir}/test.scala that contains code to compile/run
+            val testFile = new File(outDir, "test.scala")
+            NestUI.verbose("outDir:   "+outDir)
+            NestUI.verbose("logFile:  "+logFile)
+            NestUI.verbose("testFile: "+testFile)
+
+            // 3. cat {test}.scala.runner {test}.scala > testFile
+            val runnerFile = new File(dir, fileBase+".scala.runner")
+            val bodyFile   = new File(dir, fileBase+".scala")
+            val appender = StreamAppender.concat(new FileInputStream(runnerFile),
+                                                 new FileInputStream(bodyFile),
+                                                 new FileOutputStream(testFile))
+            appender.run()
+
+            try { // *catch-all*
+              // 4. compile testFile
+              if (!compileMgr.shouldCompile(testFile, kind)) {
+                NestUI.verbose("compilation of "+file+" failed\n")
+                succeeded = false
+              } else {
+                NestUI.verbose("compilation of "+testFile+"succeeded")
+                // -------- run test --------
+
+                //TODO: detect whether we have to use Runtime.exec
+                val useRuntime = true
+
+                if (useRuntime)
+                  execTest(outDir, logFile)
+                else
+                  execTestObjectRunner(file, outDir, logFile)
+                NestUI.verbose(this+" finished running "+fileBase)
+              } // successful compile
+            } catch { // *catch-all*
+              case e: Exception =>
+                NestUI.verbose("caught "+e)
+                succeeded = false
+            }
 
-          printInfoEnd(succeeded, wr)
-          wr.flush()
-          swr.flush()
-          NestUI.normal(swr.toString)
+            diff = compareOutput(dir, fileBase, kind, logFile)
+            if (!diff.equals("")) {
+              NestUI.verbose("output differs from log file\n")
+              succeeded = false
+            }
+
+            // delete log file only if test was successful
+            if (succeeded)
+              logFile.toDelete = true
+
+            if (!succeeded) {
+              errors += 1
+              NestUI.verbose("incremented errors: "+errors)
+            }
+            printInfoEnd(succeeded, wr)
+            wr.flush()
+            swr.flush()
+            NestUI.normal(swr.toString)
+
+            if (!succeeded && fileManager.showDiff) NestUI.normal(diff)
+            if (!succeeded && fileManager.showLog) showLog(logFile)
+          }
         }
         NestUI.verbose("finished testing "+kind+" with "+errors+" errors")
         NestUI.verbose("created "+compileMgr.numSeparateCompilers+" separate compilers")
         (files.length-errors, errors)
       }
+      case "script" => {
+        for (file <- files) {
+          // when option "--failed" is provided
+          // execute test only if log file is present
+          // (which means it failed before)
+          val logFile = createLogFile(file, kind)
+          if (!fileManager.failed || (logFile.exists && logFile.canRead)) {
+            val swr = new StringWriter
+            val wr = new PrintWriter(swr)
+            succeeded = true; diff = ""; log = ""
+            printInfoStart(file, wr)
+
+            val fileBase: String = basename(file.getName)
+            NestUI.verbose(this+" running test "+fileBase)
+
+            // check whether there is an args file
+            val argsFile = new File(file.getParentFile, fileBase+".args")
+            NestUI.verbose("argsFile: "+argsFile)
+            val argString = if (argsFile.exists) {
+              val swriter = new StringWriter
+              val app = StreamAppender(new BufferedReader(new FileReader(argsFile)),
+                                       swriter)
+              app.run()
+              " "+swriter.toString
+            } else ""
+
+            val proc = Runtime.getRuntime.exec(file.getAbsolutePath+argString)
+            val in = proc.getInputStream
+            val err = proc.getErrorStream
+            val writer = new PrintWriter(new FileWriter(logFile), true)
+            val inApp = new StreamAppender(new BufferedReader(new InputStreamReader(in)),
+                                           writer)
+            val errApp = new StreamAppender(new BufferedReader(new InputStreamReader(err)),
+                                            writer)
+            val async = new Thread(errApp)
+            async.start()
+            inApp.run()
+            async.join()
+
+            writer.close()
+
+            diff = compareOutput(file.getParentFile, fileBase, kind, logFile)
+            if (!diff.equals("")) {
+              NestUI.verbose("output differs from log file\n")
+              succeeded = false
+            }
+
+            // delete log file only if test was successful
+            if (succeeded)
+              logFile.toDelete = true
+
+            if (!succeeded) {
+              errors += 1
+              NestUI.verbose("incremented errors: "+errors)
+            }
+            printInfoEnd(succeeded, wr)
+            wr.flush()
+            swr.flush()
+            NestUI.normal(swr.toString)
+
+            if (!succeeded && fileManager.showDiff) NestUI.normal(diff)
+            if (!succeeded && fileManager.showLog) showLog(logFile)
+          }
+        }
+      }
+      NestUI.verbose("finished testing "+kind+" with "+errors+" errors")
+      NestUI.verbose("created "+compileMgr.numSeparateCompilers+" separate compilers")
+      (files.length-errors, errors)
     }
   }
+
+  def showLog(logFile: File) {
+    val logReader = new BufferedReader(new FileReader(logFile))
+    val strWriter = new StringWriter
+    val logWriter = new PrintWriter(strWriter, true)
+    val logAppender = new StreamAppender(logReader, logWriter)
+    logAppender.run()
+    logReader.close()
+    val log = strWriter.toString
+    NestUI.normal(log)
+  }
 }