[split] Squashed commit of the following:

commit faf3d257e7dc95d1492b7c8c6faf9da3c53adac6
Merge: d2ecdbd 53fdb61
Author: mmcbride <mccv@twitter.com>
Date:   Mon Aug 8 22:05:09 2011 -0700

    Merge branch 'master' into persistent_eval

commit d2ecdbddd288607cd1fb597ba04685628ad8b2f0
Author: mmcbride <mccv@twitter.com>
Date:   Mon Aug 8 21:40:32 2011 -0700

    two minor things from jeremy cloud

commit a17260bd1cd90bab0ac635e0257a9bea8e9da62f
Merge: 5f26ee0 e8d277e
Author: mmcbride <mccv@twitter.com>
Date:   Wed Aug 3 20:16:06 2011 -0700

    Merge branch 'master' into persistent_eval

commit 5f26ee0544ad39439c509e8b87e60c4cff0f07b9
Author: mmcbride <mccv@twitter.com>
Date:   Wed Aug 3 20:13:33 2011 -0700

    feedback from RB

commit 775c8661f485724af3be51be720b821b83b044ec
Merge: 350c313 f7d1db2
Author: mmcbride <mccv@twitter.com>
Date:   Tue Jul 26 12:41:00 2011 -0700

    Merge branch 'master' into persistent_eval

commit 350c313991b31f9e332753a62ff5b92c61ce7cae
Author: mmcbride <mccv@twitter.com>
Date:   Tue Jul 26 10:58:25 2011 -0700

    persistent compilation targets for eval

1 files changed, 147 insertions, 35 deletions

diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala
index e1da151..8f05ed1 100644
--- a/libraries/eval/Eval.scala
+++ b/libraries/eval/Eval.scala
@@ -27,10 +27,12 @@ import scala.collection.mutable
 import scala.io.Source
 import scala.tools.nsc.{Global, Settings}
 import scala.tools.nsc.interpreter.AbstractFileClassLoader
-import scala.tools.nsc.io.VirtualDirectory
+import scala.tools.nsc.io.{AbstractFile, VirtualDirectory}
 import scala.tools.nsc.reporters.AbstractReporter
 import scala.tools.nsc.util.{BatchSourceFile, Position}
 
+case class LastMod(timestamp: Long, code: String)
+
 /**
  * Evaluate a file or string and return the result.
  */
@@ -39,7 +41,35 @@ object Eval extends Eval {
   private val jvmId = java.lang.Math.abs(new Random().nextInt())
 }
 
-class Eval {
+/**
+ * evaluates files, strings or input streams, and returns the result.
+ * In all cases, code to be evaled is wrapped in an apply method in a
+ * generated class. An instance of the class is instantiated, and the
+ * result of apply is returned.
+ *
+ * If target is None, the results are compiled to memory (and are therefore
+ * ephemeral). If target is Some(path), path must point to a directory, and
+ * eval emits class files to that directory.
+ *
+ * eval also supports a limited set of preprocessors. Limited means
+ * exactly one, that supports directives of the form #include <file>.
+ *
+ * The general flow of evaluation is
+ * # convert arguments to a string
+ * # run preprocessors on that string
+ * # wrap processed code in a class
+ * # compile the class
+ * # create an instance of that class
+ * # return the results of apply()
+ */
+class Eval(target: Option[File]) {
+  /**
+   * empty constructor for backwards compatibility
+   */
+  def this() {
+    this(None)
+  }
+
   import Eval.jvmId
 
   private lazy val compilerPath = try {
@@ -57,9 +87,13 @@ class Eval {
   }
 
   /**
-   * Preprocessors to run the code through before it is passed to the Scala compiler
+   * Preprocessors to run the code through before it is passed to the Scala compiler.
+   * if you want to add new resolvers, you can do so with
+   * new Eval(...) {
+   *   lazy val preprocessors = {...}
+   * }
    */
-  private lazy val preprocessors: Seq[Preprocessor] =
+  protected lazy val preprocessors: Seq[Preprocessor] =
     Seq(
       new IncludePreprocessor(
         Seq(
@@ -70,45 +104,95 @@ class Eval {
       )
     )
 
-  private lazy val compiler = new StringCompiler(2, preprocessors)
+  private lazy val compiler = new StringCompiler(2, target)
+
+  /**
+   * run preprocessors on our string, returning a LastMod
+   * where timestamp is the last modified time of any file in that contributed
+   * to the text.
+   * Last modified is computed here because we support includes
+   */
+  def sourceForString(code: String) = {
+    preprocessors.foldLeft(LastMod(0L, code)) { (acc, p) =>
+      val processed = p(acc.code)
+      LastMod(acc.timestamp max processed.timestamp, processed.code)
+    }
+  }
 
   /**
    * Eval[Int]("1 + 1") // => 2
    */
   def apply[T](code: String, resetState: Boolean = true): T = {
-    val id = uniqueId(code)
-    val className = "Evaluator__" + id
-    val cls = compiler(wrapCodeInClass(className, code), className, id, resetState)
-    cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T]
+    val processed = sourceForString(code)
+    applyProcessed(processed.code, resetState)
   }
 
   /**
    * Eval[Int](new File("..."))
    */
   def apply[T](files: File*): T = {
-    apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"))
+    if (target.isDefined) {
+      val targetDir = target.get
+      val unprocessedSource = files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n")
+      val processed = sourceForString(unprocessedSource)
+      val oldestTarget = targetDir.listFiles.foldLeft(Long.MaxValue)((oldest, f) => f.lastModified min oldest)
+      if (processed.timestamp > oldestTarget) {
+        compiler.reset()
+      }
+      val className = "Evaluator__" + files(0).getName.split("\\.")(0)
+      applyProcessed(className, processed.code, false)
+    } else {
+      apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"), true)
+    }
   }
 
   /**
    * Eval[Int](getClass.getResourceAsStream("..."))
    */
   def apply[T](stream: InputStream): T = {
-    apply(Source.fromInputStream(stream).mkString)
+    apply(sourceForString(Source.fromInputStream(stream).mkString).code)
+  }
+
+  /**
+   * same as apply[T], but does not run preprocessors.
+   * Will generate a classname of the form Evaluater__<unique>,
+   * where unique is computed from the jvmID (a random number)
+   * and a digest of code
+   */
+  def applyProcessed[T](code: String, resetState: Boolean): T = {
+    val id = uniqueId(code)
+    val className = "Evaluator__" + id
+    applyProcessed(className, code, resetState)
   }
 
+  /**
+   * same as apply[T], but does not run preprocessors.
+   */
+  def applyProcessed[T](className: String, code: String, resetState: Boolean): T = {
+    val cls = compiler(wrapCodeInClass(className, code), className, resetState)
+    cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T]
+  }
+
+  /**
+   * converts the given file to evaluable source.
+   * delegates to toSource(code: String)
+   */
   def toSource(file: File): String = {
     toSource(scala.io.Source.fromFile(file).mkString)
   }
 
+  /**
+   * converts the given file to evaluable source.
+   */
   def toSource(code: String): String = {
-    compiler.sourceForString(code)
+    sourceForString(code).code
   }
 
   /**
    * Compile an entire source file into the virtual classloader.
    */
   def compile(code: String) {
-    compiler(code)
+    compiler(sourceForString(code).code)
   }
 
   /**
@@ -124,7 +208,7 @@ class Eval {
    * @throw CompilerException if not Eval-able.
    */
   def check(code: String) {
-    val id = uniqueId(code)
+    val id = uniqueId(sourceForString(code).code)
     val className = "Evaluator__" + id
     val wrappedCode = wrapCodeInClass(className, code)
     compile(wrappedCode) // may throw CompilerException
@@ -203,12 +287,13 @@ class Eval {
   }
 
   trait Preprocessor {
-    def apply(code: String): String
+    def apply(code: String): LastMod
   }
 
   trait Resolver {
     def resolvable(path: String): Boolean
     def get(path: String): InputStream
+    def lastModified(path: String): Long
   }
 
   class FilesystemResolver(root: File) extends Resolver {
@@ -219,6 +304,14 @@ class Eval {
     def resolvable(path: String): Boolean =
       file(path).exists
 
+    def lastModified(path: String): Long = {
+      if (resolvable(path)) {
+        file(path).lastModified
+      } else {
+        0
+      }
+    }
+
     def get(path: String): InputStream =
       new FileInputStream(file(path))
   }
@@ -230,6 +323,8 @@ class Eval {
     def resolvable(path: String): Boolean =
       clazz.getResourceAsStream(quotePath(path)) != null
 
+    def lastModified(path: String): Long = 0
+
     def get(path: String): InputStream =
       clazz.getResourceAsStream(quotePath(path))
   }
@@ -245,40 +340,51 @@ class Eval {
    * @example #include file-name.scala
    *
    * This is the only directive supported by this preprocessor.
+   *
+   * Note that it is *not* recursive. Included files cannot have includes
    */
-  private class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
-    def apply(code: String): String =
-      code.lines map { line: String =>
+  class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
+    def apply(code: String): LastMod = {
+      var lastMod = 0L
+      val lines = code.lines map { line: String =>
         val tokens = line.trim.split(' ')
         if (tokens.length == 2 && tokens(0).equals("#include")) {
           val path = tokens(1)
           resolvers find { resolver: Resolver =>
             resolver.resolvable(path)
           } match {
-            case Some(r: Resolver) =>
+            case Some(r: Resolver) => {
+              lastMod = lastMod max r.lastModified(path)
               StreamIO.buffer(r.get(path)).toString
+            }
             case _ =>
               throw new IllegalStateException("No resolver could find '%s'".format(path))
           }
         } else {
           line
         }
-      } mkString("\n")
+      }
+      val processed = lines.mkString("\n")
+      LastMod(lastMod, processed)
+    }
   }
 
   /**
    * Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep
    * around one of these and reuse it.
    */
-  private class StringCompiler(lineOffset: Int, preprocessors: Seq[Preprocessor]) {
-    val virtualDirectory = new VirtualDirectory("(memory)", None)
+  private class StringCompiler(lineOffset: Int, targetDir: Option[File]) {
+    val target = targetDir match {
+      case Some(dir) => AbstractFile.getDirectory(dir)
+      case None => new VirtualDirectory("(memory)", None)
+    }
 
     val cache = new mutable.HashMap[String, Class[_]]()
 
     val settings = new Settings
     settings.deprecation.value = true // enable detailed deprecation warnings
     settings.unchecked.value = true // enable detailed unchecked warnings
-    settings.outputDirs.setSingleOutput(virtualDirectory)
+    settings.outputDirs.setSingleOutput(target)
 
     val pathList = compilerPath ::: libPath
     settings.bootclasspath.value = pathList.mkString(File.pathSeparator)
@@ -321,12 +427,24 @@ class Eval {
      * Class loader for finding classes compiled by this StringCompiler.
      * After each reset, this class loader will not be able to find old compiled classes.
      */
-    var classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader)
+    var classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader)
 
     def reset() {
-      virtualDirectory.clear
+      targetDir match {
+        case None => {
+          target.asInstanceOf[VirtualDirectory].clear
+        }
+        case Some(t) => {
+          target.foreach { abstractFile =>
+            if (abstractFile.file == null || abstractFile.file.getName.endsWith(".class")) {
+              abstractFile.delete
+            }
+          }
+        }
+      }
+      cache.clear()
       reporter.reset
-      classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader)
+      classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader)
     }
 
     object Debug {
@@ -363,14 +481,12 @@ class Eval {
      * Compile scala code. It can be found using the above class loader.
      */
     def apply(code: String) {
-      val processedCode = sourceForString(code)
-
       if (Debug.enabled)
-        Debug.printWithLineNumbers(processedCode)
+        Debug.printWithLineNumbers(code)
 
       // if you're looking for the performance hit, it's 1/2 this line...
       val compiler = new global.Run
-      val sourceFiles = List(new BatchSourceFile("(inline)", processedCode))
+      val sourceFiles = List(new BatchSourceFile("(inline)", code))
       // ...and 1/2 this line:
       compiler.compileSources(sourceFiles)
 
@@ -379,14 +495,10 @@ class Eval {
       }
     }
 
-    def sourceForString(code: String) = {
-      preprocessors.foldLeft(code) { case (c: String, p: Preprocessor) => p(c) }
-    }
-
     /**
      * Compile a new class, load it, and return it. Thread-safe.
      */
-    def apply(code: String, className: String, id: String, resetState: Boolean = true): Class[_] = {
+    def apply(code: String, className: String, resetState: Boolean = true): Class[_] = {
       synchronized {
         if (resetState) reset()
         findClass(className).getOrElse {