[split] recompile configs based on hash instead of timestamp

2 files changed, 63 insertions, 71 deletions

diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala
index d353338..d2188cb 100644
--- a/libraries/eval/Eval.scala
+++ b/libraries/eval/Eval.scala
@@ -18,7 +18,7 @@ package com.twitter.util
 
 import com.twitter.io.StreamIO
 import com.twitter.conversions.string._
-import java.io.{File, InputStream, FileInputStream, FileNotFoundException}
+import java.io._
 import java.math.BigInteger
 import java.net.URLClassLoader
 import java.security.MessageDigest
@@ -33,8 +33,6 @@ import scala.tools.nsc.reporters.AbstractReporter
 import scala.tools.nsc.util.{BatchSourceFile, Position}
 import scala.util.matching.Regex
 
-case class LastMod(timestamp: Option[Long], code: String)
-
 /**
  * Evaluate a file or string and return the result.
  */
@@ -114,52 +112,50 @@ class Eval(target: Option[File]) {
   private lazy val compiler = new StringCompiler(2, target)
 
   /**
-   * run preprocessors on our string, returning a LastMod
-   * where timestamp is the last modified time of any file in that contributed
-   * to the text.
-   * Last modified is computed here because we support includes
+   * run preprocessors on our string, returning a String that is the processed source
    */
-  def sourceForString(code: String, lastModified: Option[Long]): LastMod = {
-    preprocessors.foldLeft(LastMod(lastModified, code)) { (acc, p) =>
-      val processed = p(acc.code, lastModified)
-
-      // timestamp of the newest processed file.
-      // if both are defined, take the max. otherwise
-      // take any defined timestamp
-      val newestProcessed = Seq(processed.timestamp, acc.timestamp).max
-      LastMod(newestProcessed, processed.code)
+  def sourceForString(code: String): String = {
+    preprocessors.foldLeft(code) { (acc, p) =>
+      p(acc)
     }
   }
 
   /**
-   * Eval[Int]("1 + 1") // => 2
+   * write the current checksum to a file
+   */
+  def writeChecksum(checksum: String, file: File) {
+    val writer = new FileWriter(file)
+    writer.write("%s".format(checksum))
+    writer.close
+  }
+
+  /**
+   * val i: Int = new Eval()("1 + 1") // => 2
    */
   def apply[T](code: String, resetState: Boolean = true): T = {
-    val processed = sourceForString(code, None)
-    applyProcessed(processed.code, resetState)
+    val processed = sourceForString(code)
+    applyProcessed(processed, resetState)
   }
 
   /**
-   * Eval[Int](new File("..."))
+   * val i: Int = new Eval()(new File("..."))
    */
   def apply[T](files: File*): T = {
     if (target.isDefined) {
       val targetDir = target.get
       val unprocessedSource = files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n")
-      val lastModified = files.foldLeft(None: Option[Long]) { (acc, f) => Seq(acc, Some(f.lastModified)).max }
-      val processed = sourceForString(unprocessedSource, lastModified)
-      val oldestTarget = targetDir.listFiles.foldLeft(Long.MaxValue) { (oldest, f) =>
-        f.lastModified min oldest
+      val processed = sourceForString(unprocessedSource)
+      val sourceChecksum = uniqueId(processed, None)
+      val checksumFile = new File(targetDir, "checksum")
+      val lastChecksum = if (checksumFile.exists) {
+        Source.fromFile(checksumFile).getLines.take(1).toList.head
+      } else {
+        -1
       }
-      processed.timestamp match {
-        // if we got a last-modified-source timestamp threaded through, use it to check compiler resets
-        case Some(newestSource) => {
-          if (newestSource > oldestTarget) {
-            compiler.reset()
-          }
-        }
-        // if there are no timestamps anywhere, just reset the compiler
-        case None => compiler.reset()
+
+      if (lastChecksum != sourceChecksum) {
+        compiler.reset()
+        writeChecksum(sourceChecksum, checksumFile)
       }
 
       // why all this nonsense? Well.
@@ -169,18 +165,18 @@ class Eval(target: Option[File]) {
       // so, clean it hash it and slap it on the end of Evaluator
       val cleanBaseName = fileToClassName(files(0))
       val className = "Evaluator__%s_%s".format(
-        cleanBaseName, uniqueId(files(0).getCanonicalPath, None))
-      applyProcessed(className, processed.code, false)
+        cleanBaseName, sourceChecksum)
+      applyProcessed(className, processed, false)
     } else {
       apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"), true)
     }
   }
 
   /**
-   * Eval[Int](getClass.getResourceAsStream("..."))
+   * val i: Int = new Eval()(getClass.getResourceAsStream("..."))
    */
   def apply[T](stream: InputStream): T = {
-    apply(sourceForString(Source.fromInputStream(stream).mkString, None).code)
+    apply(sourceForString(Source.fromInputStream(stream).mkString))
   }
 
   /**
@@ -215,14 +211,14 @@ class Eval(target: Option[File]) {
    * converts the given file to evaluable source.
    */
   def toSource(code: String): String = {
-    sourceForString(code, None).code
+    sourceForString(code)
   }
 
   /**
    * Compile an entire source file into the virtual classloader.
    */
   def compile(code: String) {
-    compiler(sourceForString(code, None).code)
+    compiler(sourceForString(code))
   }
 
   /**
@@ -238,7 +234,7 @@ class Eval(target: Option[File]) {
    * @throws CompilerException if not Eval-able.
    */
   def check(code: String) {
-    val id = uniqueId(sourceForString(code, None).code)
+    val id = uniqueId(sourceForString(code))
     val className = "Evaluator__" + id
     val wrappedCode = wrapCodeInClass(className, code)
     compile(wrappedCode) // may throw CompilerException
@@ -332,13 +328,12 @@ class Eval(target: Option[File]) {
   }
 
   trait Preprocessor {
-    def apply(code: String, lastModified: Option[Long]): LastMod
+    def apply(code: String): String
   }
 
   trait Resolver {
     def resolvable(path: String): Boolean
     def get(path: String): InputStream
-    def lastModified(path: String): Long
   }
 
   class FilesystemResolver(root: File) extends Resolver {
@@ -348,14 +343,6 @@ class Eval(target: Option[File]) {
     def resolvable(path: String): Boolean =
       file(path).exists
 
-    def lastModified(path: String): Long = {
-      if (resolvable(path)) {
-        file(path).lastModified
-      } else {
-        0
-      }
-    }
-
     def get(path: String): InputStream =
       new FileInputStream(file(path))
   }
@@ -367,8 +354,6 @@ class Eval(target: Option[File]) {
     def resolvable(path: String): Boolean =
       clazz.getResourceAsStream(quotePath(path)) != null
 
-    def lastModified(path: String): Long = 0
-
     def get(path: String): InputStream =
       clazz.getResourceAsStream(quotePath(path))
   }
@@ -390,11 +375,10 @@ class Eval(target: Option[File]) {
   class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
     def maximumRecursionDepth = 100
 
-    def apply(code: String, lastModified: Option[Long]): LastMod =
-      apply(code, lastModified, maximumRecursionDepth)
+    def apply(code: String): String =
+      apply(code, maximumRecursionDepth)
 
-    def apply(code: String, lastModified: Option[Long], maxDepth: Int): LastMod = {
-      var lastMod = lastModified
+    def apply(code: String, maxDepth: Int): String = {
       val lines = code.lines map { line: String =>
         val tokens = line.trim.split(' ')
         if (tokens.length == 2 && tokens(0).equals("#include")) {
@@ -403,14 +387,11 @@ class Eval(target: Option[File]) {
             resolver.resolvable(path)
           } match {
             case Some(r: Resolver) => {
-              lastMod = Seq(lastMod, Some(r.lastModified(path))).max
               // recursively process includes
               if (maxDepth == 0) {
                 throw new IllegalStateException("Exceeded maximum recusion depth")
               } else {
-                val subLastMod = apply(StreamIO.buffer(r.get(path)).toString, lastMod, maxDepth - 1)
-                lastMod = Seq(lastMod, subLastMod.timestamp).max
-                subLastMod.code
+                apply(StreamIO.buffer(r.get(path)).toString, maxDepth - 1)
               }
             }
             case _ =>
@@ -420,8 +401,7 @@ class Eval(target: Option[File]) {
           line
         }
       }
-      val processed = lines.mkString("\n")
-      LastMod(lastMod, processed)
+      lines.mkString("\n")
     }
   }
 
diff --git a/libraries/eval/test/EvalTest.scala b/libraries/eval/test/EvalTest.scala
index 32855de..ef5ac6a 100644
--- a/libraries/eval/test/EvalTest.scala
+++ b/libraries/eval/test/EvalTest.scala
@@ -1,7 +1,8 @@
 package com.twitter.util
 
 import org.specs.Specification
-import java.io.{File, FileOutputStream}
+import java.io.{File, FileOutputStream, FileWriter}
+import scala.io.Source
 
 import com.twitter.io.TempFile
 
@@ -32,12 +33,12 @@ object EvalSpec extends Specification {
       val res: String = e(sourceFile)
       res mustEqual "hello"
       val className = e.fileToClassName(sourceFile)
+      val processedSource = e.sourceForString(Source.fromFile(sourceFile).getLines.mkString("\n"))
       val fullClassName = "Evaluator__%s_%s.class".format(
-        className, e.uniqueId(sourceFile.getCanonicalPath, None))
+        className, e.uniqueId(processedSource, None))
       val targetFileName = f.getAbsolutePath() + File.separator + fullClassName
       val targetFile = new File(targetFileName)
       targetFile.exists must be_==(true)
-      val targetMod = targetFile.lastModified
     }
 
     "apply(new File(...) with target" in {
@@ -51,8 +52,9 @@ object EvalSpec extends Specification {
 
       // make sure it created a class file with the expected name
       val className = e.fileToClassName(sourceFile)
+      val processedSource = e.sourceForString(Source.fromFile(sourceFile).getLines.mkString("\n"))
       val fullClassName = "Evaluator__%s_%s.class".format(
-        className, e.uniqueId(sourceFile.getCanonicalPath, None))
+        className, e.uniqueId(processedSource, None))
       val targetFileName = f.getAbsolutePath() + File.separator + fullClassName
       val targetFile = new File(targetFileName)
       targetFile.exists must be_==(true)
@@ -60,21 +62,31 @@ object EvalSpec extends Specification {
 
       // eval again, make sure it works
       val res2: Int = e(sourceFile)
-      // and make sure it didn't create a new file
-      f.listFiles.length mustEqual 1
+      // and make sure it didn't create a new file (1 + checksum)
+      f.listFiles.length mustEqual 2
       // and make sure it didn't update the file
       val targetFile2 = new File(targetFileName)
       targetFile2.lastModified mustEqual targetMod
 
-      // touch source, ensure recompile
+      // touch source, ensure no-recompile (checksum hasn't changed)
       sourceFile.setLastModified(System.currentTimeMillis())
       val res3: Int = e(sourceFile)
       res3 mustEqual 2
       // and make sure it didn't create a different file
-      f.listFiles.length mustEqual 1
+      f.listFiles.length mustEqual 2
       // and make sure it updated the file
       val targetFile3 = new File(targetFileName)
-      targetFile3.lastModified must be_>=(targetMod)
+      targetFile3.lastModified mustEqual targetMod
+
+      // append a newline, altering checksum, verify recompile
+      val writer = new FileWriter(sourceFile)
+      writer.write("//a comment\n2\n")
+      writer.close
+      val res4: Int = e(sourceFile)
+      res4 mustEqual 2
+      // and make sure it created a new file
+      val targetFile4 = new File(targetFileName)
+      targetFile4.exists must beFalse
     }
 
     "apply(InputStream)" in {