[split] Squashed commit of the following:

commit ead2b12ebf8c8c9b0d191235119cea7be01a22e8
Author: wilhelm bierbaum <wilhelm@twitter.com>
Date:   Fri Apr 29 13:20:37 2011 -0700

    use StreamIO

commit 23a12f4b8c40facaad4a81a363d02a3914897267
Merge: 59da183 90a1dd0
Author: wilhelm bierbaum <wilhelm@twitter.com>
Date:   Fri Apr 29 13:20:16 2011 -0700

    Merge remote-tracking branch 'origin/master' into eval_include_directives

commit 59da18308707f6d285e4b65e7e7a33aa62af77ab
Merge: 4296624 f6ead9a
Author: wilhelm bierbaum <wilhelm@twitter.com>
Date:   Thu Apr 28 23:58:18 2011 -0700

    Merge remote-tracking branch 'origin/master' into eval_include_directives

    Conflicts:
    	util/util-eval/src/test/scala/com/twitter/util/EvaluatorSpec.scala

commit 42966245dc24d2026a8b3955e3c313445a93ac3e
Merge: a0aa60e d98d9c9
Author: Wilhelm Bierbaum <wilhelm@twitter.com>
Date:   Tue Apr 19 20:05:57 2011 -0700

    Merge branch 'master' into eval_include_directives

commit a0aa60e80df3bd0312f3349eed61e997109fa510
Author: Wilhelm Bierbaum <wilhelm@twitter.com>
Date:   Tue Apr 19 20:05:46 2011 -0700

    revert inadvertent changes

commit 931be4dce6dd82ec2ef8d09f8576c9a27f50640a
Author: Wilhelm Bierbaum <wilhelm@twitter.com>
Date:   Tue Apr 19 17:10:19 2011 -0700

    introduce a preprocessor that can #include files from the filesystem (by default rooted in ./ or ./config) or the classpath into the Eval-based config compiler

5 files changed, 125 insertions, 4 deletions

diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala
index 2b22bcb..ee45256 100644
--- a/libraries/eval/Eval.scala
+++ b/libraries/eval/Eval.scala
@@ -16,7 +16,8 @@
 
 package com.twitter.util
 
-import java.io.{File, InputStream}
+import com.twitter.io.StreamIO
+import java.io.{File, InputStream, FileInputStream, FileNotFoundException}
 import java.math.BigInteger
 import java.net.URLClassLoader
 import java.security.MessageDigest
@@ -55,7 +56,21 @@ class Eval {
       throw new RuntimeException("Unable to load scala base object from classpath (scala-library jar is missing?)", e)
   }
 
-  private lazy val compiler = new StringCompiler(2)
+  /**
+   * Preprocessors to run the code through before it is passed to the Scala compiler
+   */
+  private lazy val preprocessors: Seq[Preprocessor] =
+    Seq(
+      new IncludePreprocessor(
+        Seq(
+          new ClassScopedResolver(getClass),
+          new FilesystemResolver(new File(".")),
+          new FilesystemResolver(new File("." + File.separator + "config"))
+        )
+      )
+    )
+
+  private lazy val compiler = new StringCompiler(2, preprocessors)
 
   /**
    * Eval[Int]("1 + 1") // => 2
@@ -175,11 +190,75 @@ class Eval {
     })
   }
 
+  trait Preprocessor {
+    def apply(code: String): String
+  }
+
+  trait Resolver {
+    def resolvable(path: String): Boolean
+    def get(path: String): InputStream
+  }
+
+  class FilesystemResolver(root: File) extends Resolver {
+    private[this] def file(path: String): File =
+      new File(root.getAbsolutePath + File.separator + path)
+
+
+    def resolvable(path: String): Boolean =
+      file(path).exists
+
+    def get(path: String): InputStream =
+      new FileInputStream(file(path))
+  }
+
+  class ClassScopedResolver(clazz: Class[_]) extends Resolver {
+    private[this] def quotePath(path: String) =
+      "/" + path
+
+    def resolvable(path: String): Boolean =
+      clazz.getResourceAsStream(quotePath(path)) != null
+
+    def get(path: String): InputStream =
+      clazz.getResourceAsStream(quotePath(path))
+  }
+
+  class ResolutionFailedException(message: String) extends Exception
+
+  /*
+   * This is a preprocesor that can include files by requesting them from the given classloader
+   *
+   * Thusly, if you put FS directories on your classpath (e.g. config/ under your app root,) you
+   * mix in configuration from the filesystem.
+   *
+   * @example #include file-name.scala
+   *
+   * This is the only directive supported by this preprocessor.
+   */
+  private class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor {
+    def apply(code: String): String =
+      code.lines map { line: String =>
+        val tokens = line.trim.split(' ')
+        if (tokens.length == 2 && tokens(0).equals("#include")) {
+          val path = tokens(1)
+          resolvers find { resolver: Resolver =>
+            resolver.resolvable(path)
+          } match {
+            case Some(r: Resolver) =>
+              StreamIO.buffer(r.get(path)).toString
+            case _ =>
+              throw new IllegalStateException("No resolver could find '%s'".format(path))
+          }
+        } else {
+          line
+        }
+      } mkString("\n")
+  }
+
   /**
    * Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep
    * around one of these and reuse it.
    */
-  private class StringCompiler(lineOffset: Int) {
+  private class StringCompiler(lineOffset: Int, preprocessors: Seq[Preprocessor]) {
     val virtualDirectory = new VirtualDirectory("(memory)", None)
 
     val cache = new mutable.HashMap[String, Class[_]]()
@@ -238,12 +317,32 @@ class Eval {
       classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader)
     }
 
+    object Debug {
+      val enabled =
+        System.getProperty("eval.debug") != null
+
+      def printWithLineNumbers(code: String) {
+        printf("Code follows (%d bytes)\n", code.length)
+
+        var numLines = 0
+        code.lines foreach { line: String =>
+          numLines += 1
+          println(numLines.toString.padTo(5, ' ') + "| " + line)
+        }
+      }
+    }
+
     /**
      * Compile scala code. It can be found using the above class loader.
      */
     def apply(code: String) {
+      val processedCode = preprocessors.foldLeft(code) { case (c: String, p: Preprocessor) => p(c) }
+
+      if (Debug.enabled)
+        Debug.printWithLineNumbers(processedCode)
+
       val compiler = new global.Run
-      val sourceFiles = List(new BatchSourceFile("(inline)", code))
+      val sourceFiles = List(new BatchSourceFile("(inline)", processedCode))
       compiler.compileSources(sourceFiles)
 
       if (reporter.hasErrors || reporter.WARNING.count > 0) {
diff --git a/libraries/eval/test/EvalTest.scala b/libraries/eval/test/EvalTest.scala
index 3541381..c16de43 100644
--- a/libraries/eval/test/EvalTest.scala
+++ b/libraries/eval/test/EvalTest.scala
@@ -42,5 +42,18 @@ object EvalSpec extends Specification {
       (new Eval).check("23")      mustEqual ()
       (new Eval).check("invalid") must throwA[Eval.CompilerException]
     }
+
+    "#include" in {
+      val derived = Eval[() => String](
+        TempFile.fromResourcePath("/Base.scala"),
+        TempFile.fromResourcePath("/DerivedWithInclude.scala"))
+      derived() mustEqual "hello"
+      derived.toString mustEqual "hello, joe"
+    }
+
+    "throws a compilation error when Ruby is #included" in {
+      Eval[() => String](
+        TempFile.fromResourcePath("RubyInclude.scala")) must throwA[Throwable]
+    }
   }
 }
diff --git a/libraries/eval/test/resources/DerivedWithInclude.scala b/libraries/eval/test/resources/DerivedWithInclude.scala
new file mode 100644
index 0000000..476d60e
--- /dev/null
+++ b/libraries/eval/test/resources/DerivedWithInclude.scala
@@ -0,0 +1,3 @@
+new Base {
+#include HelloJoe.scala
+}
diff --git a/libraries/eval/test/resources/HelloJoe.scala b/libraries/eval/test/resources/HelloJoe.scala
new file mode 100644
index 0000000..c6ea42e
--- /dev/null
+++ b/libraries/eval/test/resources/HelloJoe.scala
@@ -0,0 +1,3 @@
+/* real-time declarative programming now */
+override def toString = "hello, joe"
+
diff --git a/libraries/eval/test/resources/RubyInclude.scala b/libraries/eval/test/resources/RubyInclude.scala
new file mode 100644
index 0000000..a763d52
--- /dev/null
+++ b/libraries/eval/test/resources/RubyInclude.scala
@@ -0,0 +1,3 @@
+object CodeThatIncludesSomeRuby {
+#include hello.rb
+}