From 1f9ff6569de3993cb36629f1b8983a65452690a3 Mon Sep 17 00:00:00 2001 From: wilhelm bierbaum Date: Fri, 29 Apr 2011 13:22:39 -0700 Subject: [split] Squashed commit of the following: commit ead2b12ebf8c8c9b0d191235119cea7be01a22e8 Author: wilhelm bierbaum Date: Fri Apr 29 13:20:37 2011 -0700 use StreamIO commit 23a12f4b8c40facaad4a81a363d02a3914897267 Merge: 59da183 90a1dd0 Author: wilhelm bierbaum Date: Fri Apr 29 13:20:16 2011 -0700 Merge remote-tracking branch 'origin/master' into eval_include_directives commit 59da18308707f6d285e4b65e7e7a33aa62af77ab Merge: 4296624 f6ead9a Author: wilhelm bierbaum Date: Thu Apr 28 23:58:18 2011 -0700 Merge remote-tracking branch 'origin/master' into eval_include_directives Conflicts: util/util-eval/src/test/scala/com/twitter/util/EvaluatorSpec.scala commit 42966245dc24d2026a8b3955e3c313445a93ac3e Merge: a0aa60e d98d9c9 Author: Wilhelm Bierbaum Date: Tue Apr 19 20:05:57 2011 -0700 Merge branch 'master' into eval_include_directives commit a0aa60e80df3bd0312f3349eed61e997109fa510 Author: Wilhelm Bierbaum Date: Tue Apr 19 20:05:46 2011 -0700 revert inadvertent changes commit 931be4dce6dd82ec2ef8d09f8576c9a27f50640a Author: Wilhelm Bierbaum Date: Tue Apr 19 17:10:19 2011 -0700 introduce a preprocessor that can #include files from the filesystem (by default rooted in ./ or ./config) or the classpath into the Eval-based config compiler --- libraries/eval/Eval.scala | 107 ++++++++++++++++++++- libraries/eval/test/EvalTest.scala | 13 +++ .../eval/test/resources/DerivedWithInclude.scala | 3 + libraries/eval/test/resources/HelloJoe.scala | 3 + libraries/eval/test/resources/RubyInclude.scala | 3 + 5 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 libraries/eval/test/resources/DerivedWithInclude.scala create mode 100644 libraries/eval/test/resources/HelloJoe.scala create mode 100644 libraries/eval/test/resources/RubyInclude.scala (limited to 'libraries/eval') diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala index 2b22bcb..ee45256 100644 --- a/libraries/eval/Eval.scala +++ b/libraries/eval/Eval.scala @@ -16,7 +16,8 @@ package com.twitter.util -import java.io.{File, InputStream} +import com.twitter.io.StreamIO +import java.io.{File, InputStream, FileInputStream, FileNotFoundException} import java.math.BigInteger import java.net.URLClassLoader import java.security.MessageDigest @@ -55,7 +56,21 @@ class Eval { throw new RuntimeException("Unable to load scala base object from classpath (scala-library jar is missing?)", e) } - private lazy val compiler = new StringCompiler(2) + /** + * Preprocessors to run the code through before it is passed to the Scala compiler + */ + private lazy val preprocessors: Seq[Preprocessor] = + Seq( + new IncludePreprocessor( + Seq( + new ClassScopedResolver(getClass), + new FilesystemResolver(new File(".")), + new FilesystemResolver(new File("." + File.separator + "config")) + ) + ) + ) + + private lazy val compiler = new StringCompiler(2, preprocessors) /** * Eval[Int]("1 + 1") // => 2 @@ -175,11 +190,75 @@ class Eval { }) } + trait Preprocessor { + def apply(code: String): String + } + + trait Resolver { + def resolvable(path: String): Boolean + def get(path: String): InputStream + } + + class FilesystemResolver(root: File) extends Resolver { + private[this] def file(path: String): File = + new File(root.getAbsolutePath + File.separator + path) + + + def resolvable(path: String): Boolean = + file(path).exists + + def get(path: String): InputStream = + new FileInputStream(file(path)) + } + + class ClassScopedResolver(clazz: Class[_]) extends Resolver { + private[this] def quotePath(path: String) = + "/" + path + + def resolvable(path: String): Boolean = + clazz.getResourceAsStream(quotePath(path)) != null + + def get(path: String): InputStream = + clazz.getResourceAsStream(quotePath(path)) + } + + class ResolutionFailedException(message: String) extends Exception + + /* + * This is a preprocesor that can include files by requesting them from the given classloader + * + * Thusly, if you put FS directories on your classpath (e.g. config/ under your app root,) you + * mix in configuration from the filesystem. + * + * @example #include file-name.scala + * + * This is the only directive supported by this preprocessor. + */ + private class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor { + def apply(code: String): String = + code.lines map { line: String => + val tokens = line.trim.split(' ') + if (tokens.length == 2 && tokens(0).equals("#include")) { + val path = tokens(1) + resolvers find { resolver: Resolver => + resolver.resolvable(path) + } match { + case Some(r: Resolver) => + StreamIO.buffer(r.get(path)).toString + case _ => + throw new IllegalStateException("No resolver could find '%s'".format(path)) + } + } else { + line + } + } mkString("\n") + } + /** * Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep * around one of these and reuse it. */ - private class StringCompiler(lineOffset: Int) { + private class StringCompiler(lineOffset: Int, preprocessors: Seq[Preprocessor]) { val virtualDirectory = new VirtualDirectory("(memory)", None) val cache = new mutable.HashMap[String, Class[_]]() @@ -238,12 +317,32 @@ class Eval { classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader) } + object Debug { + val enabled = + System.getProperty("eval.debug") != null + + def printWithLineNumbers(code: String) { + printf("Code follows (%d bytes)\n", code.length) + + var numLines = 0 + code.lines foreach { line: String => + numLines += 1 + println(numLines.toString.padTo(5, ' ') + "| " + line) + } + } + } + /** * Compile scala code. It can be found using the above class loader. */ def apply(code: String) { + val processedCode = preprocessors.foldLeft(code) { case (c: String, p: Preprocessor) => p(c) } + + if (Debug.enabled) + Debug.printWithLineNumbers(processedCode) + val compiler = new global.Run - val sourceFiles = List(new BatchSourceFile("(inline)", code)) + val sourceFiles = List(new BatchSourceFile("(inline)", processedCode)) compiler.compileSources(sourceFiles) if (reporter.hasErrors || reporter.WARNING.count > 0) { diff --git a/libraries/eval/test/EvalTest.scala b/libraries/eval/test/EvalTest.scala index 3541381..c16de43 100644 --- a/libraries/eval/test/EvalTest.scala +++ b/libraries/eval/test/EvalTest.scala @@ -42,5 +42,18 @@ object EvalSpec extends Specification { (new Eval).check("23") mustEqual () (new Eval).check("invalid") must throwA[Eval.CompilerException] } + + "#include" in { + val derived = Eval[() => String]( + TempFile.fromResourcePath("/Base.scala"), + TempFile.fromResourcePath("/DerivedWithInclude.scala")) + derived() mustEqual "hello" + derived.toString mustEqual "hello, joe" + } + + "throws a compilation error when Ruby is #included" in { + Eval[() => String]( + TempFile.fromResourcePath("RubyInclude.scala")) must throwA[Throwable] + } } } diff --git a/libraries/eval/test/resources/DerivedWithInclude.scala b/libraries/eval/test/resources/DerivedWithInclude.scala new file mode 100644 index 0000000..476d60e --- /dev/null +++ b/libraries/eval/test/resources/DerivedWithInclude.scala @@ -0,0 +1,3 @@ +new Base { +#include HelloJoe.scala +} diff --git a/libraries/eval/test/resources/HelloJoe.scala b/libraries/eval/test/resources/HelloJoe.scala new file mode 100644 index 0000000..c6ea42e --- /dev/null +++ b/libraries/eval/test/resources/HelloJoe.scala @@ -0,0 +1,3 @@ +/* real-time declarative programming now */ +override def toString = "hello, joe" + diff --git a/libraries/eval/test/resources/RubyInclude.scala b/libraries/eval/test/resources/RubyInclude.scala new file mode 100644 index 0000000..a763d52 --- /dev/null +++ b/libraries/eval/test/resources/RubyInclude.scala @@ -0,0 +1,3 @@ +object CodeThatIncludesSomeRuby { +#include hello.rb +} -- cgit v1.2.3