diff options
author | mmcbride <mccv@twitter.com> | 2011-08-08 22:05:56 -0700 |
---|---|---|
committer | Christopher Vogt <oss.nsp@cvogt.org> | 2016-11-07 02:08:38 -0500 |
commit | ec5d48ac3db80cf955535729d14fa36d188be130 (patch) | |
tree | 9032ea991e33fd14a5f2ad55a8fb76e0ddd64666 /libraries/eval | |
parent | 8e6a2e18a3c8ec55ce84d7820907e47fd2fb0875 (diff) | |
download | cbt-ec5d48ac3db80cf955535729d14fa36d188be130.tar.gz cbt-ec5d48ac3db80cf955535729d14fa36d188be130.tar.bz2 cbt-ec5d48ac3db80cf955535729d14fa36d188be130.zip |
[split] Squashed commit of the following:
commit faf3d257e7dc95d1492b7c8c6faf9da3c53adac6
Merge: d2ecdbd 53fdb61
Author: mmcbride <mccv@twitter.com>
Date: Mon Aug 8 22:05:09 2011 -0700
Merge branch 'master' into persistent_eval
commit d2ecdbddd288607cd1fb597ba04685628ad8b2f0
Author: mmcbride <mccv@twitter.com>
Date: Mon Aug 8 21:40:32 2011 -0700
two minor things from jeremy cloud
commit a17260bd1cd90bab0ac635e0257a9bea8e9da62f
Merge: 5f26ee0 e8d277e
Author: mmcbride <mccv@twitter.com>
Date: Wed Aug 3 20:16:06 2011 -0700
Merge branch 'master' into persistent_eval
commit 5f26ee0544ad39439c509e8b87e60c4cff0f07b9
Author: mmcbride <mccv@twitter.com>
Date: Wed Aug 3 20:13:33 2011 -0700
feedback from RB
commit 775c8661f485724af3be51be720b821b83b044ec
Merge: 350c313 f7d1db2
Author: mmcbride <mccv@twitter.com>
Date: Tue Jul 26 12:41:00 2011 -0700
Merge branch 'master' into persistent_eval
commit 350c313991b31f9e332753a62ff5b92c61ce7cae
Author: mmcbride <mccv@twitter.com>
Date: Tue Jul 26 10:58:25 2011 -0700
persistent compilation targets for eval
Diffstat (limited to 'libraries/eval')
-rw-r--r-- | libraries/eval/Eval.scala | 182 | ||||
-rw-r--r-- | libraries/eval/test/EvalTest.scala | 34 |
2 files changed, 181 insertions, 35 deletions
diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala index e1da151..8f05ed1 100644 --- a/libraries/eval/Eval.scala +++ b/libraries/eval/Eval.scala @@ -27,10 +27,12 @@ import scala.collection.mutable import scala.io.Source import scala.tools.nsc.{Global, Settings} import scala.tools.nsc.interpreter.AbstractFileClassLoader -import scala.tools.nsc.io.VirtualDirectory +import scala.tools.nsc.io.{AbstractFile, VirtualDirectory} import scala.tools.nsc.reporters.AbstractReporter import scala.tools.nsc.util.{BatchSourceFile, Position} +case class LastMod(timestamp: Long, code: String) + /** * Evaluate a file or string and return the result. */ @@ -39,7 +41,35 @@ object Eval extends Eval { private val jvmId = java.lang.Math.abs(new Random().nextInt()) } -class Eval { +/** + * evaluates files, strings or input streams, and returns the result. + * In all cases, code to be evaled is wrapped in an apply method in a + * generated class. An instance of the class is instantiated, and the + * result of apply is returned. + * + * If target is None, the results are compiled to memory (and are therefore + * ephemeral). If target is Some(path), path must point to a directory, and + * eval emits class files to that directory. + * + * eval also supports a limited set of preprocessors. Limited means + * exactly one, that supports directives of the form #include <file>. + * + * The general flow of evaluation is + * # convert arguments to a string + * # run preprocessors on that string + * # wrap processed code in a class + * # compile the class + * # create an instance of that class + * # return the results of apply() + */ +class Eval(target: Option[File]) { + /** + * empty constructor for backwards compatibility + */ + def this() { + this(None) + } + import Eval.jvmId private lazy val compilerPath = try { @@ -57,9 +87,13 @@ class Eval { } /** - * Preprocessors to run the code through before it is passed to the Scala compiler + * Preprocessors to run the code through before it is passed to the Scala compiler. + * if you want to add new resolvers, you can do so with + * new Eval(...) { + * lazy val preprocessors = {...} + * } */ - private lazy val preprocessors: Seq[Preprocessor] = + protected lazy val preprocessors: Seq[Preprocessor] = Seq( new IncludePreprocessor( Seq( @@ -70,45 +104,95 @@ class Eval { ) ) - private lazy val compiler = new StringCompiler(2, preprocessors) + private lazy val compiler = new StringCompiler(2, target) + + /** + * run preprocessors on our string, returning a LastMod + * where timestamp is the last modified time of any file in that contributed + * to the text. + * Last modified is computed here because we support includes + */ + def sourceForString(code: String) = { + preprocessors.foldLeft(LastMod(0L, code)) { (acc, p) => + val processed = p(acc.code) + LastMod(acc.timestamp max processed.timestamp, processed.code) + } + } /** * Eval[Int]("1 + 1") // => 2 */ def apply[T](code: String, resetState: Boolean = true): T = { - val id = uniqueId(code) - val className = "Evaluator__" + id - val cls = compiler(wrapCodeInClass(className, code), className, id, resetState) - cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T] + val processed = sourceForString(code) + applyProcessed(processed.code, resetState) } /** * Eval[Int](new File("...")) */ def apply[T](files: File*): T = { - apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n")) + if (target.isDefined) { + val targetDir = target.get + val unprocessedSource = files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n") + val processed = sourceForString(unprocessedSource) + val oldestTarget = targetDir.listFiles.foldLeft(Long.MaxValue)((oldest, f) => f.lastModified min oldest) + if (processed.timestamp > oldestTarget) { + compiler.reset() + } + val className = "Evaluator__" + files(0).getName.split("\\.")(0) + applyProcessed(className, processed.code, false) + } else { + apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n"), true) + } } /** * Eval[Int](getClass.getResourceAsStream("...")) */ def apply[T](stream: InputStream): T = { - apply(Source.fromInputStream(stream).mkString) + apply(sourceForString(Source.fromInputStream(stream).mkString).code) + } + + /** + * same as apply[T], but does not run preprocessors. + * Will generate a classname of the form Evaluater__<unique>, + * where unique is computed from the jvmID (a random number) + * and a digest of code + */ + def applyProcessed[T](code: String, resetState: Boolean): T = { + val id = uniqueId(code) + val className = "Evaluator__" + id + applyProcessed(className, code, resetState) } + /** + * same as apply[T], but does not run preprocessors. + */ + def applyProcessed[T](className: String, code: String, resetState: Boolean): T = { + val cls = compiler(wrapCodeInClass(className, code), className, resetState) + cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T] + } + + /** + * converts the given file to evaluable source. + * delegates to toSource(code: String) + */ def toSource(file: File): String = { toSource(scala.io.Source.fromFile(file).mkString) } + /** + * converts the given file to evaluable source. + */ def toSource(code: String): String = { - compiler.sourceForString(code) + sourceForString(code).code } /** * Compile an entire source file into the virtual classloader. */ def compile(code: String) { - compiler(code) + compiler(sourceForString(code).code) } /** @@ -124,7 +208,7 @@ class Eval { * @throw CompilerException if not Eval-able. */ def check(code: String) { - val id = uniqueId(code) + val id = uniqueId(sourceForString(code).code) val className = "Evaluator__" + id val wrappedCode = wrapCodeInClass(className, code) compile(wrappedCode) // may throw CompilerException @@ -203,12 +287,13 @@ class Eval { } trait Preprocessor { - def apply(code: String): String + def apply(code: String): LastMod } trait Resolver { def resolvable(path: String): Boolean def get(path: String): InputStream + def lastModified(path: String): Long } class FilesystemResolver(root: File) extends Resolver { @@ -219,6 +304,14 @@ class Eval { def resolvable(path: String): Boolean = file(path).exists + def lastModified(path: String): Long = { + if (resolvable(path)) { + file(path).lastModified + } else { + 0 + } + } + def get(path: String): InputStream = new FileInputStream(file(path)) } @@ -230,6 +323,8 @@ class Eval { def resolvable(path: String): Boolean = clazz.getResourceAsStream(quotePath(path)) != null + def lastModified(path: String): Long = 0 + def get(path: String): InputStream = clazz.getResourceAsStream(quotePath(path)) } @@ -245,40 +340,51 @@ class Eval { * @example #include file-name.scala * * This is the only directive supported by this preprocessor. + * + * Note that it is *not* recursive. Included files cannot have includes */ - private class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor { - def apply(code: String): String = - code.lines map { line: String => + class IncludePreprocessor(resolvers: Seq[Resolver]) extends Preprocessor { + def apply(code: String): LastMod = { + var lastMod = 0L + val lines = code.lines map { line: String => val tokens = line.trim.split(' ') if (tokens.length == 2 && tokens(0).equals("#include")) { val path = tokens(1) resolvers find { resolver: Resolver => resolver.resolvable(path) } match { - case Some(r: Resolver) => + case Some(r: Resolver) => { + lastMod = lastMod max r.lastModified(path) StreamIO.buffer(r.get(path)).toString + } case _ => throw new IllegalStateException("No resolver could find '%s'".format(path)) } } else { line } - } mkString("\n") + } + val processed = lines.mkString("\n") + LastMod(lastMod, processed) + } } /** * Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep * around one of these and reuse it. */ - private class StringCompiler(lineOffset: Int, preprocessors: Seq[Preprocessor]) { - val virtualDirectory = new VirtualDirectory("(memory)", None) + private class StringCompiler(lineOffset: Int, targetDir: Option[File]) { + val target = targetDir match { + case Some(dir) => AbstractFile.getDirectory(dir) + case None => new VirtualDirectory("(memory)", None) + } val cache = new mutable.HashMap[String, Class[_]]() val settings = new Settings settings.deprecation.value = true // enable detailed deprecation warnings settings.unchecked.value = true // enable detailed unchecked warnings - settings.outputDirs.setSingleOutput(virtualDirectory) + settings.outputDirs.setSingleOutput(target) val pathList = compilerPath ::: libPath settings.bootclasspath.value = pathList.mkString(File.pathSeparator) @@ -321,12 +427,24 @@ class Eval { * Class loader for finding classes compiled by this StringCompiler. * After each reset, this class loader will not be able to find old compiled classes. */ - var classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader) + var classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader) def reset() { - virtualDirectory.clear + targetDir match { + case None => { + target.asInstanceOf[VirtualDirectory].clear + } + case Some(t) => { + target.foreach { abstractFile => + if (abstractFile.file == null || abstractFile.file.getName.endsWith(".class")) { + abstractFile.delete + } + } + } + } + cache.clear() reporter.reset - classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader) + classLoader = new AbstractFileClassLoader(target, this.getClass.getClassLoader) } object Debug { @@ -363,14 +481,12 @@ class Eval { * Compile scala code. It can be found using the above class loader. */ def apply(code: String) { - val processedCode = sourceForString(code) - if (Debug.enabled) - Debug.printWithLineNumbers(processedCode) + Debug.printWithLineNumbers(code) // if you're looking for the performance hit, it's 1/2 this line... val compiler = new global.Run - val sourceFiles = List(new BatchSourceFile("(inline)", processedCode)) + val sourceFiles = List(new BatchSourceFile("(inline)", code)) // ...and 1/2 this line: compiler.compileSources(sourceFiles) @@ -379,14 +495,10 @@ class Eval { } } - def sourceForString(code: String) = { - preprocessors.foldLeft(code) { case (c: String, p: Preprocessor) => p(c) } - } - /** * Compile a new class, load it, and return it. Thread-safe. */ - def apply(code: String, className: String, id: String, resetState: Boolean = true): Class[_] = { + def apply(code: String, className: String, resetState: Boolean = true): Class[_] = { synchronized { if (resetState) reset() findClass(className).getOrElse { diff --git a/libraries/eval/test/EvalTest.scala b/libraries/eval/test/EvalTest.scala index a48c8ac..2fa7d44 100644 --- a/libraries/eval/test/EvalTest.scala +++ b/libraries/eval/test/EvalTest.scala @@ -22,6 +22,40 @@ object EvalSpec extends Specification { derived() mustEqual "hello" } + "apply(new File(...) with target" in { + val f = File.createTempFile("eval", "target") + f.delete() + f.mkdir() + val e = new Eval(Some(f)) + val sourceFile = TempFile.fromResourcePath("/OnePlusOne.scala") + val res: Int = e(sourceFile) + res mustEqual 2 + + // make sure it created a class file with the expected name + val targetFileName = f.getAbsolutePath() + File.separator + "Evaluator__" + sourceFile.getName + ".class" + val targetFile = new File(targetFileName) + targetFile.exists must be_==(true) + val targetMod = targetFile.lastModified + + // eval again, make sure it works + val res2: Int = e(sourceFile) + // and make sure it didn't create a new file + f.listFiles.length mustEqual 1 + // and make sure it didn't update the file + val targetFile2 = new File(targetFileName) + targetFile2.lastModified mustEqual targetMod + + // touch source, ensure recompile + sourceFile.setLastModified(System.currentTimeMillis()) + val res3: Int = e(sourceFile) + res3 mustEqual 2 + // and make sure it didn't create a different file + f.listFiles.length mustEqual 1 + // and make sure it updated the file + val targetFile3 = new File(targetFileName) + targetFile3.lastModified must be_>=(targetMod) + } + "apply(InputStream)" in { (new Eval).apply[Int](getClass.getResourceAsStream("/OnePlusOne.scala")) mustEqual 2 } |