From a219458fe9a55a60ae6a25a175b4925003eec074 Mon Sep 17 00:00:00 2001 From: Robey Pointer Date: Thu, 16 Dec 2010 18:33:57 -0800 Subject: upgrade Eval. --- libraries/eval/Eval.scala | 335 +++++++++++++++++++++++----------------------- 1 file changed, 168 insertions(+), 167 deletions(-) (limited to 'libraries/eval/Eval.scala') diff --git a/libraries/eval/Eval.scala b/libraries/eval/Eval.scala index 8ca6f89..33f7722 100644 --- a/libraries/eval/Eval.scala +++ b/libraries/eval/Eval.scala @@ -1,214 +1,215 @@ +/* + * Copyright 2010 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.twitter.util -import java.io.{File, FileWriter, InputStream} +import java.io.{File, InputStream} import java.math.BigInteger -import java.net.{URL, URLClassLoader} +import java.net.URLClassLoader import java.security.MessageDigest -import java.util.jar._ import java.util.Random +import java.util.jar.JarFile +import scala.collection.mutable import scala.io.Source -import scala.runtime._ -import scala.tools.nsc.reporters.ConsoleReporter import scala.tools.nsc.{Global, Settings} +import scala.tools.nsc.interpreter.AbstractFileClassLoader +import scala.tools.nsc.io.VirtualDirectory +import scala.tools.nsc.reporters.AbstractReporter +import scala.tools.nsc.util.{BatchSourceFile, Position} /** - * Eval is a utility function to evaluate a file and return its results. - * It is intended to be used for application configuration (rather than Configgy, XML, YAML files, etc.) - * and anything else. - * - * Consider this example. You have the following configuration trait Config.scala. - * - * package com.mycompany - * - * import com.twitter.util.Duration - * import com.twitter.util.StorageUnit - * - * trait Config { - * def myValue: Int - * def myTimeout: Duration - * def myBufferSize: StorageUnit - * } - * - * You have the following configuration file (specific values) in config/Development.scala: - * - * import com.mycompany.Config - * import com.twitter.util.TimeConversions._ - * import com.twitter.util.StorageUnitConversions._ - * - * new Config { - * val myValue = 1 - * val myTimeout = 2.seconds - * val myBufferSize = 14.kilobytes - * } - * - * And finally, in Main.scala: - * - * package com.mycompany - * - * object Main { - * def main(args: Array[String]) { - * val config = Eval[Config](new File(args(0))) - * ... - * } - * } - * - * Note that in this example there is no need for any configuration format like Configgy, YAML, etc. - * - * So how does this work? Eval takes a file or string and generates a new scala class that has an apply method that - * evaluates that string. The newly generated file is then compiled. All generated .scala and .class - * files are stored, by default, in System.getProperty("java.io.tmpdir"). - * - * After compilation, a new class loader is created with the temporary dir as the classPath. - * The generated class is loaded and then apply() is called. - * - * This implementation is inspired by - * http://scala-programming-language.1934581.n4.nabble.com/Compiler-API-td1992165.html - * + * Evaluate a file or string and return the result. */ object Eval { + // do not look at the man behind the curtain! private val compilerPath = jarPathOfClass("scala.tools.nsc.Interpreter") private val libPath = jarPathOfClass("scala.ScalaObject") + private val jvmId = java.lang.Math.abs(new Random().nextInt()) - private val md = MessageDigest.getInstance("SHA") + + val compiler = new StringCompiler(2) /** * Eval[Int]("1 + 1") // => 2 */ - def apply[T](stringToEval: String): T = synchronized { - md.reset() - val digest = md.digest(stringToEval.getBytes()) - val sha = new BigInteger(1, digest).toString(16) - - val uniqueId = sha + "_" + jvmId - val className = "Evaluator" + uniqueId - val targetDir = new File(System.getProperty("java.io.tmpdir") + File.separator + "evaluator_" + uniqueId) - ifUncompiled(targetDir, className) { targetFile => - wrapInClassAndOutput(stringToEval, className, targetFile) - compile(targetFile, targetDir) - } - val clazz = loadClass(targetDir, className) - val constructor = clazz.getConstructor() - val evaluator = constructor.newInstance().asInstanceOf[() => Any] - evaluator().asInstanceOf[T] + def apply[T](code: String): T = { + val id = uniqueId(code) + val className = "Evaluator__" + id + val cls = compiler(wrapCodeInClass(className, code), className, id) + cls.getConstructor().newInstance().asInstanceOf[() => Any].apply().asInstanceOf[T] } /** * Eval[Int](new File("...")) */ - def apply[T](fileToEval: File*): T = { - val stringToEval = List(fileToEval: _*).map(scala.io.Source.fromFile(_).mkString).mkString - apply(stringToEval) + def apply[T](files: File*): T = { + apply(files.map { scala.io.Source.fromFile(_).mkString }.mkString("\n")) } /** - * Eval[Int](getClass.getResourceAsStream("/Foo.scala")) + * Eval[Int](getClass.getResourceAsStream("...")) */ - def apply[T](stream: InputStream): T = + def apply[T](stream: InputStream): T = { apply(scala.io.Source.fromInputStream(stream).mkString) + } - private def ifUncompiled(targetDir: File, className: String)(f: File => Unit) { - targetDir.mkdirs() - targetDir.deleteOnExit() - - val targetFile = new File(targetDir, className + ".scala") - if (!targetFile.exists) { - val created = targetFile.createNewFile() - if (!created) { - // FIXME: this indicates that another jvm randomly generated the same - // integer and compiled this file. Or, more likely, this method was called - // simultaneously from two threads. - } - f(targetFile) - } + private def uniqueId(code: String): String = { + val digest = MessageDigest.getInstance("SHA-1").digest(code.getBytes()) + val sha = new BigInteger(1, digest).toString(16) + sha + "_" + jvmId + } - targetDir.listFiles().foreach { _.deleteOnExit() } + /* + * Wrap source code in a new class with an apply method. + */ + private def wrapCodeInClass(className: String, code: String) = { + "class " + className + " extends (() => Any) {\n" + + " def apply() = {\n" + + code + "\n" + + " }\n" + + "}\n" } - /** - * Wrap sourceCode in a new class that has an apply method that evaluates that sourceCode. - * Write generated (temporary) classes to targetDir + /* + * For a given FQ classname, trick the resource finder into telling us the containing jar. */ - private def wrapInClassAndOutput(sourceCode: String, className: String, targetFile: File) { - val writer = new FileWriter(targetFile) - writer.write("class " + className + " extends (() => Any) {\n") - writer.write(" def apply() = {\n") - writer.write(" " + sourceCode) - writer.write("\n }\n") - writer.write("}\n") - writer.close() + private def jarPathOfClass(className: String) = { + val resource = className.split('.').mkString("/", "/", ".class") + val path = getClass.getResource(resource).getPath + val indexOfFile = path.indexOf("file:") + 5 + val indexOfSeparator = path.lastIndexOf('!') + path.substring(indexOfFile, indexOfSeparator) + } + + /* + * Try to guess our app's classpath. + * This is probably fragile. + */ + lazy val impliedClassPath: List[String] = { + val currentClassPath = this.getClass.getClassLoader.asInstanceOf[URLClassLoader].getURLs. + map(_.toString).filter(_.startsWith("file:")).map(_.substring(5)).toList + + // if there's just one thing in the classpath, and it's a jar, assume an executable jar. + currentClassPath ::: (if (currentClassPath.size == 1 && currentClassPath(0).endsWith(".jar")) { + val jarFile = currentClassPath(0) + val relativeRoot = new File(jarFile).getParentFile() + val nestedClassPath = new JarFile(jarFile).getManifest.getMainAttributes.getValue("Class-Path") + if (nestedClassPath eq null) { + Nil + } else { + nestedClassPath.split(" ").map { f => new File(relativeRoot, f).getAbsolutePath }.toList + } + } else { + Nil + }) } - val JarFile = """\.jar$""".r /** - * Compile a given file into the targetDir + * Dynamic scala compiler. Lots of (slow) state is created, so it may be advantageous to keep + * around one of these and reuse it. */ - private def compile(file: File, targetDir: File) { - val settings = new Settings - val origBootclasspath = settings.bootclasspath.value - - // Figure out our app classpath. - // TODO: there are likely a ton of corner cases waiting here - val configulousClassLoader = this.getClass.getClassLoader.asInstanceOf[URLClassLoader] - val configulousClasspath = configulousClassLoader.getURLs.map { url => - val urlStr = url.toString - urlStr.substring(5, urlStr.length) - }.toList - - // It's not clear how many nested jars we should open. - val classPathAndClassPathsNestedInJars = configulousClasspath.flatMap { fileName => - val nestedClassPath = if (JarFile.findFirstMatchIn(fileName).isDefined) { - val nestedClassPathAttribute = new JarFile(fileName).getManifest.getMainAttributes.getValue("Class-Path") - // turns /usr/foo/bar/foo-1.0.jar into ["", "usr", "foo", "bar", "foo-1.0.jar"] - val rootDirPath = fileName.split(File.separator).toList - // and then into /usr/foo/bar - val rootDir = rootDirPath.slice(1, rootDirPath.length - 1).mkString(File.separator, File.separator, File.separator) - - if (nestedClassPathAttribute != null) { - nestedClassPathAttribute.split(" ").toList.map(rootDir + File.separator + _) - } else Nil - } else Nil - List(fileName) ::: nestedClassPath - } - val bootClassPath = origBootclasspath.split(File.pathSeparator).toList + class StringCompiler(lineOffset: Int) { + val virtualDirectory = new VirtualDirectory("(memory)", None) - // the classpath for compile is our app path + boot path + make sure we have compiler/lib there - val pathList = bootClassPath ::: (classPathAndClassPathsNestedInJars ::: List(compilerPath, libPath)) - val pathString = pathList.mkString(File.pathSeparator) - settings.bootclasspath.value = pathString - settings.classpath.value = pathString + val cache = new mutable.HashMap[String, Class[_]]() + + val settings = new Settings settings.deprecation.value = true // enable detailed deprecation warnings settings.unchecked.value = true // enable detailed unchecked warnings - settings.outdir.value = targetDir.toString + settings.outputDirs.setSingleOutput(virtualDirectory) + + val pathList = List(compilerPath, libPath) + settings.bootclasspath.value = pathList.mkString(File.pathSeparator) + settings.classpath.value = (pathList ::: impliedClassPath).mkString(File.pathSeparator) + + val reporter = new AbstractReporter { + val settings = StringCompiler.this.settings + val messages = new mutable.ListBuffer[List[String]] + + def display(pos: Position, message: String, severity: Severity) { + severity.count += 1 + val severityName = severity match { + case ERROR => "error: " + case WARNING => "warning: " + case _ => "" + } + messages += (severityName + "line " + (pos.line - lineOffset) + ": " + message) :: + (if (pos.isDefined) { + pos.inUltimateSource(pos.source).lineContent.stripLineEnd :: + (" " * (pos.column - 1) + "^") :: + Nil + } else { + Nil + }) + } - val reporter = new ConsoleReporter(settings) - val compiler = new Global(settings, reporter) - (new compiler.Run).compile(List(file.toString)) + def displayPrompt { + // no. + } - if (reporter.hasErrors || reporter.WARNING.count > 0) { - // FIXME: use proper logging - System.err.println("reporter has warnings attempting to compile") - reporter.printSummary() + override def reset { + // grumpy comment about these side-effect methods not taking parens. + super.reset + messages.clear() + } } - } - /** - * Create a new classLoader with the targetDir as the classPath. - * Load the class with className - */ - private def loadClass(targetDir: File, className: String) = { - // set up the new classloader in targetDir - val scalaClassLoader = this.getClass.getClassLoader - val targetDirURL = targetDir.toURL - val newClassLoader = URLClassLoader.newInstance(Array(targetDir.toURL), scalaClassLoader) - newClassLoader.loadClass(className) - } + val global = new Global(settings, reporter) - private def jarPathOfClass(className: String) = { - val resource = className.split('.').mkString("/", "/", ".class") - //println("resource for %s is %s".format(className, resource)) - val path = getClass.getResource(resource).getPath - val indexOfFile = path.indexOf("file:") - val indexOfSeparator = path.lastIndexOf('!') - path.substring(indexOfFile, indexOfSeparator) + /* + * Class loader for finding classes compiled by this StringCompiler. + * After each reset, this class loader will not be able to find old compiled classes. + */ + val classLoader = new AbstractFileClassLoader(virtualDirectory, this.getClass.getClassLoader) + + def reset() { + // grumpy comment about these side-effect methods not taking parens. + virtualDirectory.clear + reporter.reset + } + + /** + * Compile scala code. It can be found using the above class loader. + */ + def apply(code: String) { + val compiler = new global.Run + val sourceFiles = List(new BatchSourceFile("(inline)", code)) + compiler.compileSources(sourceFiles) + + if (reporter.hasErrors || reporter.WARNING.count > 0) { + throw new CompilerException(reporter.messages.toList) + } + } + + /** + * Reset the compiler, compile a new class, load it, and return it. Thread-safe. + */ + def apply(code: String, className: String, id: String): Class[_] = synchronized { + cache.get(id) match { + case Some(cls) => + cls + case None => + reset() + apply(code) + val cls = classLoader.loadClass(className) + cache(id) = cls + cls + } + } } + + class CompilerException(val messages: List[List[String]]) extends Exception("Compiler exception") } -- cgit v1.2.3