From c1e787f7560807ca95e021d9cb7f1406c5953c3c Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 5 Apr 2017 19:22:58 +0200 Subject: Make inter JVM communication be string based --- compiler/test/dotty/Jars.scala | 13 ++++ compiler/test/dotty/tools/vulpix/ChildJVMMain.java | 34 ++++++++ compiler/test/dotty/tools/vulpix/ChildMain.scala | 82 ------------------- .../test/dotty/tools/vulpix/ParallelTesting.scala | 49 +++++------- .../dotty/tools/vulpix/RunnerOrchestration.scala | 91 ++++++++++++++-------- compiler/test/dotty/tools/vulpix/Status.scala | 7 ++ compiler/test/dotty/tools/vulpix/Statuses.java | 25 ------ 7 files changed, 133 insertions(+), 168 deletions(-) create mode 100644 compiler/test/dotty/tools/vulpix/ChildJVMMain.java delete mode 100644 compiler/test/dotty/tools/vulpix/ChildMain.scala create mode 100644 compiler/test/dotty/tools/vulpix/Status.scala delete mode 100644 compiler/test/dotty/tools/vulpix/Statuses.java diff --git a/compiler/test/dotty/Jars.scala b/compiler/test/dotty/Jars.scala index f062f8b25..06df9c891 100644 --- a/compiler/test/dotty/Jars.scala +++ b/compiler/test/dotty/Jars.scala @@ -19,4 +19,17 @@ object Jars { val dottyTestDeps: List[String] = dottyLib :: dottyCompiler :: dottyInterfaces :: dottyExtras + + + def scalaLibraryFromRuntime: String = findJarFromRuntime("scala-library-2.") + + private def findJarFromRuntime(partialName: String) = { + val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) + urls.find(_.contains(partialName)).getOrElse { + throw new java.io.FileNotFoundException( + s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" + ) + } + } + } diff --git a/compiler/test/dotty/tools/vulpix/ChildJVMMain.java b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java new file mode 100644 index 000000000..90b795898 --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/ChildJVMMain.java @@ -0,0 +1,34 @@ +package dotty.tools.vulpix; + +import java.io.File; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.lang.reflect.Method; + +public class ChildJVMMain { + static final String MessageEnd = "##THIS IS THE END FOR ME, GOODBYE##"; + + private static void runMain(String dir) throws Exception { + ArrayList cp = new ArrayList<>(); + for (String path : dir.split(":")) + cp.add(new File(path).toURI().toURL()); + + URLClassLoader ucl = new URLClassLoader(cp.toArray(new URL[cp.size()])); + Class cls = ucl.loadClass("Test"); + Method meth = cls.getMethod("main", String[].class); + Object[] args = new Object[]{ new String[]{ "jvm" } }; + meth.invoke(null, args); + } + + public static void main(String[] args) throws Exception { + BufferedReader stdin = new BufferedReader(new InputStreamReader(System.in)); + + while (true) { + runMain(stdin.readLine()); + System.out.println(MessageEnd); + } + } +} diff --git a/compiler/test/dotty/tools/vulpix/ChildMain.scala b/compiler/test/dotty/tools/vulpix/ChildMain.scala deleted file mode 100644 index 30059a9c5..000000000 --- a/compiler/test/dotty/tools/vulpix/ChildMain.scala +++ /dev/null @@ -1,82 +0,0 @@ -package dotty.tools.vulpix - -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream, - ByteArrayOutputStream, PrintStream -} -import java.lang.reflect.InvocationTargetException - -import dotty.tools.vulpix.Statuses._ - -object ChildMain { - val realStdin = System.in - val realStderr = System.err - val realStdout = System.out - - private def runMain(dir: JFile): Status = { - def renderStackTrace(ex: Throwable): String = - ex.getStackTrace - .takeWhile(_.getMethodName != "invoke0") - .mkString(" ", "\n ", "") - - def resetOutDescriptors(): Unit = { - System.setOut(realStdout) - System.setErr(realStderr) - } - - import java.net.{ URL, URLClassLoader } - - val printStream = new ByteArrayOutputStream - - try { - // Do classloading magic and running here: - val ucl = new URLClassLoader(Array(dir.toURI.toURL)) - val cls = ucl.loadClass("Test") - val meth = cls.getMethod("main", classOf[Array[String]]) - - try { - val ps = new PrintStream(printStream) - System.setOut(ps) - System.setErr(ps) - Console.withOut(printStream) { - Console.withErr(printStream) { - // invoke main with "jvm" as arg - meth.invoke(null, Array("jvm")) - } - } - resetOutDescriptors() - } catch { - case t: Throwable => - resetOutDescriptors() - throw t - } - new Success(printStream.toString("utf-8")) - } - catch { - case ex: NoSuchMethodException => - val msg = s"test in '$dir' did not contain method: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: ClassNotFoundException => - val msg = s"test in '$dir' did not contain class: ${ex.getMessage}" - new Failure(msg, renderStackTrace(ex.getCause)) - - case ex: InvocationTargetException => - val msg = s"An exception ocurred when running main: ${ex.getCause}" - new Failure(msg, renderStackTrace(ex.getCause)) - } - } - - def main(args: Array[String]): Unit = { - val stdin = new ObjectInputStream(System.in); - val stdout = new ObjectOutputStream(System.out); - - while (true) { - val dir = stdin.readObject().asInstanceOf[JFile] - stdout.writeObject(runMain(dir)) - stdout.flush() - } - } -} diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 9b9f0a2bb..d23ab0778 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -23,8 +23,6 @@ import dotc.interfaces.Diagnostic.ERROR import dotc.util.DiffUtil import dotc.{ Driver, Compiler } -import vulpix.Statuses._ - /** A parallel testing suite whose goal is to integrate nicely with JUnit * * This trait can be mixed in to offer parallel testing to compile runs. When @@ -54,6 +52,15 @@ trait ParallelTesting extends RunnerOrchestration { self => def outDir: JFile def flags: Array[String] + def classPath: String = { + val (beforeCp, cpAndAfter) = flags.toList.span(_ != "-classpath") + if (cpAndAfter.nonEmpty) { + val (_ :: cpArg :: _) = cpAndAfter + s"${outDir.getAbsolutePath}:" + cpArg + } + else outDir.getAbsolutePath + } + def title: String = self match { case self: JointCompilationSource => @@ -294,15 +301,6 @@ trait ParallelTesting extends RunnerOrchestration { self => val files: Array[JFile] = files0.flatMap(flattenFiles) - def findJarFromRuntime(partialName: String) = { - val urls = ClassLoader.getSystemClassLoader.asInstanceOf[java.net.URLClassLoader].getURLs.map(_.getFile.toString) - urls.find(_.contains(partialName)).getOrElse { - throw new java.io.FileNotFoundException( - s"""Unable to locate $partialName on classpath:\n${urls.toList.mkString("\n")}""" - ) - } - } - def addOutDir(xs: Array[String]): Array[String] = { val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath") if (cpAndAfter.nonEmpty) { @@ -313,11 +311,10 @@ trait ParallelTesting extends RunnerOrchestration { self => } def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) { - val scalaLib = findJarFromRuntime("scala-library-2.") val fullArgs = Array( "javac", "-classpath", - s".:$scalaLib:${targetDir.getAbsolutePath}" + s".:${Jars.scalaLibraryFromRuntime}:${targetDir.getAbsolutePath}" ) ++ flags.takeRight(2) ++ fs Runtime.getRuntime.exec(fullArgs).waitFor() == 0 @@ -430,9 +427,9 @@ trait ParallelTesting extends RunnerOrchestration { self => private final class RunTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) extends Test(testSources, times, threadLimit, suppressAllOutput) { private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { - runMain(dir) match { - case success: Success => { - val outputLines = success.output.lines.toArray + runMain(testSource.classPath) match { + case Success(output) => { + val outputLines = output.lines.toArray val checkLines: Array[String] = Source.fromFile(checkFile).getLines.toArray val sourceTitle = testSource.title @@ -463,19 +460,16 @@ trait ParallelTesting extends RunnerOrchestration { self => } } - case failure: Failure => - echo(renderFailure(failure)) + case Failure(output) => + echo(output) failTestSource(testSource) - case _: Timeout => + case Timeout => echo("failed because test " + testSource.title + " timed out") failTestSource(testSource, Some("test timed out")) } } - private def renderFailure(failure: Failure): String = - failure.message + "\n" + failure.stacktrace - protected def compilationRunnable(testSource: TestSource): Runnable = new Runnable { def run(): Unit = tryCompile(testSource) { val (errorCount, warningCount, hasCheckFile, verifier: Function0[Unit]) = testSource match { @@ -519,17 +513,14 @@ trait ParallelTesting extends RunnerOrchestration { self => } if (errorCount == 0 && hasCheckFile) verifier() - else if (errorCount == 0) runMain(testSource.outDir) match { - case status: Failure => - echo(renderFailure(status)) + else if (errorCount == 0) runMain(testSource.classPath) match { + case Success(_) => // success! + case Failure(output) => failTestSource(testSource) - case _: Timeout => - echo("failed because test " + testSource.title + " timed out") + case Timeout => failTestSource(testSource, Some("test timed out")) - case _: Success => // success! } else if (errorCount > 0) { - echo(s"\nCompilation failed for: '$testSource'") val buildInstr = testSource.buildInstructions(errorCount, warningCount) addFailureInstruction(buildInstr) failTestSource(testSource) diff --git a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala index a75b1c564..22bebf714 100644 --- a/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala +++ b/compiler/test/dotty/tools/vulpix/RunnerOrchestration.scala @@ -1,11 +1,8 @@ -package dotty.tools +package dotty +package tools package vulpix -import java.io.{ - File => JFile, - InputStream, ObjectInputStream, - OutputStream, ObjectOutputStream -} +import java.io.{ File => JFile, InputStreamReader, BufferedReader, PrintStream } import java.util.concurrent.TimeoutException import scala.concurrent.duration.Duration @@ -13,8 +10,25 @@ import scala.concurrent.{ Await, Future } import scala.concurrent.ExecutionContext.Implicits.global import scala.collection.mutable -import vulpix.Statuses._ - +/** Vulpix spawns JVM subprocesses (`numberOfSlaves`) in order to run tests + * without compromising the main JVM + * + * These need to be orchestrated in a safe manner with a simple protocol. This + * interface provides just that. + * + * The protocol is defined as: + * + * - master sends classpath to for which to run `Test#main` and waits for + * `maxDuration` + * - slave invokes the method and waits until completion + * - upon completion it sends back a `RunComplete` message + * - the master checks if the child is still alive + * - child is still alive, the output was valid + * - child is dead, the output is the failure message + * + * If this whole chain of events is not completed within `maxDuration`, the + * child process is destroyed and a new child is spawned. + */ trait RunnerOrchestration { /** The maximum amount of active runners, which contain a child JVM */ @@ -29,25 +43,24 @@ trait RunnerOrchestration { def safeMode: Boolean /** Running a `Test` class's main method from the specified `dir` */ - def runMain(dir: JFile): Status = monitor.runMain(dir) + def runMain(classPath: String): Status = monitor.runMain(classPath) private[this] val monitor = new RunnerMonitor + /** Look away now, sweet child of summer */ private class RunnerMonitor { - def runMain(dir: JFile): Status = withRunner(_.runMain(dir)) + def runMain(classPath: String): Status = withRunner(_.runMain(classPath)) private class Runner(private var process: Process) { - private[this] var ois: ObjectInputStream = _ - private[this] var oos: ObjectOutputStream = _ + private[this] var childStdout: BufferedReader = _ + private[this] var childStdin: PrintStream = _ /** Checks if `process` is still alive * * When `process.exitValue()` is called on an active process the caught * exception is thrown. As such we can know if the subprocess exited or * not. - * - * @note used for debug */ def isAlive: Boolean = try { process.exitValue(); false } @@ -57,12 +70,12 @@ trait RunnerOrchestration { def kill(): Unit = { if (process ne null) process.destroy() process = null - ois = null - oos = null + childStdout = null + childStdin = null } /** Blocks less than `maxDuration` while running `Test.main` from `dir` */ - def runMain(dir: JFile): Status = { + def runMain(classPath: String): Status = { assert(process ne null, "Runner was killed and then reused without setting a new process") @@ -70,33 +83,45 @@ trait RunnerOrchestration { def respawn(): Unit = { process.destroy() process = createProcess - ois = null - oos = null + childStdout = null + childStdin = null } - if (oos eq null) oos = new ObjectOutputStream(process.getOutputStream) + if (childStdin eq null) + childStdin = new PrintStream(process.getOutputStream, /* autoFlush = */ true) // pass file to running process - oos.writeObject(dir) - oos.flush() + childStdin.println(classPath) // Create a future reading the object: - val readObject = Future { - if (ois eq null) ois = new ObjectInputStream(process.getInputStream) - ois.readObject().asInstanceOf[Status] + val readOutput = Future { + val sb = new StringBuilder + + if (childStdout eq null) + childStdout = new BufferedReader(new InputStreamReader(process.getInputStream)) + + var childOutput = childStdout.readLine() + while (childOutput != ChildJVMMain.MessageEnd && childOutput != null) { + sb.append(childOutput) + sb += '\n' + childOutput = childStdout.readLine() + } + + if (process.isAlive && childOutput != null) Success(sb.toString) + else Failure(sb.toString) } // Await result for `maxDuration` and then timout and destroy the // process: val status = - try Await.result(readObject, maxDuration) - catch { case _: TimeoutException => new Timeout() } + try Await.result(readOutput, maxDuration) + catch { case _: TimeoutException => Timeout } // Handle failure of the VM: status match { - case _ if safeMode => respawn() + case _: Success if safeMode => respawn() case _: Failure => respawn() - case _: Timeout => respawn() + case Timeout => respawn() case _ => () } status @@ -105,9 +130,11 @@ trait RunnerOrchestration { private def createProcess: Process = { val sep = sys.props("file.separator") - val cp = sys.props("java.class.path") - val java = sys.props("java.home") + sep + "bin" + sep + "java" - new ProcessBuilder(java, "-cp", cp, "dotty.tools.dotc.vulpix.ChildMain")//classOf[ChildMain].getName) + val cp = + classOf[ChildJVMMain].getProtectionDomain.getCodeSource.getLocation.getFile + ":" + + Jars.scalaLibraryFromRuntime + val javaBin = sys.props("java.home") + sep + "bin" + sep + "java" + new ProcessBuilder(javaBin, "-cp", cp, "dotty.tools.vulpix.ChildJVMMain") .redirectErrorStream(true) .redirectInput(ProcessBuilder.Redirect.PIPE) .redirectOutput(ProcessBuilder.Redirect.PIPE) diff --git a/compiler/test/dotty/tools/vulpix/Status.scala b/compiler/test/dotty/tools/vulpix/Status.scala new file mode 100644 index 000000000..3de7aff2b --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/Status.scala @@ -0,0 +1,7 @@ +package dotty.tools +package vulpix + +sealed trait Status +final case class Success(output: String) extends Status +final case class Failure(output: String) extends Status +final case object Timeout extends Status diff --git a/compiler/test/dotty/tools/vulpix/Statuses.java b/compiler/test/dotty/tools/vulpix/Statuses.java deleted file mode 100644 index 68add30eb..000000000 --- a/compiler/test/dotty/tools/vulpix/Statuses.java +++ /dev/null @@ -1,25 +0,0 @@ -package dotty.tools.vulpix; - -import java.io.Serializable; - -/** The status of each call to `main` in the test applications */ -public class Statuses { - interface Status {} - - static class Success implements Status, Serializable { - public final String output; - public Success(String output) { this.output = output; } - } - - static class Failure implements Status, Serializable { - public final String message; - public final String stacktrace; - - public Failure(String message, String stacktrace) { - this.message = message; - this.stacktrace = stacktrace; - } - } - - static class Timeout implements Status, Serializable {} -} -- cgit v1.2.3