From 0ea8b3d10ae7500426b174a33ef70d03d474ecc4 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 23 Jan 2018 01:38:20 -0800 Subject: WIP splitting `mill.scalaworker` out of `mill.scalalib` and into it's own isolated module/classloader. Most scalalib test pass, tho GenIdea is still broken --- scalalib/src/mill/scalalib/GenIdea.scala | 6 +- scalalib/src/mill/scalalib/Lib.scala | 158 ++---------------------- scalalib/src/mill/scalalib/ScalaModule.scala | 9 +- scalalib/src/mill/scalalib/ScalaWorkerApi.scala | 60 +++++++++ scalalib/src/mill/scalalib/TestRunner.scala | 156 +---------------------- 5 files changed, 76 insertions(+), 313 deletions(-) create mode 100644 scalalib/src/mill/scalalib/ScalaWorkerApi.scala (limited to 'scalalib/src') diff --git a/scalalib/src/mill/scalalib/GenIdea.scala b/scalalib/src/mill/scalalib/GenIdea.scala index 4496c8c6..0f084b2d 100644 --- a/scalalib/src/mill/scalalib/GenIdea.scala +++ b/scalalib/src/mill/scalalib/GenIdea.scala @@ -1,7 +1,7 @@ package mill.scalalib import ammonite.ops._ -import mill.define.{Segment, Segments, Target} +import mill.define.{BaseModule, Segment, Segments, Target} import mill.eval.{Evaluator, PathRef, RootModuleLoader} import mill.scalalib import mill.util.Ctx.{LoaderCtx, LogCtx} @@ -10,8 +10,8 @@ import mill.util.Strict.Agg object GenIdea { - def apply()(implicit ctx: LoaderCtx with LogCtx): Unit = { - val rootModule = ctx.load(RootModuleLoader) + def apply()(implicit ctx: LogCtx, rootModule0: BaseModule.Implicit): Unit = { + val rootModule = rootModule0.value val pp = new scala.xml.PrettyPrinter(999, 4) rm! pwd/".idea" rm! pwd/".idea_modules" diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index a038a59b..8fbbfc0f 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -2,19 +2,12 @@ package mill package scalalib import java.io.File -import java.net.URLClassLoader -import java.util.Optional import ammonite.ops._ import coursier.{Cache, Fetch, MavenRepository, Repository, Resolution, Module => CoursierModule} -import mill.define.Worker import mill.eval.{PathRef, Result} -import mill.util.{Ctx} import mill.util.Loose.Agg -import sbt.internal.inc._ -import sbt.internal.util.{ConsoleOut, MainAppender} -import sbt.util.LogExchange -import xsbti.compile.{CompilerCache => _, FileAnalysisStore => _, ScalaInstance => _, _} + object CompilationResult { implicit val jsonFormatter: upickle.default.ReadWriter[CompilationResult] = upickle.default.macroRW @@ -23,21 +16,7 @@ object CompilationResult { // analysisFile is represented by Path, so we won't break caches after file changes case class CompilationResult(analysisFile: Path, classes: PathRef) -object ZincWorker extends Worker[ZincWorker]{ - def make() = new ZincWorker -} -class ZincWorker{ - @volatile var scalaClassloaderCache = Option.empty[(Long, ClassLoader)] - @volatile var scalaInstanceCache = Option.empty[(Long, ScalaInstance)] -} object Lib{ - case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClasspathEntryLookup { - override def analysis(classpathEntry: File): Optional[CompileAnalysis] = - am(classpathEntry) - - override def definesClass(classpathEntry: File): DefinesClass = - Locate.definesClass(classpathEntry) - } def grepJar(classPath: Agg[Path], s: String) = { classPath @@ -46,134 +25,13 @@ object Lib{ .toIO } - def compileScala(zincWorker: ZincWorker, - scalaVersion: String, - sources: Agg[Path], - compileClasspath: Agg[Path], - compilerClasspath: Agg[Path], - pluginClasspath: Agg[Path], - compilerBridge: Path, - scalacOptions: Seq[String], - scalacPluginClasspath: Agg[Path], - javacOptions: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) - (implicit ctx: Ctx): CompilationResult = { - val compileClasspathFiles = compileClasspath.map(_.toIO).toArray - - val compilerJars = compilerClasspath.toArray.map(_.toIO) - val pluginJars = pluginClasspath.toArray.map(_.toIO) - - val compilerClassloaderSig = compilerClasspath.map(p => p.toString().hashCode + p.mtime.toMillis).sum - val scalaInstanceSig = - compilerClassloaderSig + pluginClasspath.map(p => p.toString().hashCode + p.mtime.toMillis).sum - - val compilerClassLoader = zincWorker.scalaClassloaderCache match{ - case Some((k, v)) if k == compilerClassloaderSig => v - case _ => - val classloader = new URLClassLoader(compilerJars.map(_.toURI.toURL), null) - zincWorker.scalaClassloaderCache = Some((compilerClassloaderSig, classloader)) - classloader - } - - val scalaInstance = zincWorker.scalaInstanceCache match{ - case Some((k, v)) if k == scalaInstanceSig => v - case _ => - val scalaInstance = new ScalaInstance( - version = scalaVersion, - loader = new URLClassLoader(pluginJars.map(_.toURI.toURL), compilerClassLoader), - libraryJar = grepJar(compilerClasspath, s"scala-library-$scalaVersion.jar"), - compilerJar = grepJar(compilerClasspath, s"scala-compiler-$scalaVersion.jar"), - allJars = compilerJars ++ pluginJars, - explicitActual = None - ) - zincWorker.scalaInstanceCache = Some((scalaInstanceSig, scalaInstance)) - scalaInstance - } - - mkdir(ctx.dest) - - val ic = new sbt.internal.inc.IncrementalCompilerImpl() - - val logger = { - val consoleAppender = MainAppender.defaultScreen(ConsoleOut.printStreamOut( - ctx.log.outputStream - )) - val l = LogExchange.logger("Hello") - LogExchange.unbindLoggerAppenders("Hello") - LogExchange.bindLoggerAppenders("Hello", (consoleAppender -> sbt.util.Level.Info) :: Nil) - l - } - - def analysisMap(f: File): Optional[CompileAnalysis] = { - if (f.isFile) { - Optional.empty[CompileAnalysis] - } else { - upstreamCompileOutput.collectFirst { - case CompilationResult(zincPath, classFiles) if classFiles.path.toNIO == f.toPath => - FileAnalysisStore.binary(zincPath.toIO).get().map[CompileAnalysis](_.getAnalysis) - }.getOrElse(Optional.empty[CompileAnalysis]) - } - } - - val lookup = MockedLookup(analysisMap) - - val zincFile = ctx.dest / 'zinc - val classesDir = ctx.dest / 'classes - - val zincIOFile = zincFile.toIO - val classesIODir = classesDir.toIO - - val store = FileAnalysisStore.binary(zincIOFile) - - val newResult = ic.compile( - ic.inputs( - classpath = classesIODir +: compileClasspathFiles, - sources = for{ - root <- sources.toArray - if exists(root) - path <- ls.rec(root) - if path.isFile && (path.ext == "scala" || path.ext == "java") - } yield path.toIO, - classesDirectory = classesIODir, - scalacOptions = (scalacPluginClasspath.map(jar => s"-Xplugin:${jar}") ++ scalacOptions).toArray, - javacOptions = javacOptions.toArray, - maxErrors = 10, - sourcePositionMappers = Array(), - order = CompileOrder.Mixed, - compilers = ic.compilers( - scalaInstance, - ClasspathOptionsUtil.boot, - None, - ZincUtil.scalaCompiler(scalaInstance, compilerBridge.toIO) - ), - setup = ic.setup( - lookup, - skip = false, - zincIOFile, - new FreshCompilerCache, - IncOptions.of(), - new ManagedLoggedReporter(10, logger), - None, - Array() - ), - pr = { - val prev = store.get() - PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup)) - } - ), - logger = logger - ) - - store.set( - AnalysisContents.create( - newResult.analysis(), - newResult.setup() - ) - ) - - CompilationResult(zincFile, PathRef(classesDir)) - } - + /** + * Resolve dependencies using Coursier. + * + * We do not bother breaking this out into the separate ScalaWorker classpath, + * because Coursier is already bundled with mill/Ammonite to support the + * `import $ivy` syntax. + */ def resolveDependencies(repositories: Seq[Repository], scalaVersion: String, scalaBinaryVersion: String, diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 87bf119c..77ebc417 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -11,8 +11,6 @@ import mill.modules.Jvm.{createAssembly, createJar, interactiveSubprocess, subpr import Lib._ import mill.define.Cross.Resolver import mill.util.Loose.Agg -import sbt.testing.Status - /** * Core configuration required to compile a single Scala compilation target */ @@ -150,9 +148,9 @@ trait ScalaModule extends mill.Module with TaskModule { outer => def resources = T.input{ Agg(PathRef(basePath / 'resources)) } def generatedSources = T { Agg.empty[PathRef] } def allSources = T{ sources() ++ generatedSources() } + def compile: T[CompilationResult] = T.persistent{ - compileScala( - ZincWorker(), + mill.scalalib.ScalaWorkerApi.scalaWorker().compileScala( scalaVersion(), allSources().map(_.path), compileDepClasspath().map(_.path), @@ -165,6 +163,7 @@ trait ScalaModule extends mill.Module with TaskModule { outer => upstreamCompileOutput() ) } + def runClasspath = T{ runDepClasspath() ++ resources() ++ Seq(compile().classes) } @@ -281,7 +280,7 @@ trait ScalaModule extends mill.Module with TaskModule { outer => object TestModule{ def handleResults(doneMsg: String, results: Seq[TestRunner.Result]) = { - if (results.count(Set(Status.Error, Status.Failure)) == 0) Result.Success((doneMsg, results)) + if (results.count(Set("Error", "Failure")) == 0) Result.Success((doneMsg, results)) else { val grouped = results.map(_.status).groupBy(x => x).mapValues(_.length).filter(_._2 != 0).toList.sorted diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala new file mode 100644 index 00000000..a032ab32 --- /dev/null +++ b/scalalib/src/mill/scalalib/ScalaWorkerApi.scala @@ -0,0 +1,60 @@ +package mill.scalalib + +import java.lang.reflect.{InvocationHandler, Method} +import java.net.URI + +import ammonite.ops.Path +import coursier.maven.MavenRepository +import mill.Agg +import mill.scalalib.TestRunner.Result +import mill.T +import mill.define.{Task, Worker} +import mill.eval.PathRef +import mill.scalalib.Lib.resolveDependencies +import mill.util.Loose + +object ScalaWorkerApi extends mill.define.BaseModule(ammonite.ops.pwd){ + def scalaWorker: Worker[ScalaWorkerApi] = T.worker{ + + val scalaWorkerJar = sys.props("MILL_SCALA_WORKER") + val scalaWorkerClasspath = + if (scalaWorkerJar != null) Loose.Agg.from(scalaWorkerJar.split(',').map(Path(_))) + else { + val mill.eval.Result.Success(v) = resolveDependencies( + Seq(MavenRepository("https://repo1.maven.org/maven2")), + "2.12.4", + "2.12", + Seq(ivy"com.lihaoyi::mill-scalaworker:0.1-SNAPSHOT") + ) + v.map(_.path) + } + + val cl = new java.net.URLClassLoader( + scalaWorkerClasspath.map(_.toNIO.toUri.toURL).toArray, + getClass.getClassLoader + ) + val cls = cl.loadClass("mill.scalaworker.ScalaWorker") + val instance = cls.getConstructor(classOf[mill.util.Ctx]).newInstance(T.ctx()) + instance.asInstanceOf[ScalaWorkerApi] + } +} + +trait ScalaWorkerApi { + def compileScala(scalaVersion: String, + sources: Agg[Path], + compileClasspath: Agg[Path], + compilerClasspath: Agg[Path], + pluginClasspath: Agg[Path], + compilerBridge: Path, + scalacOptions: Seq[String], + scalacPluginClasspath: Agg[Path], + javacOptions: Seq[String], + upstreamCompileOutput: Seq[CompilationResult]) + (implicit ctx: mill.util.Ctx): CompilationResult + + def apply(frameworkName: String, + entireClasspath: Agg[Path], + testClassfilePath: Agg[Path], + args: Seq[String]) + (implicit ctx: mill.util.Ctx): (String, Seq[Result]) +} diff --git a/scalalib/src/mill/scalalib/TestRunner.scala b/scalalib/src/mill/scalalib/TestRunner.scala index 01726022..025364be 100644 --- a/scalalib/src/mill/scalalib/TestRunner.scala +++ b/scalalib/src/mill/scalalib/TestRunner.scala @@ -1,172 +1,18 @@ package mill.scalalib - -import java.io.FileInputStream -import java.lang.annotation.Annotation -import java.net.URLClassLoader -import java.util.zip.ZipInputStream - -import ammonite.ops.{Path, ls, pwd} -import ammonite.util.Colors -import mill.modules.Jvm -import mill.util.Ctx.LogCtx -import mill.util.{PrintLogger} -import mill.util.Loose.Agg -import sbt.testing._ -import upickle.Js import mill.util.JsonFormatters._ - -import scala.collection.mutable - object TestRunner { - def listClassFiles(base: Path): Iterator[String] = { - if (base.isDir) ls.rec(base).toIterator.filter(_.ext == "class").map(_.relativeTo(base).toString) - else { - val zip = new ZipInputStream(new FileInputStream(base.toIO)) - Iterator.continually(zip.getNextEntry).takeWhile(_ != null).map(_.getName).filter(_.endsWith(".class")) - } - } - def runTests(cl: ClassLoader, framework: Framework, classpath: Agg[Path]) = { - val fingerprints = framework.fingerprints() - val testClasses = classpath.flatMap { base => - listClassFiles(base).flatMap { path => - val cls = cl.loadClass(path.stripSuffix(".class").replace('/', '.')) - fingerprints.find { - case f: SubclassFingerprint => - - (f.isModule == cls.getName.endsWith("$")) && - cl.loadClass(f.superclassName()).isAssignableFrom(cls) - case f: AnnotatedFingerprint => - (f.isModule == cls.getName.endsWith("$")) && - cls.isAnnotationPresent( - cl.loadClass(f.annotationName()).asInstanceOf[Class[Annotation]] - ) - }.map { f => (cls, f) } - } - } - testClasses - } - def main(args: Array[String]): Unit = { - try{ - val result = apply( - frameworkName = args(0), - entireClasspath = Agg.from(args(1).split(" ").map(Path(_))), - testClassfilePath = Agg.from(args(2).split(" ").map(Path(_))), - args = args(3) match{ case "" => Nil case x => x.split(" ").toList } - )(new PrintLogger( - args(5) == "true", - if(args(5) == "true") Colors.Default - else Colors.BlackWhite, - System.out, - System.err, - System.err - )) - val outputPath = args(4) - - ammonite.ops.write(Path(outputPath), upickle.default.write(result)) - }catch{case e: Throwable => - println(e) - e.printStackTrace() - } - // Tests are over, kill the JVM whether or not anyone's threads are still running - // Always return 0, even if tests fail. The caller can pick up the detailed test - // results from the outputPath - System.exit(0) - } - def apply(frameworkName: String, - entireClasspath: Agg[Path], - testClassfilePath: Agg[Path], - args: Seq[String]) - (implicit ctx: LogCtx): (String, Seq[Result]) = { - Jvm.inprocess(entireClasspath, classLoaderOverrideSbtTesting = true, cl => { - val framework = cl.loadClass(frameworkName) - .newInstance() - .asInstanceOf[sbt.testing.Framework] - - val testClasses = runTests(cl, framework, testClassfilePath) - - val runner = framework.runner(args.toArray, args.toArray, cl) - - val tasks = runner.tasks( - for ((cls, fingerprint) <- testClasses.toArray) - yield new TaskDef(cls.getName.stripSuffix("$"), fingerprint, true, Array(new SuiteSelector)) - ) - val events = mutable.Buffer.empty[Event] - for (t <- tasks) { - t.execute( - new EventHandler { - def handle(event: Event) = events.append(event) - }, - Array( - new Logger { - def debug(msg: String) = ctx.log.info(msg) - - def error(msg: String) = ctx.log.error(msg) - - def ansiCodesSupported() = true - - def warn(msg: String) = ctx.log.info(msg) - - def trace(t: Throwable) = t.printStackTrace(ctx.log.outputStream) - - def info(msg: String) = ctx.log.info(msg) - }) - ) - } - val doneMsg = runner.done() - val results = for(e <- events) yield { - val ex = if (e.throwable().isDefined) Some(e.throwable().get) else None - Result( - e.fullyQualifiedName(), - e.selector() match{ - case s: NestedSuiteSelector => s.suiteId() - case s: NestedTestSelector => s.suiteId() + "." + s.testName() - case s: SuiteSelector => s.toString - case s: TestSelector => s.testName() - case s: TestWildcardSelector => s.testWildcard() - }, - e.duration(), - e.status(), - ex.map(_.getClass.getName), - ex.map(_.getMessage), - ex.map(_.getStackTrace) - ) - } - (doneMsg, results) - }) - } - case class Result(fullyQualifiedName: String, selector: String, duration: Long, - status: Status, + status: String, exceptionName: Option[String], exceptionMsg: Option[String], exceptionTrace: Option[Seq[StackTraceElement]]) object Result{ implicit def resultRW: upickle.default.ReadWriter[Result] = upickle.default.macroRW[Result] - implicit def statusRW: upickle.default.ReadWriter[Status] = upickle.default.ReadWriter[Status]( - { - case Status.Success => Js.Str("Success") - case Status.Error => Js.Str("Error") - case Status.Failure => Js.Str("Failure") - case Status.Skipped => Js.Str("Skipped") - case Status.Ignored => Js.Str("Ignored") - case Status.Canceled => Js.Str("Canceled") - case Status.Pending => Js.Str("Pending") - }, - { - case Js.Str("Success") => Status.Success - case Js.Str("Error") => Status.Error - case Js.Str("Failure") => Status.Failure - case Js.Str("Skipped") => Status.Skipped - case Js.Str("Ignored") => Status.Ignored - case Js.Str("Canceled") => Status.Canceled - case Js.Str("Pending") => Status.Pending - } - ) } } -- cgit v1.2.3