From ea7fceb6e56f53bde3517586dfc57e10a605a524 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 12 Dec 2018 22:15:38 +0800 Subject: First pass at splitting out worker-api from mill core. (#504) This reduces the {scala,scalajs,scalanative}-worker dependency from the entirety of Mill to a much narrower `mill.api` module. This reduces the amount of classpath pollution within these workers, should mean they're much faster to download the first time, and reduces the amount of random junk they would pull in if they were to be used outside of the Mill project. The interactions between the various *Modules and their *WorkerImpls has been narrowed down to the `*.api` modules, which only depend on other `*.api` modules. A lot of things have been moved around; user code is unlikely to break, but it's possible some will if it references classes that have been moved around. Forwarders have been left for the few internal classes that Mill uses in it's own `build.sc`, to support bootstrapping. Third-party code which breaks should be a straightforward to fix just by updating imports The `*.api` modules have minimal dependencies (mostly uPickle and os-lib) and minimal code. There is still a bunch of implementation code in there: some of it defining data-types that are commonly sent across the module/worker interface (`Agg`, `PathRef`, ...), and some of it just general helper functions that are needed both in modules and workers. The latter code isn't strictly API definitions, but for now is small enough it's not worth splitting into it's own module --- scalalib/src/mill/scalalib/Dep.scala | 5 +- scalalib/src/mill/scalalib/GenIdeaImpl.scala | 2 +- scalalib/src/mill/scalalib/JavaModule.scala | 2 +- scalalib/src/mill/scalalib/Lib.scala | 40 +--------- scalalib/src/mill/scalalib/PublishModule.scala | 2 +- scalalib/src/mill/scalalib/ScalaModule.scala | 14 ++-- scalalib/src/mill/scalalib/ZincWorkerApi.scala | 87 ---------------------- scalalib/src/mill/scalalib/ZincWorkerModule.scala | 56 ++++++++++++++ .../dependency/DependencyUpdatesImpl.scala | 2 +- .../dependency/versions/VersionsFinder.scala | 2 +- .../mill/scalalib/publish/SonatypePublisher.scala | 2 +- .../mill/scalalib/scalafmt/ScalafmtWorker.scala | 2 +- 12 files changed, 75 insertions(+), 141 deletions(-) delete mode 100644 scalalib/src/mill/scalalib/ZincWorkerApi.scala create mode 100644 scalalib/src/mill/scalalib/ZincWorkerModule.scala (limited to 'scalalib/src') diff --git a/scalalib/src/mill/scalalib/Dep.scala b/scalalib/src/mill/scalalib/Dep.scala index b419462c..714fa21e 100644 --- a/scalalib/src/mill/scalalib/Dep.scala +++ b/scalalib/src/mill/scalalib/Dep.scala @@ -5,7 +5,7 @@ import upickle.default.{macroRW, ReadWriter => RW} import CrossVersion._ case class Dep(dep: coursier.Dependency, cross: CrossVersion, force: Boolean) { - import Dep.isDotty + import mill.scalalib.api.Util.isDotty def artifactName(binaryVersion: String, fullVersion: String, platformSuffix: String) = { val suffix = cross.suffixString(binaryVersion, fullVersion, platformSuffix) @@ -51,9 +51,6 @@ object Dep { val DefaultConfiguration = "default(compile)" - def isDotty(scalaVersion: String) = - scalaVersion.startsWith("0.") - implicit def parse(signature: String): Dep = { val parts = signature.split(';') val module = parts.head diff --git a/scalalib/src/mill/scalalib/GenIdeaImpl.scala b/scalalib/src/mill/scalalib/GenIdeaImpl.scala index 87b039df..2d76d804 100644 --- a/scalalib/src/mill/scalalib/GenIdeaImpl.scala +++ b/scalalib/src/mill/scalalib/GenIdeaImpl.scala @@ -4,7 +4,7 @@ import ammonite.runtime.SpecialClassLoader import coursier.{Cache, CoursierPaths, Repository} import mill.define._ import mill.eval.{Evaluator, PathRef, Result} -import mill.util.Ctx.{Home, Log} +import mill.api.Ctx.{Home, Log} import mill.util.Strict.Agg import mill.util.{Loose, Strict} import mill.{T, scalalib} diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 5cd9b600..78be8893 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -209,7 +209,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => /** * Compiles the current module to generate compiled classfiles/bytecode */ - def compile: T[CompilationResult] = T.persistent{ + def compile: T[mill.scalalib.api.CompilationResult] = T.persistent{ zincWorker.worker().compileJava( upstreamCompileOutput(), allSourceFiles().map(_.path), diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 80723637..b8b253bd 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -9,49 +9,17 @@ import javax.tools.ToolProvider import ammonite.util.Util import coursier.{Cache, Dependency, Fetch, Repository, Resolution} -import Dep.isDotty +import mill.scalalib.api.Util.isDotty import mill.Agg import mill.eval.{PathRef, Result} import mill.modules.Jvm -import mill.util.Ctx +import mill.api.Ctx import sbt.testing._ import scala.collection.mutable -object CompilationResult { - implicit val jsonFormatter: upickle.default.ReadWriter[CompilationResult] = upickle.default.macroRW -} - -// analysisFile is represented by os.Path, so we won't break caches after file changes -case class CompilationResult(analysisFile: os.Path, classes: PathRef) object Lib{ - private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r - private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r - private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r - - def scalaBinaryVersion(scalaVersion: String) = { - scalaVersion match { - case ReleaseVersion(major, minor, _) => s"$major.$minor" - case MinorSnapshotVersion(major, minor, _) => s"$major.$minor" - case DottyVersion(minor, _) => s"0.$minor" - case _ => scalaVersion - } - } - - def grepJar(classPath: Agg[os.Path], name: String, version: String, sources: Boolean = false) = { - val suffix = if (sources) "-sources" else "" - val mavenStylePath = s"$name-$version$suffix.jar" - val ivyStylePath = { - val dir = if (sources) "srcs" else "jars" - s"$version/$dir/$name$suffix.jar" - } - - classPath - .find(p => p.toString.endsWith(mavenStylePath) || p.toString.endsWith(ivyStylePath)) - .getOrElse(throw new Exception(s"Cannot find $mavenStylePath or $ivyStylePath")) - } - def depToDependencyJava(dep: Dep, platformSuffix: String = ""): Dependency = { assert(dep.cross.isConstant, s"Not a Java dependency: $dep") depToDependency(dep, "", platformSuffix) @@ -59,7 +27,7 @@ object Lib{ def depToDependency(dep: Dep, scalaVersion: String, platformSuffix: String = ""): Dependency = dep.toDependency( - binaryVersion = scalaBinaryVersion(scalaVersion), + binaryVersion = mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion), fullVersion = scalaVersion, platformSuffix = platformSuffix ) @@ -98,7 +66,7 @@ object Lib{ ) } def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String) = - if (isDotty(scalaVersion)) + if (mill.scalalib.api.Util.isDotty(scalaVersion)) Agg(ivy"$scalaOrganization::dotty-compiler:$scalaVersion".forceVersion()) else Agg( diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index a3d0856f..588781f4 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -2,7 +2,7 @@ package mill package scalalib import mill.define.{ExternalModule, Task} -import mill.eval.PathRef +import mill.api.PathRef import mill.scalalib.publish.{Artifact, SonatypePublisher} /** diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 38a5a8ea..9d669bf4 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -6,10 +6,10 @@ import mill.define.{Target, Task, TaskModule} import mill.eval.{PathRef, Result} import mill.modules.Jvm import mill.modules.Jvm.createJar -import Dep.isDotty +import mill.scalalib.api.Util.isDotty import Lib._ import mill.util.Loose.Agg -import mill.util.DummyInputStream +import mill.api.DummyInputStream /** * Core configuration required to compile a single Scala compilation target @@ -60,7 +60,7 @@ trait ScalaModule extends JavaModule { outer => publish.Artifact.fromDep( _: Dep, scalaVersion(), - Lib.scalaBinaryVersion(scalaVersion()), + mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion()), platformSuffix() ) } @@ -84,7 +84,7 @@ trait ScalaModule extends JavaModule { outer => def scalaCompilerBridgeSources = T { val (scalaVersion0, scalaBinaryVersion0) = scalaVersion() match { case Milestone213(_, _) => ("2.13.0-M2", "2.13.0-M2") - case _ => (scalaVersion(), Lib.scalaBinaryVersion(scalaVersion())) + case _ => (scalaVersion(), mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion())) } val (bridgeDep, bridgeName, bridgeVersion) = @@ -106,7 +106,7 @@ trait ScalaModule extends JavaModule { outer => Seq(bridgeDep), sources = true ).map(deps => - grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) + mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) ) } @@ -152,7 +152,7 @@ trait ScalaModule extends JavaModule { outer => resolveDeps(T.task{runIvyDeps() ++ scalaLibraryIvyDeps() ++ transitiveIvyDeps()})() } - override def compile: T[CompilationResult] = T.persistent{ + override def compile: T[mill.scalalib.api.CompilationResult] = T.persistent{ zincWorker.worker().compileMixed( upstreamCompileOutput(), allSourceFiles().map(_.path), @@ -262,7 +262,7 @@ trait ScalaModule extends JavaModule { outer => */ def artifactScalaVersion: T[String] = T { if (crossFullScalaVersion()) scalaVersion() - else Lib.scalaBinaryVersion(scalaVersion()) + else mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion()) } /** diff --git a/scalalib/src/mill/scalalib/ZincWorkerApi.scala b/scalalib/src/mill/scalalib/ZincWorkerApi.scala deleted file mode 100644 index dbcaa6da..00000000 --- a/scalalib/src/mill/scalalib/ZincWorkerApi.scala +++ /dev/null @@ -1,87 +0,0 @@ -package mill.scalalib - -import coursier.Cache -import coursier.maven.MavenRepository -import mill.Agg -import mill.T -import mill.define.{Discover, Worker} -import mill.scalalib.Lib.resolveDependencies -import mill.util.Loose -import mill.util.JsonFormatters._ - -object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ - lazy val millDiscover = Discover[this.type] -} -trait ZincWorkerModule extends mill.Module{ - def repositories = Seq( - Cache.ivy2Local, - MavenRepository("https://repo1.maven.org/maven2"), - MavenRepository("https://oss.sonatype.org/content/repositories/releases") - ) - - def classpath = T{ - mill.modules.Util.millProjectModule("MILL_SCALA_WORKER", "mill-scalalib-worker", repositories) - } - - def scalalibClasspath = T{ - mill.modules.Util.millProjectModule("MILL_SCALA_LIB", "mill-scalalib", repositories) - } - - def backgroundWrapperClasspath = T{ - mill.modules.Util.millProjectModule( - "MILL_BACKGROUNDWRAPPER", "mill-scalalib-backgroundwrapper", - repositories, artifactSuffix = "" - ) - } - - def worker: Worker[ZincWorkerApi] = T.worker{ - val cl = mill.util.ClassLoader.create( - classpath().map(_.path.toNIO.toUri.toURL).toVector, - getClass.getClassLoader - ) - val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl") - val instance = cls.getConstructor(classOf[mill.util.Ctx], classOf[Array[String]]) - .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String]) - instance.asInstanceOf[ZincWorkerApi] - } - - def compilerInterfaceClasspath = T{ - resolveDependencies( - repositories, - Lib.depToDependency(_, "2.12.4", ""), - Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}") - ) - } - -} - -trait ZincWorkerApi { - /** Compile a Java-only project */ - def compileJava(upstreamCompileOutput: Seq[CompilationResult], - sources: Agg[os.Path], - compileClasspath: Agg[os.Path], - javacOptions: Seq[String]) - (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - - /** Compile a mixed Scala/Java or Scala-only project */ - def compileMixed(upstreamCompileOutput: Seq[CompilationResult], - sources: Agg[os.Path], - compileClasspath: Agg[os.Path], - javacOptions: Seq[String], - scalaVersion: String, - scalacOptions: Seq[String], - compilerBridgeSources: os.Path, - compilerClasspath: Agg[os.Path], - scalacPluginClasspath: Agg[os.Path]) - (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - - def discoverMainClasses(compilationResult: CompilationResult) - (implicit ctx: mill.util.Ctx): Seq[String] - - def docJar(scalaVersion: String, - compilerBridgeSources: os.Path, - compilerClasspath: Agg[os.Path], - scalacPluginClasspath: Agg[os.Path], - args: Seq[String]) - (implicit ctx: mill.util.Ctx): Boolean -} diff --git a/scalalib/src/mill/scalalib/ZincWorkerModule.scala b/scalalib/src/mill/scalalib/ZincWorkerModule.scala new file mode 100644 index 00000000..5ca824ce --- /dev/null +++ b/scalalib/src/mill/scalalib/ZincWorkerModule.scala @@ -0,0 +1,56 @@ +package mill.scalalib + +import coursier.Cache +import coursier.maven.MavenRepository +import mill.Agg +import mill.T +import mill.define.{Discover, Worker} +import mill.scalalib.Lib.resolveDependencies +import mill.util.Loose +import mill.util.JsonFormatters._ + +object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ + lazy val millDiscover = Discover[this.type] +} +trait ZincWorkerModule extends mill.Module{ + def repositories = Seq( + Cache.ivy2Local, + MavenRepository("https://repo1.maven.org/maven2"), + MavenRepository("https://oss.sonatype.org/content/repositories/releases") + ) + + def classpath = T{ + mill.modules.Util.millProjectModule("MILL_SCALA_WORKER", "mill-scalalib-worker", repositories) + } + + def scalalibClasspath = T{ + mill.modules.Util.millProjectModule("MILL_SCALA_LIB", "mill-scalalib", repositories) + } + + def backgroundWrapperClasspath = T{ + mill.modules.Util.millProjectModule( + "MILL_BACKGROUNDWRAPPER", "mill-scalalib-backgroundwrapper", + repositories, artifactSuffix = "" + ) + } + + def worker: Worker[mill.scalalib.api.ZincWorkerApi] = T.worker{ + val cl = mill.api.ClassLoader.create( + classpath().map(_.path.toNIO.toUri.toURL).toVector, + getClass.getClassLoader + ) + val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl") + val instance = cls.getConstructor(classOf[mill.api.Ctx], classOf[Array[String]]) + .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String]) + instance.asInstanceOf[mill.scalalib.api.ZincWorkerApi] + } + + def compilerInterfaceClasspath = T{ + resolveDependencies( + repositories, + Lib.depToDependency(_, "2.12.4", ""), + Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}") + ) + } + +} diff --git a/scalalib/src/mill/scalalib/dependency/DependencyUpdatesImpl.scala b/scalalib/src/mill/scalalib/dependency/DependencyUpdatesImpl.scala index dc548c88..3bb94202 100644 --- a/scalalib/src/mill/scalalib/dependency/DependencyUpdatesImpl.scala +++ b/scalalib/src/mill/scalalib/dependency/DependencyUpdatesImpl.scala @@ -7,7 +7,7 @@ import mill.scalalib.dependency.updates.{ UpdatesFinder } import mill.scalalib.dependency.versions.VersionsFinder -import mill.util.Ctx.{Home, Log} +import mill.api.Ctx.{Home, Log} object DependencyUpdatesImpl { diff --git a/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala b/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala index b1033688..a831ffc3 100644 --- a/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala +++ b/scalalib/src/mill/scalalib/dependency/versions/VersionsFinder.scala @@ -4,7 +4,7 @@ import mill.define.{BaseModule, Task} import mill.eval.Evaluator import mill.scalalib.dependency.metadata.MetadataLoaderFactory import mill.scalalib.{Dep, JavaModule, Lib} -import mill.util.Ctx.{Home, Log} +import mill.api.Ctx.{Home, Log} import mill.util.{Loose, Strict} private[dependency] object VersionsFinder { diff --git a/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala b/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala index 5e596618..1843943b 100644 --- a/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala +++ b/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala @@ -3,7 +3,7 @@ package mill.scalalib.publish import java.math.BigInteger import java.security.MessageDigest -import mill.util.Logger +import mill.api.Logger import scalaj.http.HttpResponse diff --git a/scalalib/src/mill/scalalib/scalafmt/ScalafmtWorker.scala b/scalalib/src/mill/scalalib/scalafmt/ScalafmtWorker.scala index d6992fc8..47d8375f 100644 --- a/scalalib/src/mill/scalalib/scalafmt/ScalafmtWorker.scala +++ b/scalalib/src/mill/scalalib/scalafmt/ScalafmtWorker.scala @@ -3,7 +3,7 @@ package mill.scalalib.scalafmt import mill._ import mill.define.{Discover, ExternalModule, Worker} import mill.modules.Jvm -import mill.util.Ctx +import mill.api.Ctx import scala.collection.mutable -- cgit v1.2.3