diff options
author | Guillaume Martres <smarter@ubuntu.com> | 2018-08-25 18:28:59 +0900 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-25 18:28:59 +0900 |
commit | 146d58bc5d98da1d142758f974d4d5de6f5948bf (patch) | |
tree | 1395134291f0bdf474ba8e7e6d846bbdb04a126b /scalalib/src | |
parent | e4d16b38e76884c06ce74903a318b1a148c7a63d (diff) | |
download | mill-146d58bc5d98da1d142758f974d4d5de6f5948bf.tar.gz mill-146d58bc5d98da1d142758f974d4d5de6f5948bf.tar.bz2 mill-146d58bc5d98da1d142758f974d4d5de6f5948bf.zip |
Fix incremental compilation when a Scala project depends on a Java project (#414)
* Upgrade ammonite to 1.1.2-30-53edc31
This is mainly to get https://github.com/lihaoyi/Ammonite/pull/851 which
should reduce the amount of unnecessary work done by incremental
compilation in the Mill build. This requires some code changes since
this means we now depend on a more recent version of coursier, as a
side-effect this means that we do not depend on scalaz anymore.
Also use the same ammonite version in the Mill build and in
ScalaModule#ammoniteReplClasspath.
Also remove an incorrect dependency in the caffeine integration test.
This was always wrong but did not start failing until this commit,
probably due to dependencies appearing in a different order on the
classpath.
* Rename ScalaWorker to ZincWorker
Starting with the next commit, it will be used in Java-only projects
too, so the name is misleading.
* Upgrade to Zinc 1.2.1
* Fix incremental compilation when a Scala project depends on a Java project
Before this commit, JavaModule#compile simply called javac
unconditionally, thus generating new classfiles every time. But if a
Scala project depends on a Java project, this will throw off the
incremental compilation algorithm which will unnecessarily recompile
files. To avoid this we now use Zinc to compile Java projects too (as a
bonus this means that Java compilation becomes incremental). This
required some refactoring in ZincWorkerImpl to be able to compile stuff
without having to pass Scala-specific options.
The issue solved by this commit could be reproduced by running in the
Mill repository:
$ mill main.compile
$ mill -i
@ main.compile()
and observing that before this commit, the `main.compile()` call ended
up recompiling code.
Diffstat (limited to 'scalalib/src')
-rw-r--r-- | scalalib/src/mill/scalalib/JavaModule.scala | 34 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/Lib.scala | 30 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/ScalaModule.scala | 35 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/Versions.scala | 4 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/ZincWorkerApi.scala (renamed from scalalib/src/mill/scalalib/ScalaWorkerApi.scala) | 32 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala | 6 | ||||
-rw-r--r-- | scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala | 2 |
7 files changed, 58 insertions, 85 deletions
diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 259e5cd3..ab4f8795 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -17,12 +17,13 @@ import mill.util.Loose.Agg * Core configuration required to compile a single Scala compilation target */ trait JavaModule extends mill.Module with TaskModule { outer => - def scalaWorker: ScalaWorkerModule = mill.scalalib.ScalaWorkerModule + def zincWorker: ZincWorkerModule = mill.scalalib.ZincWorkerModule trait Tests extends TestModule{ override def moduleDeps = Seq(outer) override def repositories = outer.repositories override def javacOptions = outer.javacOptions + override def zincWorker = outer.zincWorker } def defaultCommandName() = "run" @@ -38,7 +39,16 @@ trait JavaModule extends mill.Module with TaskModule { outer => def finalMainClassOpt: T[Either[String, String]] = T{ mainClass() match{ case Some(m) => Right(m) - case None => Left("No main class specified or found") + case None => + zincWorker.worker().discoverMainClasses(compile())match { + case Seq() => Left("No main class specified or found") + case Seq(main) => Right(main) + case mains => + Left( + s"Multiple main classes found (${mains.mkString(",")}) " + + "please explicitly specify which one to use by overriding mainClass" + ) + } } } @@ -98,7 +108,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => } - def repositories: Seq[Repository] = scalaWorker.repositories + def repositories: Seq[Repository] = zincWorker.repositories def platformSuffix = T{ "" } @@ -133,12 +143,12 @@ trait JavaModule extends mill.Module with TaskModule { outer => } yield PathRef(path) } - def compile: T[CompilationResult] = T{ - Lib.compileJava( - allSourceFiles().map(_.path.toIO).toArray, - compileClasspath().map(_.path.toIO).toArray, - javacOptions(), - upstreamCompileOutput() + def compile: T[CompilationResult] = T.persistent{ + zincWorker.worker().compileJava( + upstreamCompileOutput(), + allSourceFiles().map(_.path), + compileClasspath().map(_.path), + javacOptions() ) } @@ -317,7 +327,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest) try Result.Success(Jvm.interactiveSubprocess( "mill.scalalib.backgroundwrapper.BackgroundWrapper", - (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path), + (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path), forkArgs(), forkEnv(), Seq(procId.toString, procTombstone.toString, token, finalMainClass()) ++ args, @@ -332,7 +342,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest) try Result.Success(Jvm.interactiveSubprocess( "mill.scalalib.backgroundwrapper.BackgroundWrapper", - (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path), + (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path), forkArgs(), forkEnv(), Seq(procId.toString, procTombstone.toString, token, mainClass) ++ args, @@ -384,7 +394,7 @@ trait TestModule extends JavaModule with TaskModule { Jvm.subprocess( mainClass = "mill.scalalib.TestRunner", - classPath = scalaWorker.scalalibClasspath().map(_.path), + classPath = zincWorker.scalalibClasspath().map(_.path), jvmArgs = forkArgs(), envArgs = forkEnv(), mainArgs = diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index e890caaf..827c6bf4 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -27,34 +27,6 @@ object CompilationResult { case class CompilationResult(analysisFile: Path, classes: PathRef) object Lib{ - def compileJava(sources: Array[java.io.File], - classpath: Array[java.io.File], - javaOpts: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) - (implicit ctx: mill.util.Ctx) = { - val javac = ToolProvider.getSystemJavaCompiler() - if (javac == null) { - throw new Exception( - "Your Java installation is not a JDK, so it can't compile Java code;" + - " Please install the JDK version of Java") - } - - rm(ctx.dest / 'classes) - mkdir(ctx.dest / 'classes) - val cpArgs = - if(classpath.isEmpty) Seq() - else Seq("-cp", classpath.mkString(File.pathSeparator)) - - val args = Seq("-d", ctx.dest / 'classes) ++ cpArgs ++ javaOpts ++ sources - - javac.run( - ctx.log.inStream, ctx.log.outputStream, ctx.log.errorStream, - args.map(_.toString):_* - ) - if (ls(ctx.dest / 'classes).isEmpty) mill.eval.Result.Failure("Compilation Failed") - else mill.eval.Result.Success(CompilationResult(ctx.dest / 'zinc, PathRef(ctx.dest / 'classes))) - } - private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r @@ -104,7 +76,7 @@ object Lib{ /** * Resolve dependencies using Coursier. * - * We do not bother breaking this out into the separate ScalaWorker classpath, + * We do not bother breaking this out into the separate ZincWorker classpath, * because Coursier is already bundled with mill/Ammonite to support the * `import $ivy` syntax. */ diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 1899ee14..3e90521e 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -24,7 +24,7 @@ trait ScalaModule extends JavaModule { outer => override def scalacPluginIvyDeps = outer.scalacPluginIvyDeps override def scalacOptions = outer.scalacOptions override def javacOptions = outer.javacOptions - override def scalaWorker = outer.scalaWorker + override def zincWorker = outer.zincWorker override def moduleDeps: Seq[JavaModule] = Seq(outer) } @@ -60,29 +60,10 @@ trait ScalaModule extends JavaModule { outer => ) } - override def finalMainClassOpt: T[Either[String, String]] = T{ - mainClass() match{ - case Some(m) => Right(m) - case None => - scalaWorker.worker().discoverMainClasses(compile())match { - case Seq() => Left("No main class specified or found") - case Seq(main) => Right(main) - case mains => - Left( - s"Multiple main classes found (${mains.mkString(",")}) " + - "please explicitly specify which one to use by overriding mainClass" - ) - } - } - } - - def scalacPluginIvyDeps = T{ Agg.empty[Dep] } def scalacOptions = T{ Seq.empty[String] } - override def repositories: Seq[Repository] = scalaWorker.repositories - private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r def scalaCompilerBridgeSources = T { @@ -142,16 +123,16 @@ trait ScalaModule extends JavaModule { outer => } override def compile: T[CompilationResult] = T.persistent{ - scalaWorker.worker().compileScala( - scalaVersion(), + zincWorker.worker().compileMixed( + upstreamCompileOutput(), allSourceFiles().map(_.path), - scalaCompilerBridgeSources(), compileClasspath().map(_.path), - scalaCompilerClasspath().map(_.path), + javacOptions(), + scalaVersion(), scalacOptions(), + scalaCompilerBridgeSources(), + scalaCompilerClasspath().map(_.path), scalacPluginClasspath().map(_.path), - javacOptions(), - upstreamCompileOutput() ) } @@ -204,7 +185,7 @@ trait ScalaModule extends JavaModule { outer => unmanagedClasspath() ++ resolveDeps(T.task{ runIvyDeps() ++ scalaLibraryIvyDeps() ++ transitiveIvyDeps() ++ - Agg(ivy"com.lihaoyi:::ammonite:1.1.2") + Agg(ivy"com.lihaoyi:::ammonite:${Versions.ammonite}") })() } diff --git a/scalalib/src/mill/scalalib/Versions.scala b/scalalib/src/mill/scalalib/Versions.scala index 6c1c385f..bfcc5495 100644 --- a/scalalib/src/mill/scalalib/Versions.scala +++ b/scalalib/src/mill/scalalib/Versions.scala @@ -1,6 +1,8 @@ package mill.scalalib object Versions { + // Keep synchronized with ammonite dependency in core in build.sc + val ammonite = "1.1.2-30-53edc31" // Keep synchronized with zinc dependency in scalalib.worker in build.sc - val zinc = "1.1.7" + val zinc = "1.2.1" } diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ZincWorkerApi.scala index 547cc5db..ebe2f071 100644 --- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerApi.scala @@ -11,10 +11,10 @@ import mill.scalalib.Lib.resolveDependencies import mill.util.Loose import mill.util.JsonFormatters._ -object ScalaWorkerModule extends mill.define.ExternalModule with ScalaWorkerModule{ +object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ lazy val millDiscover = Discover[this.type] } -trait ScalaWorkerModule extends mill.Module{ +trait ZincWorkerModule extends mill.Module{ def repositories = Seq( Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2"), @@ -36,15 +36,15 @@ trait ScalaWorkerModule extends mill.Module{ ) } - def worker: Worker[ScalaWorkerApi] = T.worker{ + def worker: Worker[ZincWorkerApi] = T.worker{ val cl = mill.util.ClassLoader.create( classpath().map(_.path.toNIO.toUri.toURL).toVector, getClass.getClassLoader ) - val cls = cl.loadClass("mill.scalalib.worker.ScalaWorkerImpl") + val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl") val instance = cls.getConstructor(classOf[mill.util.Ctx], classOf[Array[String]]) .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String]) - instance.asInstanceOf[ScalaWorkerApi] + instance.asInstanceOf[ZincWorkerApi] } def compilerInterfaceClasspath = T{ @@ -57,20 +57,26 @@ trait ScalaWorkerModule extends mill.Module{ } -trait ScalaWorkerApi { +trait ZincWorkerApi { + /** Compile a Java-only project */ + def compileJava(upstreamCompileOutput: Seq[CompilationResult], + sources: Agg[Path], + compileClasspath: Agg[Path], + javacOptions: Seq[String]) + (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - def compileScala(scalaVersion: String, + /** Compile a mixed Scala/Java or Scala-only project */ + def compileMixed(upstreamCompileOutput: Seq[CompilationResult], sources: Agg[Path], - compilerBridgeSources: Path, compileClasspath: Agg[Path], - compilerClasspath: Agg[Path], - scalacOptions: Seq[String], - scalacPluginClasspath: Agg[Path], javacOptions: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) + scalaVersion: String, + scalacOptions: Seq[String], + compilerBridgeSources: Path, + compilerClasspath: Agg[Path], + scalacPluginClasspath: Agg[Path]) (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - def discoverMainClasses(compilationResult: CompilationResult) (implicit ctx: mill.util.Ctx): Seq[String] } diff --git a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala index 7f25764b..491911bf 100644 --- a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala +++ b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala @@ -2,16 +2,18 @@ package mill.scalalib.dependency.metadata import coursier.Cache import coursier.maven.MavenRepository +import coursier.util.Task import mill.scalalib.dependency.versions.Version private[dependency] final case class MavenMetadataLoader(mavenRepo: MavenRepository) extends MetadataLoader { - private val fetch = Cache.fetch() + private val fetch = Cache.fetch[Task]() override def getVersions(module: coursier.Module): List[Version] = { + import scala.concurrent.ExecutionContext.Implicits.global // TODO fallback to 'versionsFromListing' if 'versions' doesn't work? (needs to be made public in coursier first) - val allVersions = mavenRepo.versions(module, fetch).run.unsafePerformSync + val allVersions = mavenRepo.versions(module, fetch).run.unsafeRun allVersions .map(_.available.map(Version(_))) .getOrElse(List.empty) diff --git a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala index c4d64b0e..74f64e28 100644 --- a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala +++ b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala @@ -23,7 +23,7 @@ trait ScalafmtModule extends JavaModule { def scalafmtDeps: T[Agg[PathRef]] = T { Lib.resolveDependencies( - scalaWorker.repositories, + zincWorker.repositories, Lib.depToDependency(_, "2.12.4"), Seq(ivy"com.geirsson::scalafmt-cli:${scalafmtVersion()}") ) |