From 4e3bcc9eeb8e3b6b13f29cb63360a512c0f6ed9b Mon Sep 17 00:00:00 2001 From: Guillaume R Date: Thu, 25 Jul 2019 01:14:09 +0200 Subject: Use the binary version of the compiler bridge when available (#659) * Use the binary version of the compiler bridge when available, fix #591 This also eliminates #389 for Dotty 0.13.0-RC1 and more recent * Add test for Dotty 0.16 --- scalalib/api/src/ZincWorkerApi.scala | 16 +++- scalalib/src/ZincWorkerModule.scala | 32 ++++--- .../test/resources/hello-dotty/boo/src/Main.scala | 17 ++++ scalalib/test/src/HelloWorldTests.scala | 4 + scalalib/worker/src/ZincWorkerImpl.scala | 106 ++++++++++++--------- 5 files changed, 113 insertions(+), 62 deletions(-) create mode 100644 scalalib/test/resources/hello-dotty/boo/src/Main.scala diff --git a/scalalib/api/src/ZincWorkerApi.scala b/scalalib/api/src/ZincWorkerApi.scala index 790ea274..0480e8e6 100644 --- a/scalalib/api/src/ZincWorkerApi.scala +++ b/scalalib/api/src/ZincWorkerApi.scala @@ -45,7 +45,7 @@ object CompilationResult { // analysisFile is represented by os.Path, so we won't break caches after file changes case class CompilationResult(analysisFile: os.Path, classes: PathRef) -object Util{ +object Util { def isDotty(scalaVersion: String) = scalaVersion.startsWith("0.") @@ -66,16 +66,22 @@ object Util{ private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r + private val DottyNightlyVersion = raw"""0\.(\d+)\.(\d+)-bin-(.*)-NIGHTLY""".r private val TypelevelVersion = raw"""(\d+)\.(\d+)\.(\d+)-bin-typelevel.*""".r - def scalaBinaryVersion(scalaVersion: String) = { - scalaVersion match { + def scalaBinaryVersion(scalaVersion: String) = scalaVersion match { case ReleaseVersion(major, minor, _) => s"$major.$minor" case MinorSnapshotVersion(major, minor, _) => s"$major.$minor" case DottyVersion(minor, _) => s"0.$minor" case TypelevelVersion(major, minor, _) => s"$major.$minor" case _ => scalaVersion - } } -} \ No newline at end of file + + /** @return true if the compiler bridge can be downloaded as an already compiled jar */ + def isBinaryBridgeAvailable(scalaVersion: String) = scalaVersion match { + case DottyNightlyVersion(minor, _, _) => minor.toInt >= 14 // 0.14.0-bin or more (not 0.13.0-bin) + case DottyVersion(minor, _) => minor.toInt >= 13 // 0.13.0-RC1 or more + case _ => false + } +} diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala index 4c94102c..81bfbb46 100644 --- a/scalalib/src/ZincWorkerModule.scala +++ b/scalalib/src/ZincWorkerModule.scala @@ -7,15 +7,15 @@ import mill.T import mill.api.KeyedLockedCache import mill.define.{Discover, Worker} import mill.scalalib.Lib.resolveDependencies -import mill.scalalib.api.Util.isDotty +import mill.scalalib.api.Util.{isDotty, isBinaryBridgeAvailable} import mill.scalalib.api.ZincWorkerApi import mill.api.Loose import mill.util.JsonFormatters._ -object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ +object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule { lazy val millDiscover = Discover[this.type] } -trait ZincWorkerModule extends mill.Module{ +trait ZincWorkerModule extends mill.Module { def repositories = Seq( coursier.LocalRepositories.ivy2Local, MavenRepository("https://repo1.maven.org/maven2"), @@ -37,7 +37,8 @@ trait ZincWorkerModule extends mill.Module{ ) } - def worker: Worker[mill.scalalib.api.ZincWorkerApi] = T.worker{ + def worker: Worker[mill.scalalib.api.ZincWorkerApi] = T.worker { + val cp = compilerInterfaceClasspath() val cl = mill.api.ClassLoader.create( classpath().map(_.path.toNIO.toUri.toURL).toVector, getClass.getClassLoader @@ -46,7 +47,7 @@ trait ZincWorkerModule extends mill.Module{ val instance = cls.getConstructor( classOf[ Either[ - (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path), + (ZincWorkerApi.Ctx, (String, String) => (Option[Array[os.Path]], os.Path)), String => os.Path ] ], @@ -58,8 +59,7 @@ trait ZincWorkerModule extends mill.Module{ .newInstance( Left(( T.ctx(), - compilerInterfaceClasspath().map(_.path).toArray, - (x: String, y: String) => scalaCompilerBridgeSourceJar(x, y).asSuccess.get.value + (x: String, y: String) => scalaCompilerBridgeJar(x, y, cp).asSuccess.get.value )), mill.scalalib.api.Util.grepJar(_, "scala-library", _, sources = false), mill.scalalib.api.Util.grepJar(_, "scala-compiler", _, sources = false), @@ -69,8 +69,9 @@ trait ZincWorkerModule extends mill.Module{ instance.asInstanceOf[mill.scalalib.api.ZincWorkerApi] } - def scalaCompilerBridgeSourceJar(scalaVersion: String, - scalaOrganization: String) = { + def scalaCompilerBridgeJar(scalaVersion: String, + scalaOrganization: String, + compileClassPath: Agg[mill.eval.PathRef]) = { val (scalaVersion0, scalaBinaryVersion0) = scalaVersion match { case s if s.startsWith("2.13.") => ("2.13.0-M2", "2.13.0-M2") case _ => (scalaVersion, mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion)) @@ -88,15 +89,18 @@ trait ZincWorkerModule extends mill.Module{ val version = Versions.zinc (ivy"$org::$name:$version", s"${name}_$scalaBinaryVersion0", version) } + val useSources = !isBinaryBridgeAvailable(scalaVersion) resolveDependencies( repositories, - Lib.depToDependency(_, scalaVersion0, ""), + Lib.depToDependency(_, scalaVersion0), Seq(bridgeDep), - sources = true - ).map(deps => - mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, sources = true) - ) + useSources + ).map{deps => + val cp = if (useSources) Some(compileClassPath.map(_.path).toArray) else None + val res = mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, useSources) + (cp, res) + } } def compilerInterfaceClasspath = T{ diff --git a/scalalib/test/resources/hello-dotty/boo/src/Main.scala b/scalalib/test/resources/hello-dotty/boo/src/Main.scala new file mode 100644 index 00000000..6b19f294 --- /dev/null +++ b/scalalib/test/resources/hello-dotty/boo/src/Main.scala @@ -0,0 +1,17 @@ +import cats._, cats.data._, cats.implicits._ + +trait Context + +object Main { + def foo(f: given Int => Int): Int = { + given x as Int = 1 + f + } + + def main(args: Array[String]): Unit = { + val x = Applicative[List].pure(1) + assert(x == List(1)) + val value = foo(given x => x + 1) + assert(value == 2) + } +} diff --git a/scalalib/test/src/HelloWorldTests.scala b/scalalib/test/src/HelloWorldTests.scala index d74d7d64..957709eb 100644 --- a/scalalib/test/src/HelloWorldTests.scala +++ b/scalalib/test/src/HelloWorldTests.scala @@ -264,6 +264,10 @@ object HelloWorldTests extends TestSuite { def scalaVersion = "0.9.0-RC1" def ivyDeps = Agg(ivy"org.typelevel::cats-core:1.2.0".withDottyCompat(scalaVersion())) } + object boo extends ScalaModule { + def scalaVersion = "0.16.0-RC3" + def ivyDeps = Agg(ivy"org.typelevel::cats-core:1.6.1".withDottyCompat(scalaVersion())) + } } val resourcePath = os.pwd / 'scalalib / 'test / 'resources / "hello-world" diff --git a/scalalib/worker/src/ZincWorkerImpl.scala b/scalalib/worker/src/ZincWorkerImpl.scala index 4d3de464..cf37812c 100644 --- a/scalalib/worker/src/ZincWorkerImpl.scala +++ b/scalalib/worker/src/ZincWorkerImpl.scala @@ -22,7 +22,7 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp } class ZincWorkerImpl(compilerBridge: Either[ - (ZincWorkerApi.Ctx, Array[os.Path], (String, String) => os.Path), + (ZincWorkerApi.Ctx, (String, String) => (Option[Array[os.Path]], os.Path)), String => os.Path ], libraryJarNameGrep: (Agg[os.Path], String) => os.Path, @@ -68,55 +68,75 @@ class ZincWorkerImpl(compilerBridge: Either[ scaladocMethod.invoke(scaladocClass.newInstance(), args.toArray).asInstanceOf[Boolean] } } - /** Compile the bridge if it doesn't exist yet and return the output directory. - * TODO: Proper invalidation, see #389 + + /** Compile the SBT/Zinc compiler bridge in the `compileDest` directory */ + def compileZincBridge(ctx0: ZincWorkerApi.Ctx, + workingDir: os.Path, + compileDest: os.Path, + scalaVersion: String, + compilerJars: Array[File], + compilerBridgeClasspath: Array[os.Path], + compilerBridgeSourcesJar: os.Path): Unit = { + ctx0.log.info("Compiling compiler interface...") + + os.makeDir.all(workingDir) + os.makeDir.all(compileDest) + + val sourceFolder = mill.api.IO.unpackZip(compilerBridgeSourcesJar)(workingDir) + val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0) + + val sources = os.walk(sourceFolder.path).filter(a => a.ext == "scala" || a.ext == "java") + + val argsArray = Array[String]( + "-d", compileDest.toString, + "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator) + ) ++ sources.map(_.toString) + + val allScala = sources.forall(_.ext == "scala") + val allJava = sources.forall(_.ext == "java") + if (allJava) { + import scala.sys.process._ + (Seq("javac") ++ argsArray).! + } else if (allScala) { + val compilerMain = classloader.loadClass( + if (isDotty(scalaVersion)) "dotty.tools.dotc.Main" + else "scala.tools.nsc.Main" + ) + compilerMain + .getMethod("process", classOf[Array[String]]) + .invoke(null, argsArray) + } else { + throw new IllegalArgumentException("Currently not implemented case.") + } + } + + /** If needed, compile (for Scala 2) or download (for Dotty) the compiler bridge. + * @return a path to the directory containing the compiled classes, or to the downloaded jar file */ - def compileZincBridgeIfNeeded(scalaVersion: String, scalaOrganization: String, compilerJars: Array[File]): os.Path = { - compilerBridge match{ + def compileBridgeIfNeeded(scalaVersion: String, scalaOrganization: String, compilerClasspath: Agg[os.Path]): os.Path = { + compilerBridge match { case Right(compiled) => compiled(scalaVersion) - case Left((ctx0, compilerBridgeClasspath, srcJars)) => + case Left((ctx0, bridgeProvider)) => val workingDir = ctx0.dest / scalaVersion val compiledDest = workingDir / 'compiled - if (!os.exists(workingDir)) { - ctx0.log.info("Compiling compiler interface...") - - os.makeDir.all(workingDir) - os.makeDir.all(compiledDest) - - val sourceFolder = mill.api.IO.unpackZip(srcJars(scalaVersion, scalaOrganization))(workingDir) - val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0) - - val sources = os.walk(sourceFolder.path).filter(a => a.ext == "scala" || a.ext == "java") - - val argsArray = Array[String]( - "-d", compiledDest.toString, - "-classpath", (compilerJars ++ compilerBridgeClasspath).mkString(File.pathSeparator) - ) ++ sources.map(_.toString) - - val allScala = sources.forall(_.ext == "scala") - val allJava = sources.forall(_.ext == "java") - if (allJava) { - import scala.sys.process._ - (Seq("javac") ++ argsArray).! - } else if (allScala) { - val compilerMain = classloader.loadClass( - if (isDotty(scalaVersion)) "dotty.tools.dotc.Main" - else "scala.tools.nsc.Main" - ) - compilerMain.getMethod("process", classOf[Array[String]]) - .invoke(null, argsArray) - } else { - throw new IllegalArgumentException("Currently not implemented case.") - } + if (os.exists(compiledDest)) { + compiledDest + } else { + val (cp, bridgeJar) = bridgeProvider(scalaVersion, scalaOrganization) + cp match { + case None => + bridgeJar + case Some(bridgeClasspath) => + val compilerJars = compilerClasspath.toArray.map(_.toIO) + compileZincBridge(ctx0, workingDir, compiledDest, scalaVersion, compilerJars, bridgeClasspath, bridgeJar) + compiledDest + } } - compiledDest } } - - def discoverMainClasses(compilationResult: CompilationResult) (implicit ctx: ZincWorkerApi.Ctx): Seq[String] = { def toScala[A](o: Optional[A]): Option[A] = if (o.isPresent) Some(o.get) else None @@ -238,10 +258,10 @@ class ZincWorkerImpl(compilerBridge: Either[ val combinedCompilerClasspath = compilerClasspath ++ scalacPluginClasspath val combinedCompilerJars = combinedCompilerClasspath.toArray.map(_.toIO) - val compiledCompilerBridge = compileZincBridgeIfNeeded( + val compiledCompilerBridge = compileBridgeIfNeeded( scalaVersion, scalaOrganization, - compilerClasspath.toArray.map(_.toIO) + compilerClasspath ) val compilerBridgeSig = os.mtime(compiledCompilerBridge) @@ -305,7 +325,7 @@ class ZincWorkerImpl(compilerBridge: Either[ val lookup = MockedLookup(analysisMap) val zincFile = ctx.dest / 'zinc - val classesDir = + val classesDir = if (compileToJar) ctx.dest / "classes.jar" else ctx.dest / "classes" -- cgit v1.2.3