From b2d480b2b9de6091a8e7c135dac5dd2c0f653c6b Mon Sep 17 00:00:00 2001 From: Olivier Mélois Date: Thu, 25 Apr 2019 13:59:29 +0200 Subject: Beginning of Bloop integration (#595) * Sync with latest versions, started bloop connection * BloopModule back to the bare minimum * Added first Bloop related unit-tests * More tests * Fixed global module usage. * Added resolution in bloop config * More tests, using correct repository list * revert dev change * Changed moduleSourceMap implementation * Using path-dependant trait for the module This facilitates testing by preventing the trait from referencing a global module. * Added semanticDB to bloopConfig * Added documentation * Install returns pathRefs instead of paths * bumped semanticDB * Better use of mill's cache, avoid duplication of work * addressing comments --- build.sc | 9 + .../bloop/src/mill.contrib.bloop/BloopImpl.scala | 356 +++++++++++++++++++++ .../bloop/src/mill.contrib.bloop/CirceCompat.scala | 23 ++ contrib/bloop/src/mill/contrib/Bloop.scala | 10 + .../test/src/mill/contrib/bloop/BloopTests.scala | 103 ++++++ docs/pages/9 - Contrib Modules.md | 59 ++++ main/test/src/util/TestEvaluator.scala | 4 +- 7 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala create mode 100644 contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala create mode 100644 contrib/bloop/src/mill/contrib/Bloop.scala create mode 100644 contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala diff --git a/build.sc b/build.sc index eb91022b..fd5d4e29 100755 --- a/build.sc +++ b/build.sc @@ -306,6 +306,15 @@ object contrib extends MillModule { def moduleDeps = Seq(scalalib) def ivyDeps = Agg(ivy"org.flywaydb:flyway-core:5.2.4") } + + object bloop extends MillModule { + def moduleDeps = Seq(scalalib) + def ivyDeps = Agg( + ivy"ch.epfl.scala::bloop-config:1.2.5", + ivy"com.lihaoyi::ujson-circe:0.7.4" + ) + } + } diff --git a/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala b/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala new file mode 100644 index 00000000..e99b25d5 --- /dev/null +++ b/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala @@ -0,0 +1,356 @@ +package mill.contrib.bloop + +import ammonite.ops._ +import bloop.config.ConfigEncoderDecoders._ +import bloop.config.{Config => BloopConfig} +import mill._ +import mill.api.Loose +import mill.define.{Module => MillModule, _} +import mill.eval.Evaluator +import mill.scalalib._ +import os.pwd + +/** + * Implementation of the Bloop related tasks. Inherited by the + * `mill.contrib.Bloop` object, and usable in tests by passing + * a custom evaluator. + */ +class BloopImpl(ev: () => Evaluator, wd: Path) extends ExternalModule { outer => + + /** + * Generates bloop configuration files reflecting the build, + * under pwd/.bloop. + */ + def install = T { + Task.traverse(computeModules)(_.bloop.writeConfig) + } + + /** + * Trait that can be mixed-in to quickly access the bloop config + * of the module. + * + * {{{ + * object myModule extends ScalaModule with Bloop.Module { + * ... + * } + * }}} + */ + trait Module extends MillModule with CirceCompat { self: JavaModule => + + object bloop extends MillModule { + def config = T { + new BloopOps(self).bloop.config() + } + } + } + + /** + * Extension class used to ensure that the config related tasks are + * cached alongside their respective modules, without requesting the user + * to extend a specific trait. + * + * This also ensures that we're not duplicating work between the global + * "install" task that traverse all modules in the build, and "local" tasks + * that traverse only their transitive dependencies. + */ + private implicit class BloopOps(jm: JavaModule) + extends MillModule + with CirceCompat { + override def millOuterCtx = jm.millOuterCtx + + object bloop extends MillModule { + def config = T { outer.bloopConfig(jm) } + + def writeConfig: Target[(String, PathRef)] = T { + mkdir(bloopDir) + val path = bloopConfigPath(jm) + _root_.bloop.config.write(config(), path.toNIO) + T.ctx().log.info(s"Wrote $path") + name(jm) -> PathRef(path) + } + + def writeTransitiveConfig = T { + Task.traverse(jm.transitiveModuleDeps)(_.bloop.writeConfig) + } + } + } + + private val bloopDir = wd / ".bloop" + + private def computeModules: Seq[JavaModule] = { + val eval = ev() + if (eval != null) { + val rootModule = eval.rootModule + rootModule.millInternal.segmentsToModules.values.collect { + case m: scalalib.JavaModule => m + }.toSeq + } else Seq() + } + + /** + * Computes sources files paths for the whole project. Cached in a way + * that does not get invalidated upon sourcefile change. Mainly called + * from module#sources in bloopInstall + */ + def moduleSourceMap: Target[Map[String, Seq[Path]]] = T { + val sources = Task.traverse(computeModules) { m => + m.allSources.map { paths => + m.millModuleSegments.render -> paths.map(_.path) + } + }() + sources.toMap + } + + protected def name(m: JavaModule) = m.millModuleSegments.render + + protected def bloopConfigPath(module: JavaModule): Path = + bloopDir / s"${name(module)}.json" + + ////////////////////////////////////////////////////////////////////////////// + // SemanticDB related configuration + ////////////////////////////////////////////////////////////////////////////// + + // Version of the semanticDB plugin. + def semanticDBVersion: String = "4.1.4" + + // Scala versions supported by semantic db. Needs to be updated when + // bumping semanticDBVersion. + // See [https://github.com/scalameta/metals/blob/333ab6fc00fb3542bcabd0dac51b91b72798768a/build.sbt#L121] + def semanticDBSupported = Set( + "2.12.8", + "2.12.7", + "2.12.6", + "2.12.5", + "2.12.4", + "2.11.12", + "2.11.11", + "2.11.10", + "2.11.9" + ) + + // Recommended for metals usage. + def semanticDBOptions = List( + s"-P:semanticdb:sourceroot:$pwd", + "-P:semanticdb:synthetics:on", + "-P:semanticdb:failures:warning" + ) + + ////////////////////////////////////////////////////////////////////////////// + // Computation of the bloop configuration for a specific module + ////////////////////////////////////////////////////////////////////////////// + + def bloopConfig(module: JavaModule): Task[BloopConfig.File] = { + import _root_.bloop.config.Config + def out(m: JavaModule) = bloopDir / "out" / m.millModuleSegments.render + def classes(m: JavaModule) = out(m) / "classes" + + val javaConfig = + module.javacOptions.map(opts => Some(Config.Java(options = opts.toList))) + + //////////////////////////////////////////////////////////////////////////// + // Scalac + //////////////////////////////////////////////////////////////////////////// + + val scalaConfig = module match { + case s: ScalaModule => + val semanticDb = s.resolveDeps(s.scalaVersion.map { + case scalaV if semanticDBSupported(scalaV) => + Agg(ivy"org.scalameta:semanticdb-scalac_$scalaV:$semanticDBVersion") + case _ => Agg() + }) + + T.task { + val pluginCp = semanticDb() ++ s.scalacPluginClasspath() + val pluginOptions = pluginCp.map { pathRef => + s"-Xplugin:${pathRef.path}" + } + + val allScalacOptions = + (s.scalacOptions() ++ pluginOptions ++ semanticDBOptions).toList + Some( + BloopConfig.Scala( + organization = "org.scala-lang", + name = "scala-compiler", + version = s.scalaVersion(), + options = allScalacOptions, + jars = s.scalaCompilerClasspath().map(_.path.toNIO).toList, + analysis = None, + setup = None + ) + ) + } + case _ => T.task(None) + } + + //////////////////////////////////////////////////////////////////////////// + // Platform (Jvm/Js/Native) + //////////////////////////////////////////////////////////////////////////// + + val platform = T.task { + BloopConfig.Platform.Jvm( + BloopConfig.JvmConfig( + home = T.ctx().env.get("JAVA_HOME").map(s => Path(s).toNIO), + options = module.forkArgs().toList + ), + mainClass = module.mainClass() + ) + } + + //////////////////////////////////////////////////////////////////////////// + // Tests + //////////////////////////////////////////////////////////////////////////// + + val testConfig = module match { + case m: TestModule => + T.task { + Some( + BloopConfig.Test( + frameworks = m + .testFrameworks() + .map(f => Config.TestFramework(List(f))) + .toList, + options = Config.TestOptions( + excludes = List(), + arguments = List() + ) + ) + ) + } + case _ => T.task(None) + } + + //////////////////////////////////////////////////////////////////////////// + // Ivy dependencies + sources + //////////////////////////////////////////////////////////////////////////// + + val scalaLibraryIvyDeps = module match { + case x: ScalaModule => x.scalaLibraryIvyDeps + case _ => T.task { Loose.Agg.empty[Dep] } + } + + /** + * Resolves artifacts using coursier and creates the corresponding + * bloop config. + */ + def artifacts(repos: Seq[coursier.Repository], + deps: Seq[coursier.Dependency]): List[BloopConfig.Module] = { + import coursier._ + import coursier.util._ + + def source(r: Resolution) = Resolution( + r.dependencies.map(d => + d.copy(attributes = d.attributes.copy(classifier = "sources"))) + ) + + import scala.concurrent.ExecutionContext.Implicits.global + val unresolved = Resolution(deps.toSet) + val fetch = Fetch.from(repos, Cache.fetch[Task]()) + val gatherTask = for { + resolved <- unresolved.process.run(fetch) + resolvedSources <- source(resolved).process.run(fetch) + all = resolved.dependencyArtifacts ++ resolvedSources.dependencyArtifacts + gathered <- Gather[Task].gather(all.distinct.map { + case (dep, art) => Cache.file[Task](art).run.map(dep -> _) + }) + } yield + gathered + .collect { + case (dep, Right(file)) if Path(file).ext == "jar" => + (dep.module.organization, + dep.module.name, + dep.version, + Option(dep.attributes.classifier).filter(_.nonEmpty), + file) + } + .groupBy { + case (org, mod, version, _, _) => (org, mod, version) + } + .mapValues { + _.map { + case (_, mod, _, classifier, file) => + BloopConfig.Artifact(mod, classifier, None, file.toPath) + }.toList + } + .map { + case ((org, mod, version), artifacts) => + BloopConfig.Module( + organization = org, + name = mod, + version = version, + configurations = None, + artifacts = artifacts + ) + } + + gatherTask.unsafeRun().toList + } + + val bloopResolution: Task[BloopConfig.Resolution] = T.task { + val repos = module.repositories + val allIvyDeps = module + .transitiveIvyDeps() ++ scalaLibraryIvyDeps() ++ module.compileIvyDeps() + val coursierDeps = + allIvyDeps.map(module.resolveCoursierDependency()).toList + BloopConfig.Resolution(artifacts(repos, coursierDeps)) + } + + //////////////////////////////////////////////////////////////////////////// + // Classpath + //////////////////////////////////////////////////////////////////////////// + + val ivyDepsClasspath = + module + .resolveDeps(T.task { + module.compileIvyDeps() ++ module.transitiveIvyDeps() + }) + .map(_.map(_.path).toSeq) + + def transitiveClasspath(m: JavaModule): Task[Seq[Path]] = T.task { + m.moduleDeps.map(classes) ++ + m.unmanagedClasspath().map(_.path) ++ + Task.traverse(m.moduleDeps)(transitiveClasspath)().flatten + } + + val classpath = T.task(transitiveClasspath(module)() ++ ivyDepsClasspath()) + val resources = T.task(module.resources().map(_.path.toNIO).toList) + + //////////////////////////////////////////////////////////////////////////// + // Tying up + //////////////////////////////////////////////////////////////////////////// + + val project = T.task { + val mSources = moduleSourceMap() + .get(name(module)) + .toSeq + .flatten + .map(_.toNIO) + .toList + + BloopConfig.Project( + name = name(module), + directory = module.millSourcePath.toNIO, + sources = mSources, + dependencies = module.moduleDeps.map(name).toList, + classpath = classpath().map(_.toNIO).toList, + out = out(module).toNIO, + classesDir = classes(module).toNIO, + resources = Some(resources()), + `scala` = scalaConfig(), + java = javaConfig(), + sbt = None, + test = testConfig(), + platform = Some(platform()), + resolution = Some(bloopResolution()) + ) + } + + T.task { + BloopConfig.File( + version = BloopConfig.File.LatestVersion, + project = project() + ) + } + } + + lazy val millDiscover = Discover[this.type] +} diff --git a/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala b/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala new file mode 100644 index 00000000..bfd88e07 --- /dev/null +++ b/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala @@ -0,0 +1,23 @@ +package mill.contrib.bloop + +import io.circe.{Decoder, Encoder, Json} +import upickle.core.Visitor +import upickle.default + +trait CirceCompat { + + // Converts from a Circe encoder to a uPickle one + implicit def circeWriter[T: Encoder]: default.Writer[T] = + new default.Writer[T] { + override def write0[V](out: Visitor[_, V], v: T) = + ujson.circe.CirceJson.transform(Encoder[T].apply(v), out) + } + + // Converts from a Circe decoder to a uPickle one + implicit def circeReader[T: Decoder]: default.Reader[T] = + new default.Reader.Delegate[Json, T]( + ujson.circe.CirceJson.map(Decoder[T].decodeJson).map(_.right.get)) + +} + +object CirceCompat extends CirceCompat diff --git a/contrib/bloop/src/mill/contrib/Bloop.scala b/contrib/bloop/src/mill/contrib/Bloop.scala new file mode 100644 index 00000000..9c85a308 --- /dev/null +++ b/contrib/bloop/src/mill/contrib/Bloop.scala @@ -0,0 +1,10 @@ +package mill.contrib + +import mill.eval.Evaluator +import os.pwd +import mill.contrib.bloop.BloopImpl + +/** + * Usage : `mill mill.contrib.Bloop/install` + */ +object Bloop extends BloopImpl(Evaluator.currentEvaluator.get, pwd) diff --git a/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala b/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala new file mode 100644 index 00000000..dfbb346d --- /dev/null +++ b/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala @@ -0,0 +1,103 @@ +package mill.contrib.bloop + +import bloop.config.Config.{File => BloopFile} +import bloop.config.ConfigEncoderDecoders._ +import mill._ +import mill.contrib.bloop.CirceCompat._ +import mill.scalalib._ +import mill.util.{TestEvaluator, TestUtil} +import os.Path +import upickle.default._ +import utest._ + +object BloopTests extends TestSuite { + + val workdir = os.pwd / 'target / 'workspace / "bloop" + val testEvaluator = TestEvaluator.static(build) + val testBloop = new BloopImpl(() => testEvaluator.evaluator, workdir) + + object build extends TestUtil.BaseModule { + + override def millSourcePath = BloopTests.workdir + + object scalaModule extends scalalib.ScalaModule with testBloop.Module { + def scalaVersion = "2.12.8" + val bloopVersion = "1.2.5" + override def mainClass = Some("foo.bar.Main") + + override def ivyDeps = Agg( + ivy"ch.epfl.scala::bloop-config:$bloopVersion" + ) + override def scalacOptions = Seq( + "-language:higherKinds" + ) + + object test extends super.Tests { + def testFrameworks = Seq("utest.runner.Framework") + } + } + + } + + def readBloopConf(jsonFile: String) = + read[BloopFile](os.read(workdir / ".bloop" / jsonFile)) + + def tests: Tests = Tests { + 'genBloopTests - { + + testEvaluator(testBloop.install) + val scalaModuleConfig = readBloopConf("scalaModule.json") + val testModuleConfig = readBloopConf("scalaModule.test.json") + + 'scalaModule - { + val p = scalaModuleConfig.project + val name = p.name + val sources = p.sources.map(Path(_)) + val options = p.scala.get.options + val version = p.scala.get.version + val classpath = p.classpath.map(_.toString) + val platform = p.platform.get.name + val mainCLass = p.platform.get.mainClass.get + val resolution = p.resolution.get.modules + val sdb = testBloop.semanticDBVersion + val sdbOpts = testBloop.semanticDBOptions + + assert(name == "scalaModule") + assert(sources == List(workdir / "scalaModule" / "src")) + assert(options.contains("-language:higherKinds")) + assert(options.exists(_.contains(s"semanticdb-scalac_2.12.8-$sdb.jar"))) + assert(sdbOpts.forall(options.contains)) + assert(version == "2.12.8") + assert(classpath.exists(_.contains("bloop-config_2.12-1.2.5.jar"))) + assert(platform == "jvm") + assert(mainCLass == "foo.bar.Main") + + val bloopConfigDep = resolution.find(_.name == "bloop-config_2.12").get + val artifacts = bloopConfigDep.artifacts + assert(bloopConfigDep.version == build.scalaModule.bloopVersion) + assert(bloopConfigDep.organization == "ch.epfl.scala") + assert(artifacts.map(_.name).distinct == List("bloop-config_2.12")) + assert(artifacts.flatMap(_.classifier).contains("sources")) + } + 'scalaModuleTest - { + val p = testModuleConfig.project + val name = p.name + val sources = p.sources.map(Path(_)) + val framework = p.test.get.frameworks.head.names.head + val dep = p.dependencies.head + val mainModuleClasspath = scalaModuleConfig.project.classpath + assert(name == "scalaModule.test") + assert(sources == List(workdir / "scalaModule" / "test" / "src")) + assert(framework == "utest.runner.Framework") + assert(dep == "scalaModule") + assert(mainModuleClasspath.forall(p.classpath.contains)) + } + 'configAccessTest - { + val (accessedConfig, _) = + testEvaluator(build.scalaModule.bloop.config).asSuccess.get.value.right.get + assert(accessedConfig == scalaModuleConfig) + } + } + } + +} diff --git a/docs/pages/9 - Contrib Modules.md b/docs/pages/9 - Contrib Modules.md index 1b9b55fa..0d65512b 100644 --- a/docs/pages/9 - Contrib Modules.md +++ b/docs/pages/9 - Contrib Modules.md @@ -912,3 +912,62 @@ Publishing to custom local Maven repository [40/40] project.publishM2Local Publishing to /tmp/m2repo ``` + +### Bloop + +This plugin generates [bloop](https://scalacenter.github.io/bloop/) configuration +from your build file, which lets you use the bloop CLI for compiling, and makes +your scala code editable in [Metals](https://scalameta.org/metals/) + + +#### Quickstart: +```scala +// build.sc (or any other .sc file it depends on, including predef) +// Don't forget to replace VERSION +import $ivy.`com.lihaoyi::mill-contrib-bloop:VERSION` +``` + +Then in your terminal : + +``` +> mill mill.contrib.Bloop/install +``` + +#### Mix-in + +You can mix-in the `Bloop.Module` trait with any JavaModule to quickly access +the deserialised configuration for that particular module: + +```scala +// build.sc +import mill._ +import mill.scalalib._ +import mill.contrib.Bloop + +object MyModule extends ScalaModule with Bloop.Module { + def myTask = T { bloop.config() } +} +``` + +#### Note regarding metals: + +Generating the bloop config should be enough for metals to pick it up and for +features to start working in vscode (or the bunch of other editors metals supports). +However, note that this applies only to your project sources. Your mill/ammonite related +`.sc` files are not yet supported by metals. + +The generated bloop config references the semanticDB compiler plugin required by +metals to function. If need be, the version of semanticDB can be overriden by +extending `mill.contrib.bloop.BloopImpl` in your own space. + +#### Note regarding current mill support in bloop + +The mill-bloop integration currently present in the [bloop codebase](https://github.com/scalacenter/bloop/blob/master/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala#L10) +will be deprecated in favour of this implementation. + +#### Caveats: + +At this time, only Java/ScalaModule are processed correctly. ScalaJS/ScalaNative integration will +be added in a near future. + + diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala index 97be20be..45bc41d9 100644 --- a/main/test/src/util/TestEvaluator.scala +++ b/main/test/src/util/TestEvaluator.scala @@ -12,12 +12,12 @@ object TestEvaluator{ val externalOutPath = os.pwd / 'target / 'external - def static(module: TestUtil.BaseModule)(implicit fullName: sourcecode.FullName) = { + def static(module: => TestUtil.BaseModule)(implicit fullName: sourcecode.FullName) = { new TestEvaluator(module)(fullName, TestPath(Nil)) } } -class TestEvaluator(module: TestUtil.BaseModule, failFast: Boolean = false) +class TestEvaluator(module: => TestUtil.BaseModule, failFast: Boolean = false) (implicit fullName: sourcecode.FullName, tp: TestPath){ val outPath = TestUtil.getOutPath() -- cgit v1.2.3