Beginning of Bloop integration (#595)

* Sync with latest versions, started bloop connection

* BloopModule back to the bare minimum

* Added first Bloop related unit-tests

* More tests

* Fixed global module usage.

* Added resolution in bloop config

* More tests, using correct repository list

* revert dev change

* Changed moduleSourceMap implementation

* Using path-dependant trait for the module

This facilitates testing by preventing the trait from referencing
a global module.

* Added semanticDB to bloopConfig

* Added documentation

* Install returns pathRefs instead of paths

* bumped semanticDB

* Better use of mill's cache, avoid duplication of work

* addressing comments

7 files changed, 562 insertions, 2 deletions

diff --git a/build.sc b/build.sc
index eb91022b..fd5d4e29 100755
--- a/build.sc
+++ b/build.sc
@@ -306,6 +306,15 @@ object contrib extends MillModule {
     def moduleDeps = Seq(scalalib)
     def ivyDeps = Agg(ivy"org.flywaydb:flyway-core:5.2.4")
   }
+
+  object bloop extends MillModule {
+    def moduleDeps = Seq(scalalib)
+    def ivyDeps = Agg(
+      ivy"ch.epfl.scala::bloop-config:1.2.5",
+      ivy"com.lihaoyi::ujson-circe:0.7.4"
+    )
+  }
+
 }
 
 
diff --git a/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala b/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala
new file mode 100644
index 00000000..e99b25d5
--- /dev/null
+++ b/contrib/bloop/src/mill.contrib.bloop/BloopImpl.scala
@@ -0,0 +1,356 @@
+package mill.contrib.bloop
+
+import ammonite.ops._
+import bloop.config.ConfigEncoderDecoders._
+import bloop.config.{Config => BloopConfig}
+import mill._
+import mill.api.Loose
+import mill.define.{Module => MillModule, _}
+import mill.eval.Evaluator
+import mill.scalalib._
+import os.pwd
+
+/**
+  * Implementation of the Bloop related tasks. Inherited by the
+  * `mill.contrib.Bloop` object, and usable in tests by passing
+  * a custom evaluator.
+  */
+class BloopImpl(ev: () => Evaluator, wd: Path) extends ExternalModule { outer =>
+
+  /**
+    * Generates bloop configuration files reflecting the build,
+    * under pwd/.bloop.
+    */
+  def install = T {
+    Task.traverse(computeModules)(_.bloop.writeConfig)
+  }
+
+  /**
+    * Trait that can be mixed-in to quickly access the bloop config
+    * of the module.
+    *
+    * {{{
+    * object myModule extends ScalaModule with Bloop.Module {
+    *    ...
+    * }
+    * }}}
+    */
+  trait Module extends MillModule with CirceCompat { self: JavaModule =>
+
+    object bloop extends MillModule {
+      def config = T {
+        new BloopOps(self).bloop.config()
+      }
+    }
+  }
+
+  /**
+    * Extension class used to ensure that the config related tasks are
+    * cached alongside their respective modules, without requesting the user
+    * to extend a specific trait.
+    *
+    * This also ensures that we're not duplicating work between the global
+    * "install" task that traverse all modules in the build, and "local" tasks
+    * that traverse only their transitive dependencies.
+    */
+  private implicit class BloopOps(jm: JavaModule)
+      extends MillModule
+      with CirceCompat {
+    override def millOuterCtx = jm.millOuterCtx
+
+    object bloop extends MillModule {
+      def config = T { outer.bloopConfig(jm) }
+
+      def writeConfig: Target[(String, PathRef)] = T {
+        mkdir(bloopDir)
+        val path = bloopConfigPath(jm)
+        _root_.bloop.config.write(config(), path.toNIO)
+        T.ctx().log.info(s"Wrote $path")
+        name(jm) -> PathRef(path)
+      }
+
+      def writeTransitiveConfig = T {
+        Task.traverse(jm.transitiveModuleDeps)(_.bloop.writeConfig)
+      }
+    }
+  }
+
+  private val bloopDir = wd / ".bloop"
+
+  private def computeModules: Seq[JavaModule] = {
+    val eval = ev()
+    if (eval != null) {
+      val rootModule = eval.rootModule
+      rootModule.millInternal.segmentsToModules.values.collect {
+        case m: scalalib.JavaModule => m
+      }.toSeq
+    } else Seq()
+  }
+
+  /**
+    * Computes sources files paths for the whole project. Cached in a way
+    * that does not get invalidated upon sourcefile change. Mainly called
+    * from module#sources in bloopInstall
+    */
+  def moduleSourceMap: Target[Map[String, Seq[Path]]] = T {
+    val sources = Task.traverse(computeModules) { m =>
+      m.allSources.map { paths =>
+        m.millModuleSegments.render -> paths.map(_.path)
+      }
+    }()
+    sources.toMap
+  }
+
+  protected def name(m: JavaModule) = m.millModuleSegments.render
+
+  protected def bloopConfigPath(module: JavaModule): Path =
+    bloopDir / s"${name(module)}.json"
+
+  //////////////////////////////////////////////////////////////////////////////
+  // SemanticDB related configuration
+  //////////////////////////////////////////////////////////////////////////////
+
+  // Version of the semanticDB plugin.
+  def semanticDBVersion: String = "4.1.4"
+
+  // Scala versions supported by semantic db. Needs to be updated when
+  // bumping semanticDBVersion.
+  // See [https://github.com/scalameta/metals/blob/333ab6fc00fb3542bcabd0dac51b91b72798768a/build.sbt#L121]
+  def semanticDBSupported = Set(
+    "2.12.8",
+    "2.12.7",
+    "2.12.6",
+    "2.12.5",
+    "2.12.4",
+    "2.11.12",
+    "2.11.11",
+    "2.11.10",
+    "2.11.9"
+  )
+
+  // Recommended for metals usage.
+  def semanticDBOptions = List(
+    s"-P:semanticdb:sourceroot:$pwd",
+    "-P:semanticdb:synthetics:on",
+    "-P:semanticdb:failures:warning"
+  )
+
+  //////////////////////////////////////////////////////////////////////////////
+  // Computation of the bloop configuration for a specific module
+  //////////////////////////////////////////////////////////////////////////////
+
+  def bloopConfig(module: JavaModule): Task[BloopConfig.File] = {
+    import _root_.bloop.config.Config
+    def out(m: JavaModule) = bloopDir / "out" / m.millModuleSegments.render
+    def classes(m: JavaModule) = out(m) / "classes"
+
+    val javaConfig =
+      module.javacOptions.map(opts => Some(Config.Java(options = opts.toList)))
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Scalac
+    ////////////////////////////////////////////////////////////////////////////
+
+    val scalaConfig = module match {
+      case s: ScalaModule =>
+        val semanticDb = s.resolveDeps(s.scalaVersion.map {
+          case scalaV if semanticDBSupported(scalaV) =>
+            Agg(ivy"org.scalameta:semanticdb-scalac_$scalaV:$semanticDBVersion")
+          case _ => Agg()
+        })
+
+        T.task {
+          val pluginCp = semanticDb() ++ s.scalacPluginClasspath()
+          val pluginOptions = pluginCp.map { pathRef =>
+            s"-Xplugin:${pathRef.path}"
+          }
+
+          val allScalacOptions =
+            (s.scalacOptions() ++ pluginOptions ++ semanticDBOptions).toList
+          Some(
+            BloopConfig.Scala(
+              organization = "org.scala-lang",
+              name = "scala-compiler",
+              version = s.scalaVersion(),
+              options = allScalacOptions,
+              jars = s.scalaCompilerClasspath().map(_.path.toNIO).toList,
+              analysis = None,
+              setup = None
+            )
+          )
+        }
+      case _ => T.task(None)
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Platform (Jvm/Js/Native)
+    ////////////////////////////////////////////////////////////////////////////
+
+    val platform = T.task {
+      BloopConfig.Platform.Jvm(
+        BloopConfig.JvmConfig(
+          home = T.ctx().env.get("JAVA_HOME").map(s => Path(s).toNIO),
+          options = module.forkArgs().toList
+        ),
+        mainClass = module.mainClass()
+      )
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Tests
+    ////////////////////////////////////////////////////////////////////////////
+
+    val testConfig = module match {
+      case m: TestModule =>
+        T.task {
+          Some(
+            BloopConfig.Test(
+              frameworks = m
+                .testFrameworks()
+                .map(f => Config.TestFramework(List(f)))
+                .toList,
+              options = Config.TestOptions(
+                excludes = List(),
+                arguments = List()
+              )
+            )
+          )
+        }
+      case _ => T.task(None)
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    //  Ivy dependencies + sources
+    ////////////////////////////////////////////////////////////////////////////
+
+    val scalaLibraryIvyDeps = module match {
+      case x: ScalaModule => x.scalaLibraryIvyDeps
+      case _              => T.task { Loose.Agg.empty[Dep] }
+    }
+
+    /**
+      * Resolves artifacts using coursier and creates the corresponding
+      * bloop config.
+      */
+    def artifacts(repos: Seq[coursier.Repository],
+                  deps: Seq[coursier.Dependency]): List[BloopConfig.Module] = {
+      import coursier._
+      import coursier.util._
+
+      def source(r: Resolution) = Resolution(
+        r.dependencies.map(d =>
+          d.copy(attributes = d.attributes.copy(classifier = "sources")))
+      )
+
+      import scala.concurrent.ExecutionContext.Implicits.global
+      val unresolved = Resolution(deps.toSet)
+      val fetch = Fetch.from(repos, Cache.fetch[Task]())
+      val gatherTask = for {
+        resolved <- unresolved.process.run(fetch)
+        resolvedSources <- source(resolved).process.run(fetch)
+        all = resolved.dependencyArtifacts ++ resolvedSources.dependencyArtifacts
+        gathered <- Gather[Task].gather(all.distinct.map {
+          case (dep, art) => Cache.file[Task](art).run.map(dep -> _)
+        })
+      } yield
+        gathered
+          .collect {
+            case (dep, Right(file)) if Path(file).ext == "jar" =>
+              (dep.module.organization,
+               dep.module.name,
+               dep.version,
+               Option(dep.attributes.classifier).filter(_.nonEmpty),
+               file)
+          }
+          .groupBy {
+            case (org, mod, version, _, _) => (org, mod, version)
+          }
+          .mapValues {
+            _.map {
+              case (_, mod, _, classifier, file) =>
+                BloopConfig.Artifact(mod, classifier, None, file.toPath)
+            }.toList
+          }
+          .map {
+            case ((org, mod, version), artifacts) =>
+              BloopConfig.Module(
+                organization = org,
+                name = mod,
+                version = version,
+                configurations = None,
+                artifacts = artifacts
+              )
+          }
+
+      gatherTask.unsafeRun().toList
+    }
+
+    val bloopResolution: Task[BloopConfig.Resolution] = T.task {
+      val repos = module.repositories
+      val allIvyDeps = module
+        .transitiveIvyDeps() ++ scalaLibraryIvyDeps() ++ module.compileIvyDeps()
+      val coursierDeps =
+        allIvyDeps.map(module.resolveCoursierDependency()).toList
+      BloopConfig.Resolution(artifacts(repos, coursierDeps))
+    }
+
+    ////////////////////////////////////////////////////////////////////////////
+    //  Classpath
+    ////////////////////////////////////////////////////////////////////////////
+
+    val ivyDepsClasspath =
+      module
+        .resolveDeps(T.task {
+          module.compileIvyDeps() ++ module.transitiveIvyDeps()
+        })
+        .map(_.map(_.path).toSeq)
+
+    def transitiveClasspath(m: JavaModule): Task[Seq[Path]] = T.task {
+      m.moduleDeps.map(classes) ++
+        m.unmanagedClasspath().map(_.path) ++
+        Task.traverse(m.moduleDeps)(transitiveClasspath)().flatten
+    }
+
+    val classpath = T.task(transitiveClasspath(module)() ++ ivyDepsClasspath())
+    val resources = T.task(module.resources().map(_.path.toNIO).toList)
+
+    ////////////////////////////////////////////////////////////////////////////
+    //  Tying up
+    ////////////////////////////////////////////////////////////////////////////
+
+    val project = T.task {
+      val mSources = moduleSourceMap()
+        .get(name(module))
+        .toSeq
+        .flatten
+        .map(_.toNIO)
+        .toList
+
+      BloopConfig.Project(
+        name = name(module),
+        directory = module.millSourcePath.toNIO,
+        sources = mSources,
+        dependencies = module.moduleDeps.map(name).toList,
+        classpath = classpath().map(_.toNIO).toList,
+        out = out(module).toNIO,
+        classesDir = classes(module).toNIO,
+        resources = Some(resources()),
+        `scala` = scalaConfig(),
+        java = javaConfig(),
+        sbt = None,
+        test = testConfig(),
+        platform = Some(platform()),
+        resolution = Some(bloopResolution())
+      )
+    }
+
+    T.task {
+      BloopConfig.File(
+        version = BloopConfig.File.LatestVersion,
+        project = project()
+      )
+    }
+  }
+
+  lazy val millDiscover = Discover[this.type]
+}
diff --git a/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala b/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala
new file mode 100644
index 00000000..bfd88e07
--- /dev/null
+++ b/contrib/bloop/src/mill.contrib.bloop/CirceCompat.scala
@@ -0,0 +1,23 @@
+package mill.contrib.bloop
+
+import io.circe.{Decoder, Encoder, Json}
+import upickle.core.Visitor
+import upickle.default
+
+trait CirceCompat {
+
+  // Converts from a Circe encoder to a uPickle one
+  implicit def circeWriter[T: Encoder]: default.Writer[T] =
+    new default.Writer[T] {
+      override def write0[V](out: Visitor[_, V], v: T) =
+        ujson.circe.CirceJson.transform(Encoder[T].apply(v), out)
+    }
+
+  // Converts from a Circe decoder to a uPickle one
+  implicit def circeReader[T: Decoder]: default.Reader[T] =
+    new default.Reader.Delegate[Json, T](
+      ujson.circe.CirceJson.map(Decoder[T].decodeJson).map(_.right.get))
+
+}
+
+object CirceCompat extends CirceCompat
diff --git a/contrib/bloop/src/mill/contrib/Bloop.scala b/contrib/bloop/src/mill/contrib/Bloop.scala
new file mode 100644
index 00000000..9c85a308
--- /dev/null
+++ b/contrib/bloop/src/mill/contrib/Bloop.scala
@@ -0,0 +1,10 @@
+package mill.contrib
+
+import mill.eval.Evaluator
+import os.pwd
+import mill.contrib.bloop.BloopImpl
+
+/**
+  * Usage : `mill mill.contrib.Bloop/install`
+  */
+object Bloop extends BloopImpl(Evaluator.currentEvaluator.get, pwd)
diff --git a/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala b/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala
new file mode 100644
index 00000000..dfbb346d
--- /dev/null
+++ b/contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala
@@ -0,0 +1,103 @@
+package mill.contrib.bloop
+
+import bloop.config.Config.{File => BloopFile}
+import bloop.config.ConfigEncoderDecoders._
+import mill._
+import mill.contrib.bloop.CirceCompat._
+import mill.scalalib._
+import mill.util.{TestEvaluator, TestUtil}
+import os.Path
+import upickle.default._
+import utest._
+
+object BloopTests extends TestSuite {
+
+  val workdir = os.pwd / 'target / 'workspace / "bloop"
+  val testEvaluator = TestEvaluator.static(build)
+  val testBloop = new BloopImpl(() => testEvaluator.evaluator, workdir)
+
+  object build extends TestUtil.BaseModule {
+
+    override def millSourcePath = BloopTests.workdir
+
+    object scalaModule extends scalalib.ScalaModule with testBloop.Module {
+      def scalaVersion = "2.12.8"
+      val bloopVersion = "1.2.5"
+      override def mainClass = Some("foo.bar.Main")
+
+      override def ivyDeps = Agg(
+        ivy"ch.epfl.scala::bloop-config:$bloopVersion"
+      )
+      override def scalacOptions = Seq(
+        "-language:higherKinds"
+      )
+
+      object test extends super.Tests {
+        def testFrameworks = Seq("utest.runner.Framework")
+      }
+    }
+
+  }
+
+  def readBloopConf(jsonFile: String) =
+    read[BloopFile](os.read(workdir / ".bloop" / jsonFile))
+
+  def tests: Tests = Tests {
+    'genBloopTests - {
+
+      testEvaluator(testBloop.install)
+      val scalaModuleConfig = readBloopConf("scalaModule.json")
+      val testModuleConfig = readBloopConf("scalaModule.test.json")
+
+      'scalaModule - {
+        val p = scalaModuleConfig.project
+        val name = p.name
+        val sources = p.sources.map(Path(_))
+        val options = p.scala.get.options
+        val version = p.scala.get.version
+        val classpath = p.classpath.map(_.toString)
+        val platform = p.platform.get.name
+        val mainCLass = p.platform.get.mainClass.get
+        val resolution = p.resolution.get.modules
+        val sdb = testBloop.semanticDBVersion
+        val sdbOpts = testBloop.semanticDBOptions
+
+        assert(name == "scalaModule")
+        assert(sources == List(workdir / "scalaModule" / "src"))
+        assert(options.contains("-language:higherKinds"))
+        assert(options.exists(_.contains(s"semanticdb-scalac_2.12.8-$sdb.jar")))
+        assert(sdbOpts.forall(options.contains))
+        assert(version == "2.12.8")
+        assert(classpath.exists(_.contains("bloop-config_2.12-1.2.5.jar")))
+        assert(platform == "jvm")
+        assert(mainCLass == "foo.bar.Main")
+
+        val bloopConfigDep = resolution.find(_.name == "bloop-config_2.12").get
+        val artifacts = bloopConfigDep.artifacts
+        assert(bloopConfigDep.version == build.scalaModule.bloopVersion)
+        assert(bloopConfigDep.organization == "ch.epfl.scala")
+        assert(artifacts.map(_.name).distinct == List("bloop-config_2.12"))
+        assert(artifacts.flatMap(_.classifier).contains("sources"))
+      }
+      'scalaModuleTest - {
+        val p = testModuleConfig.project
+        val name = p.name
+        val sources = p.sources.map(Path(_))
+        val framework = p.test.get.frameworks.head.names.head
+        val dep = p.dependencies.head
+        val mainModuleClasspath = scalaModuleConfig.project.classpath
+        assert(name == "scalaModule.test")
+        assert(sources == List(workdir / "scalaModule" / "test" / "src"))
+        assert(framework == "utest.runner.Framework")
+        assert(dep == "scalaModule")
+        assert(mainModuleClasspath.forall(p.classpath.contains))
+      }
+      'configAccessTest - {
+        val (accessedConfig, _) =
+          testEvaluator(build.scalaModule.bloop.config).asSuccess.get.value.right.get
+        assert(accessedConfig == scalaModuleConfig)
+      }
+    }
+  }
+
+}
diff --git a/docs/pages/9 - Contrib Modules.md b/docs/pages/9 - Contrib Modules.md
index 1b9b55fa..0d65512b 100644
--- a/docs/pages/9 - Contrib Modules.md
+++ b/docs/pages/9 - Contrib Modules.md
@@ -912,3 +912,62 @@ Publishing to custom local Maven repository
 [40/40] project.publishM2Local
 Publishing to /tmp/m2repo
 ```
+
+### Bloop
+
+This plugin generates [bloop](https://scalacenter.github.io/bloop/) configuration 
+from your build file, which lets you use the bloop CLI for compiling, and makes 
+your scala code editable in [Metals](https://scalameta.org/metals/)   
+
+
+#### Quickstart:
+```scala
+// build.sc (or any other .sc file it depends on, including predef) 
+// Don't forget to replace VERSION
+import $ivy.`com.lihaoyi::mill-contrib-bloop:VERSION`
+```
+
+Then in your terminal : 
+
+```
+> mill mill.contrib.Bloop/install 
+```
+
+#### Mix-in
+
+You can mix-in the `Bloop.Module` trait with any JavaModule to quickly access 
+the deserialised configuration for that particular module: 
+
+```scala
+// build.sc 
+import mill._ 
+import mill.scalalib._
+import mill.contrib.Bloop
+
+object MyModule extends ScalaModule with Bloop.Module {
+  def myTask = T { bloop.config() }
+}
+``` 
+
+#### Note regarding metals: 
+
+Generating the bloop config should be enough for metals to pick it up and for 
+features to start working in vscode (or the bunch of other editors metals supports). 
+However, note that this applies only to your project sources. Your mill/ammonite related
+`.sc` files are not yet supported by metals. 
+
+The generated bloop config references the semanticDB compiler plugin required by
+metals to function. If need be, the version of semanticDB can be overriden by 
+extending `mill.contrib.bloop.BloopImpl` in your own space. 
+
+#### Note regarding current mill support in bloop 
+
+The mill-bloop integration currently present in the [bloop codebase](https://github.com/scalacenter/bloop/blob/master/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala#L10)
+will be deprecated in favour of this implementation. 
+
+#### Caveats: 
+
+At this time, only Java/ScalaModule are processed correctly. ScalaJS/ScalaNative integration will 
+be added in a near future. 
+
+
diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala
index 97be20be..45bc41d9 100644
--- a/main/test/src/util/TestEvaluator.scala
+++ b/main/test/src/util/TestEvaluator.scala
@@ -12,12 +12,12 @@ object TestEvaluator{
   val externalOutPath = os.pwd / 'target / 'external
 
 
-  def static(module: TestUtil.BaseModule)(implicit fullName: sourcecode.FullName) = {
+  def static(module: => TestUtil.BaseModule)(implicit fullName: sourcecode.FullName) = {
     new TestEvaluator(module)(fullName, TestPath(Nil))
   }
 }
 
-class TestEvaluator(module: TestUtil.BaseModule, failFast: Boolean = false)
+class TestEvaluator(module: => TestUtil.BaseModule, failFast: Boolean = false)
                                             (implicit fullName: sourcecode.FullName,
                                              tp: TestPath){
   val outPath =  TestUtil.getOutPath()