Make builds able to depend on external projects (#291)

* Make builds able to depend on external projects

Builds are now able to load external projects and depend on them
as if they were local submodules.

`import $file.external.path.build`

* Disambiguate "dest" for foreign modules.

* Calling modules loaded from external directories "Foreign" to avoid
conflicting with the already existing concept of "ExternalModule".
* Amended the way `dest` is computed for foreign modules
* Added tests to check that the source paths and dest are as expected
* Added a test to show that local modules do not conflict with foreign
modules when they are named the same

* WIP windows build fail

* Added bootstrapping step in CYGWIN CI job

* * Revert externalOutPath deletion
* Add documentation for foreign-modules

* reverting appveyor changes

* Disabling Foreign modules tests against Java9

See https://github.com/lihaoyi/mill/issues/302

11 files changed, 222 insertions, 10 deletions

diff --git a/main/test/resources/examples/foreign/conflict/build.sc b/main/test/resources/examples/foreign/conflict/build.sc
new file mode 100644
index 00000000..d6c08b81
--- /dev/null
+++ b/main/test/resources/examples/foreign/conflict/build.sc
@@ -0,0 +1,24 @@
+import $file.inner.{build => innerBuild}
+import mill._
+import ammonite.ops._
+
+// In this build, we have a local module targeting
+// the 'inner sub-directory, and an imported foreign
+// module in that same directory. Their sourcePaths
+// should be the same, but their dest paths should
+// be different to avoid both modules over-writing
+// each other's caches .
+
+def checkPaths : T[Unit] = T {
+  if (innerBuild.millSourcePath != inner.millSourcePath)
+    throw new Exception("Source paths should be the same")
+}
+
+def checkDests : T[Unit] = T {
+  if (innerBuild.selfDest == inner.selfDest)
+    throw new Exception("Dest paths should be different")
+}
+
+object inner extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
diff --git a/main/test/resources/examples/foreign/conflict/inner/build.sc b/main/test/resources/examples/foreign/conflict/inner/build.sc
new file mode 100644
index 00000000..729f4f3d
--- /dev/null
+++ b/main/test/resources/examples/foreign/conflict/inner/build.sc
@@ -0,0 +1,4 @@
+import mill._
+import ammonite.ops._
+
+def selfDest = T { T.ctx().dest / up / up }
diff --git a/main/test/resources/examples/foreign/outer/build.sc b/main/test/resources/examples/foreign/outer/build.sc
new file mode 100644
index 00000000..b53cca70
--- /dev/null
+++ b/main/test/resources/examples/foreign/outer/build.sc
@@ -0,0 +1,16 @@
+import $file.inner.build
+import mill._
+import ammonite.ops._
+
+trait PathAware extends mill.Module {
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
+object sub extends PathAware with DestAware {
+  object sub extends PathAware with DestAware
+}
+
diff --git a/main/test/resources/examples/foreign/outer/inner/build.sc b/main/test/resources/examples/foreign/outer/inner/build.sc
new file mode 100644
index 00000000..2d978292
--- /dev/null
+++ b/main/test/resources/examples/foreign/outer/inner/build.sc
@@ -0,0 +1,15 @@
+import mill._
+import ammonite.ops._
+
+trait PathAware extends mill.Module {
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
+object sub extends PathAware with DestAware {
+  object sub extends PathAware with DestAware
+}
+
diff --git a/main/test/resources/examples/foreign/project/build.sc b/main/test/resources/examples/foreign/project/build.sc
new file mode 100644
index 00000000..80c2af9b
--- /dev/null
+++ b/main/test/resources/examples/foreign/project/build.sc
@@ -0,0 +1,82 @@
+import $file.^.outer.build
+import $file.inner.build
+
+import ammonite.ops._
+import mill._
+
+def assertPaths(p1 : Path, p2 : Path) : Unit = if (p1 != p2) throw new Exception(
+  s"Paths were not equal : \n- $p1 \n- $p2"
+)
+
+object sub extends PathAware with DestAware {
+
+  object sub extends PathAware with DestAware
+
+  object sub2 extends ^.outer.build.PathAware with ^.outer.build.DestAware
+
+}
+
+def checkProjectPaths = T {
+  val thisPath : Path = millSourcePath
+  assert(thisPath.last == "project")
+  assertPaths(sub.selfPath(), thisPath / 'sub)
+  assertPaths(sub.sub.selfPath(), thisPath / 'sub / 'sub)
+  assertPaths(sub.sub2.selfPath(), thisPath / 'sub / 'sub2)
+}
+
+def checkInnerPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(inner.build.millSourcePath, thisPath / 'inner )
+  assertPaths(inner.build.sub.selfPath(), thisPath / 'inner / 'sub)
+  assertPaths(inner.build.sub.sub.selfPath(), thisPath / 'inner / 'sub / 'sub)
+}
+
+def checkOuterPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(^.outer.build.millSourcePath, thisPath / up / 'outer )
+  assertPaths(^.outer.build.sub.selfPath(), thisPath / up / 'outer / 'sub)
+  assertPaths(^.outer.build.sub.sub.selfPath(), thisPath / up / 'outer / 'sub / 'sub)
+}
+
+def checkOuterInnerPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(^.outer.inner.build.millSourcePath, thisPath / up / 'outer / 'inner )
+  assertPaths(^.outer.inner.build.sub.selfPath(), thisPath / up / 'outer / 'inner /'sub)
+  assertPaths(^.outer.inner.build.sub.sub.selfPath(), thisPath / up / 'outer / 'inner / 'sub / 'sub)
+}
+
+def checkProjectDests = T {
+  val outPath : Path = millSourcePath / 'out
+  assertPaths(sub.selfDest(), outPath / 'sub)
+  assertPaths(sub.sub.selfDest(), outPath / 'sub / 'sub)
+  assertPaths(sub.sub2.selfDest(), outPath / 'sub / 'sub2)
+}
+
+def checkInnerDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(inner.build.sub.selfDest(), foreignOut / 'inner / 'sub)
+  assertPaths(inner.build.sub.sub.selfDest(), foreignOut / 'inner / 'sub / 'sub)
+}
+
+def checkOuterDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(^.outer.build.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'sub )
+  assertPaths(^.outer.build.sub.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'sub / 'sub)
+}
+
+def checkOuterInnerDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(^.outer.inner.build.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'inner / 'sub)
+  assertPaths(^.outer.inner.build.sub.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'inner / 'sub / 'sub)
+}
+
+
+trait PathAware extends mill.Module {
+
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
diff --git a/main/test/resources/examples/foreign/project/inner/build.sc b/main/test/resources/examples/foreign/project/inner/build.sc
new file mode 100644
index 00000000..2d978292
--- /dev/null
+++ b/main/test/resources/examples/foreign/project/inner/build.sc
@@ -0,0 +1,15 @@
+import mill._
+import ammonite.ops._
+
+trait PathAware extends mill.Module {
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
+object sub extends PathAware with DestAware {
+  object sub extends PathAware with DestAware
+}
+
diff --git a/main/test/src/mill/eval/ModuleTests.scala b/main/test/src/mill/eval/ModuleTests.scala
index f089a251..0b4a7c80 100644
--- a/main/test/src/mill/eval/ModuleTests.scala
+++ b/main/test/src/mill/eval/ModuleTests.scala
@@ -22,9 +22,9 @@ object ModuleTests extends TestSuite{
     rm(TestEvaluator.externalOutPath)
     'externalModuleTargetsAreNamespacedByModulePackagePath - {
       val check = new TestEvaluator(Build)
-
-      val Right((30, 1)) = check.apply(Build.z)
+      val zresult = check.apply(Build.z)
       assert(
+        zresult == Right((30, 1)),
         read(check.evaluator.outPath / 'z / "meta.json").contains("30"),
         read(TestEvaluator.externalOutPath / 'mill / 'eval / 'ModuleTests / 'ExternalModule / 'x / "meta.json").contains("13"),
         read(TestEvaluator.externalOutPath / 'mill / 'eval / 'ModuleTests / 'ExternalModule / 'inner / 'y / "meta.json").contains("17")
diff --git a/main/test/src/mill/main/ForeignBuildsTest.scala b/main/test/src/mill/main/ForeignBuildsTest.scala
new file mode 100644
index 00000000..085ada02
--- /dev/null
+++ b/main/test/src/mill/main/ForeignBuildsTest.scala
@@ -0,0 +1,31 @@
+package mill.main
+
+import ammonite.ops._
+import mill.util.ScriptTestSuite
+import utest._
+
+object ForeignBuildsTest extends ScriptTestSuite(fork = false) {
+  def workspaceSlug = "foreign-builds"
+  def scriptSourcePath =
+    pwd / 'main / 'test / 'resources / 'examples / 'foreign
+  override def buildPath = 'project / "build.sc"
+
+  val tests = Tests {
+    initWorkspace()
+    'test - {
+      // See https://github.com/lihaoyi/mill/issues/302
+      if (!ammonite.util.Util.java9OrAbove) {
+        assert(
+          eval("checkProjectPaths"),
+          eval("checkInnerPaths"),
+          eval("checkOuterPaths"),
+          eval("checkOuterInnerPaths"),
+          eval("checkProjectDests"),
+          eval("checkInnerDests"),
+          eval("checkOuterDests"),
+          eval("checkOuterInnerDests")
+        )
+      }
+    }
+  }
+}
diff --git a/main/test/src/mill/main/ForeignConflictTest.scala b/main/test/src/mill/main/ForeignConflictTest.scala
new file mode 100644
index 00000000..c5d99c9f
--- /dev/null
+++ b/main/test/src/mill/main/ForeignConflictTest.scala
@@ -0,0 +1,25 @@
+package mill.main
+
+import ammonite.ops._
+import mill.util.ScriptTestSuite
+import utest._
+
+object ForeignConflictTest extends ScriptTestSuite(fork = false) {
+  def workspaceSlug = "foreign-conflict"
+  def scriptSourcePath =
+    pwd / 'main / 'test / 'resources / 'examples / 'foreign
+  override def buildPath = 'conflict / "build.sc"
+
+  val tests = Tests {
+    initWorkspace()
+    'test - {
+      // see https://github.com/lihaoyi/mill/issues/302
+      if (!ammonite.util.Util.java9OrAbove) {
+        assert(
+          eval("checkPaths"),
+          eval("checkDests")
+        )
+      }
+    }
+  }
+}
diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala
index 53356930..e9f31ce3 100644
--- a/main/test/src/mill/util/ScriptTestSuite.scala
+++ b/main/test/src/mill/util/ScriptTestSuite.scala
@@ -8,20 +8,21 @@ import utest._
 abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{
   def workspaceSlug: String
   def scriptSourcePath: Path
+  def buildPath: RelPath = "build.sc"
 
   val workspacePath = pwd / 'target / 'workspace / workspaceSlug
+  val wd = workspacePath / buildPath / up
   val stdOutErr = new PrintStream(new ByteArrayOutputStream())
-//  val stdOutErr = new PrintStream(System.out)
   val stdIn = new ByteArrayInputStream(Array())
   lazy val runner = new mill.main.MainRunner(
-    ammonite.main.Cli.Config(wd = workspacePath),
+    ammonite.main.Cli.Config(wd = wd),
     stdOutErr, stdOutErr, stdIn, None, Map.empty
   )
   def eval(s: String*) = {
-    if (!fork) runner.runScript(workspacePath / "build.sc", s.toList)
+    if (!fork) runner.runScript(workspacePath / buildPath , s.toList)
     else{
       try {
-        %(home / "mill-release", "-i", s)(workspacePath)
+        %(home / "mill-release", "-i", s)(wd)
         true
       }catch{case e: Throwable => false}
     }
@@ -29,7 +30,7 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{
   def meta(s: String) = {
     val (List(selector), args) = ParseArgs.apply(Seq(s), multiSelect = false).right.get
 
-    read(workspacePath / "out" / selector._2.value.flatMap(_.pathSegments) / "meta.json")
+    read(wd / "out" / selector._2.value.flatMap(_.pathSegments) / "meta.json")
   }
 
 
diff --git a/main/test/src/mill/util/TestEvaluator.scala b/main/test/src/mill/util/TestEvaluator.scala
index ffff4b99..26e4b4b5 100644
--- a/main/test/src/mill/util/TestEvaluator.scala
+++ b/main/test/src/mill/util/TestEvaluator.scala
@@ -1,8 +1,7 @@
 package mill.util
 
-import ammonite.ops.{Path, pwd}
-import mill.define.Discover.applyImpl
-import mill.define.{Discover, Input, Target, Task}
+import ammonite.ops.pwd
+import mill.define.{Input, Target, Task}
 import mill.eval.Result.OuterStack
 import mill.eval.{Evaluator, Result}
 import mill.util.Strict.Agg