Fix build, split up release and assembly (#643)

* .

* .

* use non-version-changing executable for integration tests

* Update test-mill-1.sh

10 files changed, 80 insertions, 56 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 1bd0c4c7..1709b214 100755
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -21,7 +21,7 @@ environment:
     JAVA_HOME: C:\Program Files\Java\jdk10
 
 install:
-  - SET MILL_URL=https://github.com/lihaoyi/mill/releases/download/0.2.7/0.2.7-10-db8525
+  - SET MILL_URL=https://github.com/lihaoyi/mill/releases/download/0.4.2/0.4.2
 
 build_script:
   - if [%COMPILER%]==[default] (
diff --git a/.travis.yml b/.travis.yml
index e7a73872..d46cb2e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,45 +8,40 @@ git:
 matrix:
   include:
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-release.sh
+    script: ci/test-mill-release.sh
     jdk: oraclejdk8
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-release.sh
+    script: ci/test-mill-release.sh
     jdk: oraclejdk9
 
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-dev.sh
+    script: ci/test-mill-dev.sh
     jdk: oraclejdk8
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-dev.sh
+    script: ci/test-mill-dev.sh
     jdk: oraclejdk9
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-bootstrap.sh
+    script: ci/test-mill-bootstrap.sh
     jdk: oraclejdk9
 
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-0.sh
+    script: ci/test-mill-0.sh
     jdk: oraclejdk8
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-1.sh
+    script: ci/test-mill-1.sh
     jdk: oraclejdk8
   - stage: build
-    env: CI_SCRIPT=ci/test-mill-2.sh
+    script: ci/test-mill-2.sh
     jdk: oraclejdk9
 
 
   - stage: release
-    env: CI_SCRIPT="ci/on-master.py ci/release.sh"
+    script: "ci/on-master.py ci/release.sh"
     jdk: oraclejdk8
   - stage: release
-    env: CI_SCRIPT="ci/on-master.py ci/publish-docs.sh"
+    script: "ci/on-master.py ci/publish-docs.sh"
     jdk: oraclejdk8
 
-script:
-  - curl -L -o ~/bin/mill https://github.com/lihaoyi/mill/releases/download/0.2.7/0.2.7-10-db8525 && chmod +x ~/bin/mill
-  - export PATH=~/bin/mill:$PATH
-  - "$CI_SCRIPT"
-
 cache:
   directories:
   - $HOME/.coursier
diff --git a/build.sc b/build.sc
index e6aae37f..5a08960b 100755
--- a/build.sc
+++ b/build.sc
@@ -2,8 +2,7 @@ import $file.ci.shared
 import $file.ci.upload
 import java.nio.file.attribute.PosixFilePermission
 import $ivy.`org.scalaj::scalaj-http:2.4.1`
-import $ivy.`com.lihaoyi::os-lib:0.3.0`
-import ammonite.ops._
+
 import coursier.maven.MavenRepository
 import mill._
 import mill.scalalib._
@@ -62,7 +61,7 @@ trait MillModule extends MillApiModule{ outer =>
 object main extends MillModule {
   def moduleDeps = Seq(core, client)
 
-  def millBootstrap = T.sources(os.pwd / "mill-template")
+  def millBootstrap = T.sources(millSourcePath / "mill-template")
 
   def compileIvyDeps = Agg(
     ivy"org.scala-lang:scala-reflect:${scalaVersion()}"
@@ -447,13 +446,20 @@ def launcherScript(shellJvmArgs: Seq[String],
       def java(mainClass: String) =
         s"""exec $$JAVACMD $jvmArgsStr $$JAVA_OPTS -cp "${shellClassPath.mkString(":")}" $mainClass "$$@""""
 
-      val cutCount = millBootstrapGrepPrefix.length + 1
-      s"""if [ -f "$$PWD/mill" ] ; then
-         |  if [ -z "$$MILL_EXEC_PATH" ] ; then
-         |    MILL_VERSION=$$(grep -F "$millBootstrapGrepPrefix" "$$PWD/mill" | cut -c $cutCount-)
-         |    ${millBootstrapStringValue.replace("\n", "\n      ")}
-         |  fi
-         |fi
+      val bootstrapPrefix =
+        if (millBootstrapString == "") ""
+        else {
+          val cutCount = millBootstrapGrepPrefix.length + 1
+          s"""if [ -f "$$PWD/mill" ] ; then
+             |  if [ -z "$$MILL_EXEC_PATH" ] ; then
+             |    MILL_VERSION=$$(grep -F "$millBootstrapGrepPrefix" "$$PWD/mill" | cut -c $cutCount-)
+             |    ${millBootstrapStringValue.replace("\n", "\n      ")}
+             |  fi
+             |fi
+             |
+             |""".stripMargin
+        }
+      s"""$bootstrapPrefix
          |
          |if [ -z "$$JAVA_HOME" ] ; then
          |  JAVACMD="java"
@@ -516,20 +522,21 @@ object dev extends MillModule{
 
 
   // Pass dev.assembly VM options via file in Windows due to small max args limit
-  def windowsVmOptions(taskName: String, batch: Path, args: Seq[String])(implicit ctx: mill.util.Ctx) = {
+  def windowsVmOptions(taskName: String, batch: os.Path, args: Seq[String])
+                      (implicit ctx: mill.util.Ctx) = {
     if (System.getProperty("java.specification.version").startsWith("1.")) {
       throw new Error(s"$taskName in Windows is only supported using Java 9 or above")
     }
     val vmOptionsFile = T.ctx().dest / "mill.vmoptions"
     T.ctx().log.info(s"Generated $vmOptionsFile; it should be kept in the same directory as $taskName's ${batch.last}")
-    write(vmOptionsFile, args.mkString("\r\n"))
+    os.write(vmOptionsFile, args.mkString("\r\n"))
   }
 
   def launcher = T{
     val isWin = scala.util.Properties.isWin
     val outputPath = T.ctx().dest / (if (isWin) "run.bat" else "run")
 
-    write(outputPath, prependShellScript())
+    os.write(outputPath, prependShellScript())
 
     if (isWin) {
       windowsVmOptions("dev.launcher", outputPath, forkArgs())
@@ -546,7 +553,7 @@ object dev extends MillModule{
   def assembly = T{
     val isWin = scala.util.Properties.isWin
     val millPath = T.ctx().dest / (if (isWin) "mill.bat" else "mill")
-    mv(super.assembly().path, millPath)
+    os.move(super.assembly().path, millPath)
     if (isWin) windowsVmOptions("dev.launcher", millPath, forkArgs())
     PathRef(millPath)
   }
@@ -576,8 +583,8 @@ object dev extends MillModule{
     args match{
       case Nil => mill.eval.Result.Failure("Need to pass in cwd as first argument to dev.run")
       case wd0 +: rest =>
-        val wd = Path(wd0, pwd)
-        mkdir(wd)
+        val wd = os.Path(wd0, os.pwd)
+        os.makeDir(wd)
         mill.modules.Jvm.baseInteractiveSubprocess(
           Seq(launcher().path.toString) ++ rest,
           forkEnv(),
@@ -589,31 +596,49 @@ object dev extends MillModule{
   }
 }
 
-def release = T{
-  val dest = T.ctx().dest
+def releaseBase(version: String,
+                devRunClasspath: Seq[os.Path],
+                bootstrapString: String)
+               (implicit ctx: mill.api.Ctx) = {
   val filename = if (scala.util.Properties.isWin) "mill.bat" else "mill"
   val commonArgs = Seq(
-    "-DMILL_VERSION=" + publishVersion()._2,
+    "-DMILL_VERSION=" + version,
     // Workaround for Zinc/JNA bug
     // https://github.com/sbt/sbt/blame/6718803ee6023ab041b045a6988fafcfae9d15b5/main/src/main/scala/sbt/Main.scala#L130
     "-Djna.nosys=true"
   )
   val shellArgs = Seq("-DMILL_CLASSPATH=$0") ++ commonArgs
   val cmdArgs = Seq("-DMILL_CLASSPATH=%0") ++ commonArgs
-  mv(
+  os.move(
     createAssembly(
-      dev.runClasspath().map(_.path),
+      devRunClasspath,
       prependShellScript = launcherScript(
         shellArgs,
         cmdArgs,
         Agg("$0"),
         Agg("%~dpnx0"),
-        millBootstrapString()
+        bootstrapString
       )
     ).path,
-    dest / filename
+    ctx.dest / filename
+  )
+  PathRef(ctx.dest / filename)
+}
+
+def executable = T{
+  releaseBase(
+    publishVersion()._2,
+    dev.runClasspath().map(_.path),
+    ""
+  )
+}
+
+def release = T{
+  releaseBase(
+    publishVersion()._2,
+    dev.runClasspath().map(_.path),
+    millBootstrapString()
   )
-  PathRef(dest / filename)
 }
 
 val isMasterCommit = {
@@ -623,18 +648,18 @@ val isMasterCommit = {
 
 def gitHead = T.input{
   sys.env.get("TRAVIS_COMMIT").getOrElse(
-    %%('git, "rev-parse", "HEAD")(pwd).out.string.trim()
+    os.proc('git, "rev-parse", "HEAD").call().out.trim
   )
 }
 
 def publishVersion = T.input{
   val tag =
     try Option(
-      %%('git, 'describe, "--exact-match", "--tags", "--always", gitHead())(pwd).out.string.trim()
+      os.proc('git, 'describe, "--exact-match", "--tags", "--always", gitHead()).call().out.trim
     )
     catch{case e => None}
 
-  val dirtySuffix = %%('git, 'diff)(pwd).out.string.trim() match{
+  val dirtySuffix = os.proc('git, 'diff).call().out.trim match{
     case "" => ""
     case s => "-DIRTY" + Integer.toHexString(s.hashCode)
   }
@@ -642,10 +667,10 @@ def publishVersion = T.input{
   tag match{
     case Some(t) => (t, t)
     case None =>
-      val latestTaggedVersion = %%('git, 'describe, "--abbrev=0", "--always", "--tags")(pwd).out.trim
+      val latestTaggedVersion = os.proc('git, 'describe, "--abbrev=0", "--always", "--tags").call().out.trim
 
       val commitsSinceLastTag =
-        %%('git, "rev-list", gitHead(), "--not", latestTaggedVersion, "--count")(pwd).out.trim.toInt
+        os.proc('git, "rev-list", gitHead(), "--not", latestTaggedVersion, "--count").call().out.trim.toInt
 
       (latestTaggedVersion, s"$latestTaggedVersion-$commitsSinceLastTag-${gitHead().take(6)}$dirtySuffix")
   }
@@ -668,11 +693,13 @@ def uploadToGithub(authKey: String) = T.command{
       .asString
   }
 
+  upload.apply(executable().path, releaseTag, label + "-direct", authKey)
+
   upload.apply(release().path, releaseTag, label, authKey)
 
   val bootstrapScript = T.ctx().dest / "mill-bootstrap"
 
-  os.write(bootstrapScript, millBootstrapString)
+  os.write(bootstrapScript, millBootstrapString())
 
   upload.apply(bootstrapScript, releaseTag, label + "-bootstrap", authKey)
 }
diff --git a/ci/publish-local.sh b/ci/publish-local.sh
index c137340d..6342125e 100755
--- a/ci/publish-local.sh
+++ b/ci/publish-local.sh
@@ -2,6 +2,6 @@
 
 set -eux
 
-mill -i all __.publishLocal release
+./mill -i all __.publishLocal executable
 
-mv out/release/dest/mill ~/mill-release
+mv out/executable/dest/mill ~/mill-release
diff --git a/ci/release.sh b/ci/release.sh
index 28768dc0..ee0deaf8 100755
--- a/ci/release.sh
+++ b/ci/release.sh
@@ -8,7 +8,7 @@ gpg --import gpg_key
 
 rm gpg_key
 
-mill mill.scalalib.PublishModule/publishAll \
+./mill mill.scalalib.PublishModule/publishAll \
     lihaoyi:$SONATYPE_PASSWORD \
     $GPG_PASSWORD \
     __.publishArtifacts \
@@ -16,4 +16,4 @@ mill mill.scalalib.PublishModule/publishAll \
     true
 
 
-mill uploadToGithub $GITHUB_ACCESS_TOKEN
+./mill uploadToGithub $GITHUB_ACCESS_TOKEN
diff --git a/ci/test-mill-0.sh b/ci/test-mill-0.sh
index 83c361b1..e8536920 100755
--- a/ci/test-mill-0.sh
+++ b/ci/test-mill-0.sh
@@ -7,4 +7,4 @@ git clean -xdf
 
 # Run tests
 
-mill -i all {main,scalalib,scalajslib,contrib.twirllib,contrib.playlib,main.client,contrib.scalapblib,contrib.flyway,contrib.scoverage}.test
\ No newline at end of file
+./mill -i all {main,scalalib,scalajslib,contrib.twirllib,contrib.playlib,main.client,contrib.scalapblib,contrib.flyway,contrib.scoverage}.test
\ No newline at end of file
diff --git a/ci/test-mill-1.sh b/ci/test-mill-1.sh
index b0ed7bc2..df2261bd 100755
--- a/ci/test-mill-1.sh
+++ b/ci/test-mill-1.sh
@@ -6,4 +6,6 @@ set -eux
 git clean -xdf
 
 # Run tests
-mill integration.test "mill.integration.local.{JawnTests,BetterFilesTests,UpickleTests}"
+./mill integration.test "mill.integration.local.JawnTests"
+./mill integration.test "mill.integration.local.BetterFilesTests"
+./mill integration.test "mill.integration.local.UpickleTests"
diff --git a/ci/test-mill-2.sh b/ci/test-mill-2.sh
index 83eef7a3..134febc1 100755
--- a/ci/test-mill-2.sh
+++ b/ci/test-mill-2.sh
@@ -5,6 +5,6 @@ set -eux
 # Starting from scratch...
 git clean -xdf
 
-mill contrib.testng.publishLocal # Needed for CaffeineTests
+./mill contrib.testng.publishLocal # Needed for CaffeineTests
 # Run tests
-mill integration.test "mill.integration.local.{AcyclicTests,AmmoniteTests,DocAnnotationsTests}"
+./mill integration.test "mill.integration.local.{AcyclicTests,AmmoniteTests,DocAnnotationsTests}"
diff --git a/ci/test-mill-bootstrap.sh b/ci/test-mill-bootstrap.sh
index 80086df2..f5100663 100755
--- a/ci/test-mill-bootstrap.sh
+++ b/ci/test-mill-bootstrap.sh
@@ -6,7 +6,7 @@ set -eux
 git clean -xdf
 
 # First build
-mill -i all __.publishLocal release
+./mill -i all __.publishLocal release
 mv out/release/dest/mill ~/mill-1
 
 # Clean up
diff --git a/ci/test-mill-dev.sh b/ci/test-mill-dev.sh
index 1a5c5243..d81fe0df 100755
--- a/ci/test-mill-dev.sh
+++ b/ci/test-mill-dev.sh
@@ -6,7 +6,7 @@ set -eux
 git clean -xdf
 
 # Build Mill
-mill -i dev.assembly
+./mill -i dev.assembly
 
 rm -rf ~/.mill