diff options
Diffstat (limited to 'main')
-rw-r--r-- | main/src/mill/Main.scala | 20 | ||||
-rw-r--r-- | main/src/mill/main/MainModule.scala | 6 | ||||
-rw-r--r-- | main/src/mill/main/Server.scala | 210 | ||||
-rw-r--r-- | main/src/mill/modules/Jvm.scala | 62 | ||||
-rw-r--r-- | main/test/src/mill/eval/EvaluationTests.scala | 26 | ||||
-rw-r--r-- | main/test/src/mill/eval/JavaCompileJarTests.scala | 4 | ||||
-rw-r--r-- | main/test/src/mill/main/ClientServerTests.scala | 214 | ||||
-rw-r--r-- | main/test/src/mill/util/ScriptTestSuite.scala | 6 | ||||
-rw-r--r-- | main/test/src/mill/util/TestGraphs.scala | 17 |
9 files changed, 520 insertions, 45 deletions
diff --git a/main/src/mill/Main.scala b/main/src/mill/Main.scala index c9ec00ca..a349321e 100644 --- a/main/src/mill/Main.scala +++ b/main/src/mill/Main.scala @@ -8,27 +8,11 @@ import ammonite.main.Cli._ import ammonite.ops._ import ammonite.util.Util import io.github.retronym.java9rtexport.Export +import mill.client.ClientServer import mill.eval.Evaluator import mill.util.DummyInputStream -object ServerMain extends mill.clientserver.ServerMain[Evaluator.State]{ - def main0(args: Array[String], - stateCache: Option[Evaluator.State], - mainInteractive: Boolean, - stdin: InputStream, - stdout: PrintStream, - stderr: PrintStream, - env : Map[String, String]) = Main.main0( - args, - stateCache, - mainInteractive, - DummyInputStream, - stdout, - stderr, - env - ) -} object Main { def main(args: Array[String]): Unit = { @@ -126,7 +110,7 @@ object Main { env ) - if (mill.clientserver.ClientServer.isJava9OrAbove) { + if (ClientServer.isJava9OrAbove) { val rt = cliConfig.home / Export.rtJarName if (!exists(rt)) { runner.printInfo(s"Preparing Java ${System.getProperty("java.version")} runtime; this may take a minute or two ...") diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 7c84f74a..32281407 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -30,7 +30,11 @@ trait MainModule extends mill.Module{ implicit def millDiscover: mill.define.Discover[_] implicit def millScoptTasksReads[T] = new mill.main.Tasks.Scopt[T]() implicit def millScoptEvaluatorReads[T] = new mill.main.EvaluatorScopt[T]() - + def version() = mill.T.command { + val res = System.getProperty("MILL_VERSION") + println(res) + res + } /** * Resolves a mill query string and prints out the tasks it resolves to. */ diff --git a/main/src/mill/main/Server.scala b/main/src/mill/main/Server.scala new file mode 100644 index 00000000..14aade4c --- /dev/null +++ b/main/src/mill/main/Server.scala @@ -0,0 +1,210 @@ +package mill.main + +import java.io._ +import java.net.Socket + +import mill.Main +import scala.collection.JavaConverters._ +import org.scalasbt.ipcsocket._ +import mill.client._ +import mill.eval.Evaluator +import mill.util.DummyInputStream + +trait ServerMain[T]{ + var stateCache = Option.empty[T] + def main0(args: Array[String], + stateCache: Option[T], + mainInteractive: Boolean, + stdin: InputStream, + stdout: PrintStream, + stderr: PrintStream, + env : Map[String, String]): (Boolean, Option[T]) +} + +object ServerMain extends mill.main.ServerMain[Evaluator.State]{ + def main(args0: Array[String]): Unit = { + new Server( + args0(0), + this, + () => System.exit(0), + 300000, + mill.client.Locks.files(args0(0)) + ).run() + } + def main0(args: Array[String], + stateCache: Option[Evaluator.State], + mainInteractive: Boolean, + stdin: InputStream, + stdout: PrintStream, + stderr: PrintStream, + env : Map[String, String]) = Main.main0( + args, + stateCache, + mainInteractive, + DummyInputStream, + stdout, + stderr, + env + ) +} + + +class Server[T](lockBase: String, + sm: ServerMain[T], + interruptServer: () => Unit, + acceptTimeout: Int, + locks: Locks) { + + val originalStdout = System.out + def run() = { + Server.tryLockBlock(locks.processLock){ + var running = true + while (running) { + Server.lockBlock(locks.serverLock){ + val (serverSocket, socketClose) = if (ClientServer.isWindows) { + val socketName = ClientServer.WIN32_PIPE_PREFIX + new File(lockBase).getName + (new Win32NamedPipeServerSocket(socketName), () => new Win32NamedPipeSocket(socketName).close()) + } else { + val socketName = lockBase + "/io" + new File(socketName).delete() + (new UnixDomainServerSocket(socketName), () => new UnixDomainSocket(socketName).close()) + } + + val sockOpt = Server.interruptWith( + acceptTimeout, + socketClose(), + serverSocket.accept() + ) + + sockOpt match{ + case None => running = false + case Some(sock) => + try { + handleRun(sock) + serverSocket.close() + } + catch{case e: Throwable => e.printStackTrace(originalStdout) } + } + } + // Make sure you give an opportunity for the client to probe the lock + // and realize the server has released it to signal completion + Thread.sleep(10) + } + }.getOrElse(throw new Exception("PID already present")) + } + + def handleRun(clientSocket: Socket) = { + + val currentOutErr = clientSocket.getOutputStream + val socketIn = clientSocket.getInputStream + val argStream = new FileInputStream(lockBase + "/run") + val interactive = argStream.read() != 0; + val args = ClientServer.parseArgs(argStream) + val env = ClientServer.parseMap(argStream) + argStream.close() + + var done = false + val t = new Thread(() => + + try { + val stdout = new PrintStream(new ProxyOutputStream(currentOutErr, 0), true) + val stderr = new PrintStream(new ProxyOutputStream(currentOutErr, 1), true) + val (result, newStateCache) = sm.main0( + args, + sm.stateCache, + interactive, + socketIn, + stdout, + stderr, + env.asScala.toMap + ) + + sm.stateCache = newStateCache + java.nio.file.Files.write( + java.nio.file.Paths.get(lockBase + "/exitCode"), + (if (result) 0 else 1).toString.getBytes + ) + } catch{case WatchInterrupted(sc: Option[T]) => + sm.stateCache = sc + } finally{ + done = true + } + ) + + t.start() + // We cannot simply use Lock#await here, because the filesystem doesn't + // realize the clientLock/serverLock are held by different threads in the + // two processes and gives a spurious deadlock error + while(!done && !locks.clientLock.probe()) { + Thread.sleep(3) + } + + if (!done) interruptServer() + + t.interrupt() + t.stop() + + if (ClientServer.isWindows) { + // Closing Win32NamedPipeSocket can often take ~5s + // It seems OK to exit the client early and subsequently + // start up mill client again (perhaps closing the server + // socket helps speed up the process). + val t = new Thread(() => clientSocket.close()) + t.setDaemon(true) + t.start() + } else clientSocket.close() + } +} +object Server{ + def lockBlock[T](lock: Lock)(t: => T): T = { + val l = lock.lock() + try t + finally l.release() + } + def tryLockBlock[T](lock: Lock)(t: => T): Option[T] = { + lock.tryLock() match{ + case null => None + case l => + try Some(t) + finally l.release() + } + + } + def interruptWith[T](millis: Int, close: => Unit, t: => T): Option[T] = { + @volatile var interrupt = true + @volatile var interrupted = false + new Thread(() => { + Thread.sleep(millis) + if (interrupt) { + interrupted = true + close + } + }).start() + + try { + val res = + try Some(t) + catch {case e: Throwable => None} + + if (interrupted) None + else res + + } finally { + interrupt = false + } + } +} + +class ProxyOutputStream(x: => java.io.OutputStream, + key: Int) extends java.io.OutputStream { + override def write(b: Int) = x.synchronized{ + x.write(key) + x.write(b) + } +} +class ProxyInputStream(x: => java.io.InputStream) extends java.io.InputStream{ + def read() = x.read() + override def read(b: Array[Byte], off: Int, len: Int) = x.read(b, off, len) + override def read(b: Array[Byte]) = x.read(b) +} +case class WatchInterrupted[T](stateCache: Option[T]) extends Exception diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 92469988..e7fd6a79 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -1,6 +1,6 @@ package mill.modules -import java.io.{ByteArrayInputStream, FileOutputStream, File} +import java.io.{ByteArrayInputStream, File, FileOutputStream} import java.lang.reflect.Modifier import java.net.{URI, URLClassLoader} import java.nio.file.{FileSystems, Files, OpenOption, StandardOpenOption} @@ -9,7 +9,7 @@ import java.util.jar.{JarEntry, JarFile, JarOutputStream} import ammonite.ops._ import geny.Generator -import mill.clientserver.InputPumper +import mill.client.InputPumper import mill.eval.PathRef import mill.util.{Ctx, IO} import mill.util.Loose.Agg @@ -277,7 +277,7 @@ object Jvm { // Prepend shell script and make it executable if (prependShellScript.isEmpty) mv(tmp, output) else{ - val lineSep = if (isWin) "\r\n" else "\n" + val lineSep = if (!prependShellScript.endsWith("\n")) "\n\r\n" else "" val outputStream = newOutputStream(output.toNIO) IO.stream(new ByteArrayInputStream((prependShellScript + lineSep).getBytes()), outputStream) IO.stream(read.getInputStream(tmp), outputStream) @@ -319,30 +319,50 @@ object Jvm { } - def launcherShellScript(isWin: Boolean, - mainClass: String, - classPath: Agg[String], - jvmArgs: Seq[String]) = { - val cp = classPath.mkString(File.pathSeparator) - if (isWin) - s"""@echo off - | - |java ${jvmArgs.mkString(" ")} %JAVA_OPTS% -cp "$cp" $mainClass %* - """.stripMargin.split('\n').mkString("\r\n") - else - s"""#!/usr/bin/env sh - | - |exec java ${jvmArgs.mkString(" ")} $$JAVA_OPTS -cp "$cp" $mainClass "$$@" - """.stripMargin + def universalScript(shellCommands: String, + cmdCommands: String, + shebang: Boolean = false): String = { + Seq( + if (shebang) "#!/usr/bin/env sh" else "", + "@ 2>/dev/null # 2>nul & echo off & goto BOF\r", + ":", + shellCommands.replaceAll("\r\n|\n", "\n"), + "exit", + Seq( + "", + ":BOF", + "@echo off", + cmdCommands.replaceAll("\r\n|\n", "\r\n"), + "exit /B %errorlevel%", + "" + ).mkString("\r\n") + ).filterNot(_.isEmpty).mkString("\n") + } + + def launcherUniversalScript(mainClass: String, + shellClassPath: Agg[String], + cmdClassPath: Agg[String], + jvmArgs: Seq[String]) = { + universalScript( + shellCommands = + s"""exec java ${jvmArgs.mkString(" ")} $$JAVA_OPTS -cp "${shellClassPath.mkString(":")}" $mainClass "$$@"""", + cmdCommands = + s"""java ${jvmArgs.mkString(" ")} %JAVA_OPTS% -cp "${cmdClassPath.mkString(";")}" $mainClass %*""", + ) } def createLauncher(mainClass: String, classPath: Agg[Path], jvmArgs: Seq[String]) (implicit ctx: Ctx.Dest)= { val isWin = scala.util.Properties.isWin - val outputPath = ctx.dest / (if (isWin) "run.bat" else "run") - - write(outputPath, launcherShellScript(isWin, mainClass, classPath.map(_.toString), jvmArgs)) + val isBatch = isWin && + !(org.jline.utils.OSUtils.IS_CYGWIN + || org.jline.utils.OSUtils.IS_MINGW + || "MSYS" == System.getProperty("MSYSTEM")) + val outputPath = ctx.dest / (if (isBatch) "run.bat" else "run") + val classPathStrs = classPath.map(_.toString) + + write(outputPath, launcherUniversalScript(mainClass, classPathStrs, classPathStrs, jvmArgs)) if (!isWin) { val perms = Files.getPosixFilePermissions(outputPath.toNIO) diff --git a/main/test/src/mill/eval/EvaluationTests.scala b/main/test/src/mill/eval/EvaluationTests.scala index 66147963..9c215086 100644 --- a/main/test/src/mill/eval/EvaluationTests.scala +++ b/main/test/src/mill/eval/EvaluationTests.scala @@ -247,6 +247,32 @@ object EvaluationTests extends TestSuite{ !overriden.contains("object1") ) } + 'nullTasks - { + import nullTasks._ + val checker = new Checker(nullTasks) + checker(nullTarget1, null, Agg(nullTarget1), extraEvaled = -1) + checker(nullTarget1, null, Agg(), extraEvaled = -1) + checker(nullTarget2, null, Agg(nullTarget2), extraEvaled = -1) + checker(nullTarget2, null, Agg(), extraEvaled = -1) + checker(nullTarget3, null, Agg(nullTarget3), extraEvaled = -1) + checker(nullTarget3, null, Agg(), extraEvaled = -1) + checker(nullTarget4, null, Agg(nullTarget4), extraEvaled = -1) + checker(nullTarget4, null, Agg(), extraEvaled = -1) + + val nc1 = nullCommand1() + val nc2 = nullCommand2() + val nc3 = nullCommand3() + val nc4 = nullCommand4() + + checker(nc1, null, Agg(nc1), extraEvaled = -1, secondRunNoOp = false) + checker(nc1, null, Agg(nc1), extraEvaled = -1, secondRunNoOp = false) + checker(nc2, null, Agg(nc2), extraEvaled = -1, secondRunNoOp = false) + checker(nc2, null, Agg(nc2), extraEvaled = -1, secondRunNoOp = false) + checker(nc3, null, Agg(nc3), extraEvaled = -1, secondRunNoOp = false) + checker(nc3, null, Agg(nc3), extraEvaled = -1, secondRunNoOp = false) + checker(nc4, null, Agg(nc4), extraEvaled = -1, secondRunNoOp = false) + checker(nc4, null, Agg(nc4), extraEvaled = -1, secondRunNoOp = false) + } 'tasksAreUncached - { // Make sure the tasks `left` and `middle` re-compute every time, while diff --git a/main/test/src/mill/eval/JavaCompileJarTests.scala b/main/test/src/mill/eval/JavaCompileJarTests.scala index 1ac00c79..2e73b339 100644 --- a/main/test/src/mill/eval/JavaCompileJarTests.scala +++ b/main/test/src/mill/eval/JavaCompileJarTests.scala @@ -11,10 +11,10 @@ import mill.util.Strict.Agg import utest._ import mill._ object JavaCompileJarTests extends TestSuite{ - def compileAll(sources: Seq[PathRef])(implicit ctx: Dest) = { + def compileAll(sources: mill.util.Loose.Agg[PathRef])(implicit ctx: Dest) = { mkdir(ctx.dest) import ammonite.ops._ - %("javac", sources.map(_.path.toString()), "-d", ctx.dest)(wd = ctx.dest) + %("javac", sources.map(_.path.toString()).toSeq, "-d", ctx.dest)(wd = ctx.dest) PathRef(ctx.dest) } diff --git a/main/test/src/mill/main/ClientServerTests.scala b/main/test/src/mill/main/ClientServerTests.scala new file mode 100644 index 00000000..60c9c9e6 --- /dev/null +++ b/main/test/src/mill/main/ClientServerTests.scala @@ -0,0 +1,214 @@ +package mill.main +import java.io._ +import java.nio.file.Path + +import mill.client.{ClientServer, Locks} + +import scala.collection.JavaConverters._ +import utest._ +class EchoServer extends ServerMain[Int]{ + def main0(args: Array[String], + stateCache: Option[Int], + mainInteractive: Boolean, + stdin: InputStream, + stdout: PrintStream, + stderr: PrintStream, + env: Map[String, String]) = { + + val reader = new BufferedReader(new InputStreamReader(stdin)) + val str = reader.readLine() + if (args.nonEmpty){ + stdout.println(str + args(0)) + } + env.toSeq.sortBy(_._1).foreach{ + case (key, value) => stdout.println(s"$key=$value") + } + stdout.flush() + if (args.nonEmpty){ + stderr.println(str.toUpperCase + args(0)) + } + stderr.flush() + (true, None) + } +} + +object ClientServerTests extends TestSuite{ + def initStreams() = { + val in = new ByteArrayInputStream("hello\n".getBytes()) + val out = new ByteArrayOutputStream() + val err = new ByteArrayOutputStream() + (in, out, err) + } + def init() = { + val tmpDir = java.nio.file.Files.createTempDirectory("") + val locks = Locks.memory() + + (tmpDir, locks) + } + + def spawnEchoServer(tmpDir : Path, locks: Locks): Unit = { + new Thread(() => new Server( + tmpDir.toString, + new EchoServer(), + () => (), + 1000, + locks + ).run()).start() + } + + def runClientAux(tmpDir : Path, locks: Locks) + (env : Map[String, String], args: Array[String]) = { + val (in, out, err) = initStreams() + Server.lockBlock(locks.clientLock){ + mill.client.Main.run( + tmpDir.toString, + () => spawnEchoServer(tmpDir, locks), + locks, + in, + out, + err, + args, + env.asJava + ) + Thread.sleep(100) + (new String(out.toByteArray), new String(err.toByteArray)) + } + } + + def tests = Tests{ + 'hello - { + if (!ClientServer.isWindows){ + val (tmpDir, locks) = init() + def runClient(s: String) = runClientAux(tmpDir, locks)(Map.empty, Array(s)) + + // Make sure the simple "have the client start a server and + // exchange one message" workflow works from end to end. + + assert( + locks.clientLock.probe(), + locks.serverLock.probe(), + locks.processLock.probe() + ) + + val (out1, err1) = runClient("world") + + assert( + out1 == "helloworld\n", + err1 == "HELLOworld\n" + ) + + // Give a bit of time for the server to release the lock and + // re-acquire it to signal to the client that it's done + Thread.sleep(100) + + assert( + locks.clientLock.probe(), + !locks.serverLock.probe(), + !locks.processLock.probe() + ) + + // A seecond client in sequence connect to the same server + val (out2, err2) = runClient(" WORLD") + + assert( + out2 == "hello WORLD\n", + err2 == "HELLO WORLD\n" + ) + + // Make sure the server times out of not used for a while + Thread.sleep(2000) + assert( + locks.clientLock.probe(), + locks.serverLock.probe(), + locks.processLock.probe() + ) + + // Have a third client spawn/connect-to a new server at the same path + val (out3, err3) = runClient(" World") + assert( + out3 == "hello World\n", + err3 == "HELLO World\n" + ) + } + + 'envVars - { + if (!ClientServer.isWindows){ + val (tmpDir, locks) = init() + + def runClient(env : Map[String, String]) = runClientAux(tmpDir, locks)(env, Array()) + + // Make sure the simple "have the client start a server and + // exchange one message" workflow works from end to end. + + assert( + locks.clientLock.probe(), + locks.serverLock.probe(), + locks.processLock.probe() + ) + + def longString(s : String) = Array.fill(1000)(s).mkString + val b1000 = longString("b") + val c1000 = longString("c") + val a1000 = longString("a") + + val env = Map( + "a" -> a1000, + "b" -> b1000, + "c" -> c1000 + ) + + + val (out1, err1) = runClient(env) + val expected = s"a=$a1000\nb=$b1000\nc=$c1000\n" + + assert( + out1 == expected, + err1 == "" + ) + + // Give a bit of time for the server to release the lock and + // re-acquire it to signal to the client that it's done + Thread.sleep(100) + + assert( + locks.clientLock.probe(), + !locks.serverLock.probe(), + !locks.processLock.probe() + ) + + val path = List( + "/Users/foo/Library/Haskell/bin", + "/usr/local/git/bin", + "/sw/bin/", + "/usr/local/bin", + "/usr/local/", + "/usr/local/sbin", + "/usr/local/mysql/bin", + "/usr/local/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/opt/X11/bin", + "/usr/local/MacGPG2/bin", + "/Library/TeX/texbin", + "/usr/local/bin/", + "/Users/foo/bin", + "/Users/foo/go/bin", + "~/.bloop" + ) + + val pathEnvVar = path.mkString(":") + val (out2, err2) = runClient(Map("PATH" -> pathEnvVar)) + + val expected2 = s"PATH=$pathEnvVar\n" + + assert( + out2 == expected2, + err2 == "" + ) + } + } + } + } +} diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala index f88007c5..bbca5d68 100644 --- a/main/test/src/mill/util/ScriptTestSuite.scala +++ b/main/test/src/mill/util/ScriptTestSuite.scala @@ -10,8 +10,8 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{ def scriptSourcePath: Path val workspacePath = pwd / 'target / 'workspace / workspaceSlug - val stdOutErr = new PrintStream(new ByteArrayOutputStream()) -// val stdOutErr = new PrintStream(System.out) +// val stdOutErr = new PrintStream(new ByteArrayOutputStream()) + val stdOutErr = new PrintStream(System.out) val stdIn = new ByteArrayInputStream(Array()) lazy val runner = new mill.main.MainRunner( ammonite.main.Cli.Config(wd = workspacePath), @@ -21,7 +21,7 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{ if (!fork) runner.runScript(workspacePath / "build.sc", s.toList) else{ try { - %%(home / "mill-release", "-i", s)(workspacePath) + %(home / "mill-release", "-i", s)(workspacePath) true }catch{case e: Throwable => false} } diff --git a/main/test/src/mill/util/TestGraphs.scala b/main/test/src/mill/util/TestGraphs.scala index 11f72d02..83e03576 100644 --- a/main/test/src/mill/util/TestGraphs.scala +++ b/main/test/src/mill/util/TestGraphs.scala @@ -196,6 +196,23 @@ object TestGraphs{ override lazy val millDiscover: Discover[this.type] = Discover[this.type] } + object nullTasks extends TestUtil.BaseModule{ + val nullString: String = null + def nullTask1 = T.task{ nullString } + def nullTask2 = T.task{ nullTask1() } + + def nullTarget1 = T{ nullString } + def nullTarget2 = T{ nullTarget1() } + def nullTarget3 = T{ nullTask1() } + def nullTarget4 = T{ nullTask2() } + + def nullCommand1() = T.command{ nullString } + def nullCommand2() = T.command{ nullTarget1() } + def nullCommand3() = T.command{ nullTask1() } + def nullCommand4() = T.command{ nullTask2() } + + override lazy val millDiscover: Discover[this.type] = Discover[this.type] + } object singleCross extends TestUtil.BaseModule { object cross extends mill.Cross[Cross]("210", "211", "212") |