summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--client/src/mill/client/ClientServer.java83
-rw-r--r--client/src/mill/client/Main.java13
-rw-r--r--core/src/mill/eval/Evaluator.scala10
-rw-r--r--core/src/mill/util/Ctx.scala9
-rw-r--r--docs/pages/3 - Tasks.md27
-rw-r--r--main/src/mill/Main.scala11
-rw-r--r--main/src/mill/main/MainRunner.scala6
-rw-r--r--main/src/mill/main/RunScript.scala6
-rw-r--r--main/src/mill/main/Server.scala15
-rw-r--r--main/test/src/mill/main/ClientServerTests.scala154
-rw-r--r--main/test/src/mill/util/ScriptTestSuite.scala2
11 files changed, 264 insertions, 72 deletions
diff --git a/client/src/mill/client/ClientServer.java b/client/src/mill/client/ClientServer.java
index c30fc221..468f8ab3 100644
--- a/client/src/mill/client/ClientServer.java
+++ b/client/src/mill/client/ClientServer.java
@@ -4,6 +4,8 @@ package mill.client;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
public class ClientServer {
public static boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
@@ -20,23 +22,76 @@ public class ClientServer {
int argsLength = argStream.read();
String[] args = new String[argsLength];
for (int i = 0; i < args.length; i++) {
- int n = argStream.read();
- byte[] arr = new byte[n];
- argStream.read(arr);
- args[i] = new String(arr);
+ args[i] = readString(argStream);
}
return args;
}
public static void writeArgs(Boolean interactive,
String[] args,
- OutputStream argStream) throws IOException{
- argStream.write(interactive ? 1 : 0);
- argStream.write(args.length);
- int i = 0;
- while (i < args.length){
- argStream.write(args[i].length());
- argStream.write(args[i].getBytes());
- i += 1;
+ OutputStream argStream) throws IOException {
+ argStream.write(interactive ? 1 : 0);
+ argStream.write(args.length);
+ int i = 0;
+ while (i < args.length) {
+ writeString(argStream, args[i]);
+ i += 1;
+ }
+ }
+
+ /**
+ * This allows the mill client to pass the environment as he sees it to the
+ * server (as the server remains alive over the course of several runs and
+ * does not see the environment changes the client would)
+ */
+ public static void writeMap(Map<String, String> map, OutputStream argStream) throws IOException {
+ argStream.write(map.size());
+ for (Map.Entry<String, String> kv : map.entrySet()) {
+ writeString(argStream, kv.getKey());
+ writeString(argStream, kv.getValue());
+ }
}
- }
-} \ No newline at end of file
+
+ public static Map<String, String> parseMap(InputStream argStream) throws IOException {
+ Map<String, String> env = new HashMap<>();
+ int mapLength = argStream.read();
+ for (int i = 0; i < mapLength; i++) {
+ String key = readString(argStream);
+ String value = readString(argStream);
+ env.put(key, value);
+ }
+ return env;
+ }
+
+ private static String readString(InputStream inputStream) throws IOException {
+ // Result is between 0 and 255, hence the loop.
+ int read = inputStream.read();
+ int bytesToRead = read;
+ while(read == 255){
+ read = inputStream.read();
+ bytesToRead += read;
+ }
+ byte[] arr = new byte[bytesToRead];
+ int readTotal = 0;
+ while (readTotal < bytesToRead) {
+ read = inputStream.read(arr, readTotal, bytesToRead - readTotal);
+ readTotal += read;
+ }
+ return new String(arr);
+ }
+
+ private static void writeString(OutputStream outputStream, String string) throws IOException {
+ // When written, an int > 255 gets splitted. This logic performs the
+ // split beforehand so that the reading side knows that there is still
+ // more metadata to come before it's able to read the actual data.
+ // Could do with rewriting using logical masks / shifts.
+ byte[] bytes = string.getBytes();
+ int toWrite = bytes.length;
+ while(toWrite >= 255){
+ outputStream.write(255);
+ toWrite = toWrite - 255;
+ }
+ outputStream.write(toWrite);
+ outputStream.write(bytes);
+ }
+
+}
diff --git a/client/src/mill/client/Main.java b/client/src/mill/client/Main.java
index a26b653e..109a9a9d 100644
--- a/client/src/mill/client/Main.java
+++ b/client/src/mill/client/Main.java
@@ -7,10 +7,7 @@ import java.net.Socket;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.Properties;
+import java.util.*;
public class Main {
static void initServer(String lockBase, boolean setJnaNoSys) throws IOException,URISyntaxException{
@@ -52,6 +49,7 @@ public class Main {
}
public static void main(String[] args) throws Exception{
boolean setJnaNoSys = System.getProperty("jna.nosys") == null;
+ Map<String, String> env = System.getenv();
if (setJnaNoSys) {
System.setProperty("jna.nosys", "true");
}
@@ -83,7 +81,8 @@ public class Main {
System.in,
System.out,
System.err,
- args
+ args,
+ env
);
System.exit(exitCode);
}
@@ -98,10 +97,12 @@ public class Main {
InputStream stdin,
OutputStream stdout,
OutputStream stderr,
- String[] args) throws Exception{
+ String[] args,
+ Map<String, String> env) throws Exception{
FileOutputStream f = new FileOutputStream(lockBase + "/run");
ClientServer.writeArgs(System.console() != null, args, f);
+ ClientServer.writeMap(env, f);
f.close();
boolean serverInit = false;
diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala
index a1bbf142..a728ec97 100644
--- a/core/src/mill/eval/Evaluator.scala
+++ b/core/src/mill/eval/Evaluator.scala
@@ -2,6 +2,8 @@ package mill.eval
import java.net.URLClassLoader
+import scala.collection.JavaConverters._
+
import mill.util.Router.EntryPoint
import ammonite.ops._
import ammonite.runtime.SpecialClassLoader
@@ -32,7 +34,8 @@ case class Evaluator[T](home: Path,
rootModule: mill.define.BaseModule,
log: Logger,
classLoaderSig: Seq[(Either[String, Path], Long)] = Evaluator.classLoaderSig,
- workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty){
+ workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty,
+ env : Map[String, String] = Evaluator.defaultEnv){
val classLoaderSignHash = classLoaderSig.hashCode()
def evaluate(goals: Agg[Task[_]]): Evaluator.Results = {
mkdir(outPath)
@@ -271,7 +274,8 @@ case class Evaluator[T](home: Path,
}
},
multiLogger,
- home
+ home,
+ env
)
val out = System.out
@@ -335,6 +339,8 @@ object Evaluator{
// in directly) we are forced to pass it in via a ThreadLocal
val currentEvaluator = new ThreadLocal[mill.eval.Evaluator[_]]
+ val defaultEnv: Map[String, String] = System.getenv().asScala.toMap
+
case class Paths(out: Path,
dest: Path,
meta: Path,
diff --git a/core/src/mill/util/Ctx.scala b/core/src/mill/util/Ctx.scala
index 99818194..6c8b2afb 100644
--- a/core/src/mill/util/Ctx.scala
+++ b/core/src/mill/util/Ctx.scala
@@ -23,6 +23,9 @@ object Ctx{
trait Home{
def home: Path
}
+ trait Env{
+ def env: Map[String, String]
+ }
object Log{
implicit def logToCtx(l: Logger): Log = new Log { def log = l }
}
@@ -36,11 +39,13 @@ object Ctx{
class Ctx(val args: IndexedSeq[_],
dest0: () => Path,
val log: Logger,
- val home: Path)
+ val home: Path,
+ val env : Map[String, String])
extends Ctx.Dest
with Ctx.Log
with Ctx.Args
- with Ctx.Home{
+ with Ctx.Home
+ with Ctx.Env {
def dest = dest0()
def length = args.length
diff --git a/docs/pages/3 - Tasks.md b/docs/pages/3 - Tasks.md
index ca6def42..71974177 100644
--- a/docs/pages/3 - Tasks.md
+++ b/docs/pages/3 - Tasks.md
@@ -165,10 +165,10 @@ There are several APIs available to you within the body of a `T{...}` or
`T.command{...}` block to help your write the code implementing your Target or
Command:
-### mill.util.Ctx.DestCtx
+### mill.util.Ctx.Dest
- `T.ctx().dest`
-- `implicitly[mill.util.Ctx.DestCtx]`
+- `implicitly[mill.util.Ctx.Dest]`
This is the unique `out/classFiles/dest/` path or `out/run/dest/` path that is
assigned to every Target or Command. It is cleared before your task runs, and
@@ -177,10 +177,10 @@ artifacts. This is guaranteed to be unique for every `Target` or `Command`, so
you can be sure that you will not collide or interfere with anyone else writing
to those same paths.
-### mill.util.Ctx.LogCtx
+### mill.util.Ctx.Log
- `T.ctx().log`
-- `implicitly[mill.util.Ctx.LogCtx]`
+- `implicitly[mill.util.Ctx.Log]`
This is the default logger provided for every task. While your task is running,
`System.out` and `System.in` are also redirected to this logger. The logs for a
@@ -188,6 +188,25 @@ task are streamed to standard out/error as you would expect, but each task's
specific output is also streamed to a log file on disk e.g. `out/run/log` or
`out/classFiles/log` for you to inspect later.
+### mill.util.Ctx.Env
+
+- `T.ctx().env`
+- `implicitly[mill.util.Ctx.Env]`
+
+Mill keeps a long-lived JVM server to avoid paying the cost of recurrent
+classloading. Because of this, running `System.getenv` in a task might not yield
+up to date environment variables, since it will be initialised when the server
+starts, rather than when the client executes. To circumvent this, mill's client
+sends the environment variables to the server as it sees them, and the server
+makes them available as a `Map[String, String]` via the `Ctx` API.
+
+If the intent is to always pull the latest environment values, the call should
+be wrapped in an `Input` as such :
+
+```scala
+def envVar = T.input { T.ctx().env.get("ENV_VAR") }
+```
+
## Other Tasks
- [Anonymous Tasks](#anonymous-tasks), defined using `T.task{...}`
diff --git a/main/src/mill/Main.scala b/main/src/mill/Main.scala
index 734d61e2..a349321e 100644
--- a/main/src/mill/Main.scala
+++ b/main/src/mill/Main.scala
@@ -2,6 +2,8 @@ package mill
import java.io.{InputStream, PrintStream}
+import scala.collection.JavaConverters._
+
import ammonite.main.Cli._
import ammonite.ops._
import ammonite.util.Util
@@ -24,7 +26,8 @@ object Main {
ammonite.Main.isInteractive(),
System.in,
System.out,
- System.err
+ System.err,
+ System.getenv().asScala.toMap
)
System.exit(if(result) 0 else 1)
}
@@ -34,7 +37,8 @@ object Main {
mainInteractive: Boolean,
stdin: InputStream,
stdout: PrintStream,
- stderr: PrintStream): (Boolean, Option[Evaluator.State]) = {
+ stderr: PrintStream,
+ env: Map[String, String]): (Boolean, Option[Evaluator.State]) = {
import ammonite.main.Cli
val removed = Set("predef-code", "no-home-predef")
@@ -102,7 +106,8 @@ object Main {
val runner = new mill.main.MainRunner(
config.copy(colored = Some(mainInteractive)),
stdout, stderr, stdin,
- stateCache
+ stateCache,
+ env
)
if (ClientServer.isJava9OrAbove) {
diff --git a/main/src/mill/main/MainRunner.scala b/main/src/mill/main/MainRunner.scala
index efebd5a5..fed664fd 100644
--- a/main/src/mill/main/MainRunner.scala
+++ b/main/src/mill/main/MainRunner.scala
@@ -20,7 +20,8 @@ class MainRunner(val config: ammonite.main.Cli.Config,
outprintStream: PrintStream,
errPrintStream: PrintStream,
stdIn: InputStream,
- stateCache0: Option[Evaluator.State] = None)
+ stateCache0: Option[Evaluator.State] = None,
+ env : Map[String, String])
extends ammonite.MainRunner(
config, outprintStream, errPrintStream,
stdIn, outprintStream, errPrintStream
@@ -75,7 +76,8 @@ class MainRunner(val config: ammonite.main.Cli.Config,
errPrintStream,
errPrintStream,
stdIn
- )
+ ),
+ env
)
result match{
diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala
index 77930cc8..75042dea 100644
--- a/main/src/mill/main/RunScript.scala
+++ b/main/src/mill/main/RunScript.scala
@@ -29,7 +29,8 @@ object RunScript{
instantiateInterpreter: => Either[(Res.Failing, Seq[(Path, Long)]), ammonite.interp.Interpreter],
scriptArgs: Seq[String],
stateCache: Option[Evaluator.State],
- log: Logger)
+ log: Logger,
+ env : Map[String, String])
: (Res[(Evaluator[Any], Seq[PathRef], Either[String, Seq[Js.Value]])], Seq[(Path, Long)]) = {
val (evalState, interpWatched) = stateCache match{
@@ -53,7 +54,8 @@ object RunScript{
val evalRes =
for(s <- evalState)
- yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log, s.classLoaderSig, s.workerCache)
+ yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log,
+ s.classLoaderSig, s.workerCache, env)
val evaluated = for{
evaluator <- evalRes
diff --git a/main/src/mill/main/Server.scala b/main/src/mill/main/Server.scala
index c0d75c87..14aade4c 100644
--- a/main/src/mill/main/Server.scala
+++ b/main/src/mill/main/Server.scala
@@ -4,6 +4,7 @@ import java.io._
import java.net.Socket
import mill.Main
+import scala.collection.JavaConverters._
import org.scalasbt.ipcsocket._
import mill.client._
import mill.eval.Evaluator
@@ -16,7 +17,8 @@ trait ServerMain[T]{
mainInteractive: Boolean,
stdin: InputStream,
stdout: PrintStream,
- stderr: PrintStream): (Boolean, Option[T])
+ stderr: PrintStream,
+ env : Map[String, String]): (Boolean, Option[T])
}
object ServerMain extends mill.main.ServerMain[Evaluator.State]{
@@ -34,13 +36,15 @@ object ServerMain extends mill.main.ServerMain[Evaluator.State]{
mainInteractive: Boolean,
stdin: InputStream,
stdout: PrintStream,
- stderr: PrintStream) = Main.main0(
+ stderr: PrintStream,
+ env : Map[String, String]) = Main.main0(
args,
stateCache,
mainInteractive,
DummyInputStream,
stdout,
- stderr
+ stderr,
+ env
)
}
@@ -96,6 +100,7 @@ class Server[T](lockBase: String,
val argStream = new FileInputStream(lockBase + "/run")
val interactive = argStream.read() != 0;
val args = ClientServer.parseArgs(argStream)
+ val env = ClientServer.parseMap(argStream)
argStream.close()
var done = false
@@ -109,7 +114,9 @@ class Server[T](lockBase: String,
sm.stateCache,
interactive,
socketIn,
- stdout, stderr
+ stdout,
+ stderr,
+ env.asScala.toMap
)
sm.stateCache = newStateCache
diff --git a/main/test/src/mill/main/ClientServerTests.scala b/main/test/src/mill/main/ClientServerTests.scala
index e0e74fc7..60c9c9e6 100644
--- a/main/test/src/mill/main/ClientServerTests.scala
+++ b/main/test/src/mill/main/ClientServerTests.scala
@@ -1,9 +1,10 @@
package mill.main
-
import java.io._
import java.nio.file.Path
import mill.client.{ClientServer, Locks}
+
+import scala.collection.JavaConverters._
import utest._
class EchoServer extends ServerMain[Int]{
def main0(args: Array[String],
@@ -11,13 +12,21 @@ class EchoServer extends ServerMain[Int]{
mainInteractive: Boolean,
stdin: InputStream,
stdout: PrintStream,
- stderr: PrintStream) = {
+ stderr: PrintStream,
+ env: Map[String, String]) = {
val reader = new BufferedReader(new InputStreamReader(stdin))
val str = reader.readLine()
- stdout.println(str + args(0))
+ if (args.nonEmpty){
+ stdout.println(str + args(0))
+ }
+ env.toSeq.sortBy(_._1).foreach{
+ case (key, value) => stdout.println(s"$key=$value")
+ }
stdout.flush()
- stderr.println(str.toUpperCase + args(0))
+ if (args.nonEmpty){
+ stderr.println(str.toUpperCase + args(0))
+ }
stderr.flush()
(true, None)
}
@@ -37,38 +46,40 @@ object ClientServerTests extends TestSuite{
(tmpDir, locks)
}
+ def spawnEchoServer(tmpDir : Path, locks: Locks): Unit = {
+ new Thread(() => new Server(
+ tmpDir.toString,
+ new EchoServer(),
+ () => (),
+ 1000,
+ locks
+ ).run()).start()
+ }
+
+ def runClientAux(tmpDir : Path, locks: Locks)
+ (env : Map[String, String], args: Array[String]) = {
+ val (in, out, err) = initStreams()
+ Server.lockBlock(locks.clientLock){
+ mill.client.Main.run(
+ tmpDir.toString,
+ () => spawnEchoServer(tmpDir, locks),
+ locks,
+ in,
+ out,
+ err,
+ args,
+ env.asJava
+ )
+ Thread.sleep(100)
+ (new String(out.toByteArray), new String(err.toByteArray))
+ }
+ }
+
def tests = Tests{
'hello - {
if (!ClientServer.isWindows){
val (tmpDir, locks) = init()
-
- def spawnEchoServer(): Unit = {
- new Thread(() => new Server(
- tmpDir.toString,
- new EchoServer(),
- () => (),
- 1000,
- locks
- ).run()).start()
- }
-
-
- def runClient(arg: String) = {
- val (in, out, err) = initStreams()
- Server.lockBlock(locks.clientLock){
- mill.client.Main.run(
- tmpDir.toString,
- () => spawnEchoServer(),
- locks,
- in,
- out,
- err,
- Array(arg)
- )
- Thread.sleep(100)
- (new String(out.toByteArray), new String(err.toByteArray))
- }
- }
+ def runClient(s: String) = runClientAux(tmpDir, locks)(Map.empty, Array(s))
// Make sure the simple "have the client start a server and
// exchange one message" workflow works from end to end.
@@ -119,6 +130,85 @@ object ClientServerTests extends TestSuite{
err3 == "HELLO World\n"
)
}
+
+ 'envVars - {
+ if (!ClientServer.isWindows){
+ val (tmpDir, locks) = init()
+
+ def runClient(env : Map[String, String]) = runClientAux(tmpDir, locks)(env, Array())
+
+ // Make sure the simple "have the client start a server and
+ // exchange one message" workflow works from end to end.
+
+ assert(
+ locks.clientLock.probe(),
+ locks.serverLock.probe(),
+ locks.processLock.probe()
+ )
+
+ def longString(s : String) = Array.fill(1000)(s).mkString
+ val b1000 = longString("b")
+ val c1000 = longString("c")
+ val a1000 = longString("a")
+
+ val env = Map(
+ "a" -> a1000,
+ "b" -> b1000,
+ "c" -> c1000
+ )
+
+
+ val (out1, err1) = runClient(env)
+ val expected = s"a=$a1000\nb=$b1000\nc=$c1000\n"
+
+ assert(
+ out1 == expected,
+ err1 == ""
+ )
+
+ // Give a bit of time for the server to release the lock and
+ // re-acquire it to signal to the client that it's done
+ Thread.sleep(100)
+
+ assert(
+ locks.clientLock.probe(),
+ !locks.serverLock.probe(),
+ !locks.processLock.probe()
+ )
+
+ val path = List(
+ "/Users/foo/Library/Haskell/bin",
+ "/usr/local/git/bin",
+ "/sw/bin/",
+ "/usr/local/bin",
+ "/usr/local/",
+ "/usr/local/sbin",
+ "/usr/local/mysql/bin",
+ "/usr/local/bin",
+ "/usr/bin",
+ "/bin",
+ "/usr/sbin",
+ "/sbin",
+ "/opt/X11/bin",
+ "/usr/local/MacGPG2/bin",
+ "/Library/TeX/texbin",
+ "/usr/local/bin/",
+ "/Users/foo/bin",
+ "/Users/foo/go/bin",
+ "~/.bloop"
+ )
+
+ val pathEnvVar = path.mkString(":")
+ val (out2, err2) = runClient(Map("PATH" -> pathEnvVar))
+
+ val expected2 = s"PATH=$pathEnvVar\n"
+
+ assert(
+ out2 == expected2,
+ err2 == ""
+ )
+ }
+ }
}
}
}
diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala
index 2df1c0c0..bbca5d68 100644
--- a/main/test/src/mill/util/ScriptTestSuite.scala
+++ b/main/test/src/mill/util/ScriptTestSuite.scala
@@ -15,7 +15,7 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{
val stdIn = new ByteArrayInputStream(Array())
lazy val runner = new mill.main.MainRunner(
ammonite.main.Cli.Config(wd = workspacePath),
- stdOutErr, stdOutErr, stdIn
+ stdOutErr, stdOutErr, stdIn, None, Map.empty
)
def eval(s: String*) = {
if (!fork) runner.runScript(workspacePath / "build.sc", s.toList)
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-30 04:46:29 +0000