diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2018-12-12 22:15:38 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-12 22:15:38 +0800 |
commit | ea7fceb6e56f53bde3517586dfc57e10a605a524 (patch) | |
tree | cf2e4b7a403f8bcbcf3d1c4c45a8abd315cbefe6 /main | |
parent | d0e1b572e88d311e1aee23d92e0384a81de4bcb6 (diff) | |
download | mill-ea7fceb6e56f53bde3517586dfc57e10a605a524.tar.gz mill-ea7fceb6e56f53bde3517586dfc57e10a605a524.tar.bz2 mill-ea7fceb6e56f53bde3517586dfc57e10a605a524.zip |
First pass at splitting out worker-api from mill core. (#504)
This reduces the {scala,scalajs,scalanative}-worker dependency from the entirety of Mill to a much narrower `mill.api` module. This reduces the amount of classpath pollution within these workers, should mean they're much faster to download the first time, and reduces the amount of random junk they would pull in if they were to be used outside of the Mill project.
The interactions between the various *Modules and their *WorkerImpls has been narrowed down to the `*.api` modules, which only depend on other `*.api` modules.
A lot of things have been moved around; user code is unlikely to break, but it's possible some will if it references classes that have been moved around. Forwarders have been left for the few internal classes that Mill uses in it's own `build.sc`, to support bootstrapping. Third-party code which breaks should be a straightforward to fix just by updating imports
The `*.api` modules have minimal dependencies (mostly uPickle and os-lib) and minimal code. There is still a bunch of implementation code in there: some of it defining data-types that are commonly sent across the module/worker interface (`Agg`, `PathRef`, ...), and some of it just general helper functions that are needed both in modules and workers. The latter code isn't strictly API definitions, but for now is small enough it's not worth splitting into it's own module
Diffstat (limited to 'main')
33 files changed, 523 insertions, 196 deletions
diff --git a/main/api/src/io/github/retronym/java9rtexport/Copy.java b/main/api/src/io/github/retronym/java9rtexport/Copy.java new file mode 100644 index 00000000..ac3615bb --- /dev/null +++ b/main/api/src/io/github/retronym/java9rtexport/Copy.java @@ -0,0 +1,75 @@ +/* +Copyright (C) 2012-2014 EPFL +Copyright (C) 2012-2014 Typesafe, Inc. +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the EPFL nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package io.github.retronym.java9rtexport; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.*; +import java.nio.file.attribute.*; +import java.util.EnumSet; + +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +public class Copy { + public static void copyDirectory(final Path source, final Path target) + throws IOException { + Files.walkFileTree(source, EnumSet.of(FileVisitOption.FOLLOW_LINKS), + Integer.MAX_VALUE, new FileVisitor<Path>() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes sourceBasic) throws IOException { + + String relative = source.relativize(dir).toString(); + if (!Files.exists(target.getFileSystem().getPath(relative))) + Files.createDirectory(target.getFileSystem().getPath(relative)); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) throws IOException { + String relative = source.relativize(file).toString(); + Files.copy(file, target.getFileSystem().getPath(relative), COPY_ATTRIBUTES, REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + throw e; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e != null) throw e; + return FileVisitResult.CONTINUE; + } + }); + } + +}
\ No newline at end of file diff --git a/main/api/src/io/github/retronym/java9rtexport/Export.java b/main/api/src/io/github/retronym/java9rtexport/Export.java new file mode 100644 index 00000000..b71ca555 --- /dev/null +++ b/main/api/src/io/github/retronym/java9rtexport/Export.java @@ -0,0 +1,103 @@ +/* +Copyright (C) 2012-2014 EPFL +Copyright (C) 2012-2014 Typesafe, Inc. +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the EPFL nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package io.github.retronym.java9rtexport; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class Export { + private final static Object lock = new Object(); + private static File tempFile = null; + + public static String rtJarName = "rt-" + System.getProperty("java.version") + ".jar"; + + public static File rt() { + try { + synchronized (lock) { + if (tempFile == null) { + Path tempPath = Files.createTempFile("rt", ".jar"); + tempFile = tempPath.toFile(); + tempFile.deleteOnExit(); + tempFile.delete(); + FileSystem fileSystem = FileSystems.getFileSystem(URI.create("jrt:/")); + Path path = fileSystem.getPath("/modules"); + URI uri = URI.create("jar:" + tempPath.toUri()); + Map<String, String> env = new HashMap<>(); + env.put("create", "true"); + try (FileSystem zipfs = FileSystems.newFileSystem(uri, env)) { + Iterator<Path> iterator = Files.list(path).iterator(); + while (iterator.hasNext()) { + Path next = iterator.next(); + Copy.copyDirectory(next, zipfs.getPath("/")); + } + } + } + } + } catch (IOException e) { + e.printStackTrace(); + System.exit(-1); + } + return tempFile; + } + + public static boolean rtTo(File dest, boolean verbose) { + try { + if (!dest.exists()) { + if (verbose) { + System.out.println("Copying Java " + + System.getProperty("java.version") + + " runtime jar to " + + dest.getParentFile() + + " ..."); + System.out.flush(); + } + dest.getParentFile().mkdirs(); + java.nio.file.Files.copy(rt().toPath(), dest.toPath()); + return true; + } + } catch (IOException e) { + e.printStackTrace(); + System.exit(-1); + } + return false; + } + + public static File rtAt(File dir, boolean verbose) { + File f = new File(dir, rtJarName); + rtTo(f, verbose); + return f; + } + + public static File rtAt(File dir) { + return rtAt(dir, false); + } +}
\ No newline at end of file diff --git a/main/api/src/mill/api/AggWrapper.scala b/main/api/src/mill/api/AggWrapper.scala new file mode 100644 index 00000000..98d46d68 --- /dev/null +++ b/main/api/src/mill/api/AggWrapper.scala @@ -0,0 +1,120 @@ +package mill.api + + + +import scala.collection.mutable +object Strict extends AggWrapper(true) +object Loose extends AggWrapper(false) + +sealed class AggWrapper(strictUniqueness: Boolean){ + /** + * A collection with enforced uniqueness, fast contains and deterministic + * ordering. Raises an exception if a duplicate is found; call + * `toSeq.distinct` if you explicitly want to make it swallow duplicates + */ + trait Agg[V] extends TraversableOnce[V]{ + def contains(v: V): Boolean + def items: Iterator[V] + def indexed: IndexedSeq[V] + def flatMap[T](f: V => TraversableOnce[T]): Agg[T] + def map[T](f: V => T): Agg[T] + def filter(f: V => Boolean): Agg[V] + def withFilter(f: V => Boolean): Agg[V] + def collect[T](f: PartialFunction[V, T]): Agg[T] + def zipWithIndex: Agg[(V, Int)] + def reverse: Agg[V] + def zip[T](other: Agg[T]): Agg[(V, T)] + def ++[T >: V](other: TraversableOnce[T]): Agg[T] + def length: Int + } + + object Agg{ + def empty[V]: Agg[V] = new Agg.Mutable[V] + implicit def jsonFormat[T: upickle.default.ReadWriter]: upickle.default.ReadWriter[Agg[T]] = + upickle.default.readwriter[Seq[T]].bimap[Agg[T]]( + _.toList, + Agg.from(_) + ) + + def apply[V](items: V*) = from(items) + + implicit def from[V](items: TraversableOnce[V]): Agg[V] = { + val set = new Agg.Mutable[V]() + items.foreach(set.append) + set + } + + + class Mutable[V]() extends Agg[V]{ + + private[this] val set0 = mutable.LinkedHashSet.empty[V] + def contains(v: V) = set0.contains(v) + def append(v: V) = if (!contains(v)){ + set0.add(v) + + }else if (strictUniqueness){ + throw new Exception("Duplicated item inserted into OrderedSet: " + v) + } + def appendAll(vs: Seq[V]) = vs.foreach(append) + def items = set0.iterator + def indexed: IndexedSeq[V] = items.toIndexedSeq + def set: collection.Set[V] = set0 + + def map[T](f: V => T): Agg[T] = { + val output = new Agg.Mutable[T] + for(i <- items) output.append(f(i)) + output + } + def flatMap[T](f: V => TraversableOnce[T]): Agg[T] = { + val output = new Agg.Mutable[T] + for(i <- items) for(i0 <- f(i)) output.append(i0) + output + } + def filter(f: V => Boolean): Agg[V] = { + val output = new Agg.Mutable[V] + for(i <- items) if (f(i)) output.append(i) + output + } + def withFilter(f: V => Boolean): Agg[V] = filter(f) + + def collect[T](f: PartialFunction[V, T]) = this.filter(f.isDefinedAt).map(x => f(x)) + + def zipWithIndex = { + var i = 0 + this.map{ x => + i += 1 + (x, i-1) + } + } + + def reverse = Agg.from(indexed.reverseIterator) + + def zip[T](other: Agg[T]) = Agg.from(items.zip(other.items)) + def ++[T >: V](other: TraversableOnce[T]) = Agg.from(items ++ other) + def length: Int = set0.size + + // Members declared in scala.collection.GenTraversableOnce + def isTraversableAgain: Boolean = items.isTraversableAgain + def toIterator: Iterator[V] = items.toIterator + def toStream: Stream[V] = items.toStream + + // Members declared in scala.collection.TraversableOnce + def copyToArray[B >: V](xs: Array[B], start: Int,len: Int): Unit = items.copyToArray(xs, start, len) + def exists(p: V => Boolean): Boolean = items.exists(p) + def find(p: V => Boolean): Option[V] = items.find(p) + def forall(p: V => Boolean): Boolean = items.forall(p) + def foreach[U](f: V => U): Unit = items.foreach(f) + def hasDefiniteSize: Boolean = items.hasDefiniteSize + def isEmpty: Boolean = items.isEmpty + def seq: scala.collection.TraversableOnce[V] = items + def toTraversable: Traversable[V] = items.toTraversable + + override def hashCode() = items.map(_.hashCode()).sum + override def equals(other: Any) = other match{ + case s: Agg[_] => items.sameElements(s.items) + case _ => super.equals(other) + } + override def toString = items.mkString("Agg(", ", ", ")") + } + } +} diff --git a/main/core/src/mill/util/ClassLoader.scala b/main/api/src/mill/api/ClassLoader.scala index 07ab1ca9..198cbf6c 100644 --- a/main/core/src/mill/util/ClassLoader.scala +++ b/main/api/src/mill/api/ClassLoader.scala @@ -1,4 +1,4 @@ -package mill.util +package mill.api import java.net.{URL, URLClassLoader} @@ -8,6 +8,7 @@ import io.github.retronym.java9rtexport.Export import scala.util.Try object ClassLoader { + def java9OrAbove = !System.getProperty("java.specification.version").startsWith("1.") def create(urls: Seq[URL], parent: java.lang.ClassLoader) (implicit ctx: Ctx.Home): URLClassLoader = { @@ -38,7 +39,7 @@ object ClassLoader { * mill could be compiled only with jdk 9 or above. We don't want to introduce this restriction now. */ private def refinePlatformParent(parent: java.lang.ClassLoader): ClassLoader = { - if (!ammonite.util.Util.java9OrAbove || parent != null) parent + if (!java9OrAbove || parent != null) parent else { // Make sure when `parent == null`, we only delegate java.* classes // to the parent getPlatformClassLoader. This is necessary because @@ -53,7 +54,7 @@ object ClassLoader { } private def makeUrls(urls: Seq[URL])(implicit ctx: Ctx.Home): Seq[URL] = { - if (ammonite.util.Util.java9OrAbove) { + if (java9OrAbove) { urls :+ Export.rtAt(ctx.home.toIO).toURI.toURL } else { urls diff --git a/main/core/src/mill/util/Ctx.scala b/main/api/src/mill/api/Ctx.scala index bbc243b7..567da003 100644 --- a/main/core/src/mill/util/Ctx.scala +++ b/main/api/src/mill/api/Ctx.scala @@ -1,8 +1,7 @@ -package mill.util +package mill.api -import mill.define.Applicative.ImplicitStub -import scala.annotation.compileTimeOnly +import scala.annotation.{StaticAnnotation, compileTimeOnly} import scala.language.implicitConversions object Ctx{ @@ -32,8 +31,9 @@ object Ctx{ def args: IndexedSeq[_] } - def defaultHome = ammonite.ops.home / ".mill" / "ammonite" + def defaultHome = os.home / ".mill" / "ammonite" + class ImplicitStub extends StaticAnnotation } class Ctx(val args: IndexedSeq[_], dest0: () => os.Path, diff --git a/main/api/src/mill/api/JsonFormatters.scala b/main/api/src/mill/api/JsonFormatters.scala new file mode 100644 index 00000000..918fa693 --- /dev/null +++ b/main/api/src/mill/api/JsonFormatters.scala @@ -0,0 +1,44 @@ +package mill.api + +import upickle.default.{ReadWriter => RW} +import scala.util.matching.Regex +object JsonFormatters extends JsonFormatters +trait JsonFormatters { + implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String] + .bimap[os.Path]( + _.toString, + os.Path(_) + ) + + implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String] + .bimap[Regex]( + _.pattern.toString, + _.r + ) + + implicit val bytesReadWrite: RW[os.Bytes] = upickle.default.readwriter[String] + .bimap( + o => java.util.Base64.getEncoder.encodeToString(o.array), + str => new os.Bytes(java.util.Base64.getDecoder.decode(str)) + ) + + + implicit lazy val crFormat: RW[os.CommandResult] = upickle.default.macroRW + + implicit val stackTraceRW = upickle.default.readwriter[ujson.Obj].bimap[StackTraceElement]( + ste => ujson.Obj( + "declaringClass" -> ujson.Str(ste.getClassName), + "methodName" -> ujson.Str(ste.getMethodName), + "fileName" -> ujson.Str(ste.getFileName), + "lineNumber" -> ujson.Num(ste.getLineNumber) + ), + {case json: ujson.Obj => + new StackTraceElement( + json("declaringClass").str.toString, + json("methodName").str.toString, + json("fileName").str.toString, + json("lineNumber").num.toInt + ) + } + ) +} diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala new file mode 100644 index 00000000..4ae6e74d --- /dev/null +++ b/main/api/src/mill/api/Logger.scala @@ -0,0 +1,41 @@ +package mill.api + +import java.io._ + +/** + * The standard logging interface of the Mill build tool. + * + * Contains these primary logging methods, in order of increasing importance: + * + * - `debug` : internal debug messages normally not shown to the user; + * mostly useful when debugging issues + * + * - `ticker`: short-lived logging output where consecutive lines over-write + * each other; useful for information which is transient and disposable + * + * - `info`: miscellaneous logging output which isn't part of the main output + * a user is looking for, but useful to provide context on what Mill is doing + * + * - `error`: logging output which represents problems the user should care + * about + * + * + * Also contains the two forwarded stdout and stderr streams, for code executed + * by Mill to use directly. Typically these correspond to the stdout and stderr, + * but when `show` is used both are forwarded to stderr and stdout is only + * used to display the final `show` output for easy piping. + */ +trait Logger { + def colored: Boolean + + val errorStream: PrintStream + val outputStream: PrintStream + val inStream: InputStream + + def info(s: String): Unit + def error(s: String): Unit + def ticker(s: String): Unit + def debug(s: String): Unit + + def close(): Unit = () +} diff --git a/main/core/src/mill/eval/PathRef.scala b/main/api/src/mill/api/PathRef.scala index 92ef8d24..24f3627e 100644 --- a/main/core/src/mill/eval/PathRef.scala +++ b/main/api/src/mill/api/PathRef.scala @@ -1,4 +1,4 @@ -package mill.eval +package mill.api import java.io.IOException import java.nio.file.attribute.BasicFileAttributes @@ -7,7 +7,6 @@ import java.nio.{file => jnio} import java.security.{DigestOutputStream, MessageDigest} import upickle.default.{ReadWriter => RW} -import mill.util.{DummyOutputStream, IO, JsonFormatters} /** @@ -67,3 +66,57 @@ object PathRef{ } ) } + + +import java.io.{InputStream, OutputStream} + +/** + * Misc IO utilities, eventually probably should be pushed upstream into + * ammonite-ops + */ +object IO { + def stream(src: InputStream, dest: OutputStream) = { + val buffer = new Array[Byte](4096) + while ( { + src.read(buffer) match { + case -1 => false + case n => + dest.write(buffer, 0, n) + true + } + }) () + } + + + def unpackZip(src: os.Path, dest: os.RelPath = "unpacked") + (implicit ctx: Ctx.Dest) = { + + val byteStream = os.read.inputStream(src) + val zipStream = new java.util.zip.ZipInputStream(byteStream) + while({ + zipStream.getNextEntry match{ + case null => false + case entry => + if (!entry.isDirectory) { + val entryDest = ctx.dest / dest / os.RelPath(entry.getName) + os.makeDir.all(entryDest / os.up) + val fileOut = new java.io.FileOutputStream(entryDest.toString) + IO.stream(zipStream, fileOut) + fileOut.close() + } + zipStream.closeEntry() + true + } + })() + PathRef(ctx.dest / dest) + } +} + +import java.io.{ByteArrayInputStream, OutputStream} + +object DummyInputStream extends ByteArrayInputStream(Array()) +object DummyOutputStream extends java.io.OutputStream{ + override def write(b: Int) = () + override def write(b: Array[Byte]) = () + override def write(b: Array[Byte], off: Int, len: Int) = () +} diff --git a/main/core/src/mill/eval/Result.scala b/main/api/src/mill/api/Result.scala index d0400599..b4071a99 100644 --- a/main/core/src/mill/eval/Result.scala +++ b/main/api/src/mill/api/Result.scala @@ -1,4 +1,4 @@ -package mill.eval +package mill.api sealed trait Result[+T]{ def map[V](f: T => V): Result[V] diff --git a/main/core/src/mill/define/Applicative.scala b/main/core/src/mill/define/Applicative.scala index 69c506f7..5e63b1cc 100644 --- a/main/core/src/mill/define/Applicative.scala +++ b/main/core/src/mill/define/Applicative.scala @@ -25,7 +25,7 @@ object Applicative { def self: M[T] def apply()(implicit handler: ApplyHandler[M]): T = handler(self) } - class ImplicitStub extends StaticAnnotation + type Id[+T] = T trait Applyer[W[_], T[_], Z[_], Ctx] extends ApplyerGenerated[T, Z, Ctx] { @@ -81,7 +81,7 @@ object Applicative { tempIdent case (t, api) if t.symbol != null - && t.symbol.annotations.exists(_.tree.tpe =:= typeOf[ImplicitStub]) => + && t.symbol.annotations.exists(_.tree.tpe =:= typeOf[mill.api.Ctx.ImplicitStub]) => val tempIdent = Ident(ctxSym) c.internal.setType(tempIdent, t.tpe) diff --git a/main/core/src/mill/define/Task.scala b/main/core/src/mill/define/Task.scala index 07576724..a464bf18 100644 --- a/main/core/src/mill/define/Task.scala +++ b/main/core/src/mill/define/Task.scala @@ -27,7 +27,7 @@ abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T]{ /** * Evaluate this target */ - def evaluate(args: mill.util.Ctx): Result[T] + def evaluate(args: mill.api.Ctx): Result[T] /** * Even if this target's inputs did not change, does it need to re-evaluate @@ -53,7 +53,7 @@ trait Target[+T] extends NamedTask[T]{ def readWrite: RW[_] } -object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Result, mill.util.Ctx] { +object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Result, mill.api.Ctx] { implicit def apply[T](t: T) (implicit rw: RW[T], @@ -64,7 +64,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul (rw: c.Expr[RW[T]], ctx: c.Expr[mill.define.Ctx]): c.Expr[Target[T]] = { import c.universe._ - val lhs = Applicative.impl0[Task, T, mill.util.Ctx](c)(reify(Result.Success(t.splice)).tree) + val lhs = Applicative.impl0[Task, T, mill.api.Ctx](c)(reify(Result.Success(t.splice)).tree) mill.moduledefs.Cacher.impl0[TargetImpl[T]](c)( reify( @@ -85,7 +85,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul mill.moduledefs.Cacher.impl0[Target[T]](c)( reify( new TargetImpl[T]( - Applicative.impl0[Task, T, mill.util.Ctx](c)(t.tree).splice, + Applicative.impl0[Task, T, mill.api.Ctx](c)(t.tree).splice, ctx.splice, rw.splice ) @@ -118,7 +118,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul import c.universe._ val wrapped = for (value <- values.toList) - yield Applicative.impl0[Task, PathRef, mill.util.Ctx](c)( + yield Applicative.impl0[Task, PathRef, mill.api.Ctx](c)( reify(value.splice.map(PathRef(_))).tree ).tree @@ -144,7 +144,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul mill.moduledefs.Cacher.impl0[Sources](c)( reify( new Sources( - Applicative.impl0[Task, Seq[PathRef], mill.util.Ctx](c)(values.tree).splice, + Applicative.impl0[Task, Seq[PathRef], mill.api.Ctx](c)(values.tree).splice, ctx.splice ) ) @@ -163,7 +163,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul mill.moduledefs.Cacher.impl0[Input[T]](c)( reify( new Input[T]( - Applicative.impl[Task, T, mill.util.Ctx](c)(value).splice, + Applicative.impl[Task, T, mill.api.Ctx](c)(value).splice, ctx.splice, rw.splice ) @@ -194,7 +194,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul import c.universe._ reify( new Command[T]( - Applicative.impl[Task, T, mill.util.Ctx](c)(t).splice, + Applicative.impl[Task, T, mill.api.Ctx](c)(t).splice, ctx.splice, w.splice, cls.splice.value, @@ -214,11 +214,11 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul (ctx: c.Expr[mill.define.Ctx]): c.Expr[Worker[T]] = { import c.universe._ reify( - new Worker[T](Applicative.impl[Task, T, mill.util.Ctx](c)(t).splice, ctx.splice) + new Worker[T](Applicative.impl[Task, T, mill.api.Ctx](c)(t).splice, ctx.splice) ) } - def task[T](t: Result[T]): Task[T] = macro Applicative.impl[Task, T, mill.util.Ctx] + def task[T](t: Result[T]): Task[T] = macro Applicative.impl[Task, T, mill.api.Ctx] def persistent[T](t: Result[T])(implicit rw: RW[T], ctx: mill.define.Ctx): Persistent[T] = macro persistentImpl[T] @@ -233,7 +233,7 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul mill.moduledefs.Cacher.impl0[Persistent[T]](c)( reify( new Persistent[T]( - Applicative.impl[Task, T, mill.util.Ctx](c)(t).splice, + Applicative.impl[Task, T, mill.api.Ctx](c)(t).splice, ctx.splice, rw.splice ) @@ -242,20 +242,20 @@ object Target extends TargetGenerated with Applicative.Applyer[Task, Task, Resul } type TT[+X] = Task[X] - def makeT[X](inputs0: Seq[TT[_]], evaluate0: mill.util.Ctx => Result[X]) = new Task[X] { + def makeT[X](inputs0: Seq[TT[_]], evaluate0: mill.api.Ctx => Result[X]) = new Task[X] { val inputs = inputs0 - def evaluate(x: mill.util.Ctx) = evaluate0(x) + def evaluate(x: mill.api.Ctx) = evaluate0(x) } def underlying[A](v: Task[A]) = v - def mapCtx[A, B](t: Task[A])(f: (A, mill.util.Ctx) => Result[B]) = t.mapDest(f) + def mapCtx[A, B](t: Task[A])(f: (A, mill.api.Ctx) => Result[B]) = t.mapDest(f) def zip() = new Task.Task0(()) def zip[A](a: Task[A]) = a.map(Tuple1(_)) def zip[A, B](a: Task[A], b: Task[B]) = a.zip(b) } abstract class NamedTaskImpl[+T](ctx0: mill.define.Ctx, t: Task[T]) extends NamedTask[T]{ - def evaluate(args: mill.util.Ctx) = args[T](0) + def evaluate(args: mill.api.Ctx) = args[T](0) val ctx = ctx0.copy(segments = ctx0.segments ++ Seq(ctx0.segment)) val inputs = Seq(t) } @@ -303,12 +303,12 @@ object Task { class Task0[T](t: T) extends Task[T]{ lazy val t0 = t val inputs = Nil - def evaluate(args: mill.util.Ctx) = t0 + def evaluate(args: mill.api.Ctx) = t0 } abstract class Ops[+T]{ this: Task[T] => def map[V](f: T => V) = new Task.Mapped(this, f) - def mapDest[V](f: (T, mill.util.Ctx) => Result[V]) = new Task.MappedDest(this, f) + def mapDest[V](f: (T, mill.api.Ctx) => Result[V]) = new Task.MappedDest(this, f) def filter(f: T => Boolean) = this def withFilter(f: T => Boolean) = this @@ -323,22 +323,22 @@ object Task { class Sequence[+T](inputs0: Seq[Task[T]]) extends Task[Seq[T]]{ val inputs = inputs0 - def evaluate(args: mill.util.Ctx) = { + def evaluate(args: mill.api.Ctx) = { for (i <- 0 until args.length) yield args(i).asInstanceOf[T] } } class Mapped[+T, +V](source: Task[T], f: T => V) extends Task[V]{ - def evaluate(args: mill.util.Ctx) = f(args(0)) + def evaluate(args: mill.api.Ctx) = f(args(0)) val inputs = List(source) } - class MappedDest[+T, +V](source: Task[T], f: (T, mill.util.Ctx) => Result[V]) extends Task[V]{ - def evaluate(args: mill.util.Ctx) = f(args(0), args) + class MappedDest[+T, +V](source: Task[T], f: (T, mill.api.Ctx) => Result[V]) extends Task[V]{ + def evaluate(args: mill.api.Ctx) = f(args(0), args) val inputs = List(source) } class Zipped[+T, +V](source1: Task[T], source2: Task[V]) extends Task[(T, V)]{ - def evaluate(args: mill.util.Ctx) = (args(0), args(1)) + def evaluate(args: mill.api.Ctx) = (args(0), args(1)) val inputs = List(source1, source2) } } diff --git a/main/core/src/mill/eval/Evaluator.scala b/main/core/src/mill/eval/Evaluator.scala index 2aafdb7a..8709064e 100644 --- a/main/core/src/mill/eval/Evaluator.scala +++ b/main/core/src/mill/eval/Evaluator.scala @@ -7,7 +7,7 @@ import scala.collection.JavaConverters._ import mill.util.Router.EntryPoint import ammonite.runtime.SpecialClassLoader import mill.define.{Ctx => _, _} -import mill.eval.Result.OuterStack +import mill.api.Result.OuterStack import mill.util import mill.util._ import mill.util.Strict.Agg diff --git a/main/core/src/mill/eval/package.scala b/main/core/src/mill/eval/package.scala new file mode 100644 index 00000000..433f9074 --- /dev/null +++ b/main/core/src/mill/eval/package.scala @@ -0,0 +1,12 @@ +package mill + +package object eval { + // Backwards compatibility forwarders + val Result = mill.api.Result + type Result[+T] = mill.api.Result[T] + + val PathRef = mill.api.PathRef + type PathRef = mill.api.PathRef + + type Logger = mill.api.Logger +} diff --git a/main/core/src/mill/util/IO.scala b/main/core/src/mill/util/IO.scala deleted file mode 100644 index 833e52c7..00000000 --- a/main/core/src/mill/util/IO.scala +++ /dev/null @@ -1,32 +0,0 @@ -package mill.util - -import java.io.{InputStream, OutputStream} - -import scala.tools.nsc.interpreter.OutputStream - -/** - * Misc IO utilities, eventually probably should be pushed upstream into - * ammonite-ops - */ -object IO { - def stream(src: InputStream, dest: OutputStream) = { - val buffer = new Array[Byte](4096) - while ( { - src.read(buffer) match { - case -1 => false - case n => - dest.write(buffer, 0, n) - true - } - }) () - } -} - -import java.io.{ByteArrayInputStream, OutputStream} - -object DummyInputStream extends ByteArrayInputStream(Array()) -object DummyOutputStream extends OutputStream{ - override def write(b: Int) = () - override def write(b: Array[Byte]) = () - override def write(b: Array[Byte], off: Int, len: Int) = () -} diff --git a/main/core/src/mill/util/JsonFormatters.scala b/main/core/src/mill/util/JsonFormatters.scala index dba599f7..830782c6 100644 --- a/main/core/src/mill/util/JsonFormatters.scala +++ b/main/core/src/mill/util/JsonFormatters.scala @@ -1,49 +1,10 @@ package mill.util import upickle.default.{ReadWriter => RW} -import scala.util.matching.Regex -object JsonFormatters extends JsonFormatters -trait JsonFormatters { - implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String] - .bimap[os.Path]( - _.toString, - os.Path(_) - ) - - implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String] - .bimap[Regex]( - _.pattern.toString, - _.r - ) - - implicit val bytesReadWrite: RW[os.Bytes] = upickle.default.readwriter[String] - .bimap( - o => java.util.Base64.getEncoder.encodeToString(o.array), - str => new os.Bytes(java.util.Base64.getDecoder.decode(str)) - ) - - - implicit lazy val crFormat: RW[os.CommandResult] = upickle.default.macroRW +trait JsonFormatters extends mill.api.JsonFormatters{ implicit lazy val modFormat: RW[coursier.Module] = upickle.default.macroRW implicit lazy val depFormat: RW[coursier.Dependency]= upickle.default.macroRW implicit lazy val attrFormat: RW[coursier.Attributes] = upickle.default.macroRW - implicit val stackTraceRW = upickle.default.readwriter[ujson.Obj].bimap[StackTraceElement]( - ste => ujson.Obj( - "declaringClass" -> ujson.Str(ste.getClassName), - "methodName" -> ujson.Str(ste.getMethodName), - "fileName" -> ujson.Str(ste.getFileName), - "lineNumber" -> ujson.Num(ste.getLineNumber) - ), - {case json: ujson.Obj => - new StackTraceElement( - json("declaringClass").str.toString, - json("methodName").str.toString, - json("fileName").str.toString, - json("lineNumber").num.toInt - ) - } - ) - - } +object JsonFormatters extends JsonFormatters diff --git a/main/core/src/mill/util/Logger.scala b/main/core/src/mill/util/Loggers.scala index 4857953d..aab1a324 100644 --- a/main/core/src/mill/util/Logger.scala +++ b/main/core/src/mill/util/Loggers.scala @@ -1,46 +1,7 @@ package mill.util import java.io._ - -import ammonite.util.Colors - -/** - * The standard logging interface of the Mill build tool. - * - * Contains these primary logging methods, in order of increasing importance: - * - * - `debug` : internal debug messages normally not shown to the user; - * mostly useful when debugging issues - * - * - `ticker`: short-lived logging output where consecutive lines over-write - * each other; useful for information which is transient and disposable - * - * - `info`: miscellaneous logging output which isn't part of the main output - * a user is looking for, but useful to provide context on what Mill is doing - * - * - `error`: logging output which represents problems the user should care - * about - * - * - * Also contains the two forwarded stdout and stderr streams, for code executed - * by Mill to use directly. Typically these correspond to the stdout and stderr, - * but when `show` is used both are forwarded to stderr and stdout is only - * used to display the final `show` output for easy piping. - */ -trait Logger { - def colored: Boolean - - val errorStream: PrintStream - val outputStream: PrintStream - val inStream: InputStream - - def info(s: String): Unit - def error(s: String): Unit - def ticker(s: String): Unit - def debug(s: String): Unit - - def close(): Unit = () -} +import mill.api.Logger object DummyLogger extends Logger { def colored = false @@ -160,7 +121,7 @@ case class FileLogger(colored: Boolean, file: os.Path, debugEnabled: Boolean) ex def error(s: String) = outputStream.println(s) def ticker(s: String) = outputStream.println(s) def debug(s: String) = if (debugEnabled) outputStream.println(s) - val inStream: InputStream = DummyInputStream + val inStream: InputStream = mill.api.DummyInputStream override def close() = { if (outputStreamUsed) outputStream.close() diff --git a/main/core/src/mill/util/Watched.scala b/main/core/src/mill/util/Watched.scala index f1ef4fee..29be53c3 100644 --- a/main/core/src/mill/util/Watched.scala +++ b/main/core/src/mill/util/Watched.scala @@ -1,6 +1,6 @@ package mill.util -import mill.eval.PathRef +import mill.api.PathRef case class Watched[T](value: T, watched: Seq[PathRef]) object Watched{ diff --git a/main/core/src/mill/util/package.scala b/main/core/src/mill/util/package.scala new file mode 100644 index 00000000..ec5d2efc --- /dev/null +++ b/main/core/src/mill/util/package.scala @@ -0,0 +1,7 @@ +package mill + +package object util { + // Backwards compat stubs + val Ctx = mill.api.Ctx + type Ctx = mill.api.Ctx +} diff --git a/main/src/mill/MillMain.scala b/main/src/mill/MillMain.scala index e030fdc7..e953e65d 100644 --- a/main/src/mill/MillMain.scala +++ b/main/src/mill/MillMain.scala @@ -6,7 +6,7 @@ import scala.collection.JavaConverters._ import ammonite.main.Cli._ import io.github.retronym.java9rtexport.Export import mill.eval.Evaluator -import mill.util.DummyInputStream +import mill.api.DummyInputStream object MillMain { @@ -38,7 +38,7 @@ object MillMain { setIdle: Boolean => Unit): (Boolean, Option[Evaluator.State]) = { import ammonite.main.Cli - val millHome = mill.util.Ctx.defaultHome + val millHome = mill.api.Ctx.defaultHome val removed = Set("predef-code", "no-home-predef") var interactive = false diff --git a/main/src/mill/main/MainScopts.scala b/main/src/mill/main/MainScopts.scala index 5cc4d7ba..718a30e6 100644 --- a/main/src/mill/main/MainScopts.scala +++ b/main/src/mill/main/MainScopts.scala @@ -25,7 +25,7 @@ object Tasks{ class EvaluatorScopt[T]() extends scopt.Read[mill.eval.Evaluator]{ def arity = 0 - def reads = s => try{ + def reads = s => { Evaluator.currentEvaluator.get.asInstanceOf[mill.eval.Evaluator] } } diff --git a/main/src/mill/main/MillServerMain.scala b/main/src/mill/main/MillServerMain.scala index 5ced75eb..26ca99e6 100644 --- a/main/src/mill/main/MillServerMain.scala +++ b/main/src/mill/main/MillServerMain.scala @@ -9,7 +9,7 @@ import scala.collection.JavaConverters._ import org.scalasbt.ipcsocket._ import mill.main.client._ import mill.eval.Evaluator -import mill.util.DummyInputStream +import mill.api.DummyInputStream import sun.misc.{Signal, SignalHandler} trait MillServerMain[T]{ diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 119ac2aa..47526631 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -9,7 +9,8 @@ import ammonite.util.{Name, Res, Util} import mill.define import mill.define._ import mill.eval.{Evaluator, PathRef, Result} -import mill.util.{EitherOps, Logger, ParseArgs, Watched} +import mill.util.{EitherOps, ParseArgs, Watched} +import mill.api.Logger import mill.util.Strict.Agg import scala.collection.mutable diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index 71b9fc22..e950973f 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -37,7 +37,7 @@ trait VisualizeModule extends mill.define.TaskModule{ val in = new LinkedBlockingQueue[(Seq[_], Seq[_], os.Path)]() val out = new LinkedBlockingQueue[Result[Seq[PathRef]]]() - val cl = mill.util.ClassLoader.create( + val cl = mill.api.ClassLoader.create( classpath().map(_.path.toNIO.toUri.toURL).toVector, getClass.getClassLoader ) diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 8d2c4de4..1a51ed8b 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -13,7 +13,8 @@ import coursier.util.{Gather, Task} import geny.Generator import mill.main.client.InputPumper import mill.eval.{PathRef, Result} -import mill.util.{Ctx, IO} +import mill.util.Ctx +import mill.api.IO import mill.util.Loose.Agg import scala.collection.mutable @@ -165,15 +166,15 @@ object Jvm { val urls = classPath.map(_.toIO.toURI.toURL) val cl = if (classLoaderOverrideSbtTesting) { val outerClassLoader = getClass.getClassLoader - mill.util.ClassLoader.create(urls.toVector, null, customFindClass = { name => + mill.api.ClassLoader.create(urls.toVector, null, customFindClass = { name => if (name.startsWith("sbt.testing.")) Some(outerClassLoader.loadClass(name)) else None }) } else if (isolated) { - mill.util.ClassLoader.create(urls.toVector, null) + mill.api.ClassLoader.create(urls.toVector, null) } else { - mill.util.ClassLoader.create(urls.toVector, getClass.getClassLoader) + mill.api.ClassLoader.create(urls.toVector, getClass.getClassLoader) } val oldCl = Thread.currentThread().getContextClassLoader diff --git a/main/src/mill/modules/Util.scala b/main/src/mill/modules/Util.scala index 2f57595e..2b98a304 100644 --- a/main/src/mill/modules/Util.scala +++ b/main/src/mill/modules/Util.scala @@ -2,8 +2,9 @@ package mill.modules import coursier.Repository -import mill.eval.PathRef -import mill.util.{Ctx, IO, Loose} +import mill.api.{PathRef, IO} +import mill.util.{Ctx, Loose} + object Util { def cleanupScaladoc(v: String) = { @@ -42,31 +43,9 @@ object Util { val tmpName = if (dest == os.rel / "tmp.zip") "tmp2.zip" else "tmp.zip" val downloaded = download(url, tmpName) - unpackZip(downloaded.path, dest) + IO.unpackZip(downloaded.path, dest) } - def unpackZip(src: os.Path, dest: os.RelPath = "unpacked") - (implicit ctx: Ctx.Dest) = { - - val byteStream = os.read.inputStream(src) - val zipStream = new java.util.zip.ZipInputStream(byteStream) - while({ - zipStream.getNextEntry match{ - case null => false - case entry => - if (!entry.isDirectory) { - val entryDest = ctx.dest / dest / os.RelPath(entry.getName) - os.makeDir.all(entryDest / ammonite.ops.up) - val fileOut = new java.io.FileOutputStream(entryDest.toString) - IO.stream(zipStream, fileOut) - fileOut.close() - } - zipStream.closeEntry() - true - } - })() - PathRef(ctx.dest / dest) - } def millProjectModule(key: String, artifact: String, @@ -75,7 +54,7 @@ object Util { artifactSuffix: String = "_2.12") = { val localPath = sys.props(key) if (localPath != null) { - mill.eval.Result.Success( + mill.api.Result.Success( Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) ) } else { diff --git a/main/src/mill/package.scala b/main/src/mill/package.scala index 93916c8b..0ccd094f 100644 --- a/main/src/mill/package.scala +++ b/main/src/mill/package.scala @@ -3,8 +3,8 @@ import mill.util.JsonFormatters package object mill extends JsonFormatters{ val T = define.Target type T[T] = define.Target[T] - val PathRef = mill.eval.PathRef - type PathRef = mill.eval.PathRef + val PathRef = mill.api.PathRef + type PathRef = mill.api.PathRef type Module = define.Module type Cross[T] = define.Cross[T] type Agg[T] = util.Loose.Agg[T] diff --git a/main/test/resources/examples/javac/build.sc b/main/test/resources/examples/javac/build.sc index c45a6e2d..2ed9f915 100644 --- a/main/test/resources/examples/javac/build.sc +++ b/main/test/resources/examples/javac/build.sc @@ -1,6 +1,6 @@ import mill.T import mill.eval.JavaCompileJarTests.compileAll -import mill.eval.PathRef +import mill.api.PathRef import mill.modules.Jvm import mill.util.Loose diff --git a/main/test/src/mill/define/ApplicativeTests.scala b/main/test/src/mill/define/ApplicativeTests.scala index 72b715bb..9dd2132f 100644 --- a/main/test/src/mill/define/ApplicativeTests.scala +++ b/main/test/src/mill/define/ApplicativeTests.scala @@ -1,6 +1,6 @@ package mill.define -import mill.define.Applicative.ImplicitStub +import mill.api.Ctx.ImplicitStub import utest._ import scala.annotation.compileTimeOnly diff --git a/main/test/src/mill/define/CacherTests.scala b/main/test/src/mill/define/CacherTests.scala index 84a8d840..59ebf3f6 100644 --- a/main/test/src/mill/define/CacherTests.scala +++ b/main/test/src/mill/define/CacherTests.scala @@ -3,7 +3,7 @@ package mill.define import mill.util.{DummyLogger, TestEvaluator, TestUtil} import mill.util.Strict.Agg import mill.T -import mill.eval.Result.Success +import mill.api.Result.Success import utest._ import utest.framework.TestPath diff --git a/main/test/src/mill/eval/FailureTests.scala b/main/test/src/mill/eval/FailureTests.scala index bf45119c..dcfbcb60 100644 --- a/main/test/src/mill/eval/FailureTests.scala +++ b/main/test/src/mill/eval/FailureTests.scala @@ -1,7 +1,7 @@ package mill.eval import mill.T import mill.util.{TestEvaluator, TestUtil} -import mill.eval.Result.OuterStack +import mill.api.Result.OuterStack import utest._ import utest.framework.TestPath diff --git a/main/test/src/mill/eval/JavaCompileJarTests.scala b/main/test/src/mill/eval/JavaCompileJarTests.scala index 8e931747..426c6ea6 100644 --- a/main/test/src/mill/eval/JavaCompileJarTests.scala +++ b/main/test/src/mill/eval/JavaCompileJarTests.scala @@ -2,7 +2,7 @@ package mill.eval import mill.define.{Discover, Input, Target, Task} import mill.modules.Jvm -import mill.util.Ctx.Dest +import mill.api.Ctx.Dest import mill.{Module, T} import mill.util.{DummyLogger, Loose, TestEvaluator, TestUtil} import mill.util.Strict.Agg diff --git a/main/test/src/mill/util/TestEvaluator.scala b/main/test/src/mill/util/TestEvaluator.scala index 67ba96dc..9a235679 100644 --- a/main/test/src/mill/util/TestEvaluator.scala +++ b/main/test/src/mill/util/TestEvaluator.scala @@ -1,7 +1,7 @@ package mill.util import mill.define.{Input, Target, Task} -import mill.eval.Result.OuterStack +import mill.api.Result.OuterStack import mill.eval.{Evaluator, Result} import mill.util.Strict.Agg import utest.assert diff --git a/main/test/src/mill/util/TestUtil.scala b/main/test/src/mill/util/TestUtil.scala index e5fe906e..baab2992 100644 --- a/main/test/src/mill/util/TestUtil.scala +++ b/main/test/src/mill/util/TestUtil.scala @@ -2,8 +2,8 @@ package mill.util import mill.util.Router.Overrides import mill.define._ -import mill.eval.Result -import mill.eval.Result.OuterStack +import mill.api.Result +import mill.api.Result.OuterStack import utest.assert import mill.util.Strict.Agg import utest.framework.TestPath |