diff options
10 files changed, 457 insertions, 63 deletions
@@ -1,6 +1,5 @@ import $file.shared import $file.upload -import java.io.File import java.nio.file.attribute.PosixFilePermission import ammonite.ops._ @@ -9,7 +8,6 @@ import mill._ import mill.scalalib._ import publish._ import mill.modules.Jvm.createAssembly - import upickle.Js trait MillPublishModule extends PublishModule{ diff --git a/main/src/mill/modules/Assembly.scala b/main/src/mill/modules/Assembly.scala new file mode 100644 index 00000000..b7b91248 --- /dev/null +++ b/main/src/mill/modules/Assembly.scala @@ -0,0 +1,127 @@ +package mill.modules + +import java.io.InputStream +import java.util.jar.JarFile +import java.util.regex.Pattern + +import ammonite.ops._ +import geny.Generator +import mill.Agg + +import scala.collection.JavaConverters._ + +object Assembly { + + val defaultRules: Seq[Rule] = Seq( + Rule.Append("reference.conf"), + Rule.Exclude(JarFile.MANIFEST_NAME), + Rule.ExcludePattern(".*\\.[sS][fF]"), + Rule.ExcludePattern(".*\\.[dD][sS][aA]"), + Rule.ExcludePattern(".*\\.[rR][sS][aA]") + ) + + sealed trait Rule extends Product with Serializable + object Rule { + case class Append(path: String) extends Rule + + object AppendPattern { + def apply(pattern: String): AppendPattern = AppendPattern(Pattern.compile(pattern)) + } + case class AppendPattern(pattern: Pattern) extends Rule + + case class Exclude(path: String) extends Rule + + object ExcludePattern { + def apply(pattern: String): ExcludePattern = ExcludePattern(Pattern.compile(pattern)) + } + case class ExcludePattern(pattern: Pattern) extends Rule + } + + def groupAssemblyEntries(inputPaths: Agg[Path], assemblyRules: Seq[Assembly.Rule]): Map[String, GroupedEntry] = { + val rulesMap = assemblyRules.collect { + case r@Rule.Append(path) => path -> r + case r@Rule.Exclude(path) => path -> r + }.toMap + + val appendPatterns = assemblyRules.collect { + case Rule.AppendPattern(pattern) => pattern.asPredicate().test(_) + } + + val excludePatterns = assemblyRules.collect { + case Rule.ExcludePattern(pattern) => pattern.asPredicate().test(_) + } + + classpathIterator(inputPaths).foldLeft(Map.empty[String, GroupedEntry]) { + case (entries, entry) => + val mapping = entry.mapping + + rulesMap.get(mapping) match { + case Some(_: Assembly.Rule.Exclude) => + entries + case Some(_: Assembly.Rule.Append) => + val newEntry = entries.getOrElse(mapping, AppendEntry.empty).append(entry) + entries + (mapping -> newEntry) + + case _ if excludePatterns.exists(_(mapping)) => + entries + case _ if appendPatterns.exists(_(mapping)) => + val newEntry = entries.getOrElse(mapping, AppendEntry.empty).append(entry) + entries + (mapping -> newEntry) + + case _ if !entries.contains(mapping) => + entries + (mapping -> WriteOnceEntry(entry)) + case _ => + entries + } + } + } + + private def classpathIterator(inputPaths: Agg[Path]): Generator[AssemblyEntry] = { + Generator.from(inputPaths) + .filter(exists) + .flatMap { + p => + if (p.isFile) { + val jf = new JarFile(p.toIO) + Generator.from( + for(entry <- jf.entries().asScala if !entry.isDirectory) + yield JarFileEntry(entry.getName, () => jf.getInputStream(entry)) + ) + } + else { + ls.rec.iter(p) + .filter(_.isFile) + .map(sub => PathEntry(sub.relativeTo(p).toString, sub)) + } + } + } +} + +private[modules] sealed trait GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry +} + +private[modules] object AppendEntry { + val empty: AppendEntry = AppendEntry(Nil) +} + +private[modules] case class AppendEntry(entries: List[AssemblyEntry]) extends GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry = copy(entries = entry :: this.entries) +} + +private[modules] case class WriteOnceEntry(entry: AssemblyEntry) extends GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry = this +} + +private[this] sealed trait AssemblyEntry { + def mapping: String + def inputStream: InputStream +} + +private[this] case class PathEntry(mapping: String, path: Path) extends AssemblyEntry { + def inputStream: InputStream = read.getInputStream(path) +} + +private[this] case class JarFileEntry(mapping: String, getIs: () => InputStream) extends AssemblyEntry { + def inputStream: InputStream = getIs() +} diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 1a28189f..be683e4a 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -1,14 +1,14 @@ package mill.modules -import java.io.{ByteArrayInputStream, File, FileOutputStream} +import java.io._ import java.lang.reflect.Modifier -import java.net.{URI, URLClassLoader} -import java.nio.file.{FileSystems, Files, OpenOption, StandardOpenOption} +import java.net.URI +import java.nio.file.{FileSystems, Files, StandardOpenOption} import java.nio.file.attribute.PosixFilePermission +import java.util.Collections import java.util.jar.{JarEntry, JarFile, JarOutputStream} import ammonite.ops._ -import ammonite.util.Util import coursier.{Cache, Dependency, Fetch, Repository, Resolution} import geny.Generator import mill.main.client.InputPumper @@ -17,7 +17,7 @@ import mill.util.{Ctx, IO} import mill.util.Loose.Agg import scala.collection.mutable - +import scala.collection.JavaConverters._ object Jvm { @@ -232,18 +232,20 @@ object Jvm { PathRef(outputPath) } - def newOutputStream(p: java.nio.file.Path) = Files.newOutputStream( - p, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE - ) + def newOutputStream(p: java.nio.file.Path, append: Boolean = false) = { + val options = + if(append) Seq(StandardOpenOption.APPEND) + else Seq(StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) + Files.newOutputStream(p, options:_*) + } def createAssembly(inputPaths: Agg[Path], mainClass: Option[String] = None, prependShellScript: String = "", base: Option[Path] = None, - isWin: Boolean = scala.util.Properties.isWin) - (implicit ctx: Ctx.Dest) = { + assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules) + (implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { + val tmp = ctx.dest / "out-tmp.jar" val baseUri = "jar:" + tmp.toIO.getCanonicalFile.toURI.toASCIIString @@ -263,20 +265,22 @@ object Jvm { manifest.write(manifestOut) manifestOut.close() - def isSignatureFile(mapping: String): Boolean = - Set("sf", "rsa", "dsa").exists(ext => mapping.toLowerCase.endsWith(s".$ext")) - - for(v <- classpathIterator(inputPaths)){ - val (file, mapping) = v - val p = zipFs.getPath(mapping) - if (p.getParent != null) Files.createDirectories(p.getParent) - if (!isSignatureFile(mapping)) { - val outputStream = newOutputStream(p) - IO.stream(file, outputStream) - outputStream.close() + Assembly.groupAssemblyEntries(inputPaths, assemblyRules).view + .map { + case (mapping, aggregate) => + zipFs.getPath(mapping) -> aggregate } - file.close() - } + .foreach { + case (path, AppendEntry(entries)) => + val concatenated = new SequenceInputStream( + Collections.enumeration(entries.map(_.inputStream).asJava)) + writeEntry(path, concatenated, append = Files.exists(path)) + case (path, WriteOnceEntry(entry)) => + if (Files.notExists(path)) { + writeEntry(path, entry.inputStream, append = false) + } + } + zipFs.close() val output = ctx.dest / "out.jar" @@ -301,28 +305,14 @@ object Jvm { PathRef(output) } + private def writeEntry(p: java.nio.file.Path, is: InputStream, append: Boolean): Unit = { + if (p.getParent != null) Files.createDirectories(p.getParent) + val outputStream = newOutputStream(p, append) - def classpathIterator(inputPaths: Agg[Path]) = { - Generator.from(inputPaths) - .filter(exists) - .flatMap{ - p => - if (p.isFile) { - val jf = new JarFile(p.toIO) - import collection.JavaConverters._ - Generator.selfClosing(( - for(entry <- jf.entries().asScala if !entry.isDirectory) - yield (jf.getInputStream(entry), entry.getName), - () => jf.close() - )) - } - else { - ls.rec.iter(p) - .filter(_.isFile) - .map(sub => read.getInputStream(sub) -> sub.relativeTo(p).toString) - } - } + IO.stream(is, outputStream) + outputStream.close() + is.close() } def universalScript(shellCommands: String, diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 26c58a8b..79fad9aa 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -7,7 +7,7 @@ import coursier.Repository import mill.define.Task import mill.define.TaskModule import mill.eval.{PathRef, Result} -import mill.modules.Jvm +import mill.modules.{Assembly, Jvm} import mill.modules.Jvm.{createAssembly, createJar} import Lib._ import mill.scalalib.publish.{Artifact, Scope} @@ -108,6 +108,8 @@ trait JavaModule extends mill.Module with TaskModule { outer => } } + def assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules + def sources = T.sources{ millSourcePath / 'src } def resources = T.sources{ millSourcePath / 'resources } def generatedSources = T{ Seq.empty[PathRef] } @@ -130,6 +132,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => upstreamCompileOutput() ) } + def localClasspath = T{ resources() ++ Agg(compile().classes) } @@ -158,7 +161,11 @@ trait JavaModule extends mill.Module with TaskModule { outer => * upstream dependencies do not change */ def upstreamAssembly = T{ - createAssembly(upstreamAssemblyClasspath().map(_.path), mainClass()) + createAssembly( + upstreamAssemblyClasspath().map(_.path), + mainClass(), + assemblyRules = assemblyRules + ) } def assembly = T{ @@ -166,11 +173,11 @@ trait JavaModule extends mill.Module with TaskModule { outer => Agg.from(localClasspath().map(_.path)), mainClass(), prependShellScript(), - Some(upstreamAssembly().path) + Some(upstreamAssembly().path), + assemblyRules ) } - def jar = T{ createJar( localClasspath().map(_.path).filter(exists), diff --git a/scalalib/test/resources/hello-world-multi/core/resources/reference.conf b/scalalib/test/resources/hello-world-multi/core/resources/reference.conf new file mode 100644 index 00000000..afb44467 --- /dev/null +++ b/scalalib/test/resources/hello-world-multi/core/resources/reference.conf @@ -0,0 +1,4 @@ +############################## +# Core Reference Config File # +############################## +bar.baz=hello diff --git a/scalalib/test/resources/hello-world-multi/core/src/Main.scala b/scalalib/test/resources/hello-world-multi/core/src/Main.scala new file mode 100644 index 00000000..5cbb75cf --- /dev/null +++ b/scalalib/test/resources/hello-world-multi/core/src/Main.scala @@ -0,0 +1,5 @@ +object Main extends App { + val person = Person.fromString("rockjam:25") + println(s"hello ${person.name}, your age is: ${person.age}") +} + diff --git a/scalalib/test/resources/hello-world-multi/model/resources/reference.conf b/scalalib/test/resources/hello-world-multi/model/resources/reference.conf new file mode 100644 index 00000000..8e7ed298 --- /dev/null +++ b/scalalib/test/resources/hello-world-multi/model/resources/reference.conf @@ -0,0 +1,4 @@ +############################### +# Model Reference Config File # +############################### +foo.bar=2 diff --git a/scalalib/test/resources/hello-world-multi/model/src/Person.scala b/scalalib/test/resources/hello-world-multi/model/src/Person.scala new file mode 100644 index 00000000..23e3821e --- /dev/null +++ b/scalalib/test/resources/hello-world-multi/model/src/Person.scala @@ -0,0 +1,8 @@ +object Person { + def fromString(s: String): Person = { + val Array(name, age) = s.split(":") + Person(name, age.toInt) + } +} +case class Person(name: String, age: Int) + diff --git a/scalalib/test/resources/hello-world/core/resources/reference.conf b/scalalib/test/resources/hello-world/core/resources/reference.conf new file mode 100644 index 00000000..bf0f66ae --- /dev/null +++ b/scalalib/test/resources/hello-world/core/resources/reference.conf @@ -0,0 +1,4 @@ +######################################## +# My application Reference Config File # +######################################## +akka.http.client.user-agent-header="hello-world-client" diff --git a/scalalib/test/src/mill/scalalib/HelloWorldTests.scala b/scalalib/test/src/mill/scalalib/HelloWorldTests.scala index 46cf82de..81a5b8a6 100644 --- a/scalalib/test/src/mill/scalalib/HelloWorldTests.scala +++ b/scalalib/test/src/mill/scalalib/HelloWorldTests.scala @@ -3,15 +3,14 @@ package mill.scalalib import java.util.jar.JarFile import ammonite.ops._ -import ammonite.ops.ImplicitWd._ import mill._ -import mill.define.{Discover, Target} +import mill.define.Target import mill.eval.{Evaluator, Result} +import mill.modules.Assembly import mill.scalalib.publish._ import mill.util.{TestEvaluator, TestUtil} import mill.scalalib.publish.VersionControl import utest._ - import utest.framework.TestPath import scala.collection.JavaConverters._ @@ -21,10 +20,14 @@ object HelloWorldTests extends TestSuite { trait HelloBase extends TestUtil.BaseModule{ def millSourcePath = TestUtil.getSrcPathBase() / millOuterCtx.enclosing.split('.') } + trait HelloWorldModule extends scalalib.ScalaModule { def scalaVersion = "2.12.4" } + trait HelloWorldModuleWithMain extends HelloWorldModule { + def mainClass = Some("Main") + } object HelloWorld extends HelloBase { object core extends HelloWorldModule @@ -45,11 +48,96 @@ object HelloWorldTests extends TestSuite { } object HelloWorldWithMain extends HelloBase { - object core extends HelloWorldModule{ - def mainClass = Some("Main") + object core extends HelloWorldModuleWithMain + } + + val akkaHttpDeps = Agg(ivy"com.typesafe.akka::akka-http:10.0.13") + + object HelloWorldAkkaHttpAppend extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) } } + object HelloWorldAkkaHttpExclude extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) + } + } + + object HelloWorldAkkaHttpAppendPattern extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) + } + } + + object HelloWorldAkkaHttpExcludePattern extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) + } + } + + object HelloWorldAkkaHttpNoRules extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq.empty + } + } + + object HelloWorldMultiAppend extends HelloBase { + object core extends HelloWorldModuleWithMain { + def moduleDeps = Seq(model) + + def assemblyRules = Seq(Assembly.Rule.Append("reference.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiExclude extends HelloBase { + object core extends HelloWorldModuleWithMain { + def moduleDeps = Seq(model) + + def assemblyRules = Seq(Assembly.Rule.Exclude("reference.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiAppendPattern extends HelloBase { + object core extends HelloWorldModuleWithMain { + def moduleDeps = Seq(model) + + def assemblyRules = Seq(Assembly.Rule.AppendPattern(".*.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiExcludePattern extends HelloBase { + object core extends HelloWorldModuleWithMain { + def moduleDeps = Seq(model) + + def assemblyRules = Seq(Assembly.Rule.ExcludePattern(".*.conf")) + } + object model extends HelloWorldModule + } + + object HelloWorldMultiNoRules extends HelloBase { + object core extends HelloWorldModuleWithMain { + def moduleDeps = Seq(model) + + def assemblyRules = Seq.empty + } + object model extends HelloWorldModule + } + object HelloWorldWarnUnused extends HelloBase{ object core extends HelloWorldModule { def scalacOptions = T(Seq("-Ywarn-unused")) @@ -158,6 +246,15 @@ object HelloWorldTests extends TestSuite { attrs.get(Name.MAIN_CLASS).map(_.asInstanceOf[String]) } + def jarEntries(jar: JarFile): Set[String] = { + jar.entries().asScala.map(_.getName).toSet + } + + def readFileFromJar(jar: JarFile, name: String): String = { + val is = jar.getInputStream(jar.getEntry(name)) + read(is) + } + def compileClassfiles = Seq[RelPath]( "Main.class", "Main$.class", @@ -178,14 +275,14 @@ object HelloWorldTests extends TestSuite { cp(resourcePath, m.millSourcePath) t(eval) } - + def tests: Tests = Tests { 'scalaVersion - { - - 'fromBuild - workspaceTest(HelloWorld){eval => + + 'fromBuild - workspaceTest(HelloWorld){eval => val Right((result, evalCount)) = eval.apply(HelloWorld.core.scalaVersion) assert( @@ -421,10 +518,11 @@ object HelloWorldTests extends TestSuite { val jarFile = new JarFile(result.path.toIO) val entries = jarFile.entries().asScala.map(_.getName).toSet - val manifestFiles = Seq[RelPath]( - "META-INF" / "MANIFEST.MF" + val otherFiles = Seq[RelPath]( + "META-INF" / "MANIFEST.MF", + "reference.conf" ) - val expectedFiles = compileClassfiles ++ manifestFiles + val expectedFiles = compileClassfiles ++ otherFiles assert( entries.nonEmpty, @@ -452,7 +550,7 @@ object HelloWorldTests extends TestSuite { evalCount > 0 ) val jarFile = new JarFile(result.path.toIO) - val entries = jarFile.entries().asScala.map(_.getName).toSet + val entries = jarEntries(jarFile) val mainPresent = entries.contains("Main.class") assert(mainPresent) @@ -462,6 +560,155 @@ object HelloWorldTests extends TestSuite { assert(mainClass.contains("Main")) } + 'assemblyRules - { + def checkAppend[M <: TestUtil.BaseModule](module: M, + target: Target[PathRef]) = + workspaceTest(module) { eval => + val Right((result, _)) = eval.apply(target) + + val jarFile = new JarFile(result.path.toIO) + + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + // akka modules configs are present + referenceContent.contains("akka-http Reference Config File"), + referenceContent.contains("akka-http-core Reference Config File"), + referenceContent.contains("Akka Actor Reference Config File"), + referenceContent.contains("Akka Stream Reference Config File"), + // our application config is present too + referenceContent.contains("My application Reference Config File"), + referenceContent.contains( + """akka.http.client.user-agent-header="hello-world-client"""" + ) + ) + } + + val helloWorldMultiResourcePath = pwd / 'scalalib / 'test / 'resources / "hello-world-multi" + + def checkAppendMulti[M <: TestUtil.BaseModule]( + module: M, + target: Target[PathRef]) = + workspaceTest( + module, + resourcePath = helloWorldMultiResourcePath + ) { eval => + val Right((result, _)) = eval.apply(target) + + val jarFile = new JarFile(result.path.toIO) + + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + // reference config from core module + referenceContent.contains("Core Reference Config File"), + // reference config from model module + referenceContent.contains("Model Reference Config File"), + // concatenated content + referenceContent.contains("bar.baz=hello"), + referenceContent.contains("foo.bar=2") + ) + } + + 'appendWithDeps - checkAppend( + HelloWorldAkkaHttpAppend, + HelloWorldAkkaHttpAppend.core.assembly + ) + 'appendMultiModule - checkAppendMulti( + HelloWorldMultiAppend, + HelloWorldMultiAppend.core.assembly + ) + 'appendPatternWithDeps - checkAppend( + HelloWorldAkkaHttpAppendPattern, + HelloWorldAkkaHttpAppendPattern.core.assembly + ) + 'appendPatternMultiModule - checkAppendMulti( + HelloWorldMultiAppendPattern, + HelloWorldMultiAppendPattern.core.assembly + ) + + def checkExclude[M <: TestUtil.BaseModule](module: M, + target: Target[PathRef], + resourcePath: Path = resourcePath + ) = + workspaceTest(module, resourcePath) { eval => + val Right((result, _)) = eval.apply(target) + + val jarFile = new JarFile(result.path.toIO) + + assert(!jarEntries(jarFile).contains("reference.conf")) + } + + 'excludeWithDeps - checkExclude( + HelloWorldAkkaHttpExclude, + HelloWorldAkkaHttpExclude.core.assembly + ) + 'excludeMultiModule - checkExclude( + HelloWorldMultiExclude, + HelloWorldMultiExclude.core.assembly, + resourcePath = helloWorldMultiResourcePath + + ) + 'excludePatternWithDeps - checkExclude( + HelloWorldAkkaHttpExcludePattern, + HelloWorldAkkaHttpExcludePattern.core.assembly + ) + 'excludePatternMultiModule - checkExclude( + HelloWorldMultiExcludePattern, + HelloWorldMultiExcludePattern.core.assembly, + resourcePath = helloWorldMultiResourcePath + ) + + 'writeFirstWhenNoRule - { + 'withDeps - workspaceTest(HelloWorldAkkaHttpNoRules) { eval => + val Right((result, _)) = eval.apply(HelloWorldAkkaHttpNoRules.core.assembly) + + val jarFile = new JarFile(result.path.toIO) + + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + val allOccurrences = Seq( + referenceContent.contains("akka-http Reference Config File"), + referenceContent.contains("akka-http-core Reference Config File"), + referenceContent.contains("Akka Actor Reference Config File"), + referenceContent.contains("Akka Stream Reference Config File"), + referenceContent.contains("My application Reference Config File") + ) + + val timesOcccurres = allOccurrences.find(identity).size + + assert(timesOcccurres == 1) + } + + 'multiModule - workspaceTest( + HelloWorldMultiNoRules, + resourcePath = helloWorldMultiResourcePath + ) { eval => + val Right((result, _)) = eval.apply(HelloWorldMultiNoRules.core.assembly) + + val jarFile = new JarFile(result.path.toIO) + + assert(jarEntries(jarFile).contains("reference.conf")) + + val referenceContent = readFileFromJar(jarFile, "reference.conf") + + assert( + referenceContent.contains("Model Reference Config File"), + referenceContent.contains("foo.bar=2"), + + !referenceContent.contains("Core Reference Config File"), + !referenceContent.contains("bar.baz=hello") + ) + } + } + } + 'run - workspaceTest(HelloWorldWithMain){eval => val Right((result, evalCount)) = eval.apply(HelloWorldWithMain.core.assembly) |