From 9ba4cb69331386dfde9bac69dc2d5b22401face3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 12 Dec 2018 16:56:02 -0800 Subject: collapse boilerplate folder structure within src/ folders (#505) * collapse boilerplate folder structure within src/ folders * . --- main/src/modules/Assembly.scala | 126 +++++++++++ main/src/modules/Jvm.scala | 481 ++++++++++++++++++++++++++++++++++++++++ main/src/modules/Util.scala | 73 ++++++ 3 files changed, 680 insertions(+) create mode 100644 main/src/modules/Assembly.scala create mode 100644 main/src/modules/Jvm.scala create mode 100644 main/src/modules/Util.scala (limited to 'main/src/modules') diff --git a/main/src/modules/Assembly.scala b/main/src/modules/Assembly.scala new file mode 100644 index 00000000..141bc226 --- /dev/null +++ b/main/src/modules/Assembly.scala @@ -0,0 +1,126 @@ +package mill.modules + +import java.io.InputStream +import java.util.jar.JarFile +import java.util.regex.Pattern + +import geny.Generator +import mill.Agg + +import scala.collection.JavaConverters._ + +object Assembly { + + val defaultRules: Seq[Rule] = Seq( + Rule.Append("reference.conf"), + Rule.Exclude(JarFile.MANIFEST_NAME), + Rule.ExcludePattern(".*\\.[sS][fF]"), + Rule.ExcludePattern(".*\\.[dD][sS][aA]"), + Rule.ExcludePattern(".*\\.[rR][sS][aA]") + ) + + sealed trait Rule extends Product with Serializable + object Rule { + case class Append(path: String) extends Rule + + object AppendPattern { + def apply(pattern: String): AppendPattern = AppendPattern(Pattern.compile(pattern)) + } + case class AppendPattern(pattern: Pattern) extends Rule + + case class Exclude(path: String) extends Rule + + object ExcludePattern { + def apply(pattern: String): ExcludePattern = ExcludePattern(Pattern.compile(pattern)) + } + case class ExcludePattern(pattern: Pattern) extends Rule + } + + def groupAssemblyEntries(inputPaths: Agg[os.Path], assemblyRules: Seq[Assembly.Rule]): Map[String, GroupedEntry] = { + val rulesMap = assemblyRules.collect { + case r@Rule.Append(path) => path -> r + case r@Rule.Exclude(path) => path -> r + }.toMap + + val appendPatterns = assemblyRules.collect { + case Rule.AppendPattern(pattern) => pattern.asPredicate().test(_) + } + + val excludePatterns = assemblyRules.collect { + case Rule.ExcludePattern(pattern) => pattern.asPredicate().test(_) + } + + classpathIterator(inputPaths).foldLeft(Map.empty[String, GroupedEntry]) { + case (entries, entry) => + val mapping = entry.mapping + + rulesMap.get(mapping) match { + case Some(_: Assembly.Rule.Exclude) => + entries + case Some(_: Assembly.Rule.Append) => + val newEntry = entries.getOrElse(mapping, AppendEntry.empty).append(entry) + entries + (mapping -> newEntry) + + case _ if excludePatterns.exists(_(mapping)) => + entries + case _ if appendPatterns.exists(_(mapping)) => + val newEntry = entries.getOrElse(mapping, AppendEntry.empty).append(entry) + entries + (mapping -> newEntry) + + case _ if !entries.contains(mapping) => + entries + (mapping -> WriteOnceEntry(entry)) + case _ => + entries + } + } + } + + private def classpathIterator(inputPaths: Agg[os.Path]): Generator[AssemblyEntry] = { + Generator.from(inputPaths) + .filter(os.exists) + .flatMap { + p => + if (os.isFile(p)) { + val jf = new JarFile(p.toIO) + Generator.from( + for(entry <- jf.entries().asScala if !entry.isDirectory) + yield JarFileEntry(entry.getName, () => jf.getInputStream(entry)) + ) + } + else { + os.walk.stream(p) + .filter(os.isFile) + .map(sub => PathEntry(sub.relativeTo(p).toString, sub)) + } + } + } +} + +private[modules] sealed trait GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry +} + +private[modules] object AppendEntry { + val empty: AppendEntry = AppendEntry(Nil) +} + +private[modules] case class AppendEntry(entries: List[AssemblyEntry]) extends GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry = copy(entries = entry :: this.entries) +} + +private[modules] case class WriteOnceEntry(entry: AssemblyEntry) extends GroupedEntry { + def append(entry: AssemblyEntry): GroupedEntry = this +} + +private[this] sealed trait AssemblyEntry { + def mapping: String + def inputStream: InputStream +} + +private[this] case class PathEntry(mapping: String, path: os.Path) extends AssemblyEntry { + def inputStream: InputStream = os.read.inputStream(path) +} + +private[this] case class JarFileEntry(mapping: String, getIs: () => InputStream) extends AssemblyEntry { + def inputStream: InputStream = getIs() +} diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala new file mode 100644 index 00000000..1a51ed8b --- /dev/null +++ b/main/src/modules/Jvm.scala @@ -0,0 +1,481 @@ +package mill.modules + +import java.io._ +import java.lang.reflect.Modifier +import java.net.URI +import java.nio.file.{FileSystems, Files, StandardOpenOption} +import java.nio.file.attribute.PosixFilePermission +import java.util.Collections +import java.util.jar.{JarEntry, JarFile, JarOutputStream} + +import coursier.{Cache, Dependency, Fetch, Repository, Resolution} +import coursier.util.{Gather, Task} +import geny.Generator +import mill.main.client.InputPumper +import mill.eval.{PathRef, Result} +import mill.util.Ctx +import mill.api.IO +import mill.util.Loose.Agg + +import scala.collection.mutable +import scala.collection.JavaConverters._ + +object Jvm { + /** + * Runs a JVM subprocess with the given configuration and returns a + * [[os.CommandResult]] with it's aggregated output and error streams + */ + def callSubprocess(mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String] = Seq.empty, + envArgs: Map[String, String] = Map.empty, + mainArgs: Seq[String] = Seq.empty, + workingDir: os.Path = null, + streamOut: Boolean = true) + (implicit ctx: Ctx) = { + + val commandArgs = + Vector("java") ++ + jvmArgs ++ + Vector("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ + mainArgs + + val workingDir1 = Option(workingDir).getOrElse(ctx.dest) + os.makeDir.all(workingDir1) + + os.proc(commandArgs).call(cwd = workingDir1, env = envArgs) + } + + /** + * Runs a JVM subprocess with the given configuration and streams + * it's stdout and stderr to the console. + */ + def runSubprocess(mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String] = Seq.empty, + envArgs: Map[String, String] = Map.empty, + mainArgs: Seq[String] = Seq.empty, + workingDir: os.Path = null, + background: Boolean = false): Unit = { + val args = + Vector("java") ++ + jvmArgs ++ + Vector("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ + mainArgs + + if (background) spawnSubprocess(args, envArgs, workingDir) + else runSubprocess(args, envArgs, workingDir) + } + + @deprecated("Use runSubprocess instead") + def baseInteractiveSubprocess(commandArgs: Seq[String], + envArgs: Map[String, String], + workingDir: os.Path) = { + runSubprocess(commandArgs, envArgs, workingDir) + } + + /** + * Runs a generic subprocess and waits for it to terminate. + */ + def runSubprocess(commandArgs: Seq[String], + envArgs: Map[String, String], + workingDir: os.Path) = { + val process = spawnSubprocess(commandArgs, envArgs, workingDir) + + process.waitFor() + if (process.exitCode() == 0) () + else throw new Exception("Interactive Subprocess Failed") + } + + /** + * Spawns a generic subprocess, streaming the stdout and stderr to the + * console. If the System.out/System.err have been substituted, makes sure + * that the subprocess's stdout and stderr streams go to the subtituted + * streams + */ + def spawnSubprocess(commandArgs: Seq[String], + envArgs: Map[String, String], + workingDir: os.Path) = { + // If System.in is fake, then we pump output manually rather than relying + // on `os.Inherit`. That is because `os.Inherit` does not follow changes + // to System.in/System.out/System.err, so the subprocess's streams get sent + // to the parent process's origin outputs even if we want to direct them + // elsewhere + if (System.in.isInstanceOf[ByteArrayInputStream]){ + val process = os.proc(commandArgs).spawn( + cwd = workingDir, + env = envArgs, + stdin = os.Pipe, + stdout = os.Pipe, + stderr = os.Pipe + ) + + val sources = Seq( + process.stdout -> System.out, + process.stderr -> System.err, + System.in -> process.stdin + ) + + for((std, dest) <- sources){ + new Thread(new InputPumper(std, dest, false)).start() + } + + process + }else{ + os.proc(commandArgs).spawn( + cwd = workingDir, + env = envArgs, + stdin = os.Inherit, + stdout = os.Inherit, + stderr = os.Inherit + ) + } + } + + + def runLocal(mainClass: String, + classPath: Agg[os.Path], + mainArgs: Seq[String] = Seq.empty) + (implicit ctx: Ctx): Unit = { + inprocess(classPath, classLoaderOverrideSbtTesting = false, isolated = true, closeContextClassLoaderWhenDone = true, cl => { + getMainMethod(mainClass, cl).invoke(null, mainArgs.toArray) + }) + } + + private def getMainMethod(mainClassName: String, cl: ClassLoader) = { + val mainClass = cl.loadClass(mainClassName) + val method = mainClass.getMethod("main", classOf[Array[String]]) + // jvm allows the actual main class to be non-public and to run a method in the non-public class, + // we need to make it accessible + method.setAccessible(true) + val modifiers = method.getModifiers + if (!Modifier.isPublic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not public") + if (!Modifier.isStatic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not static") + method + } + + + def inprocess[T](classPath: Agg[os.Path], + classLoaderOverrideSbtTesting: Boolean, + isolated: Boolean, + closeContextClassLoaderWhenDone: Boolean, + body: ClassLoader => T) + (implicit ctx: Ctx.Home): T = { + val urls = classPath.map(_.toIO.toURI.toURL) + val cl = if (classLoaderOverrideSbtTesting) { + val outerClassLoader = getClass.getClassLoader + mill.api.ClassLoader.create(urls.toVector, null, customFindClass = { name => + if (name.startsWith("sbt.testing.")) + Some(outerClassLoader.loadClass(name)) + else None + }) + } else if (isolated) { + mill.api.ClassLoader.create(urls.toVector, null) + } else { + mill.api.ClassLoader.create(urls.toVector, getClass.getClassLoader) + } + + val oldCl = Thread.currentThread().getContextClassLoader + Thread.currentThread().setContextClassLoader(cl) + try { + body(cl) + } finally { + if (closeContextClassLoaderWhenDone) { + Thread.currentThread().setContextClassLoader(oldCl) + cl.close() + } + } + } + + + private def createManifest(mainClass: Option[String]) = { + val m = new java.util.jar.Manifest() + m.getMainAttributes.put(java.util.jar.Attributes.Name.MANIFEST_VERSION, "1.0") + m.getMainAttributes.putValue( "Created-By", "Scala mill" ) + mainClass.foreach( + m.getMainAttributes.put(java.util.jar.Attributes.Name.MAIN_CLASS, _) + ) + m + } + + /** + * Create a jar file containing all files from the specified input Paths, + * called out.jar in the implicit ctx.dest folder. An optional main class may + * be provided for the jar. An optional filter function may also be provided to + * selectively include/exclude specific files. + * @param inputPaths - `Agg` of `os.Path`s containing files to be included in the jar + * @param mainClass - optional main class for the jar + * @param fileFilter - optional file filter to select files to be included. + * Given a `os.Path` (from inputPaths) and a `os.RelPath` for the individual file, + * return true if the file is to be included in the jar. + * @param ctx - implicit `Ctx.Dest` used to determine the output directory for the jar. + * @return - a `PathRef` for the created jar. + */ + def createJar(inputPaths: Agg[os.Path], + mainClass: Option[String] = None, + fileFilter: (os.Path, os.RelPath) => Boolean = (p: os.Path, r: os.RelPath) => true) + (implicit ctx: Ctx.Dest): PathRef = { + val outputPath = ctx.dest / "out.jar" + os.remove.all(outputPath) + + val seen = mutable.Set.empty[os.RelPath] + seen.add(os.rel / "META-INF" / "MANIFEST.MF") + val jar = new JarOutputStream( + new FileOutputStream(outputPath.toIO), + createManifest(mainClass) + ) + + try{ + assert(inputPaths.forall(os.exists(_))) + for{ + p <- inputPaths + (file, mapping) <- + if (os.isFile(p)) Iterator(p -> os.rel / p.last) + else os.walk(p).filter(os.isFile).map(sub => sub -> sub.relativeTo(p)).sorted + if !seen(mapping) && fileFilter(p, mapping) + } { + seen.add(mapping) + val entry = new JarEntry(mapping.toString) + entry.setTime(os.mtime(file)) + jar.putNextEntry(entry) + jar.write(os.read.bytes(file)) + jar.closeEntry() + } + } finally { + jar.close() + } + + PathRef(outputPath) + } + + def createAssembly(inputPaths: Agg[os.Path], + mainClass: Option[String] = None, + prependShellScript: String = "", + base: Option[os.Path] = None, + assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules) + (implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { + + val tmp = ctx.dest / "out-tmp.jar" + + val baseUri = "jar:" + tmp.toIO.getCanonicalFile.toURI.toASCIIString + val hm = new java.util.HashMap[String, String]() + + base match{ + case Some(b) => os.copy(b, tmp) + case None => hm.put("create", "true") + } + + val zipFs = FileSystems.newFileSystem(URI.create(baseUri), hm) + + val manifest = createManifest(mainClass) + val manifestPath = zipFs.getPath(JarFile.MANIFEST_NAME) + Files.createDirectories(manifestPath.getParent) + val manifestOut = Files.newOutputStream( + manifestPath, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ) + manifest.write(manifestOut) + manifestOut.close() + + Assembly.groupAssemblyEntries(inputPaths, assemblyRules).view + .foreach { + case (mapping, AppendEntry(entries)) => + val path = zipFs.getPath(mapping).toAbsolutePath + val concatenated = new SequenceInputStream( + Collections.enumeration(entries.map(_.inputStream).asJava)) + writeEntry(path, concatenated, append = true) + case (mapping, WriteOnceEntry(entry)) => + val path = zipFs.getPath(mapping).toAbsolutePath + if (Files.notExists(path)) { + writeEntry(path, entry.inputStream, append = false) + } + } + + zipFs.close() + val output = ctx.dest / "out.jar" + + // Prepend shell script and make it executable + if (prependShellScript.isEmpty) os.move(tmp, output) + else{ + val lineSep = if (!prependShellScript.endsWith("\n")) "\n\r\n" else "" + os.write( + output, + Seq[os.Source]( + prependShellScript + lineSep, + os.read.inputStream(tmp) + ) + ) + + if (!scala.util.Properties.isWin) { + os.perms.set( + output, + os.perms(output) + + PosixFilePermission.GROUP_EXECUTE + + PosixFilePermission.OWNER_EXECUTE + + PosixFilePermission.OTHERS_EXECUTE + ) + } + } + + PathRef(output) + } + + private def writeEntry(p: java.nio.file.Path, is: InputStream, append: Boolean): Unit = { + if (p.getParent != null) Files.createDirectories(p.getParent) + val options = + if(append) Seq(StandardOpenOption.APPEND, StandardOpenOption.CREATE) + else Seq(StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) + + val outputStream = java.nio.file.Files.newOutputStream(p, options:_*) + IO.stream(is, outputStream) + outputStream.close() + is.close() + } + def universalScript(shellCommands: String, + cmdCommands: String, + shebang: Boolean = false): String = { + Seq( + if (shebang) "#!/usr/bin/env sh" else "", + "@ 2>/dev/null # 2>nul & echo off & goto BOF\r", + ":", + shellCommands.replaceAll("\r\n|\n", "\n"), + "exit", + Seq( + "", + ":BOF", + "@echo off", + cmdCommands.replaceAll("\r\n|\n", "\r\n"), + "exit /B %errorlevel%", + "" + ).mkString("\r\n") + ).filterNot(_.isEmpty).mkString("\n") + } + + def launcherUniversalScript(mainClass: String, + shellClassPath: Agg[String], + cmdClassPath: Agg[String], + jvmArgs: Seq[String], + shebang: Boolean = false) = { + universalScript( + shellCommands = + s"""exec java ${jvmArgs.mkString(" ")} $$JAVA_OPTS -cp "${shellClassPath.mkString(":")}" $mainClass "$$@"""", + cmdCommands = + s"""java ${jvmArgs.mkString(" ")} %JAVA_OPTS% -cp "${cmdClassPath.mkString(";")}" $mainClass %*""", + shebang = shebang + ) + } + def createLauncher(mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String]) + (implicit ctx: Ctx.Dest)= { + val isWin = scala.util.Properties.isWin + val isBatch = isWin && + !(org.jline.utils.OSUtils.IS_CYGWIN + || org.jline.utils.OSUtils.IS_MINGW + || "MSYS" == System.getProperty("MSYSTEM")) + val outputPath = ctx.dest / (if (isBatch) "run.bat" else "run") + val classPathStrs = classPath.map(_.toString) + + os.write(outputPath, launcherUniversalScript(mainClass, classPathStrs, classPathStrs, jvmArgs)) + + if (!isWin) { + val perms = Files.getPosixFilePermissions(outputPath.toNIO) + perms.add(PosixFilePermission.GROUP_EXECUTE) + perms.add(PosixFilePermission.OWNER_EXECUTE) + perms.add(PosixFilePermission.OTHERS_EXECUTE) + Files.setPosixFilePermissions(outputPath.toNIO, perms) + } + PathRef(outputPath) + } + + /** + * Resolve dependencies using Coursier. + * + * We do not bother breaking this out into the separate ZincWorkerApi classpath, + * because Coursier is already bundled with mill/Ammonite to support the + * `import $ivy` syntax. + */ + def resolveDependencies(repositories: Seq[Repository], + deps: TraversableOnce[coursier.Dependency], + force: TraversableOnce[coursier.Dependency], + sources: Boolean = false, + mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = { + + val (_, resolution) = resolveDependenciesMetadata( + repositories, deps, force, mapDependencies + ) + val errs = resolution.metadataErrors + if(errs.nonEmpty) { + val header = + s"""| + |Resolution failed for ${errs.length} modules: + |-------------------------------------------- + |""".stripMargin + + val errLines = errs.map { + case ((module, vsn), errMsgs) => s" ${module.trim}:$vsn \n\t" + errMsgs.mkString("\n\t") + }.mkString("\n") + val msg = header + errLines + "\n" + Result.Failure(msg) + } else { + + def load(artifacts: Seq[coursier.Artifact]) = { + val logger = None + + import scala.concurrent.ExecutionContext.Implicits.global + val loadedArtifacts = Gather[Task].gather( + for (a <- artifacts) + yield coursier.Cache.file[Task](a, logger = logger).run + .map(a.isOptional -> _) + ).unsafeRun + + val errors = loadedArtifacts.collect { + case (false, Left(x)) => x + case (true, Left(x)) if !x.notFound => x + } + val successes = loadedArtifacts.collect { case (_, Right(x)) => x } + (errors, successes) + } + + val sourceOrJar = + if (sources) resolution.classifiersArtifacts(Seq("sources")) + else resolution.artifacts(true) + val (errors, successes) = load(sourceOrJar) + if(errors.isEmpty){ + mill.Agg.from( + successes.map(p => PathRef(os.Path(p), quick = true)).filter(_.path.ext == "jar") + ) + }else{ + val errorDetails = errors.map(e => s"${ammonite.util.Util.newLine} ${e.describe}").mkString + Result.Failure("Failed to load source dependencies" + errorDetails) + } + } + } + + + def resolveDependenciesMetadata(repositories: Seq[Repository], + deps: TraversableOnce[coursier.Dependency], + force: TraversableOnce[coursier.Dependency], + mapDependencies: Option[Dependency => Dependency] = None) = { + + val forceVersions = force + .map(mapDependencies.getOrElse(identity[Dependency](_))) + .map{d => d.module -> d.version} + .toMap + + val start = Resolution( + deps.map(mapDependencies.getOrElse(identity[Dependency](_))).toSet, + forceVersions = forceVersions, + mapDependencies = mapDependencies + ) + + val fetch = Fetch.from(repositories, Cache.fetch[Task]()) + + import scala.concurrent.ExecutionContext.Implicits.global + val resolution = start.process.run(fetch).unsafeRun() + (deps.toSeq, resolution) + } +} diff --git a/main/src/modules/Util.scala b/main/src/modules/Util.scala new file mode 100644 index 00000000..2b98a304 --- /dev/null +++ b/main/src/modules/Util.scala @@ -0,0 +1,73 @@ +package mill.modules + + +import coursier.Repository +import mill.api.{PathRef, IO} +import mill.util.{Ctx, Loose} + + +object Util { + def cleanupScaladoc(v: String) = { + v.linesIterator.map( + _.dropWhile(_.isWhitespace) + .stripPrefix("/**") + .stripPrefix("*/") + .stripPrefix("*") + .dropWhile(_.isWhitespace) + ).toArray + .dropWhile(_.isEmpty) + .reverse + .dropWhile(_.isEmpty) + .reverse + } + def download(url: String, dest: os.RelPath = "download")(implicit ctx: Ctx.Dest) = { + val out = ctx.dest / dest + + val website = new java.net.URI(url).toURL + val rbc = java.nio.channels.Channels.newChannel(website.openStream) + try{ + val fos = new java.io.FileOutputStream(out.toIO) + try{ + fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE) + PathRef(out) + } finally{ + fos.close() + } + } finally{ + rbc.close() + } + } + + def downloadUnpackZip(url: String, dest: os.RelPath = "unpacked") + (implicit ctx: Ctx.Dest) = { + + val tmpName = if (dest == os.rel / "tmp.zip") "tmp2.zip" else "tmp.zip" + val downloaded = download(url, tmpName) + IO.unpackZip(downloaded.path, dest) + } + + + def millProjectModule(key: String, + artifact: String, + repositories: Seq[Repository], + resolveFilter: os.Path => Boolean = _ => true, + artifactSuffix: String = "_2.12") = { + val localPath = sys.props(key) + if (localPath != null) { + mill.api.Result.Success( + Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true))) + ) + } else { + mill.modules.Jvm.resolveDependencies( + repositories, + Seq( + coursier.Dependency( + coursier.Module("com.lihaoyi", artifact + artifactSuffix), + sys.props("MILL_VERSION") + ) + ), + Nil + ).map(_.filter(x => resolveFilter(x.path))) + } + } +} -- cgit v1.2.3