diff options
20 files changed, 630 insertions, 276 deletions
@@ -1,9 +1,23 @@ import ammonite.ops._ import coursier.maven.MavenRepository import mill._ -import mill.scalaplugin._ +import mill.scalaplugin._, publish._ import mill.modules.Jvm.createAssembly +trait MillPublishModule extends PublishModule { + def publishVersion = "0.0.1" + + def publishWithFullScalaVersion = true + + def pomSettings = PomSettings( + organization = "com.lihaoyi", + description = publishName(), + developers = Seq(Developer("lihaoyi", "Li Haoyi", "https://github.com/lihaoyi/mill")), + licenses = Seq(License("MIT License", "https://spdx.org/licenses/MIT.html#licenseText")), + scm = SCM("https://github.com/lihaoyi/mill", "scm:git:https://github.com/lihaoyi/mill.git"), + url = "https://github.com/lihaoyi/mill" + ) +} object CompilerPlugin extends SbtScalaModule{ def scalaVersion = "2.12.4" @@ -40,9 +54,11 @@ trait MillModule extends SbtScalaModule{ outer => } } -object Core extends MillModule { +object Core extends MillModule with MillPublishModule { def projectDeps = Seq(CompilerPlugin) + def publishName = "mill-core" + def compileIvyDeps = Seq( Dep.Java("org.scala-lang", "scala-reflect", scalaVersion()) ) @@ -72,11 +88,6 @@ object Core extends MillModule { PathRef(dest) } } - - override def organization = "com.lihaoyi" - override def name = "mill" - override def version = "0.0.1" - override def useFullScalaVersionForPublish = true } val bridgeVersions = Seq("2.10.6", "2.11.8", "2.11.11", "2.12.3", "2.12.4") @@ -106,7 +117,8 @@ val bridges = for(crossVersion <- mill.define.Cross(bridgeVersions:_*)) yield ne ) } -object ScalaPlugin extends MillModule { +object ScalaPlugin extends MillModule with MillPublishModule { + def publishName = "mill-scala" def projectDeps = Seq(Core) def basePath = pwd / 'scalaplugin diff --git a/core/src/main/scala/mill/define/Applicative.scala b/core/src/main/scala/mill/define/Applicative.scala index 32806643..ddbcccdc 100644 --- a/core/src/main/scala/mill/define/Applicative.scala +++ b/core/src/main/scala/mill/define/Applicative.scala @@ -53,6 +53,9 @@ object Applicative { def zipMap[A, B, C, D, E, F, G, H, I, J, R](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H], i: T[I], j: T[J]) (cb: (A, B, C, D, E, F, G, H, I, J, Ctx) => Z[R]) = mapCtx(zip(a, b, c, d, e, f, g, h, i, j)){case ((a, b, c, d, e, f, g, h, i, j), x) => cb(a, b, c, d, e, f, g, h, i, j, x)} + def zipMap[A, B, C, D, E, F, G, H, I, J, K, R](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H], i: T[I], j: T[J], k: T[K]) + (cb: (A, B, C, D, E, F, G, H, I, J, K, Ctx) => Z[R]) = mapCtx(zip(a, b, c, d, e, f, g, h, i, j, k)){case ((a, b, c, d, e, f, g, h, i, j, k), x) => cb(a, b, c, d, e, f, g, h, i, j, k, x)} + def zip(): T[Unit] def zip[A](a: T[A]): T[Tuple1[A]] def zip[A, B](a: T[A], b: T[B]): T[(A, B)] @@ -64,6 +67,7 @@ object Applicative { def zip[A, B, C, D, E, F, G, H](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H]): T[(A, B, C, D, E, F, G, H)] def zip[A, B, C, D, E, F, G, H, I](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H], i: T[I]): T[(A, B, C, D, E, F, G, H, I)] def zip[A, B, C, D, E, F, G, H, I, J](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H], i: T[I], j: T[J]): T[(A, B, C, D, E, F, G, H, I, J)] + def zip[A, B, C, D, E, F, G, H, I, J, K](a: T[A], b: T[B], c: T[C], d: T[D], e: T[E], f: T[F], g: T[G], h: T[H], i: T[I], j: T[J], k: T[K]): T[(A, B, C, D, E, F, G, H, I, J, K)] } diff --git a/core/src/main/scala/mill/define/Task.scala b/core/src/main/scala/mill/define/Task.scala index 59782f3d..17392725 100644 --- a/core/src/main/scala/mill/define/Task.scala +++ b/core/src/main/scala/mill/define/Task.scala @@ -125,6 +125,10 @@ object Target extends Applicative.Applyer[Task, Task, Result, Ctx]{ val inputs = Seq(a, b, c, d, e, f, g, h, i, j) def evaluate(args: Ctx) = (args[A](0), args[B](1), args[C](2), args[D](3), args[E](4), args[F](5), args[G](6), args[H](7), args[I](8), args[J](9)) } + def zip[A, B, C, D, E, F, G, H, I, J, K](a: Task[A], b: Task[B], c: Task[C], d: Task[D], e: Task[E], f: Task[F], g: Task[G], h: Task[H], i: Task[I], j: Task[J], k: Task[K]) = new Task[(A, B, C, D, E, F, G, H, I, J, K)]{ + val inputs = Seq(a, b, c, d, e, f, g, h, i, j, k) + def evaluate(args: Ctx) = (args[A](0), args[B](1), args[C](2), args[D](3), args[E](4), args[F](5), args[G](6), args[H](7), args[I](8), args[J](9), args[K](10)) + } } class TargetImpl[+T](t: Task[T], enclosing: String) extends Target[T] { val inputs = Seq(t) diff --git a/core/src/main/scala/mill/modules/Jvm.scala b/core/src/main/scala/mill/modules/Jvm.scala index ae13ccb2..f366a0e0 100644 --- a/core/src/main/scala/mill/modules/Jvm.scala +++ b/core/src/main/scala/mill/modules/Jvm.scala @@ -92,9 +92,9 @@ object Jvm { m } - // TODO: outputPath default to ctx.dest - def createJar(outputPath: Path, inputPaths: Seq[Path], mainClass: Option[String] = None) + def createJar(inputPaths: Seq[Path], mainClass: Option[String] = None) (implicit ctx: Ctx.DestCtx): PathRef = { + val outputPath = ctx.dest rm(outputPath) if(inputPaths.nonEmpty) { mkdir(outputPath/up) @@ -126,13 +126,11 @@ object Jvm { PathRef(outputPath) } - - // TODO: outputPath default to ctx.dest - def createAssembly(outputPath: Path, - inputPaths: Seq[Path], + def createAssembly(inputPaths: Seq[Path], mainClass: Option[String] = None, prependShellScript: String = "") (implicit ctx: Ctx.DestCtx): PathRef = { + val outputPath = ctx.dest rm(outputPath) if(inputPaths.nonEmpty) { diff --git a/core/src/main/scala/mill/util/Ctx.scala b/core/src/main/scala/mill/util/Ctx.scala index 38a8aee3..6549531f 100644 --- a/core/src/main/scala/mill/util/Ctx.scala +++ b/core/src/main/scala/mill/util/Ctx.scala @@ -5,12 +5,17 @@ import mill.define.Applicative.ImplicitStub import mill.util.Ctx.{ArgCtx, DestCtx, LoaderCtx, LogCtx} import scala.annotation.compileTimeOnly +import scala.language.implicitConversions object Ctx{ @compileTimeOnly("Target.ctx() can only be used with a T{...} block") @ImplicitStub implicit def taskCtx: Ctx = ??? + object DestCtx { + implicit def pathToCtx(path: Path): DestCtx = + new DestCtx { def dest: Path = path } + } trait DestCtx{ def dest: Path } diff --git a/core/src/test/scala/mill/define/ApplicativeTests.scala b/core/src/test/scala/mill/define/ApplicativeTests.scala index 0dbc91a0..c34ed62a 100644 --- a/core/src/test/scala/mill/define/ApplicativeTests.scala +++ b/core/src/test/scala/mill/define/ApplicativeTests.scala @@ -47,6 +47,9 @@ object ApplicativeTests extends TestSuite { def zip[A, B, C, D, E, F, G, H, I, J](a: O[A], b: O[B], c: O[C], d: O[D], e: O[E], f: O[F], g: O[G], h: O[H], i: O[I], j: O[J]) = { for(a <- a; b <- b; c <- c; d <- d; e <- e; f <- f; g <- g; h <- h; i <- i; j <- j) yield (a, b, c, d, e, f, g, h, i, j) } + def zip[A, B, C, D, E, F, G, H, I, J, K](a: O[A], b: O[B], c: O[C], d: O[D], e: O[E], f: O[F], g: O[G], h: O[H], i: O[I], j: O[J], k: O[K]) = { + for(a <- a; b <- b; c <- c; d <- d; e <- e; f <- f; g <- g; h <- h; i <- i; j <- j; k <- k) yield (a, b, c, d, e, f, g, h, i, j, k) + } } class Counter{ var value = 0 diff --git a/core/src/test/scala/mill/eval/JavaCompileJarTests.scala b/core/src/test/scala/mill/eval/JavaCompileJarTests.scala index ee24b04e..7639d8e2 100644 --- a/core/src/test/scala/mill/eval/JavaCompileJarTests.scala +++ b/core/src/test/scala/mill/eval/JavaCompileJarTests.scala @@ -40,7 +40,7 @@ object JavaCompileJarTests extends TestSuite{ def resourceRoot = T.source{ resourceRootPath } def allSources = T{ ls.rec(sourceRoot().path).map(PathRef(_)) } def classFiles = T{ compileAll(allSources()) } - def jar = T{ Jvm.createJar(T.ctx().dest, Seq(resourceRoot().path, classFiles().path)) } + def jar = T{ Jvm.createJar(Seq(resourceRoot().path, classFiles().path)) } def run(mainClsName: String) = T.command{ %%('java, "-cp", classFiles().path, mainClsName) diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/ScalaModule.scala b/scalaplugin/src/main/scala/mill/scalaplugin/ScalaModule.scala index f7cb8963..949682ca 100644 --- a/scalaplugin/src/main/scala/mill/scalaplugin/ScalaModule.scala +++ b/scalaplugin/src/main/scala/mill/scalaplugin/ScalaModule.scala @@ -3,11 +3,12 @@ package scalaplugin import ammonite.ops._ import coursier.{Cache, MavenRepository, Repository} -import mill.define.{Source, Task} +import mill.define.Task import mill.define.Task.{Module, TaskModule} import mill.eval.{PathRef, Result} import mill.modules.Jvm import mill.modules.Jvm.{createAssembly, createJar, interactiveSubprocess, subprocess} + import Lib._ trait TestScalaModule extends ScalaModule with TaskModule { override def defaultCommandName() = "test" @@ -48,7 +49,8 @@ trait TestScalaModule extends ScalaModule with TaskModule { } } } -trait ScalaModule extends Module with TaskModule{ outer => + +trait ScalaModule extends Module with TaskModule { outer => def defaultCommandName() = "run" trait Tests extends TestScalaModule{ def scalaVersion = outer.scalaVersion() @@ -197,38 +199,42 @@ trait ScalaModule extends Module with TaskModule{ outer => (runDepClasspath().filter(_.path.ext != "pom") ++ Seq(resources(), compile().classes)).map(_.path).filter(exists) } + def assembly = T{ - val outDir = T.ctx().dest/up - val n = name() - val v = version() - val jarName = s"${n}-${v}.jar" - val dest = outDir/jarName - createAssembly(dest, assemblyClasspath(), prependShellScript = prependShellScript()) + createAssembly(assemblyClasspath(), prependShellScript = prependShellScript()) } def classpath = T{ Seq(resources(), compile().classes) } def jar = T{ - val outDir = T.ctx().dest/up - val n = name() - val v = version() - val jarName = s"${n}-${v}.jar" - val dest = outDir/jarName - createJar(dest, Seq(resources(), compile().classes).map(_.path).filter(exists), mainClass()) - PathRef(dest) + createJar( + Seq(resources(), compile().classes).map(_.path).filter(exists), + mainClass() + ) } - def sourcesJar = T{ - val outDir = T.ctx().dest/up - val n = name() - val v = version() - val jarName = s"${n}-${v}-sources.jar" - val dest = outDir/jarName + def docsJar = T { + val outDir = T.ctx().dest - val inputs = Seq(sources(), resources()).map(_.path).filter(exists) + val javadocDir = outDir / 'javadoc + mkdir(javadocDir) + + val options = { + val files = ls.rec(sources().path).filter(_.isFile).map(_.toNIO.toString) + files ++ Seq("-d", javadocDir.toNIO.toString, "-usejavacp") + } + + subprocess( + "scala.tools.nsc.ScalaDoc", + compileDepClasspath().filterNot(_.path.ext == "pom").map(_.path), + options = options + ) - createJar(dest, inputs) - PathRef(dest) + createJar(Seq(javadocDir))(outDir / "javadoc.jar") + } + + def sourcesJar = T { + createJar(Seq(sources(), resources()).map(_.path).filter(exists))(T.ctx().dest / "sources.jar") } def run() = T.command{ @@ -247,28 +253,80 @@ trait ScalaModule extends Module with TaskModule{ outer => options = Seq("-usejavacp") ) } +} + +trait PublishModule extends ScalaModule { outer => + import mill.scalaplugin.publish._ + + def publishName: T[String] = basePath.last.toString + def publishVersion: T[String] = "0.0.1-SNAPSHOT" + def pomSettings: T[PomSettings] + + // publish artifact with name "mill_2.12.4" instead of "mill_2.12" + def publishWithFullScalaVersion: Boolean = false + + def artifactScalaVersion: T[String] = T { + if (publishWithFullScalaVersion) scalaVersion() + else scalaBinaryVersion() + } + + def pom = T { + val dependencies = + ivyDeps().map(Artifact.fromDep(_, scalaVersion(), scalaBinaryVersion())) + val pom = Pom(artifact(), dependencies, publishName(), pomSettings()) + + val pomPath = T.ctx().dest / s"${publishName()}_${artifactScalaVersion()}-${publishVersion()}.pom" + write.over(pomPath, pom) + PathRef(pomPath) + } + + def ivy = T { + val dependencies = + ivyDeps().map(Artifact.fromDep(_, scalaVersion(), scalaBinaryVersion())) + val ivy = Ivy(artifact(), dependencies) + val ivyPath = T.ctx().dest / "ivy.xml" + write.over(ivyPath, ivy) + PathRef(ivyPath) + } + + def artifact: T[Artifact] = T { + Artifact(pomSettings().organization, s"${publishName()}_${artifactScalaVersion()}", publishVersion()) + } + + def publishLocal(): define.Command[Unit] = T.command { + LocalPublisher.publish( + jar = jar().path, + sourcesJar = sourcesJar().path, + docsJar = docsJar().path, + pom = pom().path, + ivy = ivy().path, + artifact = artifact() + ) + } + + def sonatypeUri: String = "https://oss.sonatype.org/service/local" - def organization: T[String] = "acme" - def name: T[String] = pwd.last.toString - def version: T[String] = "0.0.1-SNAPSHOT" - - // build artifact name as "mill-2.12.4" instead of "mill-2.12" - def useFullScalaVersionForPublish: Boolean = false - - def publishLocal() = T.command { - import publish._ - val file = jar() - val scalaFull = scalaVersion() - val scalaBin = scalaBinaryVersion() - val useFullVersion = useFullScalaVersionForPublish - val deps = ivyDeps() - val dependencies = deps.map(d => Artifact.fromDep(d, scalaFull, scalaBin)) - val artScalaVersion = if (useFullVersion) scalaFull else scalaBin - val artifact = ScalaArtifact(organization(), name(), version(), artScalaVersion) - LocalPublisher.publish(file, artifact, dependencies) + def sonatypeSnapshotUri: String = "https://oss.sonatype.org/content/repositories/snapshots" + + def publish(credentials: String, gpgPassphrase: String): define.Command[Unit] = T.command { + val baseName = s"${publishName()}_${artifactScalaVersion()}-${publishVersion()}" + val artifacts = Seq( + jar().path -> s"${baseName}.jar", + sourcesJar().path -> s"${baseName}-sources.jar", + docsJar().path -> s"${baseName}-javadoc.jar", + pom().path -> s"${baseName}.pom" + ) + new SonatypePublisher( + sonatypeUri, + sonatypeSnapshotUri, + credentials, + gpgPassphrase, + T.ctx().log + ).publish(artifacts, artifact()) } } + trait SbtScalaModule extends ScalaModule { outer => def basePath: Path override def sources = T.source{ basePath / 'src / 'main / 'scala } diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/Ivy.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/Ivy.scala new file mode 100644 index 00000000..5b2276e1 --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/Ivy.scala @@ -0,0 +1,53 @@ +package mill.scalaplugin.publish + +import scala.xml.PrettyPrinter + +object Ivy { + + val head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + def apply( + artifact: Artifact, + dependencies: Seq[Dependency] + ): String = { + val xml = + <ivy-module version="2.0" xmlns:e="http://ant.apache.org/ivy/extra"> + <info + organisation={artifact.group} module={artifact.id} revision={artifact.version} status="release"> + <description/> + </info> + <configurations> + <conf name="pom" visibility="public" description=""/> + <conf extends="runtime" name="test" visibility="public" description=""/> + <conf name="provided" visibility="public" description=""/> + <conf name="optional" visibility="public" description=""/> + <conf name="compile" visibility="public" description=""/> + <conf extends="compile" name="runtime" visibility="public" description=""/> + </configurations> + + <publications> + <artifact name={artifact.id} type="pom" ext="pom" conf="pom"/> + <artifact name={artifact.id} type="jar" ext="jar" conf="compile"/> + <artifact name={artifact.id} type="src" ext="jar" conf="compile" e:classifier="sources"/> + <artifact name={artifact.id} type="doc" ext="jar" conf="compile" e:classifier="javadoc"/> + </publications> + <dependencies>{dependencies.map(renderDependency)}</dependencies> + </ivy-module> + + val pp = new PrettyPrinter(120, 4) + head + pp.format(xml).replaceAll(">", ">") + } + + private def renderDependency(dep: Dependency) = { + val scope = scopeToConf(dep.scope) + <dependency org={dep.artifact.group} name={dep.artifact.id} rev={dep.artifact.version} conf={s"$scope->default(compile)"}></dependency> + } + + private def scopeToConf(s: Scope): String = s match { + case Scope.Compile => "compile" + case Scope.Provided => "provided" + case Scope.Test => "test" + case Scope.Runtime => "runtime" + } + +} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/IvyFile.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/IvyFile.scala deleted file mode 100644 index c9f36ebe..00000000 --- a/scalaplugin/src/main/scala/mill/scalaplugin/publish/IvyFile.scala +++ /dev/null @@ -1,63 +0,0 @@ -package mill.scalaplugin.publish - -trait IvyFile { - - val head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" - - def generateIvy( - artifact: Artifact, - dependencies: Seq[Dependency], - pomSettings: PomSettings - ): String = { - val xml = ivyXml(artifact, dependencies, pomSettings) - head + xml - } - - // can't use scala-xml - // it escapes '->' inside dependency conf - def ivyXml( - artifact: Artifact, - dependencies: Seq[Dependency], - pomSettings: PomSettings - ): String = { - val deps = dependencies.map(d => { - import d.artifact._ - val scope = scopeToConf(d.scope) - s""" <dependency org="${group}" name="${id}" rev="${version}" conf="${scope}->default(compile)"> - | </dependency> """.stripMargin - }).mkString("\n") - s"""<ivy-module version="2.0" xmlns:e="http://ant.apache.org/ivy/extra"> - | <info organisation="${artifact.group}" module="${artifact.id}" revision="${artifact.version}" - | status="release"> - | <description/> - | </info> - | <configurations> - | <conf name="pom" visibility="public" description=""/> - | <conf extends="runtime" name="test" visibility="public" description=""/> - | <conf name="provided" visibility="public" description=""/> - | <conf name="optional" visibility="public" description=""/> - | <conf name="compile" visibility="public" description=""/> - | <conf extends="compile" name="runtime" visibility="public" description=""/> - | </configurations> - | - | <publications> - | <artifact name="${artifact.id}" type="pom" ext="pom" conf="pom"/> - | <artifact name="${artifact.id}" type="jar" ext="jar" conf="compile"/> - | </publications> - | <dependencies> - |${deps} - | </dependencies> - |</ivy-module> - """.stripMargin - } - - private def scopeToConf(s: Scope): String = s match { - case Scope.Compile => "compile" - case Scope.Provided => "provided" - case Scope.Test => "test" - case Scope.Runtime => "runtime" - } - -} - -object IvyFile extends IvyFile diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/JsonFormatters.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/JsonFormatters.scala new file mode 100644 index 00000000..e4aed0ea --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/JsonFormatters.scala @@ -0,0 +1,11 @@ +package mill.scalaplugin.publish + +import upickle.default.{ReadWriter => RW} + +trait JsonFormatters { + implicit lazy val artifactFormat: RW[Artifact] = upickle.default.macroRW + implicit lazy val developerFormat: RW[Developer] = upickle.default.macroRW + implicit lazy val licenseFormat: RW[License] = upickle.default.macroRW + implicit lazy val scmFormat: RW[SCM] = upickle.default.macroRW + implicit lazy val pomSettingsFormat: RW[PomSettings] = upickle.default.macroRW +} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/LocalPublisher.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/LocalPublisher.scala new file mode 100644 index 00000000..acec6249 --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/LocalPublisher.scala @@ -0,0 +1,33 @@ +package mill.scalaplugin.publish + +import ammonite.ops._ + +object LocalPublisher { + + private val root: Path = home / ".ivy2" / "local" + + def publish(jar: Path, + sourcesJar: Path, + docsJar: Path, + pom: Path, + ivy: Path, + artifact: Artifact): Unit = { + val releaseDir = root / artifact.group / artifact.id / artifact.version + writeFiles( + jar -> releaseDir / "jars" / s"${artifact.id}.jar", + sourcesJar -> releaseDir / "srcs" / s"${artifact.id}-sources.jar", + docsJar -> releaseDir / "docs" / s"${artifact.id}-javadoc.jar", + pom -> releaseDir / "poms" / s"${artifact.id}.pom", + ivy -> releaseDir / "ivys" / "ivy.xml" + ) + } + + private def writeFiles(fromTo: (Path, Path)*): Unit = { + fromTo.foreach { + case (from, to) => + mkdir(to / up) + cp.over(from, to) + } + } + +} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/PomFile.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/Pom.scala index 95f05761..fab6c624 100644 --- a/scalaplugin/src/main/scala/mill/scalaplugin/publish/PomFile.scala +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/Pom.scala @@ -1,19 +1,16 @@ package mill.scalaplugin.publish -import scala.xml.{Elem, NodeSeq} +import scala.xml.{Elem, NodeSeq, PrettyPrinter} - -trait PomFile { +object Pom { val head = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" //TODO - not only jar packaging support? - //TODO - description - def generatePom( - artifact: Artifact, - dependencies: Seq[Dependency], - pomSettings: PomSettings - ): String = { + def apply(artifact: Artifact, + dependencies: Seq[Dependency], + name: String, + pomSettings: PomSettings): String = { val xml = <project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" @@ -21,9 +18,11 @@ trait PomFile { xmlns ="http://maven.apache.org/POM/4.0.0"> <modelVersion>4.0.0</modelVersion> + <name>{name}</name> + <groupId>{artifact.group}</groupId> <artifactId>{artifact.id}</artifactId> <packaging>jar</packaging> - <description></description> + <description>{pomSettings.description}</description> <version>{artifact.version}</version> <url>{pomSettings.url}</url> @@ -42,49 +41,48 @@ trait PomFile { </dependencies> </project> - val pp = new scala.xml.PrettyPrinter(120, 4) - val data = pp.format(xml) - head + data + val pp = new PrettyPrinter(120, 4) + head + pp.format(xml) } private def renderLicense(l: License): Elem = { - import l._ <license> - <name>{name}</name> - <url>{url}</url> - <distribution>{distribution}</distribution> + <name>{l.name}</name> + <url>{l.url}</url> + <distribution>{l.distribution}</distribution> </license> } private def renderDeveloper(d: Developer): Elem = { - import d._ <developer> - <id>{id}</id> - <name>{name}</name> - <organization>{organization}</organization> - <organizationUrl>{organizationUrl}</organizationUrl> + <id>{d.id}</id> + <name>{d.name}</name> + { + d.organization.map { org => + <organization>{org}</organization> + }.getOrElse(NodeSeq.Empty) + } + { + d.organizationUrl.map { orgUrl => + <organizationUrl>{orgUrl}</organizationUrl> + }.getOrElse(NodeSeq.Empty) + } </developer> } private def renderDependency(d: Dependency): Elem = { - import d._ - import artifact._ - val scope = d.scope match { - case Scope.Compile => NodeSeq.Empty + case Scope.Compile => NodeSeq.Empty case Scope.Provided => <scope>provided</scope> - case Scope.Test => <scope>test</scope> - case Scope.Runtime => <scope>runtime</scope> + case Scope.Test => <scope>test</scope> + case Scope.Runtime => <scope>runtime</scope> } <dependency> - <groupId>{group}</groupId> - <artifactId>{id}</artifactId> - <version>{version}</version> + <groupId>{d.artifact.group}</groupId> + <artifactId>{d.artifact.id}</artifactId> + <version>{d.artifact.version}</version> {scope} </dependency> } } - -object PomFile extends PomFile - diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/Publisher.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/Publisher.scala deleted file mode 100644 index 99cd13b5..00000000 --- a/scalaplugin/src/main/scala/mill/scalaplugin/publish/Publisher.scala +++ /dev/null @@ -1,62 +0,0 @@ -package mill.scalaplugin.publish - -import java.io.File -import java.nio.file.{Files, StandardCopyOption, StandardOpenOption} -import java.security.MessageDigest - -import ammonite.ops._ -import mill.eval.PathRef - -object LocalPublisher { - - val root: Path = { - val ivy2 = { - // a bit touchy on Windows... - don't try to manually write down the URI with s"file://..." - val str = new File(sys.props("user.home") + "/.ivy2/").toString - if (str.endsWith("/")) str else str + "/" - } - Path(ivy2 + "local/") - } - - def publish( - file: PathRef, - artifact: Artifact, - dependencies: Seq[Dependency] - ): String = { - val sett = PomSettings("mill", "url", Seq.empty, SCM("", ""), Seq.empty) - val f = file.path - val pomData = PomFile.generatePom(artifact, dependencies, sett) - val ivyData = IvyFile.generateIvy(artifact, dependencies, sett) - - - val dir = root/artifact.group/artifact.id/artifact.version - - val jars = dir/"jars" - val poms = dir/"poms" - val ivys = dir/"ivys" - - val fileName = artifact match { - case j: JavaArtifact => j.name - case sa: ScalaArtifact => - val postfix = { - val arr = sa.scalaVersion.split('.') - val erased = if (arr.length > 2) arr.dropRight(1) else arr - erased.mkString(".") - } - s"${sa.name}_$postfix" - } - - Seq(dir, jars, poms, ivys).foreach(d => if (!d.toIO.exists()) mkdir(d)) - - Files.copy(f.toNIO, (dir/"jars"/s"$fileName.jar").toNIO, StandardCopyOption.REPLACE_EXISTING) - Files.write((dir/"poms"/s"$fileName.pom").toNIO, pomData.getBytes(), - StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING - ) - Files.write((dir/"ivys"/"ivy.xml").toNIO, ivyData.getBytes(), - StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING - ) - - dir.toString() - } - -} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypeHttpApi.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypeHttpApi.scala new file mode 100644 index 00000000..abf65cf6 --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypeHttpApi.scala @@ -0,0 +1,130 @@ +package mill.scalaplugin.publish + +import java.util.Base64 + +import upickle.json + +import scala.concurrent.duration._ +import scalaj.http.{BaseHttp, HttpOptions, HttpRequest, HttpResponse} + +object PatientHttp + extends BaseHttp( + options = Seq( + HttpOptions.connTimeout(5.seconds.toMillis.toInt), + HttpOptions.readTimeout(1.minute.toMillis.toInt), + HttpOptions.followRedirects(false) + ) + ) + +class SonatypeHttpApi(uri: String, credentials: String) { + + private val base64Creds = base64(credentials) + + private val commonHeaders = Seq( + "Authorization" -> s"Basic ${base64Creds}", + "Accept" -> "application/json", + "Content-Type" -> "application/json" + ) + + // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles.html + def getStagingProfileUri(groupId: String): String = { + val response = withRetry( + PatientHttp(s"${uri}/staging/profiles").headers(commonHeaders)) + + val resourceUri = + json + .read(response.body)("data") + .arr + .find(profile => profile("name").str == groupId) + .map(_("resourceURI").str.toString) + + resourceUri.getOrElse( + throw new RuntimeException( + s"Could not find staging profile for groupId: ${groupId}") + ) + } + + def getStagingRepoState(stagingRepoId: String): String = { + val response = PatientHttp(s"${uri}/staging/repository/${stagingRepoId}") + .option(HttpOptions.readTimeout(60000)) + .headers(commonHeaders) + .asString + + json.read(response.body)("type").str.toString + } + + // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_start.html + def createStagingRepo(profileUri: String, groupId: String): String = { + val response = withRetry(PatientHttp(s"${profileUri}/start") + .headers(commonHeaders) + .postData( + s"""{"data": {"description": "fresh staging profile for ${groupId}"}}""")) + + json.read(response.body)("data")("stagedRepositoryId").str.toString + } + + // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_finish.html + def closeStagingRepo(profileUri: String, repositoryId: String): Boolean = { + val response = withRetry( + PatientHttp(s"${profileUri}/finish") + .headers(commonHeaders) + .postData( + s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "closing staging repository"}}""" + )) + + response.code == 201 + } + + // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_promote.html + def promoteStagingRepo(profileUri: String, repositoryId: String): Boolean = { + val response = withRetry( + PatientHttp(s"${profileUri}/promote") + .headers(commonHeaders) + .postData( + s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "promote staging repository"}}""" + )) + + response.code == 201 + } + + // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_drop.html + def dropStagingRepo(profileUri: String, repositoryId: String): Boolean = { + val response = withRetry( + PatientHttp(s"${profileUri}/drop") + .headers(commonHeaders) + .postData( + s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "drop staging repository"}}""" + )) + + response.code == 201 + } + + private val uploadTimeout = 5.minutes.toMillis.toInt + + def upload(uri: String, data: Array[Byte]): HttpResponse[String] = { + PatientHttp(uri) + .option(HttpOptions.readTimeout(uploadTimeout)) + .method("PUT") + .headers( + "Content-Type" -> "application/binary", + "Authorization" -> s"Basic ${base64Creds}" + ) + .put(data) + .asString + } + + private def withRetry(request: HttpRequest, + retries: Int = 10): HttpResponse[String] = { + val resp = request.asString + if (resp.is5xx && retries > 0) { + Thread.sleep(500) + withRetry(request, retries - 1) + } else { + resp + } + } + + private def base64(s: String) = + new String(Base64.getEncoder.encode(s.getBytes)) + +} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypePublisher.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypePublisher.scala new file mode 100644 index 00000000..90a39745 --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/SonatypePublisher.scala @@ -0,0 +1,148 @@ +package mill.scalaplugin.publish + +import java.math.BigInteger +import java.security.MessageDigest + +import ammonite.ops._ +import mill.util.Logger + +import scalaj.http.HttpResponse + +class SonatypePublisher(uri: String, + snapshotUri: String, + credentials: String, + gpgPassphrase: String, + log: Logger) { + + private val api = new SonatypeHttpApi(uri, credentials) + + def publish(artifacts: Seq[(Path, String)], artifact: Artifact): Unit = { + val signedArtifacts = artifacts ++ artifacts.map { + case (file, name) => + poorMansSign(file, gpgPassphrase) -> s"${name}.asc" + } + + val signedArtifactsWithDigest = signedArtifacts.flatMap { + case (file, name) => + val content = read.bytes(file) + + Seq( + name -> content, + (name + ".md5") -> md5hex(content), + (name + ".sha1") -> sha1hex(content) + ) + } + + val publishPath = Seq( + artifact.group.replace(".", "/"), + artifact.id, + artifact.version + ).mkString("/") + + if (artifact.isSnapshot) + publishSnapshot(publishPath, signedArtifactsWithDigest, artifact) + else + publishRelease(publishPath, signedArtifactsWithDigest, artifact) + } + + private def publishSnapshot(publishPath: String, + payloads: Seq[(String, Array[Byte])], + artifact: Artifact): Unit = { + val baseUri: String = snapshotUri + "/" + publishPath + + val publishResults = payloads.map { + case (fileName, data) => + log.info(s"Uploading ${fileName}") + val resp = api.upload(s"${baseUri}/${fileName}", data) + resp + } + reportPublishResults(publishResults, artifact) + } + + private def publishRelease(publishPath: String, + payloads: Seq[(String, Array[Byte])], + artifact: Artifact): Unit = { + val profileUri = api.getStagingProfileUri(artifact.group) + val stagingRepoId = + api.createStagingRepo(profileUri, artifact.group) + val baseUri = + s"${uri}/staging/deployByRepositoryId/${stagingRepoId}/${publishPath}" + + val publishResults = payloads.map { + case (fileName, data) => + log.info(s"Uploading ${fileName}") + api.upload(s"${baseUri}/${fileName}", data) + } + reportPublishResults(publishResults, artifact) + + log.info("Closing staging repository") + api.closeStagingRepo(profileUri, stagingRepoId) + + log.info("Waiting for staging repository to close") + awaitRepoStatus("closed", stagingRepoId) + + log.info("Promoting staging repository") + api.promoteStagingRepo(profileUri, stagingRepoId) + + log.info("Waiting for staging repository to release") + awaitRepoStatus("released", stagingRepoId) + + log.info("Dropping staging repository") + api.dropStagingRepo(profileUri, stagingRepoId) + + log.info(s"Published ${artifact.id} successfully") + } + + private def reportPublishResults(publishResults: Seq[HttpResponse[String]], + artifact: Artifact) = { + if (publishResults.forall(_.is2xx)) { + log.info(s"Published ${artifact.id} to Sonatype") + } else { + val errors = publishResults.filterNot(_.is2xx).map { response => + s"Code: ${response.code}, message: ${response.body}" + } + throw new RuntimeException( + s"Failed to publish ${artifact.id} to Sonatype. Errors: \n${errors.mkString("\n")}" + ) + } + } + + private def awaitRepoStatus(status: String, + stagingRepoId: String, + attempts: Int = 20): Unit = { + def isRightStatus = + api.getStagingRepoState(stagingRepoId).equalsIgnoreCase(status) + var attemptsLeft = attempts + + while (attemptsLeft > 0 && !isRightStatus) { + Thread.sleep(3000) + attemptsLeft -= 1 + if (attemptsLeft == 0) { + throw new RuntimeException( + s"Couldn't wait for staging repository to be ${status}. Failing") + } + } + } + + // http://central.sonatype.org/pages/working-with-pgp-signatures.html#signing-a-file + private def poorMansSign(file: Path, passphrase: String): Path = { + val fileName = file.toString + import ammonite.ops.ImplicitWd._ + %("gpg", "--yes", "-a", "-b", "--passphrase", passphrase, fileName) + Path(fileName + ".asc") + } + + private def md5hex(bytes: Array[Byte]): Array[Byte] = + hexArray(md5.digest(bytes)).getBytes + + private def sha1hex(bytes: Array[Byte]): Array[Byte] = + hexArray(sha1.digest(bytes)).getBytes + + private def md5 = MessageDigest.getInstance("md5") + + private def sha1 = MessageDigest.getInstance("sha1") + + private def hexArray(arr: Array[Byte]) = + String.format("%0" + (arr.length << 1) + "x", new BigInteger(1, arr)) + +} diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/package.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/package.scala new file mode 100644 index 00000000..1b405b90 --- /dev/null +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/package.scala @@ -0,0 +1,3 @@ +package mill.scalaplugin + +package object publish extends JsonFormatters diff --git a/scalaplugin/src/main/scala/mill/scalaplugin/publish/settings.scala b/scalaplugin/src/main/scala/mill/scalaplugin/publish/settings.scala index ece7fbd1..e13825ab 100644 --- a/scalaplugin/src/main/scala/mill/scalaplugin/publish/settings.scala +++ b/scalaplugin/src/main/scala/mill/scalaplugin/publish/settings.scala @@ -1,53 +1,33 @@ package mill.scalaplugin.publish import mill.scalaplugin.Dep -import mill.util.JsonFormatters -trait Artifact extends Serializable with Product { - def group: String - def name: String - def id: String - def version: String +case class Artifact(group: String, id: String, version: String) { + def isSnapshot: Boolean = version.endsWith("-SNAPSHOT") } object Artifact { def fromDep(dep: Dep, scalaFull: String, scalaBin: String): Dependency = { dep match { - case d: Dep.Java => - import d.dep._ - val art = JavaArtifact(module.organization, module.name, version) - Dependency(art, Scope.Compile) - case d: Dep.Scala => - import d.dep._ - val art = ScalaArtifact(module.organization, module.name, version, scalaBin) - Dependency(art, Scope.Compile) - case d: Dep.Point => - import d.dep._ - val art = ScalaArtifact(module.organization, module.name, version, scalaFull) - Dependency(art, Scope.Compile) + case Dep.Java(dep) => + Dependency( + Artifact(dep.module.organization, dep.module.name, dep.version), + Scope.Compile) + case Dep.Scala(dep) => + Dependency(Artifact(dep.module.organization, + s"${dep.module.name}_${scalaBin}", + dep.version), + Scope.Compile) + case Dep.Point(dep) => + Dependency(Artifact(dep.module.organization, + s"${dep.module.name}_${scalaFull}", + dep.version), + Scope.Compile) } } } -case class JavaArtifact( - group: String, - name: String, - version: String -) extends Artifact { - override def id: String = name -} - -case class ScalaArtifact( - group: String, - name: String, - version: String, - scalaVersion: String, -) extends Artifact { - - override def id: String = s"${name}_$scalaVersion" -} - sealed trait Scope object Scope { case object Compile extends Scope @@ -57,34 +37,34 @@ object Scope { } case class Dependency( - artifact: Artifact, - scope: Scope + artifact: Artifact, + scope: Scope ) case class License( - name: String, - url: String, - distribution: String = "repo" + name: String, + url: String, + distribution: String = "repo" ) case class SCM( - url: String, - connection: String + url: String, + connection: String ) case class Developer( - id: String, - name: String, - url: String, - organization: String, - organizationUrl: String + id: String, + name: String, + url: String, + organization: Option[String] = None, + organizationUrl: Option[String] = None ) case class PomSettings( - organization: String, - url: String, - licenses: Seq[License], - scm: SCM, - developers: Seq[Developer] + description: String, + organization: String, + url: String, + licenses: Seq[License], + scm: SCM, + developers: Seq[Developer] ) - diff --git a/scalaplugin/src/test/scala/mill/scalaplugin/AcyclicTests.scala b/scalaplugin/src/test/scala/mill/scalaplugin/AcyclicTests.scala index 18f20852..84ff5407 100644 --- a/scalaplugin/src/test/scala/mill/scalaplugin/AcyclicTests.scala +++ b/scalaplugin/src/test/scala/mill/scalaplugin/AcyclicTests.scala @@ -2,19 +2,36 @@ package mill.scalaplugin import ammonite.ops.ImplicitWd._ import ammonite.ops._ -import mill.define.{Cross,Task} +import mill.define.{Cross, Task} import mill.discover.Discovered import mill.eval.Result +import mill.scalaplugin.publish._ import utest._ import mill.util.JsonFormatters._ object AcyclicBuild{ val acyclic = for(crossVersion <- Cross("2.10.6", "2.11.8", "2.12.3", "2.12.4")) - yield new SbtScalaModule{outer => + yield new SbtScalaModule with PublishModule {outer => def basePath = AcyclicTests.workspacePath - override def organization = "com.lihaoyi" - override def name = "acyclic" - override def version = "0.1.7" + def publishName = "acyclic" + def publishVersion = "0.1.7" + + def pomSettings = PomSettings( + description = publishName(), + organization = "com.lihaoyi", + url = "https://github.com/lihaoyi/acyclic", + licenses = Seq( + License("MIT license", "http://www.opensource.org/licenses/mit-license.php") + ), + scm = SCM( + "git://github.com/lihaoyi/acyclic.git", + "scm:git://github.com/lihaoyi/acyclic.git" + ), + developers = Seq( + Developer("lihaoyi", "Li Haoyi","https://github.com/lihaoyi") + ) + ) + def scalaVersion = crossVersion def ivyDeps = Seq( Dep.Java("org.scala-lang", "scala-compiler", scalaVersion()) diff --git a/scalaplugin/src/test/scala/mill/scalaplugin/HelloWorldTests.scala b/scalaplugin/src/test/scala/mill/scalaplugin/HelloWorldTests.scala index 239f6871..8d535a35 100644 --- a/scalaplugin/src/test/scala/mill/scalaplugin/HelloWorldTests.scala +++ b/scalaplugin/src/test/scala/mill/scalaplugin/HelloWorldTests.scala @@ -7,6 +7,7 @@ import mill.define.{Target, Task} import mill.discover.Discovered import mill.discover.Mirror.LabelledTarget import mill.eval.Result +import mill.scalaplugin.publish._ import sbt.internal.inc.CompileFailed import utest._ @@ -29,6 +30,26 @@ object HelloWorldFatalWarnings extends HelloWorldModule { def scalacOptions = T(Seq("-Ywarn-unused", "-Xfatal-warnings")) } +object HelloWorldWithPublish extends HelloWorldModule with PublishModule { + def publishName = "hello-world" + def publishVersion = "0.0.1" + + def pomSettings = PomSettings( + organization = "com.lihaoyi", + description = "hello world ready for real world publishing", + url = "https://github.com/lihaoyi/hello-world-publish", + licenses = Seq( + License("Apache License, Version 2.0", + "http://www.apache.org/licenses/LICENSE-2.0")), + scm = SCM( + "https://github.com/lihaoyi/hello-world-publish", + "scm:git:https://github.com/lihaoyi/hello-world-publish" + ), + developers = + Seq(Developer("lihaoyi", "Li Haoyi", "https://github.com/lihaoyi")) + ) +} + object HelloWorldTests extends TestSuite { val srcPath = pwd / 'scalaplugin / 'src / 'test / 'resource / "hello-world" @@ -98,7 +119,8 @@ object HelloWorldTests extends TestSuite { val outPath = result.classes.path val analysisFile = result.analysisFile val outputFiles = ls.rec(outPath) - val expectedClassfiles = compileClassfiles(outputPath / 'compile / 'classes) + val expectedClassfiles = + compileClassfiles(outputPath / 'compile / 'classes) assert( outPath == outputPath / 'compile / 'classes, exists(analysisFile), |