From 1e6baf44761267a0588561a59402f012a2d51e16 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 28 Dec 2017 00:16:03 -0800 Subject: Move all the source generation logic into a `shared.sc` file to avoid duplication DRY it up internally Move the Bridge downloading logic into `shared.sc` as well, and swap the subprocesses for in-memory processing using scalaj-http and ZipInputStream --- build.sbt | 147 ++++++++------------------------------------------------------ build.sc | 129 +++++++----------------------------------------------- shared.sc | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 242 deletions(-) create mode 100644 shared.sc diff --git a/build.sbt b/build.sbt index 0907cddd..a15acc5a 100644 --- a/build.sbt +++ b/build.sbt @@ -12,124 +12,28 @@ val sharedSettings = Seq( resolvers += Resolver.sonatypeRepo("releases"), scalacOptions += "-P:acyclic:force", autoCompilerPlugins := true, - addCompilerPlugin("com.lihaoyi" %% "acyclic" % "0.1.7"), - libraryDependencies += "com.lihaoyi" % "ammonite" % "1.0.3-20-75e58ac" % "test" cross CrossVersion.full, - - sourceGenerators in Test += Def.task { - val file = (sourceManaged in Test).value / "amm.scala" - IO.write(file, """object amm extends App { ammonite.Main().run() }""") - Seq(file) - }.taskValue + addCompilerPlugin("com.lihaoyi" %% "acyclic" % "0.1.7") ) val coreSettings = Seq( sourceGenerators in Compile += Def.task { - object CodeGenerator { - private def generateLetters(n: Int) = { - val base = 'A'.toInt - (0 until n).map(i => (i + base).toChar) - } - - def generateApplyer(dir: File) = { - def generate(n: Int) = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val zipArgs = lowercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") - - val body = s"mapCtx(zip($zipArgs)) { case (($zipArgs), z) => cb($zipArgs, z) }" - val zipmap = s"def zipMap[$typeArgs, Res]($parameters)(cb: ($typeArgs, Ctx) => Z[Res]) = $body" - val zip = s"def zip[$typeArgs]($parameters): TT[($typeArgs)]" - - if (n < 22) List(zipmap, zip).mkString(System.lineSeparator) else zip - } - val output = List( - "package mill.define", - "import scala.language.higherKinds", - "trait ApplyerGenerated[TT[_], Z[_], Ctx] {", - "def mapCtx[A, B](a: TT[A])(f: (A, Ctx) => Z[B]): TT[B]", - (2 to 22).map(generate).mkString(System.lineSeparator), - "}").mkString(System.lineSeparator) - - val file = dir / "ApplicativeGenerated.scala" - IO.write(file, output) - file - } - - def generateTarget(dir: File) = { - def generate(n: Int) = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val args = lowercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") - val body = uppercases.zipWithIndex.map { case (t, i) => s"args[$t]($i)" }.mkString(", ") - - s"def zip[$typeArgs]($parameters) = makeT[($typeArgs)](Seq($args), (args: Ctx) => ($body))" - } - - val output = List( - "package mill.define", - "import scala.language.higherKinds", - "import mill.eval.Result", - "import mill.util.Ctx", - "trait TargetGenerated {", - "type TT[+X]", - "def makeT[X](inputs: Seq[TT[_]], evaluate: Ctx => Result[X]): TT[X]", - (3 to 22).map(generate).mkString(System.lineSeparator), - "}").mkString(System.lineSeparator) - - val file = dir / "TaskGenerated.scala" - IO.write(file, output) - file - } - - def generateSources(dir: File) = { - Seq(generateApplyer(dir), generateTarget(dir)) - } - } + import sys.process._ val dir = (sourceManaged in Compile).value - CodeGenerator.generateSources(dir) + if (!dir.exists()) { + IO.createDirectory(dir) + Seq("amm", "shared.sc", "generateSources", dir.toString).! + } + IO.listFiles(dir).toSeq }.taskValue, sourceGenerators in Test += Def.task { - object CodeGenerator { - private def generateLetters(n: Int) = { - val base = 'A'.toInt - (0 until n).map(i => (i + base).toChar) - } - - def generateApplicativeTest(dir: File) = { - def generate(n: Int): String = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: Option[$upper]" }.mkString(", ") - val result = lowercases.mkString(", ") - val forArgs = lowercases.map(i => s"$i <- $i").mkString("; ") - s"def zip[$typeArgs]($parameters) = { for ($forArgs) yield ($result) }" - } - - val output = List( - "package mill.define", - "trait OptGenerated {", - (2 to 22).map(generate).mkString(System.lineSeparator), - "}" - ).mkString(System.lineSeparator) - - val file = dir / "ApplicativeTestsGenerated.scala" - IO.write(file, output) - file - } - - def generateTests(dir: File) = { - Seq(generateApplicativeTest(dir)) - } - } - + import sys.process._ val dir = (sourceManaged in Test).value - CodeGenerator.generateTests(dir) + if (!dir.exists()) { + IO.createDirectory(dir) + Seq("amm", "shared.sc", "generateTests", dir.toString).! + } + IO.listFiles(dir).toSeq }.taskValue ) @@ -158,27 +62,12 @@ def bridge(bridgeVersion: String) = Project( ), (sourceGenerators in Compile) += Def.task{ import sys.process._ - import collection.JavaConverters._ - val v = scalaBinaryVersion.value - val url = - s"http://repo1.maven.org/maven2/org/scala-sbt/compiler-bridge_$v/1.0.5/compiler-bridge_$v-1.0.5-sources.jar" - val curlDest = java.nio.file.Paths.get(target.value.toString, "sources") - - if (!java.nio.file.Files.exists(curlDest)) { - Seq("rm", "-rf", curlDest.toString).! - java.nio.file.Files.createDirectories(curlDest) - - Seq("curl", "-L", "-o", curlDest.resolve("bridge.jar").toString, url).! - Seq("unzip", curlDest.resolve("bridge.jar").toString, "-d", curlDest.toString).! + val dir = (sourceManaged in Compile).value + if (!dir.exists()) { + IO.createDirectory(dir) + Seq("amm", "shared.sc", "downloadBridgeSource", dir.toString, bridgeVersion).! } - val sources = java.nio.file.Files.walk(curlDest) - .iterator - .asScala - .filter(_.toString.endsWith(".scala")) - .map(_.toFile) - .toSeq - - sources + (dir ** "*.scala").get }.taskValue ) ) diff --git a/build.sc b/build.sc index 07909ebf..9ee3001d 100755 --- a/build.sc +++ b/build.sc @@ -1,3 +1,4 @@ +import $file.shared import ammonite.ops._ import coursier.maven.MavenRepository import mill._ @@ -39,7 +40,8 @@ trait MillModule extends SbtScalaModule{ outer => def testArgs = T{ Seq.empty[String] } - object test extends this.Tests{ + val test = new Tests + class Tests extends super.Tests{ def defaultCommandName() = "forkTest" def forkArgs = T{ testArgs() } def projectDeps = @@ -69,107 +71,21 @@ object Core extends MillModule { def basePath = pwd / 'core - object CodeGenerator { - private def generateLetters(n: Int) = { - val base = 'A'.toInt - (0 until n).map(i => (i + base).toChar) - } - - private def write(dir: String, filename: String, s: String) = { - import java.io.{BufferedWriter, FileWriter} - - val path = java.nio.file.Paths.get(dir, filename) - val w = new BufferedWriter(new FileWriter(path.toFile)) - w.write(s) - w.close() - } - - def generateApplyer(dir: String) = { - def generate(n: Int) = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val zipArgs = lowercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") - - val body = s"mapCtx(zip($zipArgs)) { case (($zipArgs), z) => cb($zipArgs, z) }" - val zipmap = s"def zipMap[$typeArgs, Res]($parameters)(cb: ($typeArgs, Ctx) => Z[Res]) = $body" - val zip = s"def zip[$typeArgs]($parameters): TT[($typeArgs)]" - - if (n < 22) List(zipmap, zip).mkString(System.lineSeparator) else zip - } - val output = List( - "package mill.define", - "import scala.language.higherKinds", - "trait ApplyerGenerated[TT[_], Z[_], Ctx] {", - "def mapCtx[A, B](a: TT[A])(f: (A, Ctx) => Z[B]): TT[B]", - (2 to 22).map(generate).mkString(System.lineSeparator), - "}").mkString(System.lineSeparator) - - write(dir, "ApplicativeGenerated.scala", output) - } - - def generateTarget(dir: String) = { - def generate(n: Int) = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val args = lowercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") - val body = uppercases.zipWithIndex.map { case (t, i) => s"args[$t]($i)" }.mkString(", ") - - s"def zip[$typeArgs]($parameters) = makeT[($typeArgs)](Seq($args), (args: Ctx) => ($body))" - } - - val output = List( - "package mill.define", - "import scala.language.higherKinds", - "import mill.eval.Result", - "import mill.util.Ctx", - "trait TargetGenerated {", - "type TT[+X]", - "def makeT[X](inputs: Seq[TT[_]], evaluate: Ctx => Result[X]): TT[X]", - (3 to 22).map(generate).mkString(System.lineSeparator), - "}").mkString(System.lineSeparator) - write(dir, "TaskGenerated.scala", output) - } - - def generateApplicativeTest(dir: String) = { - def generate(n: Int): String = { - val uppercases = generateLetters(n) - val lowercases = uppercases.map(Character.toLowerCase) - val typeArgs = uppercases.mkString(", ") - val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: Option[$upper]" }.mkString(", ") - val result = lowercases.mkString(", ") - val forArgs = lowercases.map(i => s"$i <- $i").mkString("; ") - s"def zip[$typeArgs]($parameters) = { for ($forArgs) yield ($result) }" - } - - val output = List( - "package mill.define", - "trait OptGenerated {", - (2 to 22).map(generate).mkString(System.lineSeparator), - "}" - ).mkString(System.lineSeparator) - - write(dir, "ApplicativeTestsGenerated.scala", output) - } + def generatedSources = T{ + mkdir(T.ctx().dest) + shared.generateSources(T.ctx().dest) + PathRef(T.ctx().dest) + } - def generateSources(p: Path) = { - val dir = p.toString() - generateApplyer(dir) - generateTarget(dir) - } + def allSources = super.allSources() ++ Seq(generatedSources()) + val test = new Tests{ + def generatedSources = T{ + mkdir(T.ctx().dest) + shared.generateTests(T.ctx().dest) + PathRef(T.ctx().dest) - def generateTests(p: Path) = { - generateApplicativeTest(p.toString()) } - } - - def sources = { - CodeGenerator.generateSources(this.basePath / 'src / 'main / 'scala / 'mill / 'define) - CodeGenerator.generateTests(pwd / 'core / 'src / 'test / 'scala / 'mill / 'define) - super.sources + def allSources = super.allSources() ++ Seq(generatedSources()) } val cross = @@ -202,20 +118,7 @@ val bridges = for(crossVersion <- mill.define.Cross(bridgeVersions:_*)) yield ne path } def allSources = T{ - - val v = crossVersion.split('.').dropRight(1).mkString(".") - val url = - s"http://repo1.maven.org/maven2/org/scala-sbt/compiler-bridge_$v/1.0.5/compiler-bridge_$v-1.0.5-sources.jar" - val curlDest = T.ctx().dest - implicit val pwd = curlDest - mkdir(curlDest) - rm(curlDest/"bridge.jar") - - %("curl", "-L", "-o", curlDest / "bridge.jar", url) - %("unzip", curlDest / "bridge.jar" , "-d", curlDest / 'classes) - - - Seq(PathRef(curlDest / 'classes)) + Seq(PathRef(shared.downloadBridgeSource(T.ctx().dest, crossVersion))) } def ivyDeps = Seq( Dep.Java("org.scala-lang", "scala-compiler", crossVersion), diff --git a/shared.sc b/shared.sc new file mode 100644 index 00000000..17b82595 --- /dev/null +++ b/shared.sc @@ -0,0 +1,125 @@ +/** + * Utility code that is shared between our SBT build and our Mill build. SBT + * calls this by shelling out to Ammonite in a subprocess, while Mill loads it + * via import $file + */ + +import ammonite.ops.{write, Path, mkdir, RelPath, up} + +def argNames(n: Int) = { + val uppercases = (0 until n).map("T" + _) + val lowercases = uppercases.map(_.toLowerCase) + val typeArgs = uppercases.mkString(", ") + val zipArgs = lowercases.mkString(", ") + (lowercases, uppercases, typeArgs, zipArgs) +} +def generateApplyer(dir: Path) = { + def generate(n: Int) = { + val (lowercases, uppercases, typeArgs, zipArgs) = argNames(n) + val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") + + val body = s"mapCtx(zip($zipArgs)) { case (($zipArgs), z) => cb($zipArgs, z) }" + val zipmap = s"def zipMap[$typeArgs, Res]($parameters)(cb: ($typeArgs, Ctx) => Z[Res]) = $body" + val zip = s"def zip[$typeArgs]($parameters): TT[($typeArgs)]" + + if (n < 22) List(zipmap, zip).mkString("\n") else zip + } + write( + dir / "ApplicativeGenerated.scala", + s"""package mill.define + |import scala.language.higherKinds + |trait ApplyerGenerated[TT[_], Z[_], Ctx] { + | def mapCtx[A, B](a: TT[A])(f: (A, Ctx) => Z[B]): TT[B] + | ${(2 to 22).map(generate).mkString("\n")} + |}""".stripMargin + ) +} + +def generateTarget(dir: Path) = { + def generate(n: Int) = { + val (lowercases, uppercases, typeArgs, zipArgs) = argNames(n) + val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: TT[$upper]" }.mkString(", ") + val body = uppercases.zipWithIndex.map { case (t, i) => s"args[$t]($i)" }.mkString(", ") + + s"def zip[$typeArgs]($parameters) = makeT[($typeArgs)](Seq($zipArgs), (args: mill.util.Ctx) => ($body))" + } + + write( + dir / "TaskGenerated.scala", + s"""package mill.define + |import scala.language.higherKinds + |trait TargetGenerated { + | type TT[+X] + | def makeT[X](inputs: Seq[TT[_]], evaluate: mill.util.Ctx => mill.eval.Result[X]): TT[X] + | ${(3 to 22).map(generate).mkString("\n")} + |}""".stripMargin + ) +} + +def generateApplicativeTest(dir: Path) = { + def generate(n: Int): String = { + val (lowercases, uppercases, typeArgs, zipArgs) = argNames(n) + val parameters = lowercases.zip(uppercases).map { case (lower, upper) => s"$lower: Option[$upper]" }.mkString(", ") + val forArgs = lowercases.map(i => s"$i <- $i").mkString("; ") + s"def zip[$typeArgs]($parameters) = { for ($forArgs) yield ($zipArgs) }" + } + + write( + dir / "ApplicativeTestsGenerated.scala", + s"""package mill.define + |trait OptGenerated { + | ${(2 to 22).map(generate).mkString("\n")} + |} + """.stripMargin + ) +} + +@main +def generateSources(p: Path) = { + generateApplyer(p) + generateTarget(p) +} + +@main +def generateTests(p: Path) = { + generateApplicativeTest(p) +} + +@main +def downloadBridgeSource(curlDest: Path, crossVersion: String) = { + val v = crossVersion.split('.').dropRight(1).mkString(".") + val url = + s"http://repo1.maven.org/maven2/org/scala-sbt/compiler-bridge_$v/1.0.5/compiler-bridge_$v-1.0.5-sources.jar" + + mkdir(curlDest) + + val bytes = scalaj.http.Http.apply(url).asBytes.body + val byteStream = new java.io.ByteArrayInputStream(bytes) + val zipStream = new java.util.zip.ZipInputStream(byteStream) + while({ + zipStream.getNextEntry match{ + case null => false + case entry => + if (!entry.isDirectory) { + val dest = curlDest / RelPath(entry.getName) + mkdir(dest / up) + val fileOut = new java.io.FileOutputStream(dest.toString) + val buffer = new Array[Byte](4096) + while ( { + zipStream.read(buffer) match { + case -1 => false + case n => + fileOut.write(buffer, 0, n) + true + } + }) () + fileOut.close() + } + zipStream.closeEntry() + true + } + })() + + + curlDest +} \ No newline at end of file -- cgit v1.2.3