diff options
author | Andrew Richards <richards.aj@gmail.com> | 2018-07-18 08:50:09 +0100 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2018-07-18 15:50:09 +0800 |
commit | 1829391c0de0efcb96b1187fe35a0e9127e00d29 (patch) | |
tree | 4216bde1516e75fccbc8a8c6a0d1eebb5b929364 /scalanativelib/src | |
parent | 2d779b8ccd0a2f29a7d14dc193b4ad093142b521 (diff) | |
download | mill-1829391c0de0efcb96b1187fe35a0e9127e00d29.tar.gz mill-1829391c0de0efcb96b1187fe35a0e9127e00d29.tar.bz2 mill-1829391c0de0efcb96b1187fe35a0e9127e00d29.zip |
WIP: Scala native (#206)
* add scala-native PR#1143 as submodule
* first pass at integrating scala-native build into mill
including worker/bridge
* add the native libraries to the compile and run classpath
* sssshhh don't be so noisy
* update scala-native to latest build WIP
* update mill to latest scala-native build-api code
* add test interface from scala-native
this code is not published ornot published at the correct scala version so copy it in for now
* implement tests for scala-native
very messy at the moment
also correct bridge version as much as possible with out a scala-native release
* update to scala-native/master
scala-native #1143 now merged
* Remove scala-native submodule
* updates for scala-native 0.3.7 release
* fixes after rebase
* make test framework agnostic and tidy dependencies
* add robust method of getting JVM classpath for running tests
support for multiple test frameworks
tidy up
* rebase fixes for 0.2.0
* add SbtNativeModule and tidy
* rebase fixes
* fix building of compile / run Classpath (via transitiveIvyDeps)
better method of loading JVM test frameworks
* add tests for build, run, utest, scalatest
* move native tests into it own trait which can be extended/overidden
* change release mode to a sealed trait instead of boolean
* add logLevel to ScalaNativeModule and plumb in
propagate release and log levels to test projects
* use test-runner from scala-native instead of including project source
add ability easily compile against scala-native snapshots
* add some docs
* update to 0.3.8
Diffstat (limited to 'scalanativelib/src')
-rw-r--r-- | scalanativelib/src/mill/scalanativelib/ScalaNativeBridge.scala | 77 | ||||
-rw-r--r-- | scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala | 312 |
2 files changed, 389 insertions, 0 deletions
diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeBridge.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeBridge.scala new file mode 100644 index 00000000..a7777fad --- /dev/null +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeBridge.scala @@ -0,0 +1,77 @@ +package mill.scalanativelib + +import java.io.File +import java.net.URLClassLoader + +import ammonite.ops.Path +import mill.define.{Discover, Worker} +import mill.{Agg, T} +import sbt.testing.Framework + + +class ScalaNativeWorker { + private var scalaInstanceCache = Option.empty[(Long, ScalaNativeBridge)] + + def bridge(toolsClasspath: Agg[Path]): ScalaNativeBridge = { + val classloaderSig = toolsClasspath.map(p => p.toString().hashCode + p.mtime.toMillis).sum + scalaInstanceCache match { + case Some((sig, bridge)) if sig == classloaderSig => bridge + case _ => + val cl = new URLClassLoader( + toolsClasspath.map(_.toIO.toURI.toURL).toArray, + getClass.getClassLoader + ) + try { + val bridge = cl + .loadClass("mill.scalanativelib.bridge.ScalaNativeBridge") + .getDeclaredConstructor() + .newInstance() + .asInstanceOf[ScalaNativeBridge] + scalaInstanceCache = Some((classloaderSig, bridge)) + bridge + } + catch { + case e: Exception => + e.printStackTrace() + throw e + } + } + } +} + + +// result wrapper to preserve some type safety +case class NativeConfig(config: Any) + +trait ScalaNativeBridge { + def discoverClang: Path + def discoverClangPP: Path + def discoverTarget(clang: Path, workDir: Path): String + def discoverCompileOptions: Seq[String] + def discoverLinkingOptions: Seq[String] + + def config(nativeLibJar: Path, + mainClass: String, + classpath: Seq[Path], + nativeWorkdir: Path, + nativeClang: Path, + nativeClangPP: Path, + nativeTarget: String, + nativeCompileOptions: Seq[String], + nativeLinkingOptions: Seq[String], + nativeGC: String, + nativeLinkStubs: Boolean, + releaseMode: ReleaseMode, + logLevel: NativeLogLevel): NativeConfig + + def defaultGarbageCollector: String + def nativeLink(nativeConfig: NativeConfig, outPath: Path): Path + + def newScalaNativeFrameWork(framework: Framework, id: Int, testBinary: File, + logLevel: NativeLogLevel, envVars: Map[String, String]): Framework +} + +object ScalaNativeBridge extends mill.define.ExternalModule { + def scalaNativeBridge: Worker[ScalaNativeWorker] = T.worker { new ScalaNativeWorker() } + lazy val millDiscover = Discover[this.type] +} diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala new file mode 100644 index 00000000..d4d2e050 --- /dev/null +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -0,0 +1,312 @@ +package mill +package scalanativelib + +import java.net.URLClassLoader + +import ammonite.ops.Path +import coursier.Cache +import coursier.maven.MavenRepository +import mill.define.{Target, Task} +import mill.eval.Result +import mill.modules.Jvm +import mill.scalalib.{Dep, DepSyntax, Lib, SbtModule, ScalaModule, TestModule, TestRunner} +import mill.util.Loose.Agg +import sbt.testing.{AnnotatedFingerprint, SubclassFingerprint} +import sbt.testing.Fingerprint +import upickle.default.{ReadWriter => RW, macroRW} + + +sealed abstract class NativeLogLevel(val level: Int) extends Ordered[NativeLogLevel] { + def compare(that: NativeLogLevel) = this.level - that.level +} + +object NativeLogLevel { + case object Error extends NativeLogLevel(200) + case object Warn extends NativeLogLevel(300) + case object Info extends NativeLogLevel(400) + case object Debug extends NativeLogLevel(500) + case object Trace extends NativeLogLevel(600) + + implicit def rw: RW[NativeLogLevel] = macroRW +} + +sealed abstract class ReleaseMode(val name: String) + +object ReleaseMode { + case object Debug extends ReleaseMode("debug") + case object Release extends ReleaseMode("release") + + implicit def rw: RW[ReleaseMode] = macroRW +} + + +trait ScalaNativeModule extends ScalaModule { outer => + def scalaNativeVersion: T[String] + override def platformSuffix = s"_native${scalaNativeBinaryVersion()}" + override def artifactSuffix: T[String] = s"${platformSuffix()}_${artifactScalaVersion()}" + + trait Tests extends TestScalaNativeModule { + override def scalaWorker = outer.scalaWorker + override def scalaVersion = outer.scalaVersion() + override def scalaNativeVersion = outer.scalaNativeVersion() + override def releaseMode = outer.releaseMode() + override def logLevel = outer.logLevel() + override def moduleDeps = Seq(outer) + } + + def scalaNativeBinaryVersion = T{ scalaNativeVersion().split('.').take(2).mkString(".") } + + // This allows compilation and testing versus SNAPSHOT versions of scala-native + def scalaNativeToolsVersion = T{ + if (scalaNativeVersion().endsWith("-SNAPSHOT")) + scalaNativeVersion() + else + scalaNativeBinaryVersion() + } + + def bridge = T.task{ ScalaNativeBridge.scalaNativeBridge().bridge(bridgeFullClassPath()) } + + def scalaNativeBridgeClasspath = T { + val snBridgeKey = "MILL_SCALANATIVE_BRIDGE_" + scalaNativeBinaryVersion().replace('.', '_').replace('-', '_') + val snBridgePath = sys.props(snBridgeKey) + if (snBridgePath != null) + Result.Success(Agg(snBridgePath.split(',').map(p => PathRef(Path(p), quick = true)): _*)) + else + Lib.resolveDependencies( + Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")), + Lib.depToDependency(_, "2.12.4", ""), + Seq(ivy"com.lihaoyi::mill-scalanativelib-scalanativebridges-${scalaNativeBinaryVersion()}:${sys.props("MILL_VERSION")}") + ) + } + + def toolsIvyDeps = T{ + Seq( + ivy"org.scala-native:tools_2.12:${scalaNativeVersion()}", + ivy"org.scala-native:util_2.12:${scalaNativeVersion()}", + ivy"org.scala-native:nir_2.12:${scalaNativeVersion()}" + ) + } + + override def transitiveIvyDeps: T[Agg[Dep]] = T{ + ivyDeps() ++ nativeIvyDeps() ++ Task.traverse(moduleDeps)(_.transitiveIvyDeps)().flatten + } + + def nativeLibIvy = T{ ivy"org.scala-native::nativelib_native${scalaNativeToolsVersion()}:${scalaNativeVersion()}" } + + def nativeIvyDeps = T{ + Seq(nativeLibIvy()) ++ + Seq( + ivy"org.scala-native::javalib_native${scalaNativeToolsVersion()}:${scalaNativeVersion()}", + ivy"org.scala-native::auxlib_native${scalaNativeToolsVersion()}:${scalaNativeVersion()}", + ivy"org.scala-native::scalalib_native${scalaNativeToolsVersion()}:${scalaNativeVersion()}" + ) + } + + def bridgeFullClassPath = T { + Lib.resolveDependencies( + Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")), + Lib.depToDependency(_, scalaVersion(), platformSuffix()), + toolsIvyDeps() + ).map(t => (scalaNativeBridgeClasspath().toSeq ++ t.toSeq).map(_.path)) + } + + override def scalacPluginIvyDeps = super.scalacPluginIvyDeps() ++ + Agg(ivy"org.scala-native:nscplugin_${scalaVersion()}:${scalaNativeVersion()}") + + def logLevel: Target[NativeLogLevel] = T{ NativeLogLevel.Info } + + def releaseMode: Target[ReleaseMode] = T { ReleaseMode.Debug } + + def nativeWorkdir = T{ T.ctx().dest } + + // Location of the clang compiler + def nativeClang = T{ bridge().discoverClang } + + // Location of the clang++ compiler + def nativeClangPP = T{ bridge().discoverClangPP } + + // GC choice, either "none", "boehm" or "immix" + def nativeGC = T{ + Option(System.getenv.get("SCALANATIVE_GC")) + .getOrElse(bridge().defaultGarbageCollector) + } + + def nativeTarget = T{ bridge().discoverTarget(nativeClang(), nativeWorkdir()) } + + // Options that are passed to clang during compilation + def nativeCompileOptions = T{ bridge().discoverCompileOptions } + + // Options that are passed to clang during linking + def nativeLinkingOptions = T{ bridge().discoverLinkingOptions } + + // Whether to link `@stub` methods, or ignore them + def nativeLinkStubs = T { false } + + + def nativeLibJar = T{ + resolveDeps(T.task{Agg(nativeLibIvy())})() + .filter{p => p.toString.contains("scala-native") && p.toString.contains("nativelib")} + .toList + .head + } + + def nativeConfig = T.task { + val classpath = runClasspath().map(_.path).filter(_.toIO.exists).toList + + bridge().config( + nativeLibJar().path, + finalMainClass(), + classpath, + nativeWorkdir(), + nativeClang(), + nativeClangPP(), + nativeTarget(), + nativeCompileOptions(), + nativeLinkingOptions(), + nativeGC(), + nativeLinkStubs(), + releaseMode(), + logLevel()) + } + + // Generates native binary + def nativeLink = T{ bridge().nativeLink(nativeConfig(), (T.ctx().dest / 'out)) } + + // Runs the native binary + override def run(args: String*) = T.command{ + Jvm.baseInteractiveSubprocess( + Vector(nativeLink().toString) ++ args, + forkEnv(), + workingDir = ammonite.ops.pwd) + } +} + + +trait TestScalaNativeModule extends ScalaNativeModule with TestModule { testOuter => + case class TestDefinition(framework: String, clazz: Class[_], fingerprint: Fingerprint) { + def name = clazz.getName.reverse.dropWhile(_ == '$').reverse + } + + override def testLocal(args: String*) = T.command { test(args:_*) } + + override def test(args: String*) = T.command{ + val outputPath = T.ctx().dest / "out.json" + + // The test frameworks run under the JVM and communicate with the native binary over a socket + // therefore the test framework is loaded from a JVM classloader + val testClassloader = + new URLClassLoader(testClasspathJvm().map(_.path.toIO.toURI.toURL).toArray, + this.getClass.getClassLoader) + val frameworkInstances = TestRunner.frameworks(testFrameworks())(testClassloader) + val testBinary = testRunnerNative.nativeLink().toIO + val envVars = forkEnv() + + val nativeFrameworks = (cl: ClassLoader) => + frameworkInstances.zipWithIndex.map { case (f, id) => + bridge().newScalaNativeFrameWork(f, id, testBinary, logLevel(), envVars) + } + + val (doneMsg, results) = TestRunner.runTests( + nativeFrameworks, + testClasspathJvm().map(_.path), + Agg(compile().classes.path), + args + ) + + TestModule.handleResults(doneMsg, results) + } + + private val supportedTestFrameworks = Set("utest", "scalatest") + + // get the JVM classpath entries for supported test frameworks + def testFrameworksJvmClasspath = T{ + Lib.resolveDependencies( + repositories, + Lib.depToDependency(_, scalaVersion(), ""), + transitiveIvyDeps().collect{ case x: Dep.Scala => x }.filter(d => supportedTestFrameworks(d.dep.module.name)) + ) + } + + def testClasspathJvm = T{ + localClasspath() ++ + transitiveLocalClasspath() ++ + unmanagedClasspath() ++ + testFrameworksJvmClasspath() + } + + // creates a specific binary used for running tests - has a different (generated) main class + // which knows the names of all the tests and references to invoke them + object testRunnerNative extends ScalaNativeModule { + override def scalaWorker = testOuter.scalaWorker + override def scalaVersion = testOuter.scalaVersion() + override def scalaNativeVersion = testOuter.scalaNativeVersion() + override def moduleDeps = Seq(testOuter) + override def releaseMode = testOuter.releaseMode() + override def logLevel = testOuter.logLevel() + override def nativeLinkStubs = true + + override def ivyDeps = testOuter.ivyDeps() ++ Agg( + ivy"org.scala-native::test-interface_native${scalaNativeToolsVersion()}:${scalaNativeVersion()}" + ) + + override def mainClass = Some("scala.scalanative.testinterface.TestMain") + + override def generatedSources = T { + val outDir = T.ctx().dest + ammonite.ops.write.over(outDir / "TestMain.scala", makeTestMain()) + Seq(PathRef(outDir)) + } + } + + // generate a main class for the tests + def makeTestMain = T{ + val frameworkInstances = TestRunner.frameworks(testFrameworks()) _ + + val testClasses = + Jvm.inprocess(testClasspathJvm().map(_.path), classLoaderOverrideSbtTesting = true, isolated = true, + cl => { + frameworkInstances(cl).flatMap { framework => + val df = Lib.discoverTests(cl, framework, Agg(compile().classes.path)) + df.map(d => TestDefinition(framework.getClass.getName, d._1, d._2)) + } + } + ) + + val frameworks = testClasses.map(_.framework).distinct + + val frameworksList = + if (frameworks.isEmpty) + "Nil" + else + frameworks.mkString("List(new _root_.", ", new _root_.", ")") + + val testsMap = makeTestsMap(testClasses) + + s"""package scala.scalanative.testinterface + |object TestMain extends TestMainBase { + | override val frameworks = $frameworksList + | override val tests = Map[String, AnyRef]($testsMap) + | def main(args: Array[String]): Unit = + | testMain(args) + |}""".stripMargin + } + + private def makeTestsMap(tests: Seq[TestDefinition]): String = { + tests + .map { t => + val isModule = t.fingerprint match { + case af: AnnotatedFingerprint => af.isModule + case sf: SubclassFingerprint => sf.isModule + } + + val inst = + if (isModule) s"_root_.${t.name}" else s"new _root_.${t.name}" + s""""${t.name}" -> $inst""" + } + .mkString(", ") + } +} + + +trait SbtNativeModule extends ScalaNativeModule with SbtModule + |