From 791cb363b77332e3abdf4039102dfcdb863ce6c3 Mon Sep 17 00:00:00 2001 From: Jakob Odersky Date: Mon, 2 May 2016 05:19:07 -0700 Subject: Use macro annotation to load native library This also removes the need for third projects to depend on a "loader library". --- .../jodersky/sbt/jni/build/Autotools.scala.notyet | 28 ++++ .../ch/jodersky/sbt/jni/build/BuildTool.scala | 68 +++++++++ .../scala/ch/jodersky/sbt/jni/build/CMake.scala | 33 +++++ .../sbt/jni/build/ConfigureMakeInstall.scala | 55 +++++++ .../ch/jodersky/sbt/jni/plugins/JniJavah.scala | 68 +++++++++ .../ch/jodersky/sbt/jni/plugins/JniLoad.scala | 25 ++++ .../ch/jodersky/sbt/jni/plugins/JniNative.scala | 159 +++++++++++++++++++++ .../ch/jodersky/sbt/jni/plugins/JniPackage.scala | 95 ++++++++++++ .../ch/jodersky/sbt/jni/util/BytecodeUtil.scala | 62 ++++++++ 9 files changed, 593 insertions(+) create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/build/Autotools.scala.notyet create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/build/BuildTool.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/build/CMake.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/build/ConfigureMakeInstall.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniJavah.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniLoad.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniNative.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniPackage.scala create mode 100644 plugin/src/main/scala/ch/jodersky/sbt/jni/util/BytecodeUtil.scala (limited to 'plugin/src/main/scala/ch/jodersky/sbt') diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/build/Autotools.scala.notyet b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/Autotools.scala.notyet new file mode 100644 index 0000000..7d81858 --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/Autotools.scala.notyet @@ -0,0 +1,28 @@ +package ch.jodersky.sbt.jni +package build + +import java.io.File +import sbt._ + +object Autotools extends BuildTool with ConfigureMakeInstall { + + val name = "Autotools" + + def detect(baseDirectory: File) = baseDirectory.list().contains("configure") + + override def getInstance(baseDir: File, buildDir: File, logger: Logger) = new Instance { + + override def log = logger + override def baseDirectory = baseDir + override def buildDirectory = buildDir + + override def configure(target: File) = Process( + s"${base.getAbsolutePath}/configure " + + s"--prefix=${target.getAbsolutePath} " + + s"--libdir=${target.getAbsolutePath} " + + "--disable-versioned-lib", + build + ) + } + +} \ No newline at end of file diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/build/BuildTool.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/BuildTool.scala new file mode 100644 index 0000000..3d0deb1 --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/BuildTool.scala @@ -0,0 +1,68 @@ +package ch.jodersky.sbt.jni +package build + +import java.io.{ File, InputStream } +import java.nio.file.Files + +import scala.io.Source +import sbt.Logger + +trait BuildTool { + + /** Name of this build tool. */ + def name: String + + /** Detect if this build tool is configured in the given directory. + * E.g. for the Make build tool, this would return true if a Makefile is present + * in the given directory. + */ + def detect(baseDirectory: File): Boolean + + protected def templateMappings: Seq[(String, String)] + + /** Initialize the given directory with a minimal, functioning configuration for + * this build tool. E.g. for the Make build tool, this would create a Makefile in + * the given directory that is compatible with sbt-jni. + * @return all created files + */ + def initTemplate(baseDirectory: File, projectName: String): Seq[File] = + for ((resource, name) <- templateMappings) yield { + val resourceStream = this.getClass.getResourceAsStream(resource) + + if (resourceStream == null) sys.error(s"Template for $name not found.") + + val raw = Source.fromInputStream(resourceStream).mkString("") + val replaced = raw.replaceAll("\\{\\{project\\}\\}", projectName) + + baseDirectory.mkdir() + val out = baseDirectory.toPath().resolve(name) + Files.write(out, replaced.getBytes) + out.toFile() + } + + /** Actual tasks that can be perfomed on a specific configuration, such as + * configured in a Makefile. + */ + trait Instance { + + /** Invokes the native build tool's clean task */ + def clean(): Unit + + /** Invokes the native build tool's main task, resulting in a single shared + * library file. + * @param baseDirectory the directory where the native project is located + * @param buildDirectory a directory from where the build is called, it may be used + * to store temporary files + * @param targetDirectory the directory into which the native library is copied + * @return the native library file + */ + def library( + targetDirectory: File + ): File + + } + + /** Get an instance (build configuration) of this tool, in the specified directory. */ + def getInstance(baseDirectory: File, buildDirectory: File, logger: Logger): Instance + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/build/CMake.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/CMake.scala new file mode 100644 index 0000000..eb7be1e --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/CMake.scala @@ -0,0 +1,33 @@ +package ch.jodersky.sbt.jni +package build + +import sbt._ + +object CMake extends BuildTool with ConfigureMakeInstall { + + override val name = "CMake" + + override def detect(baseDirectory: File) = baseDirectory.list().contains("CMakeLists.txt") + + override protected def templateMappings = Seq( + "/ch/jodersky/sbt/jni/templates/CMakeLists.txt" -> "CMakeLists.txt" + ) + + override def getInstance(baseDir: File, buildDir: File, logger: Logger) = new Instance { + + override def log = logger + override def baseDirectory = baseDir + override def buildDirectory = buildDir + + override def configure(target: File) = Process( + // disable producing versioned library files, not needed for fat jars + "cmake " + + s"-DCMAKE_INSTALL_PREFIX:PATH=${target.getAbsolutePath} " + + s"-DLIB_INSTALL_DIR:PATH=${target.getAbsolutePath} " + + "-DLIB_ENABLE_MINOR_VERSIONS:BOOLEAN=OFF " + + baseDirectory.getAbsolutePath, + buildDirectory + ) + } + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/build/ConfigureMakeInstall.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/ConfigureMakeInstall.scala new file mode 100644 index 0000000..7bfedae --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/build/ConfigureMakeInstall.scala @@ -0,0 +1,55 @@ +package ch.jodersky.sbt.jni +package build + +import java.io.File +import sbt._ + +trait ConfigureMakeInstall { self: BuildTool => + + /* API for native build tools that use a standard 'configure && make && make install' process, + * where the configure step is left ab + stract. */ + trait Instance extends self.Instance { + + def log: Logger + def baseDirectory: File + def buildDirectory: File + + def clean() = Process("make clean", buildDirectory) ! log + + def configure(targetDirectory: File): ProcessBuilder + + def make(): ProcessBuilder = Process("make", buildDirectory) + + def install(): ProcessBuilder = Process("make install", buildDirectory) + + def library( + targetDirectory: File + ): File = { + + val ev: Int = ( + configure(targetDirectory) #&& make() #&& install() + ) ! log + + if (ev != 0) sys.error(s"Building native library failed. Exit code: ${ev}") + + val products: List[File] = + (targetDirectory ** ("*.so" | "*.dylib")).get.filter(_.isFile).toList + + // only one produced library is expected + products match { + case Nil => + sys.error(s"No files were created during compilation, " + + "something went wrong with the ${name} configuration.") + case head :: Nil => + head + case head :: tail => + log.warn("More than one file was created during compilation, " + + s"only the first one (${head.getAbsolutePath}) will be used.") + head + } + } + } + +} + diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniJavah.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniJavah.scala new file mode 100644 index 0000000..edda1fc --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniJavah.scala @@ -0,0 +1,68 @@ +package ch.jodersky.sbt.jni +package plugins + +import sbt._ +import sbt.Keys._ +import util.BytecodeUtil + +/** Adds `javah` header-generation functionality to projects. */ +object JniJavah extends AutoPlugin { + + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + object autoImport { + + val javahClasses = taskKey[Set[String]]( + "Finds fully qualified names of classes containing native declarations." + ) + + val javah = taskKey[File]( + "Generate JNI headers. Returns the directory containing generated headers." + ) + + } + import autoImport._ + + lazy val mainSettings: Seq[Setting[_]] = Seq( + + javahClasses in javah := { + val compiled: inc.Analysis = (compile in Compile).value + val classFiles: Set[File] = compiled.relations.allProducts.toSet + val nativeClasses = classFiles flatMap { file => + BytecodeUtil.nativeClasses(file) + } + nativeClasses + }, + + target in javah := target.value / "native" / "include", + + javah := { + val out = (target in javah).value + val jcp: Seq[File] = { (compile in Compile).value; Seq((classDirectory in Compile).value) } + val cp = jcp.mkString(sys.props("path.separator")) + val log = streams.value.log + + val classes = (javahClasses in javah).value + if (!classes.isEmpty) { + log.info("Headers will be generated to " + out.getAbsolutePath) + } + for (clazz <- classes) { + log.info("Generating header for " + clazz) + val parts = Seq( + "javah", + "-d", out.getAbsolutePath, + "-classpath", cp, + clazz + ) + val cmd = parts.mkString(" ") + val ev = Process(cmd) ! streams.value.log + if (ev != 0) sys.error(s"Error occured running javah. Exit code: ${ev}") + } + out + } + ) + + override lazy val projectSettings = mainSettings + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniLoad.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniLoad.scala new file mode 100644 index 0000000..51d9c7b --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniLoad.scala @@ -0,0 +1,25 @@ +package ch.jodersky.sbt.jni +package plugins + +import sbt._ +import sbt.Keys._ + +object JniLoad extends AutoPlugin { + + override def requires = empty + override def trigger = allRequirements + + lazy val settings: Seq[Setting[_]] = Seq( + // Macro Paradise plugin and dependencies are needed to expand annotation macros. + // Once expanded however, downstream projects don't need these dependencies anymore + // (hence the "Provided" configuration). + addCompilerPlugin( + "org.scalamacros" % "paradise" % ProjectVersion.MacrosParadise cross CrossVersion.full + ), + libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % Provided, + libraryDependencies += "ch.jodersky" %% "sbt-jni-macros" % ProjectVersion.Macros % Provided + ) + + override def projectSettings = settings + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniNative.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniNative.scala new file mode 100644 index 0000000..6192a08 --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniNative.scala @@ -0,0 +1,159 @@ +package ch.jodersky.sbt.jni +package plugins + +import build._ +import sbt._ +import sbt.Keys._ + +/** Wraps a native build system in sbt tasks. */ +object JniNative extends AutoPlugin { + + object autoImport { + + //Main task, inspect this first + val nativeCompile = taskKey[File]( + "Builds a native library by calling the native build tool." + ) + + val nativePlatform = settingKey[String]( + "Platform (architecture-kernel) of the system this build is running on." + ) + + val nativeBuildTool = taskKey[BuildTool]( + "The build tool to be used when building a native library." + ) + + val nativeInit = inputKey[Seq[File]]( + "Initialize a native build script from a template." + ) + + } + import autoImport._ + + val nativeBuildToolInstance = taskKey[BuildTool#Instance]("Get an instance of the current native build tool.") + + lazy val settings: Seq[Setting[_]] = Seq( + + // the value retruned must match that of `ch.jodersky.jni.PlatformMacros#current()` of project `macros` + nativePlatform := { + try { + val lines = Process("uname -sm").lines + if (lines.length == 0) { + sys.error("Error occured trying to run `uname`") + } + // uname -sm returns " " + val parts = lines.head.split(" ") + if (parts.length != 2) { + sys.error("'uname -sm' returned unexpected string: " + lines.head) + } else { + val arch = parts(1).toLowerCase.replaceAll("\\s", "") + val kernel = parts(0).toLowerCase.replaceAll("\\s", "") + arch + "-" + kernel + } + } catch { + case ex: Exception => + sLog.value.error("Error trying to determine platform.") + sLog.value.warn("Cannot determine platform! It will be set to 'unknown'.") + "unknown-unknown" + } + }, + + sourceDirectory in nativeCompile := sourceDirectory.value / "native", + + target in nativeCompile := target.value / "native" / (nativePlatform).value, + + nativeBuildTool := { + val tools = Seq(CMake) + + val src = (sourceDirectory in nativeCompile).value + + val tool = if (src.exists && src.isDirectory) { + tools.find(t => t detect src) + } else { + None + } + tool getOrElse sys.error("No supported native build tool detected. " + + s"Check that the setting 'sourceDirectory in nativeCompile' (currently set to$src) " + + "points to a directory containing a supported build script. Supported build tools are: " + + tools.map(_.name).mkString(",") + ) + + }, + + nativeBuildToolInstance := { + val tool = nativeBuildTool.value + val srcDir = (sourceDirectory in nativeCompile).value + val buildDir = (target in nativeCompile).value / "build" + IO.createDirectory(buildDir) + tool.getInstance( + baseDirectory = srcDir, + buildDirectory = buildDir, + logger = streams.value.log + ) + }, + + clean in nativeCompile := { + val log = streams.value.log + + log.debug("Cleaning native build") + try { + val toolInstance = nativeBuildToolInstance.value + toolInstance.clean() + } catch { + case ex: Exception => + log.debug(s"Native cleaning failed: $ex") + } + + }, + + nativeCompile := { + val tool = nativeBuildTool.value + val toolInstance = nativeBuildToolInstance.value + val targetDir = (target in nativeCompile).value / "bin" + val log = streams.value.log + + IO.createDirectory(targetDir) + + log.info(s"Building library with native build tool ${tool.name}") + val lib = toolInstance.library(targetDir) + log.success(s"Library built in ${lib.getAbsolutePath}") + lib + }, + + // also clean native sources + clean := { + (clean in nativeCompile).value + clean.value + }, + + nativeInit := { + import complete.DefaultParsers._ + + val log = streams.value.log + + def getTool(toolName: String): BuildTool = toolName.toLowerCase match { + case "cmake" => CMake + case _ => sys.error("Unsupported build tool: " + toolName) + } + + val args = spaceDelimited(" []").parsed.toList + + val (tool: BuildTool, lib: String) = args match { + case Nil => sys.error("Invalid arguments.") + case tool :: Nil => (getTool(tool), name.value) + case tool :: lib :: other => (getTool(tool), lib) + } + + log.info(s"Initializing native build with ${tool.name} configuration") + val files = tool.initTemplate((sourceDirectory in nativeCompile).value, lib) + files foreach { file => + log.info("Wrote to " + file.getAbsolutePath) + } + files + } + + ) + + override lazy val projectSettings = settings + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniPackage.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniPackage.scala new file mode 100644 index 0000000..b6b5100 --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/plugins/JniPackage.scala @@ -0,0 +1,95 @@ +package ch.jodersky.sbt.jni +package plugins + +import sbt._ +import sbt.Keys._ +import java.io.File + +/** Packages libraries built with JniNative. */ +object JniPackage extends AutoPlugin { + + // JvmPlugin is required or else it will override resource generators when first included + override def requires = JniNative && plugins.JvmPlugin + override def trigger = allRequirements + + object autoImport { + + val enableNativeCompilation = settingKey[Boolean]( + "Determines if native compilation is enabled. If not enabled, only pre-compiled libraries in " + + "`unmanagedNativeDirectories` will be packaged." + ) + + val unmanagedNativeDirectories = settingKey[Seq[File]]( + "Unmanaged directories containing native libraries. The libraries must be regular files " + + "contained in a subdirectory corresponding to a platform. For example " + + "`/x86_64-linux/libfoo.so` is an unmanaged library for machines having " + + "the x86_64 architecture and running the Linux kernel." + ) + + val unmanagedNativeLibraries = taskKey[Seq[(File, String)]]( + "Reads `unmanagedNativeDirectories` and maps platforms to library files specified theirin." + ) + + val managedNativeLibraries = taskKey[Seq[(File, String)]]( + "Maps locally built, platform-dependant libraries." + ) + + val nativeLibraries = taskKey[Seq[(File, String)]]( + "All native libraries, managed and unmanaged." + ) + + } + import autoImport._ + import JniNative.autoImport._ + + lazy val settings: Seq[Setting[_]] = Seq( + + enableNativeCompilation := true, + + unmanagedNativeDirectories := Seq(baseDirectory.value / "lib_native"), + + unmanagedNativeLibraries := { + val baseDirs: Seq[File] = unmanagedNativeDirectories.value + val mappings: Seq[(File, String)] = unmanagedNativeDirectories.value.flatMap { dir => + val files: Seq[File] = (dir ** "*").get + files pair rebase(dir, "/native") + } + mappings + }, + + managedNativeLibraries := Def.taskDyn[Seq[(File, String)]] { + val enableManaged = (enableNativeCompilation).value + if (enableManaged) Def.task { + val library: File = nativeCompile.value + val platform = nativePlatform.value + + Seq(library -> s"/native/$platform/${library.name}") + } + else Def.task { + Seq.empty + } + }.value, + + nativeLibraries := unmanagedNativeLibraries.value ++ managedNativeLibraries.value, + + resourceGenerators += Def.task { + val libraries: Seq[(File, String)] = nativeLibraries.value + val resources: Seq[File] = for ((file, path) <- libraries) yield { + + // native library as a managed resource file + val resource = resourceManaged.value / path + + // copy native library to a managed resource, so that it is always available + // on the classpath, even when not packaged as a jar + IO.copyFile(file, resource) + resource + } + resources + }.taskValue + + ) + + override lazy val projectSettings = inConfig(Compile)(settings) ++ + Seq(crossPaths := false) // don't add scala version to native jars + +} diff --git a/plugin/src/main/scala/ch/jodersky/sbt/jni/util/BytecodeUtil.scala b/plugin/src/main/scala/ch/jodersky/sbt/jni/util/BytecodeUtil.scala new file mode 100644 index 0000000..fe728a6 --- /dev/null +++ b/plugin/src/main/scala/ch/jodersky/sbt/jni/util/BytecodeUtil.scala @@ -0,0 +1,62 @@ +package ch.jodersky.sbt.jni +package util + +import java.io.{ File, FileInputStream, Closeable } +import scala.collection.mutable.{ HashSet } + +import org.objectweb.asm.{ ClassReader, ClassVisitor, MethodVisitor, Opcodes } + +object BytecodeUtil { + + private class NativeFinder extends ClassVisitor(Opcodes.ASM5) { + + // classes found to contain at least one @native def + val _nativeClasses = new HashSet[String] + def nativeClasses = _nativeClasses.toSet + + private var fullyQualifiedName: String = "" + + override def visit(version: Int, access: Int, name: String, signature: String, + superName: String, interfaces: Array[String]): Unit = { + fullyQualifiedName = name.replaceAll("/", ".") + } + + override def visitMethod(access: Int, name: String, desc: String, + signature: String, exceptions: Array[String]): MethodVisitor = { + + val isNative = (access & Opcodes.ACC_NATIVE) != 0 + + if (isNative) { + _nativeClasses += fullyQualifiedName + } + + null //return null, do not visit method further + } + + } + + private def using[A >: Null <: Closeable, R](mkStream: => A)(action: A => R): R = { + var stream: A = null + try { + stream = mkStream + action(stream) + } finally { + if (stream != null) { + stream.close() + } + } + } + + /** Finds classes containing native implementations. + * @param classFile java class file from which classes are read + * @return all fully qualified names of classes that contain at least one member annotated + * with @native + */ + def nativeClasses(classFile: File): Set[String] = using(new FileInputStream(classFile)) { in => + val reader = new ClassReader(in) + val finder = new NativeFinder + reader.accept(finder, 0) + finder.nativeClasses + } + +} -- cgit v1.2.3