From 5c94ee017051d51f51e06a61a8bc4e70a15e36da Mon Sep 17 00:00:00 2001 From: Jakob Odersky Date: Mon, 13 Jan 2014 17:40:34 +0100 Subject: enable easier cross-compilation --- project/FlowBuild.scala | 216 +++++++++++++++++---------------------- project/jni.scala | 27 ++--- project/native.scala | 207 ++++++++++++++++++------------------- project/nativefat.scala.disabled | 47 +++++++++ 4 files changed, 249 insertions(+), 248 deletions(-) create mode 100644 project/nativefat.scala.disabled (limited to 'project') diff --git a/project/FlowBuild.scala b/project/FlowBuild.scala index 087ff6f..fcf4d10 100644 --- a/project/FlowBuild.scala +++ b/project/FlowBuild.scala @@ -1,21 +1,25 @@ import sbt._ import Keys._ - -import NativeKeys._ -import NativeDefaults._ import JniKeys._ +import NativeKeys._ object FlowBuild extends Build { val Organization = "com.github.jodersky" - val Version = "1.0.1-SNAPSHOT" //version of flow library - val BinaryMajorVersion = 2 //binary major version used to select shared libraries when publishing (needs to be incremented if API changes are made to flow.h or NativeSerial.java) val ScalaVersion = "2.10.3" - //see native settings down below + val Version = "1.1.1" //version of flow library + val NativeMajorVersion = 2 //major version of native API + val NativeMinorVersionPosix = 0 //minor version of native posix implementation + val NativeVersionPosix = NativeMajorVersion + "." + NativeMinorVersionPosix + + val release = settingKey[Boolean]("Indicates if this build is a release.") + val gitHeadCommitSha = settingKey[String]("Current commit sha.") lazy val commonSettings: Seq[Setting[_]] = Seq( organization := Organization, - version := Version, scalaVersion := ScalaVersion, + release in ThisBuild := sys.props("release") == true, + gitHeadCommitSha in ThisBuild := Process("git rev-parse HEAD").lines.head, + version in ThisBuild:= { if (release.value ) Version else Version + "-" + gitHeadCommitSha.value }, licenses := Seq(("BSD-3-Clause", url("http://opensource.org/licenses/BSD-3-Clause"))), resolvers += "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/", scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature")) @@ -25,138 +29,106 @@ object FlowBuild extends Build { connectInput in run := true, outputStrategy := Some(StdoutOutput) ) + + lazy val root: Project = ( + Project("root", file(".")).aggregate(flow) + settings( + publish := (), + publishLocal := () + ) + ) - lazy val main: Project = ( - Project("flow", file("flow-main")) + lazy val flow: Project = ( + Project("flow", file("flow")) settings (commonSettings: _*) - settings ( + settings (JniDefaults.settings: _*) + settings (NativeDefaults.settings: _*) + settings (selectHost().settings: _*) + settings( + nativeIncludeDirectories += (sourceDirectory in Compile).value / "native" / "include", + nativeIncludeDirectories ++= jdkHome.value.map(jdk => jdk / "include").toSeq, + javahClasses := Seq("com.github.jodersky.flow.internal.NativeSerial"), + javahHeaderDirectory := (sourceDirectory in Compile).value / "native" / "include", + compileOrder in Compile := CompileOrder.Mixed, libraryDependencies ++= Seq( Dependencies.akkaActor, Dependencies.ioCore, - Dependencies.ioFile), - compileOrder in Compile := CompileOrder.Mixed, - resourceGenerators in Compile <+= (resourceManaged in Compile, binariesDirectory in ThisBuild) map { (resDir, binDir) => - val binaries: Seq[(File, File)] = getLatestBinaries(binDir, BinaryMajorVersion) - val resources = for (binary <- binaries) yield { - val versionedBinary = binary._1 - val unversionedBinary = binary._2 - - val relative = (unversionedBinary relativeTo binDir).get.getPath - - val resource = resDir / "native" / relative - IO.copyFile(versionedBinary, resource) - resource - } - resources - } + Dependencies.ioFile) ) ) - //returns latest, major version-compatible binaries for every OS and architecture combination - //tuples returned (file with binary version appended, file without version string) - def getLatestBinaries(base: File, majorVersion: Int): Seq[(File, File)] = { - def latest(platform: File) = { - val Pattern = "(.+)\\.(\\d+)\\.(\\d+)".r - val MajorVersion = majorVersion.toString - val majorCompatible = platform.listFiles.map(_.getAbsolutePath) collect { - case path @ Pattern(strippedPath, MajorVersion, minorVersion) => (path, strippedPath, minorVersion) - } - val latestMinor = majorCompatible.sortBy(_._3).lastOption - latestMinor map { case (path, strippedPath, _) => - (file(path), file(strippedPath)) - } - } - - val oSs = IO.listFiles(base).filter(file => !file.name.startsWith(".")) - val platforms = oSs.flatMap(_.listFiles) - - platforms.flatMap(latest(_)) - } - lazy val samplesTerminal = ( - Project("flow-samples-terminal", file("flow-samples") / "terminal") + Project("flow-samples-terminal", file("flow-samples") / "flow-samples-terminal") settings(commonSettings: _*) settings(runSettings: _*) - dependsOn(main) - ) - - - //--- native settings and projects -------------------------------------------------- - - val binariesDirectory = settingKey[File]("Directory containing published native binaries.") - override lazy val settings = super.settings ++ Seq( - (binariesDirectory in ThisBuild) := (baseDirectory in ThisBuild).value / "flow-binaries" + dependsOn(flow) ) - - def canonicalBinaryPath(base: File, binaryName: String) = { - base / sys.props("os.name").toLowerCase.filter(_ != ' ') / sys.props("os.arch").toLowerCase / binaryName - } - val publishNative = taskKey[File]("Publish native binary compiled on current OS to flow-binaries project so that it may be packaged in a distribution of flow.") - val publishNativeImpl = Def.task{ - val in = (link in Native).value - val out = canonicalBinaryPath((binariesDirectory in ThisBuild).value, in.getName) - IO.copyFile(in, out) - out - } - lazy val commonNativeSettings: Seq[Setting[_]] = Seq( - nativeSource in Native := baseDirectory.value / "src", - includeDirectories in Native += file("flow-native") / "shared" / "include", - nativeCompile in Native := ((nativeCompile in Native) dependsOn (compile in Compile in main)).value, - publishNative := publishNativeImpl.value, - javahClasspath := Seq((classDirectory in Compile in main).value), - javahClasses := Seq("com.github.jodersky.flow.internal.NativeSerial")) ++ JniDefaults.defaultSettings - - //--- native unix-like settings and projects ---------------------------------------- - - val UnixBinaryMinorVersion = 1 - - lazy val unixNativeSettings: Seq[Setting[_]] = commonNativeSettings ++ Seq( - flags in Native := Seq("-fPIC", "-O2") + //the current operating system used to run the native compile + trait Host{ def settings: Seq[Setting[_]] } + + object Linux extends Host { + + val compiler = "gcc" + val linker = compiler + val cFlags = Seq("-O2", "-fPIC") + val linkerFlags = Seq("-shared", s"-Wl,-soname,libflow.so.${NativeMajorVersion}") + val binary = s"libflow.so" + + val builds = List( + NativeBuild("amd64-linux", "gcc", cFlags :+ "-m64", "gcc", linkerFlags :+ "-m64", binary), + NativeBuild("x86-linux", "gcc", cFlags :+ "-m32", "gcc", linkerFlags :+ "-m32", binary), + NativeBuild("arm-linux", "arm-linux-gnueabi-gcc", cFlags, "arm-linux-gnueabi-gcc", linkerFlags, binary) + //add other build configurations here or adapt existing ones to your needs ) - lazy val nativeLinux = ( - NativeProject("flow-native-linux", file("flow-native") / "unix") - settings (unixNativeSettings: _*) - settings ( - target := baseDirectory.value / "target" / "linux", - includeDirectories in Native ++= jdkHome.value.map(jdk => jdk / "include" / "linux").toSeq, - linkFlags in Native ++= Seq("-shared", s"-Wl,-soname,libflow.so.${BinaryMajorVersion}"), - binaryName in Native := s"libflow.so.${BinaryMajorVersion}.${UnixBinaryMinorVersion}" + lazy val settings = Seq( + nativeVersion := NativeVersionPosix, + nativeIncludeDirectories ++= jdkHome.value.map(jdk => jdk / "include" / "linux").toSeq, + nativeSource := nativeSource.value / "posix", + nativeBuilds := builds ) - dependsOn(main) - ) - - lazy val nativeMacOSX = ( - NativeProject("flow-native-macosx", file("flow-native") / "unix") - settings (unixNativeSettings: _*) - settings ( - target := baseDirectory.value / "target" / "macosx", - includeDirectories in Native += file("/System/Library/Frameworks/JavaVM.framework/Headers/jni.h"), - includeDirectories in Native += file("/Developer/SDKs/MacOSX10.6.sdk/System/Library/Frameworks/JavaVM.framework/Versions/A/Headers"), - linkFlags in Native ++= Seq("-dynamiclib"), - binaryName in Native := s"libflow.jnilib.${BinaryMajorVersion}.${UnixBinaryMinorVersion}" + + } + + //stub, not sure if this works + object MacOSX extends Host { + + val compiler = "gcc" + val linker = compiler + val cFlags = Seq("-O2", "-fPIC") + val linkerFlags = Seq("-dynamiclib") + val binary = s"libflow.jnilib" + + val localBuild = NativeBuild( + "amd64-macosx", + compiler, + cFlags, + linker, + linkerFlags, + binary ) - dependsOn (main) - ) - - - /* stub for native project on windows, I don't know if this would actually work... - * - * val WindowsBinaryMinorVersion = 0 - * - lazy val nativeWindows = ( - NativeProject("flow-native-windows", file("flow-native") / "windows") - settings ( - //windows is not a unix-like OS, several default settings need to be changed - cCompiler in Native := "???", - flags in Native := Seq("-fPIC", "-O2"), - linkFlags in Native ++= ???, - binaryName in Native := s"flow.dll.${BinaryMajorVersion}.${WindowsBinaryMinorVersion}" - includeDirectories in Native += jdkHome.value / "include" / "windows" + + lazy val settings = Seq( + nativeVersion := NativeVersionPosix, + nativeIncludeDirectories += file("/System/Library/Frameworks/JavaVM.framework/Headers/jni.h"), + nativeIncludeDirectories += file("/Developer/SDKs/MacOSX10.6.sdk/System/Library/Frameworks/JavaVM.framework/Versions/A/Headers"), + nativeSource := nativeSource.value / "posix", + nativeBuilds := Seq(localBuild) ) - dependsOn (main) - )*/ - + + } + + private def osError = throw new RuntimeException("Sorry, native compilation under the current OS is not supported.") + def selectHost() = System.getProperty("os.name").toLowerCase match { + case "linux" => Linux + case "macosx" => MacOSX + case _ => new Host{ + val settings = Seq( + nativeCompile := osError, + nativeLink := osError + ) + } + } } diff --git a/project/jni.scala b/project/jni.scala index 5e248f2..e87f0c7 100644 --- a/project/jni.scala +++ b/project/jni.scala @@ -1,11 +1,9 @@ import sbt._ import Keys._ -import NativeKeys._ import scala.util.Try object JniKeys { val jdkHome = settingKey[Option[File]]("Home of JDK.") - val jdkHomeNotFound = taskKey[Unit]("Utility task that informs user that no JDK was found.") val javahHeaderDirectory = settingKey[File]("Directory where generated javah header files are placed.") val javahClasses = settingKey[Seq[String]]("Fully qualified names of classes containing native declarations.") val javahClasspath = taskKey[Seq[File]]("Classpath to use in javah.") @@ -15,27 +13,16 @@ object JniKeys { object JniDefaults { import JniKeys._ - val defaultSettings: Seq[Setting[_]] = Seq( + val settings: Seq[Setting[_]] = Seq( jdkHome := Try(file(sys.env("JAVA_HOME"))).toOption, - jdkHomeNotFound := jdkHomeNotFoundImpl.value, - nativeCompile in Native := ((nativeCompile in Native) dependsOn jdkHomeNotFound).value, - javahHeaderDirectory := (sourceManaged in Native).value / "javah", - javah := javahImpl.value, - sourceGenerators in Native <+= javah map { headers => headers}, - includeDirectories in Native += javahHeaderDirectory.value, - includeDirectories in Native ++= jdkHome.value.map( jdk => jdk / "include").toSeq) + javahHeaderDirectory := baseDirectory.value, + javahClasspath := Seq((classDirectory in Compile).value), + javah := javahImpl.value + ) - def jdkHomeNotFoundImpl = Def.task { - if (jdkHome.value == None) { - streams.value.log.warn( - "No JDK home directory found, any native code using JNI may not compile. Please set JAVA_HOME environment variable.") - } - () - } - def javahImpl = Def.task { - val cps = javahClasspath.value - val cp = cps.mkString(":") + val jcp = javahClasspath.value + val cp = jcp.mkString(":") for (clazz <- javahClasses.value) { val parts = Seq( "javah", diff --git a/project/native.scala b/project/native.scala index 5c84f11..68c970e 100644 --- a/project/native.scala +++ b/project/native.scala @@ -3,32 +3,35 @@ import Keys._ import java.io.File import scala.collection.mutable.HashSet +case class NativeBuild( + name: String, + cCompiler: String, + cFlags: Seq[String], + linker: String, + linkerFlags: Seq[String], + binary: String) + object NativeKeys { - val Native = config("native") - - //compilation - val cCompiler = settingKey[String]("Default c compiler.") - val cppCompiler = settingKey[String]("Default c++ comppiler.") - val cFlags = settingKey[Seq[String]]("Default flags for c compiler.") - val cppFlags = settingKey[Seq[String]]("Default flags for c++ compiler.") - val flags = settingKey[Seq[String]]("Default flags prepended to c and c++ flags.") - val nativeCompile = taskKey[Seq[File]]("Compile native sources.") - - val linker = settingKey[String]("Linker used in project.") - val linkFlags = settingKey[Seq[String]]("Default options for linker.") - val libraryNames = settingKey[Seq[String]]("Default names of libraries to use during linking.") - val binaryName = settingKey[String]("Final binary product.") - val link = taskKey[File]("Link compiled objects to a final product.") + //build settings + val nativeBuilds = taskKey[Seq[NativeBuild]]("All native build configurations, including cross-compilation.") + val nativeVersion = settingKey[String]("Version of native binary") + //compile settings + val nativeIncludeDirectories = settingKey[Seq[File]]("Directories to include during build (gcc -I option)") + + //link settings + val nativeLibraries = settingKey[Seq[String]]("Default names of libraries to use during linking.") + val nativeLibraryDirectories = settingKey[Seq[File]]("Directories to search for libraries (gcc -L option)") + //directories val nativeSource = settingKey[File]("Lowest level directory containing all native sources.") - val cSources = taskKey[Seq[File]]("All c source files, managed and unmanaged.") - val cppSources = taskKey[Seq[File]]("All c++ source files, managed and unmanaged.") - val includeDirectories = settingKey[Seq[File]]("Directories to include during build (gcc -I option)") - val libraryDirectories = settingKey[Seq[File]]("Directories to search for libraries (gcc -L option)") - val objectDirectory = settingKey[File]("Directory containing all compiled objects.") - //val objects = taskKey[Seq[File]]("Object files generated from source files. Note: there should be a one-to-one mapping between source and object files.") + val nativeCSources = taskKey[Seq[File]]("All c source files.") + val nativeTargetDirectory = settingKey[File]("Directory containing all compiled and linked files.") + + //tasks + val nativeCompile = taskKey[Map[NativeBuild, Seq[File]]]("Compile all native build configurations.") + val nativeLink = taskKey[Map[NativeBuild, File]]("Link all native build configurations.") } @@ -37,102 +40,94 @@ object NativeDefaults { private def generate(generators: SettingKey[Seq[Task[Seq[File]]]]) = generators { _.join.map(_.flatten) } - def compileImpl = Def.task { - implicit val log = streams.value.log - - def compile(compiler: String, flags: Seq[String], src: File) = { - val obj = objectDirectory.value / (src.base + ".o") - IO.createDirectory(obj.getParentFile()) - val parts: Seq[String] = Seq(compiler) ++ - flags ++ - includeDirectories.value.map("-I" + _.getAbsolutePath) ++ - Seq("-o", obj.getAbsolutePath()) ++ - Seq("-c", src.getAbsolutePath()) - - val cmd = parts.mkString(" ") - log.info(cmd) - val ev = Process(cmd) ! log - if (ev != 0) throw new RuntimeException(s"compilation of ${src.getAbsoluteFile()} failed") - obj - } + private def compile(logger: Logger, compiler: String, flags: Seq[String], includeDirectories: Seq[File], src: File, out: File): File = { + IO.createDirectory(out.getParentFile) + val parts: Seq[String] = + Seq(compiler) ++ + flags ++ + includeDirectories.map("-I" + _.getAbsolutePath) ++ + Seq("-o", out.getAbsolutePath) ++ + Seq("-c", src.getAbsolutePath) - cSources.value.map { src => - compile(cCompiler.value, cFlags.value, src) - } ++ - cppSources.value.map { src => - compile(cppCompiler.value, cppFlags.value, src) - } + val cmd = parts.mkString(" ") + logger.info(cmd) + val ev = Process(cmd) ! logger + if (ev != 0) throw new RuntimeException(s"Compilation of ${src.getAbsoluteFile()} failed.") + out } - def linkImpl = Def.task { - implicit val log = streams.value.log - - val out = (target.value / binaryName.value) - val parts: Seq[String] = Seq(linker.value) ++ - linkFlags.value ++ + private def link(logger: Logger, linker: String, flags: Seq[String], libraryDirectories: Seq[File], libraries: Seq[String], in: Seq[File], out: File): File = { + val parts: Seq[String] = + Seq(linker) ++ + flags ++ Seq("-o", out.getAbsolutePath) ++ - nativeCompile.value.map(_.getAbsolutePath) ++ - libraryDirectories.value.map("-L" + _.getAbsolutePath) ++ - libraryNames.value.map("-l" + _) + in.map(_.getAbsolutePath) ++ + libraryDirectories.map("-L" + _.getAbsolutePath) ++ + libraries.map("-l" + _) val cmd = parts.mkString(" ") - log.info(cmd) - val ev = Process(cmd) ! log - if (ev != 0) throw new RuntimeException(s"linking of ${out.getAbsoluteFile()} failed") + logger.info(cmd) + val ev = Process(cmd) ! logger + if (ev != 0) throw new RuntimeException(s"Linking of ${out.getAbsoluteFile()} failed.") out } - val compileSettings: Seq[Setting[_]] = inConfig(Native)(Seq( - cCompiler := "gcc", - cppCompiler := "g++", - cFlags := flags.value, - cppFlags := flags.value, - flags := Seq("-fPIC", "-O2"), - nativeCompile := compileImpl.value, - sourceGenerators := Seq())) - - val linkSettings: Seq[Setting[_]] = inConfig(Native)(Seq( - linker := "gcc", - linkFlags := Seq(), - libraryNames := Seq(), - binaryName := binaryName.value, - link := linkImpl.value)) - - val fileSettings: Seq[Setting[_]] = inConfig(Native)(Seq( - target := (target in Compile).value / "native", - nativeSource := (sourceDirectory in Compile).value / "native", - sourceManaged := target.value / "src_managed", - unmanagedSources := (nativeSource.value ** (includeFilter in unmanagedSources).value).get, - managedSources := (generate(sourceGenerators).value ** (includeFilter in managedSources).value).get, - sources := unmanagedSources.value ++ managedSources.value, - cSources := sources.value.filter(src => (includeFilter in cSources).value accept src), - cppSources := sources.value.filter(src => (includeFilter in cppSources).value accept src), - includeFilter in (unmanagedSources) := ("*.c" || "*.cpp" || "*.cxx" || "*.cc"), - includeFilter in (managedSources) := (includeFilter in unmanagedSources).value, - includeFilter in cSources := "*.c", - includeFilter in cppSources := "*.cpp" || "*.cc" || "*.cxx", - objectDirectory := target.value / "objects", - includeDirectories := Seq(nativeSource.value, sourceManaged.value), - libraryDirectories := Seq(), - binaryName := (name in Compile).value)) - - val defaultSettings: Seq[Setting[_]] = inConfig(Native)( - compileSettings ++ - linkSettings ++ - fileSettings) - - implicit class RichNativeProject(project: Project) { - def dependsOnNative(other: Project): Project = { - val newSettings: Seq[Setting[_]] = inConfig(Native)(Seq( - link in project := ((link in project) dependsOn (link in other)).value, - includeDirectories in project ++= (includeDirectories in other).value, - libraryDirectories in project += (target in other).value - )) - project.settings(newSettings: _*) + + def nativeCompileImpl() = Def.task { + val logger = streams.value.log + val builds = nativeBuilds.value + val outDir = nativeTargetDirectory.value + val includeDirs = nativeIncludeDirectories.value + val csrcs = nativeCSources.value + + val compilations = for (build <- builds) yield { + logger.info("Compiling configuration " + build.name) + val objects = for (src <- csrcs) yield { + compile(logger, build.cCompiler, build.cFlags, includeDirs, src, outDir / build.name / "objects" / (src.base + ".o")) + } + build -> objects + } + compilations.toMap + } + + lazy val nativeLinkImpl = Def.task { + val logger = streams.value.log + val builds = nativeBuilds.value + val outDir = nativeTargetDirectory.value + val libDirs = nativeLibraryDirectories.value + val libs = nativeLibraries.value + val compilations = nativeCompile.value + val version = nativeVersion.value + + val linkages = for (build <- builds) yield { + logger.info("Linking configuration " + build.name) + val objects = compilations(build) + val binary = link(logger, build.linker, build.linkerFlags, libDirs, libs, objects, outDir / build.name / build.binary) + build -> binary } - + linkages.toMap } - def NativeProject(id: String, base: File) = Project(id, base).settings(NativeDefaults.defaultSettings: _*) + def localPlatform = try { + Process("gcc -dumpmachine").lines.headOption + } catch { + case ex: Exception => None + } + + + val settings: Seq[Setting[_]] = Seq( + //nativeBuilds := + + nativeSource := (sourceDirectory in Compile).value / "native", + includeFilter in nativeCSources := "*.c", + nativeCSources := (nativeSource.value ** (includeFilter in nativeCSources).value).get, + nativeTargetDirectory := target.value / "native", + + nativeIncludeDirectories := Seq(nativeSource.value, nativeSource.value / "include"), + nativeLibraries := Seq(), + nativeLibraryDirectories := Seq(), + nativeCompile := nativeCompileImpl.value, + nativeLink := nativeLinkImpl.value + ) } diff --git a/project/nativefat.scala.disabled b/project/nativefat.scala.disabled new file mode 100644 index 0000000..e1cfcd3 --- /dev/null +++ b/project/nativefat.scala.disabled @@ -0,0 +1,47 @@ +import sbt._ +import Keys._ +import NativeKeys._ +import java.io.File +import scala.collection.mutable.HashSet + +object NativeFatKeys { + + val NativeFat = config("nativeFat") + + +} + +object NativeFatDefaults { + import NativeFatKeys._ + + def highest(files: Seq[File], separator: String) = { + files.sortBy{f => + f.getName.split(separator) match { + case Array(name, version) => version + case _ => throw new RuntimeException("Error parsing file version string: " + f.getName) + } + }.last + } + + val gen = taskKey[Seq[File]]("") + + val genImpl = Def.task { + val resDir = (resourceManaged).value + val binDir = nativeTargetDirectory.value + val binarises = nativeLink.value + + val resources = for ((_, binary) <- binarises) yield { + val relative = (binary relativeTo binDir).get.getPath + val resource = resDir / "native" / relative + IO.copyFile(binary, resource) + resource + } + resources.toSeq + } + + val settings = Seq( + gen := genImpl.value, + resourceGenerators in NativeFat := Seq(genImpl) + ) + +} \ No newline at end of file -- cgit v1.2.3