From 47fb0d381022bda800b7f8ee234224aa20020e4a Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Mon, 5 Dec 2011 15:52:10 -0500 Subject: Adding SHA resolve to the SBT build so that we don't need the push/pull binary libs script anymore. Only pull is implemented. --- project/ShaResolve.scala | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 project/ShaResolve.scala (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala new file mode 100644 index 0000000000..2b7fafd576 --- /dev/null +++ b/project/ShaResolve.scala @@ -0,0 +1,57 @@ +import sbt._ + +import Build._ +import Keys._ +import Project.Initialize +import scala.collection.{ mutable, immutable } + + + + +/** Helpers to resolve SHA artifacts from typesafe repo. */ +object ShaResolve { + import dispatch.{Http,url} + val remote_urlbase="http://typesafe.artifactoryonline.com/typesafe/scala-sha-bootstrap/org/scala-lang/bootstrap" + + val pullBinaryLibs = TaskKey[Unit]("pull-binary-libs", "Pulls binary libs by the SHA key.") + val pushBinaryLibs = TaskKey[Unit]("push-binary-libs", "Pushes binary libs whose SHA has changed.") + + + def settings: Seq[Setting[_]] = Seq( + pullBinaryLibs in ThisBuild <<= (baseDirectory, streams) map resolveLibs + ) + + def resolveLibs(dir: File, s: TaskStreams): Unit = { + for { + (file, name) <- dir ** "*.desired.sha1" x relativeTo(dir) + uri = name.dropRight(13) + jar = dir / uri + if !jar.exists || !isValidSha(file) + sha = getShaFromShafile(file) + } pullFile(jar, sha + "/" + uri, s) + } + + def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file) + + + def isValidSha(file: File): Boolean = + try (Process(Seq("shasum", "-p", "--check", file.getAbsolutePath), Some(file.getParentFile)).!! contains "OK") + catch { + case t: Exception => false + } + + + def pullFile(file: File, uri: String, s: TaskStreams): Unit = { + val url = remote_urlbase + "/" + uri + val fous = new java.io.FileOutputStream(file) + s.log.info("Pulling [" + url + "] to [" + file + "]") + try Http(dispatch.url(url) >>> fous) finally fous.close() + } + + def pushFile(file: File, uri: String, user: String, pw: String): Unit = { + val url = remote_urlbase + "/" + uri + val sender = dispatch.url(url).PUT.as(user,pw) <<< (file, "application/java-archive") + // TODO - output to logger. + Http(sender >>> System.out) + } +} -- cgit v1.2.3 From 16ad68fa3aa5d1ccd55865c3d06fec142cf9d630 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 8 Dec 2011 14:05:56 -0500 Subject: Added local cache for project jars. .desired.sha1 files now resolve into a local repository before being copied into the main repo. If the local repository exists and has a file, then there is no download necessary. --- project/ShaResolve.scala | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index 2b7fafd576..4d9579b07d 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -12,23 +12,24 @@ import scala.collection.{ mutable, immutable } object ShaResolve { import dispatch.{Http,url} val remote_urlbase="http://typesafe.artifactoryonline.com/typesafe/scala-sha-bootstrap/org/scala-lang/bootstrap" - + val pullBinaryLibs = TaskKey[Unit]("pull-binary-libs", "Pulls binary libs by the SHA key.") val pushBinaryLibs = TaskKey[Unit]("push-binary-libs", "Pushes binary libs whose SHA has changed.") - + val binaryLibCache = SettingKey[File]("binary-lib-cache", "Location of the cache of binary libs for this scala build.") def settings: Seq[Setting[_]] = Seq( - pullBinaryLibs in ThisBuild <<= (baseDirectory, streams) map resolveLibs + binaryLibCache in ThisBuild := file(System.getProperty("user.home")) / ".sbt" / "cache" / "scala", + pullBinaryLibs in ThisBuild <<= (baseDirectory, binaryLibCache, streams) map resolveLibs ) - def resolveLibs(dir: File, s: TaskStreams): Unit = { + def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = { for { (file, name) <- dir ** "*.desired.sha1" x relativeTo(dir) uri = name.dropRight(13) jar = dir / uri if !jar.exists || !isValidSha(file) sha = getShaFromShafile(file) - } pullFile(jar, sha + "/" + uri, s) + } pullFile(jar, sha + "/" + uri, cacheDir, s) } def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file) @@ -41,11 +42,18 @@ object ShaResolve { } - def pullFile(file: File, uri: String, s: TaskStreams): Unit = { - val url = remote_urlbase + "/" + uri - val fous = new java.io.FileOutputStream(file) - s.log.info("Pulling [" + url + "] to [" + file + "]") - try Http(dispatch.url(url) >>> fous) finally fous.close() + def pullFile(file: File, uri: String, cacheDir: File, s: TaskStreams): Unit = { + val cachedFile = cacheDir / uri + if (!cachedFile.exists) { + // Ensure the directory for the cache exists. + cachedFile.getParentFile.mkdirs() + val url = remote_urlbase + "/" + uri + val fous = new java.io.FileOutputStream(cachedFile) + s.log.info("Pulling [" + cachedFile + "] to cache") + try Http(dispatch.url(url) >>> fous) finally fous.close() + } + s.log.info("Pulling [" + file + "] from local cache") + IO.copyFile(cachedFile, file) } def pushFile(file: File, uri: String, user: String, pw: String): Unit = { -- cgit v1.2.3 From 47e14ed768606cf70e12996072452825919877c7 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 8 Dec 2011 14:35:07 -0500 Subject: Fixed SHA resolver to only pull necessary JAR files and ignore other directories. --- project/ShaResolve.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index 4d9579b07d..fe804b8199 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -23,8 +23,9 @@ object ShaResolve { ) def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = { + val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1") for { - (file, name) <- dir ** "*.desired.sha1" x relativeTo(dir) + (file, name) <- files x relativeTo(dir) uri = name.dropRight(13) jar = dir / uri if !jar.exists || !isValidSha(file) -- cgit v1.2.3 From 47d7bde18831de58b7099a1edbb684c7142bb163 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Fri, 9 Dec 2011 21:44:31 -0500 Subject: Parallelize sha resolution. Retreiving STARR from SHA repo is now parallel by default. YAY for parallel collections. --- project/ShaResolve.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index fe804b8199..c6034bbf01 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -25,7 +25,7 @@ object ShaResolve { def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = { val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1") for { - (file, name) <- files x relativeTo(dir) + (file, name) <- (files x relativeTo(dir)).par uri = name.dropRight(13) jar = dir / uri if !jar.exists || !isValidSha(file) -- cgit v1.2.3 From 182e6947c6bd27cf66e435292df2858609139782 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 14 Dec 2011 13:51:55 -0500 Subject: Improved error output. * One SHA resolve, now outputs lots of stack traces if parallel execution failure. --- project/ShaResolve.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index c6034bbf01..82139ee591 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -4,7 +4,7 @@ import Build._ import Keys._ import Project.Initialize import scala.collection.{ mutable, immutable } - +import scala.collection.parallel.CompositeThrowable @@ -22,7 +22,7 @@ object ShaResolve { pullBinaryLibs in ThisBuild <<= (baseDirectory, binaryLibCache, streams) map resolveLibs ) - def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = { + def resolveLibs(dir: File, cacheDir: File, s: TaskStreams): Unit = loggingParallelExceptions(s) { val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1") for { (file, name) <- (files x relativeTo(dir)).par @@ -33,6 +33,13 @@ object ShaResolve { } pullFile(jar, sha + "/" + uri, cacheDir, s) } + @inline final def loggingParallelExceptions[U](s: TaskStreams)(f: => U): U = try f catch { + case t: CompositeThrowable => + s.log.error("Error during parallel execution, GET READ FOR STACK TRACES!!") + t.throwables foreach (t2 => s.log.trace(t2)) + throw t + } + def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file) -- cgit v1.2.3 From 249be0ce81c89302b06aa3ed6444aa5bd55659db Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 14 Dec 2011 16:58:11 -0500 Subject: Fixes windows issues. --- project/ShaResolve.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'project/ShaResolve.scala') diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index 82139ee591..5cc99fd9cf 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -26,7 +26,7 @@ object ShaResolve { val files = (dir / "test" / "files" ** "*.desired.sha1") +++ (dir / "lib" ** "*.desired.sha1") for { (file, name) <- (files x relativeTo(dir)).par - uri = name.dropRight(13) + uri = name.dropRight(13).replace('\\', '/') jar = dir / uri if !jar.exists || !isValidSha(file) sha = getShaFromShafile(file) -- cgit v1.2.3 From e5961909c0775682d840aa18baabea9755c82822 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 14 Dec 2011 17:44:51 -0500 Subject: Native SHA1 calculations. * SHA1 sum calculations are now done in pure Scala. * Cache jar SHAs are checked for validity. --- .gitignore | 4 ++++ project/ShaResolve.scala | 60 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 8 deletions(-) (limited to 'project/ShaResolve.scala') diff --git a/.gitignore b/.gitignore index d392f0e82c..8b8b6978b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ *.jar +*.obj +*.log +*~ +target/ diff --git a/project/ShaResolve.scala b/project/ShaResolve.scala index 5cc99fd9cf..f54e96c0c6 100644 --- a/project/ShaResolve.scala +++ b/project/ShaResolve.scala @@ -5,7 +5,7 @@ import Keys._ import Project.Initialize import scala.collection.{ mutable, immutable } import scala.collection.parallel.CompositeThrowable - +import java.security.MessageDigest /** Helpers to resolve SHA artifacts from typesafe repo. */ @@ -30,29 +30,73 @@ object ShaResolve { jar = dir / uri if !jar.exists || !isValidSha(file) sha = getShaFromShafile(file) - } pullFile(jar, sha + "/" + uri, cacheDir, s) + } pullFile(jar, sha + "/" + uri, cacheDir, sha, s) } @inline final def loggingParallelExceptions[U](s: TaskStreams)(f: => U): U = try f catch { case t: CompositeThrowable => - s.log.error("Error during parallel execution, GET READ FOR STACK TRACES!!") + s.log.error("Error during parallel execution, GET READY FOR STACK TRACES!!") t.throwables foreach (t2 => s.log.trace(t2)) throw t } - def getShaFromShafile(file: File): String = (IO read file split "\\s" headOption) getOrElse error("No SHA found for " + file) + def getShaFromShafile(file: File): String = parseShaFile(file)._2 + + // This should calculate the SHA sum of a file the same as the linux process. + def calculateSha(file: File): String = { + val digest = MessageDigest.getInstance("SHA1") + val in = new java.io.FileInputStream(file); + val buffer = new Array[Byte](8192) + try { + def read(): Unit = in.read(buffer) match { + case x if x <= 0 => () + case size => digest.update(buffer, 0, size); read() + } + read() + } finally in.close() + val sha = convertToHex(digest.digest()) + sha + } + + // TODO - Prettier way of doing this... + private def convertToHex(data: Array[Byte]): String = { + val buf = new StringBuffer + for (i <- 0 until data.length) { + var halfbyte = (data(i) >>> 4) & 0x0F; + var two_halfs = 0; + while(two_halfs < 2) { + if ((0 <= halfbyte) && (halfbyte <= 9)) + buf.append(('0' + halfbyte).toChar) + else + buf.append(('a' + (halfbyte - 10)).toChar); + halfbyte = data(i) & 0x0F; + two_halfs += 1 + } + } + return buf.toString + } + // Parses a sha file into a file and a sha. + def parseShaFile(file: File): (File, String) = + IO.read(file).split("\\s") match { + case Array(sha, filename) if filename.startsWith("?") => (new File(file.getParentFile, filename.drop(1)), sha) + case Array(sha, filename) => (new File(file.getParentFile, filename), sha) + case _ => error(file.getAbsolutePath + " is an invalid sha file") + } + def isValidSha(file: File): Boolean = - try (Process(Seq("shasum", "-p", "--check", file.getAbsolutePath), Some(file.getParentFile)).!! contains "OK") - catch { + try { + val (jar, sha) = parseShaFile(file) + jar.exists && calculateSha(jar) == sha + } catch { case t: Exception => false } - def pullFile(file: File, uri: String, cacheDir: File, s: TaskStreams): Unit = { + def pullFile(file: File, uri: String, cacheDir: File, sha: String, s: TaskStreams): Unit = { val cachedFile = cacheDir / uri - if (!cachedFile.exists) { + if (!cachedFile.exists || calculateSha(cachedFile) != sha) { // Ensure the directory for the cache exists. cachedFile.getParentFile.mkdirs() val url = remote_urlbase + "/" + uri -- cgit v1.2.3