From 4f4a5ce29d240e700137e16fdb9156417ae63be1 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 10:55:37 -0400 Subject: this seems to fix some bug with nailgun not restarting fast enough --- cbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbt b/cbt index 5224744..867276a 100755 --- a/cbt +++ b/cbt @@ -152,8 +152,10 @@ stage1 () { if [ $use_nailgun -eq 0 ]; then echo "Stopping nailgun" 1>&2 $NG ng-stop >> $nailgun_out 2>> $nailgun_err & + sleep 1 echo "Restarting nailgun" 1>&2 ng-server 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & + sleep 1 fi fi -- cgit v1.2.3 From 99f624011b3ae0e6088ffcb86dc414e3750ad5c9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 10:57:09 -0400 Subject: compute logged times during startup as delta from beginning --- cbt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cbt b/cbt index 867276a..a893dff 100755 --- a/cbt +++ b/cbt @@ -8,6 +8,19 @@ # - reduction of dependencies # - performance improvements +start_seconds=$(gdate +"%s") +start_nanos=1$(gdate +"%N") + +time_taken() { + i=$(( $(gdate +"%s") - start_seconds )) + n=$(( $(( 1$(gdate +"%N") - start_nanos )) / 1000000 )) + if [[ ( "$n" < 0 ) ]]; then + i=$(( i-1 )) + n=$(( n+1000 )) + fi + echo "$i.$n" +} + # utility function to log message to stderr with stating the time log () { msg=$1 @@ -23,8 +36,8 @@ log () { which gdate 2>&1 > /dev/null gdate_installed=$? if [ $gdate_installed -eq 0 ]; then - i=`gdate +"%S.%N"` - echo "[$i] $msg" 1>&2 + delta=$(time_taken) + echo "[$delta] $msg" 1>&2 fi fi } -- cgit v1.2.3 From 51a353881de307aec8aa7c7f8eb8a83856132633 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 10:58:02 -0400 Subject: Do not check javac version (took > 0.1 seconds each time). This will lead to compile errors with java6, but those should be easy to google. --- cbt | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cbt b/cbt index a893dff..b83d7bc 100755 --- a/cbt +++ b/cbt @@ -47,16 +47,22 @@ log "Checking for dependencies" $* which javac 2>&1 > /dev/null javac_installed=$? if [ ! $javac_installed -eq 0 ]; then - echo "You need to install javac! CBT needs it to bootstrap from Java sources into Scala." 1>&2 - exit 1 -fi -javac_version=$(javac -version 2>&1 | cut -d ' ' -f 2) -javac_version_minor=$(echo -n $javac_version | cut -d '.' -f 2) -if [ ! "$javac_version_minor" -ge "7" ]; then - echo "You need to install javac version 1.7 or greater!" 2>&1 - echo "Current javac version is $javac_version" 2>&1 + echo "You need to install javac 1.7 or later! CBT needs it to bootstrap from Java sources into Scala." 1>&2 exit 1 fi + +# log "cutting javac version" $* +# javac_version=$(javac -version 2>&1) # e.g. "javac 1.8.0_u60" +# javac_version_update=${javac_version/javac 1./} # e.g. "8.0_u60" +# javac_version_minor_pointed=${javac_version_update%_*} # e.g. "8.0" +# javac_version_minor=${javac_version_minor_pointed%.*} # e.g. "8" +# log "cutting javac version done" $* +# if [ ! "$javac_version_minor" -ge "7" ]; then +# echo "You need to install javac version 1.7 or greater!" 2>&1 +# echo "Current javac version is $javac_version" 2>&1 +# exit 1 +# fi + which ng 2>&1 > /dev/null ng_installed=$? which ng-server 2>&1 > /dev/null -- cgit v1.2.3 From 13c0f445f48fbea1398ca6340ea4cdd8dcb6bfb0 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 18:55:44 -0400 Subject: add important comment --- stage1/resolver.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 694bd83..b19255a 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -149,9 +149,11 @@ abstract class Dependency{ new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) } - def linearize(deps: Seq[Dependency]): Seq[Dependency] = + private def linearize(deps: Seq[Dependency]): Seq[Dependency] = + // Order is important here in order to generate the correct lineraized dependency order for EarlyDependencies + // (and maybe this as well in case we want to get rid of MultiClassLoader) if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) - + private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ -- cgit v1.2.3 From 56d780d05604cb8bb49c691889c398f1e570cb91 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 18:58:04 -0400 Subject: Speedup transitiveDependencies computation by removing duplicate versions and doing less string processing --- stage1/resolver.scala | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index b19255a..ab2a18f 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -157,10 +157,16 @@ abstract class Dependency{ private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - val deps = linearize(dependencies) - val hasInfo = deps.collect{ case d:ArtifactInfo => d } - val noInfo = deps.filter{ - case _:ArtifactInfo => false + // FIXME: this is probably wrong too eager. + // We should consider replacing versions during traversals already + // not just replace after traversals, because that could mean we + // pulled down dependencies current versions don't even rely + // on anymore. + + val deps: Seq[Dependency] = linearize(dependencies).reverse.distinct.reverse + val hasInfo: Seq[Dependency with ArtifactInfo] = deps.collect{ case d:Dependency with ArtifactInfo => d } + val noInfo: Seq[Dependency] = deps.filter{ + case _:Dependency with ArtifactInfo => false case _ => true } noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct @@ -429,9 +435,9 @@ case class BoundMavenDependency( } object BoundMavenDependency{ def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java - def semanticVersionLessThan(left: String, right: String) = { + def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = { // FIXME: this ignores ends when different size - val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt) + val zipped = left zip right val res = zipped.map { case (Left(i),Left(j)) => i compare j case (Right(i),Right(j)) => i compare j @@ -447,13 +453,16 @@ object BoundMavenDependency{ } /* this obviously should be overridable somehow */ def updateOutdated( - deps: Seq[ArtifactInfo], - versionLessThan: (String, String) => Boolean = semanticVersionLessThan - )(implicit logger: Logger): Seq[ArtifactInfo] = { + deps: Seq[Dependency with ArtifactInfo], + versionLessThan: (Array[Either[Int,String]], Array[Either[Int,String]]) => Boolean = semanticVersionLessThan + )(implicit logger: Logger): Seq[Dependency with ArtifactInfo] = { val latest = deps .groupBy( d => (d.groupId, d.artifactId) ) .mapValues( - _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) + _.groupBy(_.version) // remove duplicates + .map( _._2.head ) + .toVector + .sortBy( _.version.split("\\.|\\-").map(toInt) )( Ordering.fromLessThan(versionLessThan) ) .last ) deps.map{ -- cgit v1.2.3 From 2277875205e69b6e4f527f341e1dcf16a8495326 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Tue, 26 Apr 2016 19:00:07 -0400 Subject: remove dead code --- stage1/resolver.scala | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/stage1/resolver.scala b/stage1/resolver.scala index ab2a18f..c2855c5 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -9,14 +9,6 @@ import paths._ import scala.concurrent._ import scala.concurrent.duration._ -private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ - lazy val children = computeChildren - def linearize: Seq[Dependency] = root +: children.flatMap(_.linearize) - def show(indent: Int = 0): Stream[Char] = { - (" " * indent ++ root.show ++ "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _) - } -} - trait ArtifactInfo extends Dependency{ def artifactId: String def groupId: String @@ -144,11 +136,6 @@ abstract class Dependency{ def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) ) def dependencies: Seq[Dependency] - private def resolveRecursive(parents: List[Dependency] = List()): Tree = { - // diff removes circular dependencies - new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) - } - private def linearize(deps: Seq[Dependency]): Seq[Dependency] = // Order is important here in order to generate the correct lineraized dependency order for EarlyDependencies // (and maybe this as well in case we want to get rid of MultiClassLoader) -- cgit v1.2.3 From 232de4be3fb2169070eeb72804c5a4d5500094f9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:17:13 -0400 Subject: add commented out test case for resolving maven version ranges --- test/simple/build/build.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index 173c1da..5782c2d 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -17,6 +17,9 @@ class Build(context: cbt.Context) extends BasicBuild(context){ MavenDependency("com.spotify", "missinglink-core", "0.1.1"), // the below tests pom inheritance with variable substitution being parts of strings MavenDependency("cc.factorie","factorie_2.11","1.2") + // the dependency below uses a maven version range. Currently not supported. + // TODO: put in a proper error message for version range not supported + //MavenDependency("com.github.nikita-volkov", "sext", "0.2.4") ), MavenRepository.combine( MavenRepository.central, -- cgit v1.2.3 From a8b87e23767f6fc7022152eb3b6784d42c93a33b Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:19:57 -0400 Subject: Wrong use of classifier --- stage2/SbtDependencyDsl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/SbtDependencyDsl.scala b/stage2/SbtDependencyDsl.scala index 4fd4250..1ecf72c 100644 --- a/stage2/SbtDependencyDsl.scala +++ b/stage2/SbtDependencyDsl.scala @@ -13,7 +13,7 @@ trait SbtDependencyDsl{ self: Build => def %(artifactId: String) = new DependencyBuilder2( groupId, artifactId, None ) } implicit class DependencyBuilder3(d: MavenDependency){ - def %(classifier: String) = d.copy(classifier = Classifier(Some(classifier))) + def %(classifier: String): MavenDependency = d//.copy(classifier = Classifier(Some(classifier))) } /* -- cgit v1.2.3 From 63d1076783f3d39876c829108c335661948010b5 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:32:35 -0400 Subject: Make sonatype credentials customizable from Build class rather than hard coded --- stage2/Lib.scala | 17 ++++++++--------- stage2/PublishBuild.scala | 17 +++++++++++++++-- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/stage2/Lib.scala b/stage2/Lib.scala index cebdb92..6894073 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -340,14 +340,14 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ else items.map(projection) } - def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = { if(sourceFiles.nonEmpty){ val files = artifacts.map(nameAndContents) - uploadAll(url, files) + uploadAll(url, files, credentials) } } - def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = { // TODO: make concurrency configurable here if(sourceFiles.nonEmpty){ val files = (artifacts ++ artifacts.map(sign)).map(nameAndContents) @@ -358,15 +358,15 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ ) } val all = (files ++ checksums) - uploadAll(url, all) + uploadAll(url, all, credentials) } } - def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])]): Unit = - nameAndContents.map{ case(name, content) => upload(name, content, url) } + def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])], credentials: String ): Unit = + nameAndContents.map{ case(name, content) => upload(name, content, url, credentials: String ) } - def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL): Unit = { + def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL, credentials: String): Unit = { import java.net._ import java.io._ logger.task("uploading "++fileName) @@ -374,8 +374,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val httpCon = url.openConnection.asInstanceOf[HttpURLConnection] httpCon.setDoOutput(true) httpCon.setRequestMethod("PUT") - val userPassword = new String(readAllBytes(sonatypeLogin.toPath)).trim - val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes) + val encoding = new sun.misc.BASE64Encoder().encode(credentials.getBytes) httpCon.setRequestProperty("Authorization", "Basic " ++ encoding) httpCon.setRequestProperty("Content-Type", "application/binary") httpCon.getOutputStream.write( diff --git a/stage2/PublishBuild.scala b/stage2/PublishBuild.scala index 6b85b22..cc4f5e5 100644 --- a/stage2/PublishBuild.scala +++ b/stage2/PublishBuild.scala @@ -1,6 +1,7 @@ package cbt import java.io.File import java.net.URL +import java.nio.file.Files.readAllBytes import scala.collection.immutable.Seq abstract class PublishBuild(context: Context) extends PackageBuild(context){ @@ -37,10 +38,22 @@ abstract class PublishBuild(context: Context) extends PackageBuild(context){ def snapshotUrl = new URL("https://oss.sonatype.org/content/repositories/snapshots") def releaseUrl = new URL("https://oss.sonatype.org/service/local/staging/deploy/maven2") override def copy(context: Context) = super.copy(context).asInstanceOf[PublishBuild] + + protected def sonatypeCredentials = { + // FIXME: this should probably not use cbtHome, but some reference to the system's host cbt + new String(readAllBytes((context.cbtHome ++ "/sonatype.login").toPath)).trim + } + def publishSnapshot: Unit = { val snapshotBuild = copy( context.copy(version = Some(version+"-SNAPSHOT")) ) val files = snapshotBuild.pom +: snapshotBuild.`package` - lib.publishSnapshot(sourceFiles, files, snapshotUrl ++ releaseFolder ) + lib.publishSnapshot( + sourceFiles, files, snapshotUrl ++ releaseFolder, sonatypeCredentials + ) + } + def publishSigned: Unit = { + lib.publishSigned( + sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder, sonatypeCredentials + ) } - def publishSigned: Unit = lib.publishSigned(sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder ) } -- cgit v1.2.3 From 74ddcf55ccf0fb4c87b4b0ea06f95a6a419f262f Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:34:52 -0400 Subject: move soure file filterin logic into library for reusability --- stage2/BasicBuild.scala | 10 ++-------- stage2/Lib.scala | 9 +++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 9f9dbdc..46f9d5a 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -63,16 +63,10 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe def compileStatusFile: File = compileTarget ++ ".last-success" /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */ - def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter) - - /** Which file endings to consider being source files. */ - def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") + def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(lib.sourceFileFilter) /** Absolute path names for all individual files found in sources directly or contained in directories. */ - final def sourceFiles: Seq[File] = for { - base <- sources.filter(_.exists).map(lib.realpath) - file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file) - } yield file + final def sourceFiles: Seq[File] = lib.sourceFiles(sources) protected def assertSourceDirectories(): Unit = { val nonExisting = diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 6894073..3c5ff3a 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -209,6 +209,15 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ def basename(path: File): String = path.toString.stripSuffix("/").split("/").last def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/")) def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString)) + /** Which file endings to consider being source files. */ + def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") + + def sourceFiles( sources: Seq[File], sourceFileFilter: File => Boolean = sourceFileFilter ): Seq[File] = { + for { + base <- sources.filter(_.exists).map(lib.realpath) + file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file) + } yield file + } def jarFile( jarFile: File, files: Seq[File] ): Option[File] = { if( files.isEmpty ){ -- cgit v1.2.3 From 820e06bc132dc4363a6cc7869ccc96c830e17018 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:35:17 -0400 Subject: from from File to Path directly --- stage2/Lib.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 3c5ff3a..aa98e7c 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -208,7 +208,8 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ // file system helpers def basename(path: File): String = path.toString.stripSuffix("/").split("/").last def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/")) - def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString)) + def nameAndContents(file: File) = basename(file) -> readAllBytes(file.toPath) + /** Which file endings to consider being source files. */ def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") @@ -238,7 +239,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val entry = new JarEntry( name ) entry.setTime(file.lastModified) jar.putNextEntry(entry) - jar.write( readAllBytes( Paths.get(file.toString) ) ) + jar.write( readAllBytes( file.toPath ) ) jar.closeEntry name } -- cgit v1.2.3 From 797855bbad9df5bda34884a4c518b6145130a9bd Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:38:03 -0400 Subject: make cacheabilty of git dependency depend on cacheability of it's build. Also separate out regex, so we can use it to detect cbt version urls later --- stage2/GitDependency.scala | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index 16423df..333d81c 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -5,25 +5,26 @@ import scala.collection.immutable.Seq import org.eclipse.jgit.api._ import org.eclipse.jgit.lib.Ref +object GitDependency{ + val GitUrl = "(git:|https:|file:/)//([^/]+)/(.+)".r +} case class GitDependency( url: String, ref: String // example: git://github.com/cvogt/cbt.git# )(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends Dependency{ + import GitDependency._ override def lib = new Lib(logger) - override def canBeCached = true + override def canBeCached = dependency.canBeCached // TODO: add support for authentication via ssh and/or https // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ + private val GitUrl( _, domain, path ) = url + - private val GitUrl = "(git|https)://([^/]+)/(.+)".r - private val GitUrl( _, domain, path ) = url - - private object dependenciesCache extends Cache[Seq[Dependency]] - def dependencies = dependenciesCache{ - val checkoutDirectory = paths.cbtHome ++ s"/cache/git/$domain/$path/$ref" + def checkout: File = { + val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" if(checkoutDirectory.exists){ logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") } else { - logger.git(s"Cloning $url into $checkoutDirectory") val git = Git.cloneRepository() @@ -35,14 +36,16 @@ case class GitDependency( git.checkout() .setName(ref) .call() - } - val managedBuild = lib.loadDynamic( - context.copy( projectDirectory = checkoutDirectory, args = Seq() ) - ) - Seq( managedBuild ) + checkoutDirectory + } + private object dependencyCache extends Cache[Dependency] + def dependency = dependencyCache{ + BuildDependency( context.copy( projectDirectory = checkout ) ) } + def dependencies = Seq(dependency) + def exportedClasspath = ClassPath(Seq()) def exportedJars = Seq() private[cbt] def targetClasspath = exportedClasspath -- cgit v1.2.3 From 373a16fef9dd00dbbd9b1fa973c1bd5f51994fc6 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:40:05 -0400 Subject: Build mixin with recommended settings --- stage2/BasicBuild.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 46f9d5a..b489947 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -11,6 +11,17 @@ import java.util.jar._ import scala.collection.immutable.Seq import scala.util._ +trait Recommended extends BasicBuild{ + override def scalacOptions = super.scalacOptions ++ Seq( + "-feature", + "-deprecation", + "-unchecked", + "-language:postfixOps", + "-language:implicitConversions", + "-language:higherKinds", + "-language:existentials" + ) +} class BasicBuild( context: Context ) extends Build( context ) class Build(val context: Context) extends Dependency with TriggerLoop with SbtDependencyDsl{ // library available to builds @@ -107,7 +118,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe // ========== compile, run, test ========== /** scalac options used for zinc and scaladoc */ - def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" ) + def scalacOptions: Seq[String] = Seq() private object needsUpdateCache extends Cache[Boolean] def needsUpdate: Boolean = needsUpdateCache( -- cgit v1.2.3 From b54f3fd4c4d9601a5f5bfc6d4ed881a15f37a472 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:41:55 -0400 Subject: only run tests if there are actually tests to run (useful for recursively running tests in nested projects) --- stage2/BasicBuild.scala | 4 +++- stage2/Lib.scala | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index b489947..65db8a4 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -139,7 +139,9 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe def runClass: String = "Main" def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader(context.classLoaderCache) ) - def test: ExitCode = lib.test(context) + def test: Option[ExitCode] = { + lib.test(context) + } /* context.logger.composition(">"*80) diff --git a/stage2/Lib.scala b/stage2/Lib.scala index aa98e7c..d7456e1 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -108,19 +108,21 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } } - def test( context: Context ): ExitCode = { - val loggers = logger.enabledLoggers.mkString(",") - // FIXME: this is a hack to pass logger args on to the tests. - // should probably have a more structured way - val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None - - logger.lib(s"invoke testDefault( $context )") - val exitCode: ExitCode = loadDynamic( - context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), - new Build(_) with mixins.Test - ).run - logger.lib(s"return testDefault( $context )") - exitCode + def test( context: Context ): Option[ExitCode] = { + if((context.projectDirectory ++ "/test").exists){ + val loggers = logger.enabledLoggers.mkString(",") + // FIXME: this is a hack to pass logger args on to the tests. + // should probably have a more structured way + val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None + + logger.lib(s"invoke testDefault( $context )") + val exitCode: ExitCode = loadDynamic( + context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), + new BasicBuild(_) with mixins.Test + ).run.asInstanceOf[ExitCode] // FIXME + logger.lib(s"return testDefault( $context )") + Some(exitCode) + } else None } // task reflection helpers -- cgit v1.2.3 From 3ccf29c263e6e449b3b5d86c3910840c467261d9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:43:19 -0400 Subject: print null as root class loader in URLClassLoader --- stage1/URLClassLoader.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stage1/URLClassLoader.scala b/stage1/URLClassLoader.scala index 9e96992..5014cee 100644 --- a/stage1/URLClassLoader.scala +++ b/stage1/URLClassLoader.scala @@ -17,8 +17,8 @@ class URLClassLoader( classPath: ClassPath, parent: ClassLoader )( implicit val ++ ( getURLs.map(_.toString).sorted.mkString(",\n") ++ ( - if(getParent() != ClassLoader.getSystemClassLoader()) - ",\n" ++ getParent().toString + if(getParent() != ClassLoader.getSystemClassLoader().getParent()) + ",\n" ++ Option(getParent()).map(_.toString).getOrElse("null") else "" ) ).split("\n").map(" "++_).mkString("\n") -- cgit v1.2.3 From 17da8b5c94c3de3053654d83948df630ffe0d0c2 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:44:11 -0400 Subject: minor cleanup, better logging for downloads --- stage1/Stage1Lib.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 985200e..cb84310 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -67,9 +67,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def download(url: URL, target: File, sha1: Option[String]): Boolean = { if( target.exists ){ + logger.resolver(green("found ") ++ url.string) true } else { - val incomplete = Paths.get( target.string ++ ".incomplete" ); + val incomplete = ( target ++ ".incomplete" ).toPath; val connection = url.openConnection.asInstanceOf[HttpURLConnection] if(connection.getResponseCode != HttpURLConnection.HTTP_OK){ logger.resolver(blue("not found: ") ++ url.string) @@ -91,7 +92,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ assert( expected == actual, s"$expected == $actual" ) logger.resolver( green("verified") ++ " checksum for " ++ target.string) } - Files.move(incomplete, Paths.get(target.string), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + Files.move(incomplete, target.toPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); true } } -- cgit v1.2.3 From b37ccc12d3d6e8fc5fcf2730fc498f5bb8b08655 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:45:56 -0400 Subject: if compiler crashed, print commands to reproduce it outside of cbt --- stage1/Stage1Lib.scala | 57 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index cb84310..f484bbf 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -193,21 +193,50 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val start = System.currentTimeMillis - val code = redirectOutToErr{ - lib.runMain( - "com.typesafe.zinc.Main", - Seq( - "-scala-compiler", scalaCompiler.toString, - "-scala-library", scalaLibrary.toString, - "-sbt-interface", sbtInterface.toString, - "-compiler-interface", compilerInterface.toString, - "-scala-extra", scalaReflect.toString, - "-cp", cp, - "-d", compileTarget.toString - ) ++ scalacOptions.map("-S"++_) ++ files.map(_.toString), - zinc.classLoader(classLoaderCache) + val _class = "com.typesafe.zinc.Main" + val dualArgs = + Seq( + "-scala-compiler", scalaCompiler.toString, + "-scala-library", scalaLibrary.toString, + "-sbt-interface", sbtInterface.toString, + "-compiler-interface", compilerInterface.toString, + "-scala-extra", scalaReflect.toString, + "-d", compileTarget.toString ) - } + val singleArgs = scalacOptions.map( "-S" ++ _ ) + + val code = + try{ + redirectOutToErr{ + lib.runMain( + _class, + dualArgs ++ singleArgs ++ Seq( + "-cp", cp // let's put cp last. It so long + ) ++ files.map(_.toString), + zinc.classLoader(classLoaderCache) + ) + } + } catch { + case e: Exception => + System.err.println(red("The Scala compiler crashed. Try running it by hand:")) + System.out.println(s""" +java -cp \\ +${zinc.classpath.strings.mkString(":\\\n")} \\ +\\ +${_class} \\ +\\ +${dualArgs.grouped(2).map(_.mkString(" ")).mkString(" \\\n")} \\ +\\ +${singleArgs.mkString(" \\\n")} \\ +\\ +-cp \\ +${classpath.strings.mkString(":\\\n")} \\ +\\ +${files.sorted.mkString(" \\\n")} +""" + ) + ExitCode.Failure + } if(code == ExitCode.Success){ // write version and when last compilation started so we can trigger -- cgit v1.2.3 From 1de728fc751669fb08b5263d5ba4df11b64afda0 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:51:14 -0400 Subject: for better performance do not generate exceptions during class loading for control flow, but use null instead --- nailgun_launcher/CBTUrlClassLoader.java | 4 ++++ stage1/CachingClassLoader.scala | 9 +++++++-- stage1/MultiClassLoader.scala | 5 ++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java index c05391b..0ba8a61 100644 --- a/nailgun_launcher/CBTUrlClassLoader.java +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -3,6 +3,7 @@ import java.io.*; import java.net.*; import java.util.*; import static cbt.Stage0Lib.*; +import java.util.concurrent.ConcurrentHashMap; class CbtURLClassLoader extends java.net.URLClassLoader{ public String toString(){ return ( @@ -15,6 +16,9 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ ); } public Class loadClass(String name) throws ClassNotFoundException{ + Class _class = super.loadClass(name); + if(_class == null) throw new ClassNotFoundException(name); + else return _class; //System.out.println("loadClass("+name+") on \n"+this); return super.loadClass(name); } diff --git a/stage1/CachingClassLoader.scala b/stage1/CachingClassLoader.scala index e75f14c..4ddebda 100644 --- a/stage1/CachingClassLoader.scala +++ b/stage1/CachingClassLoader.scala @@ -5,8 +5,13 @@ import scala.util.Try trait CachingClassLoader extends ClassLoader{ def logger: Logger - val cache = new KeyLockedLazyCache[String,Try[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) + val cache = new KeyLockedLazyCache[String,Option[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) override def loadClass(name: String, resolve: Boolean) = { - cache.get( name, Try(super.loadClass(name, resolve)) ).get + cache.get( name, Try(super.loadClass(name, resolve)).toOption ).getOrElse(null) + } + override def loadClass(name: String) = { + val _class = super.loadClass(name) + if(_class == null) throw new ClassNotFoundException(name) + else _class } } diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index 5a93a63..cce4cd5 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -8,14 +8,13 @@ class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) e override def findClass(name: String) = { parents.find( parent => try{ - parent.loadClass(name) - true + null != parent.loadClass(name) // FIXME: is it correct to just ignore the resolve argument here? } catch { case _:ClassNotFoundException => false } ).map( _.loadClass(name) - ).getOrElse( throw new ClassNotFoundException(name) ) + ).getOrElse( null ) } override def toString = ( scala.Console.BLUE -- cgit v1.2.3 From 7f9380157a5cf95e767f74d6f7c52da24ea8f51f Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:52:21 -0400 Subject: add caching to CBTUrlClassLoader (so we can eventually wrap the system class loader into it to not produce multiple exceptions in a classloader tree, where a class is not in the systemclassloader) --- nailgun_launcher/CBTUrlClassLoader.java | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java index 0ba8a61..9c41978 100644 --- a/nailgun_launcher/CBTUrlClassLoader.java +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -15,12 +15,31 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ + "\n)" ); } + ClassLoaderCache2 cache = new ClassLoaderCache2( + new ConcurrentHashMap(), + new ConcurrentHashMap() + ); public Class loadClass(String name) throws ClassNotFoundException{ Class _class = super.loadClass(name); if(_class == null) throw new ClassNotFoundException(name); else return _class; + } + public Class loadClass(String name, Boolean resolve) throws ClassNotFoundException{ //System.out.println("loadClass("+name+") on \n"+this); - return super.loadClass(name); + if(!cache.contains(name)) + try{ + cache.put(super.loadClass(name, resolve), name); + } catch (ClassNotFoundException e){ + cache.put(Object.class, name); + } + Class _class = cache.get(name); + if(_class == Object.class){ + if( name == "java.lang.Object" ) + return Object.class; + else return null; + } else { + return _class; + } } void assertExist(URL[] urls){ for(URL url: urls){ -- cgit v1.2.3 From f946be87b81cc9cb040ab5e3f8a5195a915ad1e4 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:55:00 -0400 Subject: minor improvements to docs, imports, visibilities --- DEVELOPER_GUIDE.txt | 2 +- README.md | 6 +++--- stage1/Cache.scala | 2 +- stage1/MultiClassLoader.scala | 1 - stage1/cbt.scala | 2 ++ stage1/resolver.scala | 11 ++++++----- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/DEVELOPER_GUIDE.txt b/DEVELOPER_GUIDE.txt index 0e20339..e42277e 100644 --- a/DEVELOPER_GUIDE.txt +++ b/DEVELOPER_GUIDE.txt @@ -1,6 +1,6 @@ Welcome developer. -CBT has a very easy code base that you can fully master it in an afternoon. +CBT has a very easy code base that you can fully master in very little time. Don't shy away from submiting PRs :). And because CBT bootstraps from source you already have the code there. diff --git a/README.md b/README.md index 84259da..2485518 100644 --- a/README.md +++ b/README.md @@ -71,16 +71,16 @@ You can see how your build is configured via overrides. call `cbt` to see a full list of available commands for this build. -Look into the class `DefaultBuild` in CBT's source code to see their +Look into the class PackageBuild (and it's super class BasicBuild) in CBT's source code to see their details. The source code is really simple. Don't shy away from looking, even as a beginner. No crazy stuff, I promise ;). You -can find the relevant code in CBT's `stage2/DefaultBuild.scala` +can find the relevant code in CBT's stage2/BasicBuild.scala I order to keep executing the same command triggered by file changes use `cbt loop `. You can find example builds in CBT's own `test/` folder. Not all of them have a build file, in which case CBT uses the default -`cbt.DefaultBuild`. +cbt.BasicBuild. A folder `build/` can have its own folder `build/` inside in order to add source or maven dependencies to your build. Eventually diff --git a/stage1/Cache.scala b/stage1/Cache.scala index 6e6b9eb..a8036e5 100644 --- a/stage1/Cache.scala +++ b/stage1/Cache.scala @@ -1,6 +1,6 @@ package cbt /** -Caches exactly one value +Caches exactly one value. Is there a less boiler-platy way to achieve this, that doesn't require creating an instance for each thing you want to cache? */ diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index cce4cd5..74e65aa 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -1,6 +1,5 @@ package cbt import java.net._ -import scala.util.Try import scala.collection.immutable.Seq // do not make this a case class, required object identity equality diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 01af0d5..7b8b632 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -2,6 +2,8 @@ package cbt import java.io._ import java.nio.file._ import java.net._ +import scala.collection.immutable.Seq + object `package`{ private val lib = new BaseLib implicit class FileExtensionMethods( file: File ){ diff --git a/stage1/resolver.scala b/stage1/resolver.scala index c2855c5..ad6df23 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -159,7 +159,7 @@ abstract class Dependency{ noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct } - def show: String = this.getClass.getSimpleName + override def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = dependencyTreeRecursion() private def dependencyTreeRecursion(indent: Int = 0): String = ( @@ -296,6 +296,7 @@ case class BoundMavenDependency( import scala.collection.JavaConversions._ private def resolve(suffix: String, hash: Option[String]): File = { + logger.resolver("Resolving "+this) val file = mavenCache ++ basePath ++ "." ++ suffix val urls = repositories.map(_ ++ basePath ++ "." ++ suffix) urls.find( @@ -325,10 +326,10 @@ case class BoundMavenDependency( private object pomCache extends Cache[File] def pom: File = pomCache{ resolve("pom", Some(pomSha1)) } - def pomXml = XML.loadFile(pom.string) + private def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== - lazy val transitivePom: Seq[BoundMavenDependency] = { + private lazy val transitivePom: Seq[BoundMavenDependency] = { (pomXml \ "parent").collect{ case parent => BoundMavenDependency( @@ -342,7 +343,7 @@ case class BoundMavenDependency( }.flatMap(_.transitivePom) :+ this } - lazy val properties: Map[String, String] = ( + private lazy val properties: Map[String, String] = ( transitivePom.flatMap{ d => val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text @@ -352,7 +353,7 @@ case class BoundMavenDependency( } ).toMap - lazy val dependencyVersions: Map[String, (String,String)] = + private lazy val dependencyVersions: Map[String, (String,String)] = transitivePom.flatMap( p => (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ -- cgit v1.2.3 From e9db891b2c5b9979b20f347387705d9997e71fb8 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:56:23 -0400 Subject: propagate time taken from bash script into logger --- cbt | 4 ++-- nailgun_launcher/NailgunLauncher.java | 4 +++- stage1/logger.scala | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cbt b/cbt index b83d7bc..5d02917 100755 --- a/cbt +++ b/cbt @@ -184,7 +184,7 @@ stage1 () { then log "Running JVM directly" $* # -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:5005 - java -cp $NAILGUN$TARGET cbt.NailgunLauncher "$CWD" $* + java -Xmx6072m -Xss10M -cp $NAILGUN$TARGET cbt.NailgunLauncher $(time_taken) "$CWD" $* else log "Running via nailgun." $* for i in 0 1 2 3 4 5 6 7 8 9; do @@ -207,7 +207,7 @@ stage1 () { sleep 0.3 done log "Running CBT via Nailgun." $* - $NG cbt.NailgunLauncher "$CWD" $* + $NG cbt.NailgunLauncher $(time_taken) "$CWD" $* fi exitCode=$? log "Done running CBT." $* diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index a12d059..d3c7825 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -44,7 +44,6 @@ public class NailgunLauncher{ MalformedURLException, IOException, NoSuchAlgorithmException { - long start = System.currentTimeMillis(); //System.err.println("ClassLoader: "+stage1classLoader); //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); //System.err.println("now: "+now); @@ -54,11 +53,14 @@ public class NailgunLauncher{ _assert(TARGET != null, TARGET); _assert(STAGE1 != null, STAGE1); + Long _start = System.currentTimeMillis(); if(args[0].equals("check-alive")){ System.exit(33); return; } + String[] diff = args[0].split("\\."); + long start = _start - (Long.parseLong(diff[0]) * 1000L) - Long.parseLong(diff[1]); List stage1SourceFiles = new ArrayList(); for( File f: new File(STAGE1).listFiles() ){ if( f.isFile() && f.toString().endsWith(".scala") ){ diff --git a/stage1/logger.scala b/stage1/logger.scala index c21dc86..1e0a693 100644 --- a/stage1/logger.scala +++ b/stage1/logger.scala @@ -12,7 +12,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { def log(name: String, msg: => String) = { val timeTaken = ((System.currentTimeMillis.toDouble - start) / 1000).toString - System.err.println( s"[${" "*(6-timeTaken.size)}$timeTaken][$name] $msg" ) + System.err.println( s"[$timeTaken][$name] $msg" ) } def showInvocation(method: String, args: Any) = method ++ "( " ++ args.toString ++ " )" -- cgit v1.2.3 From f515fc42cc95f8638e39d0c1c99882192118c9e2 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 09:56:53 -0400 Subject: move version numbers to Scala land (to reduce Java code size) --- nailgun_launcher/NailgunLauncher.java | 3 --- stage1/constants.scala | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index d3c7825..8838543 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -13,9 +13,6 @@ import static cbt.Stage0Lib.*; * dependencies outside the JDK. */ public class NailgunLauncher{ - public static String SCALA_VERSION = "2.11.8"; - public static String SCALA_XML_VERSION = "1.0.5"; - public static String ZINC_VERSION = "0.3.9"; public static String CBT_HOME = System.getenv("CBT_HOME"); public static String NAILGUN = System.getenv("NAILGUN"); diff --git a/stage1/constants.scala b/stage1/constants.scala index 4c39237..437cf19 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,7 +1,7 @@ package cbt object constants{ - val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION - val scalaVersion = NailgunLauncher.SCALA_VERSION - val zincVersion = NailgunLauncher.ZINC_VERSION + val scalaXmlVersion = "1.0.5" + val scalaVersion = "2.11.8" + val zincVersion = "0.3.9" val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } -- cgit v1.2.3 From 9951f3f3e65337d2ca567ffb1760a6545fe14998 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 22:30:42 -0400 Subject: Add debugging tips to the dev docs --- DEVELOPER_GUIDE.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/DEVELOPER_GUIDE.txt b/DEVELOPER_GUIDE.txt index e42277e..a26d760 100644 --- a/DEVELOPER_GUIDE.txt +++ b/DEVELOPER_GUIDE.txt @@ -1,17 +1,28 @@ Welcome developer. -CBT has a very easy code base that you can fully master in very little time. +CBT has a very easy code base that's easy to master. Don't shy away from submiting PRs :). And because CBT bootstraps from source you already have the code there. +The only tricky parts are class loading and cache invalidation. Most changes +will not need to interfere with this though. + The ./cbt bash script starts the process. You currently need javac, nailgun, gpg and realpath or gcc installed. +If you have any troubles with class not found, method not found, +abstract method error, NullPointerException, etc. +Try `killall -KILL java`. To restart nailgun. +Or try `cbt direct ` to circumvent nailgun. +It can also help to delete all target folders `find .|grep target\$|xargs rm -rf` +inside of CBT. Or (almost never) the `cache/` directory. + CBT's directory structure cbt Shell script launching cbt. Can be symlinked. -bootstrap_scala/ Self-contained downloader for the core Scala jars. Allows bootstrapping from Java into Scala. +compatibility/ Java interfaces that all CBT versions are source compatible to. For communication + between composed builds of different versions. nailgun_launcher/ Self-contained helper that allows using Nailgun with minimal permanent classpath. (Is this actually needed?) realpath/ Self-contained realpath source code to correctly figure our CBTs home directory. (Open for replacement ideas.) stage1/ CBT's code that only relies only on Scala/Java built-ins. Contains a Maven resolver to download libs for stage2. -- cgit v1.2.3 From 53247b5610b0168a3dd93d3d8f1224b78995ecde Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 23:35:58 -0400 Subject: Reproducible builds, composing different CBT version and various improvements One large commit, because it is was hard to do these things in isolation or to separate them now. CBT now knows how to load other versions of itself - Support for reproducible builds (!), by providing a CBT git URL and hash to tie build to - Support for composing builds using different CBT versions (!) - introduce (in compatibility/) Java interfaces all CBT versions need to stay compatible with, so they can talk to each other. And put extension methods to these interfaces in cbt package object Class loading - add some sanity checks for class loading - improve class loader invalidation to fix bugs - implement caching in Java land class loaders. In particular to prevent the system class loader to repeatedly generate ClassNotFound exceptions in each sink of the class loader DAG for non JDK classes (meaning major speed up for projects with many classes). - getting rid of transient class loader cache unifying into "persistent" one instead (which is still wrong as invalidation eventually needs to invalidate entire sub graphs of the class loading DAG, not single class loaders. Seems like we'll have to abandon the hashmap based approach and tie caching to dependency objects) Other Caching - cache dependencies extracted from xml files, which was one major time killer, but invalidate cache when cbt changed (maven dependency user facing api needs simplification now!) - memorize last successful compile time in the file system rather than memory, to guard against unnecessary recompiling even across launches (or when using cbt direct) Structural improvements - Factor out ClassLoaderCache on Java land into its own class. - Port MultiClassLoader to Java land, to better compose classloaders in NailgunLauncher. - Remove many global constants and variables (in object paths and in NailgunLauncher) and pass them through instead. Needed for composing of builds. - move more code from resolver into Lib for less entanglement with classes (needed to compatibility interfaces) and better re-usability - remove canBeCached. Everything can be cached now, but we need to be careful about correct invalidation. - remove build announcing produced jars. We can add if ever needed. - change callNullary to return exit code instead of Unit as preparation for next commit introducing "recursive" ScalaTest - Makes ScalaTest support work (still a bit too inflexible, but mostly works well) --- README.md | 2 +- build/build.scala | 14 +- compatibility/ArtifactInfo.java | 7 + compatibility/BuildInterface.java | 11 ++ compatibility/Context.java | 21 +++ compatibility/Dependency.java | 10 ++ compatibility/Result.java | 11 ++ coursier/Coursier.scala | 2 +- nailgun_launcher/CBTUrlClassLoader.java | 4 +- nailgun_launcher/ClassLoaderCache2.java | 37 +++++ nailgun_launcher/EarlyDependencies.java | 167 +++++++++++++-------- nailgun_launcher/MultiClassLoader2.java | 29 ++++ nailgun_launcher/NailgunLauncher.java | 196 ++++++++++++++++--------- nailgun_launcher/Stage0Lib.java | 108 ++++++++------ plugins/scalatest/ScalaTest.scala | 66 +++++++++ plugins/scalatest/build/build.scala | 13 ++ stage1/ClassLoaderCache.scala | 19 +-- stage1/ContextImplementation.scala | 22 +++ stage1/KeyLockedLazyCache.scala | 14 +- stage1/MavenRepository.scala | 27 ++-- stage1/MultiClassLoader.scala | 2 +- stage1/Stage1.scala | 149 +++++++++++++++---- stage1/Stage1Lib.scala | 119 ++++++++++++--- stage1/cbt.scala | 68 +++++++++ stage1/paths.scala | 16 +- stage1/resolver.scala | 249 ++++++++++++-------------------- stage2/AdminStage2.scala | 2 +- stage2/AdminTasks.scala | 69 ++++++--- stage2/BasicBuild.scala | 38 ++--- stage2/BuildBuild.scala | 72 ++++++--- stage2/BuildDependency.scala | 5 +- stage2/GitDependency.scala | 4 +- stage2/Lib.scala | 56 ++++--- stage2/PackageBuild.scala | 10 +- stage2/SbtDependencyDsl.scala | 2 +- stage2/Stage2.scala | 36 ++++- stage2/mixins.scala | 22 +-- test/build/build.scala | 2 +- test/simple-fixed/Main.scala | 6 + test/simple-fixed/build/build.scala | 34 +++++ test/simple/build/build.scala | 15 +- test/test.scala | 51 +++++-- 42 files changed, 1237 insertions(+), 570 deletions(-) create mode 100644 compatibility/ArtifactInfo.java create mode 100644 compatibility/BuildInterface.java create mode 100644 compatibility/Context.java create mode 100644 compatibility/Dependency.java create mode 100644 compatibility/Result.java create mode 100644 nailgun_launcher/ClassLoaderCache2.java create mode 100644 nailgun_launcher/MultiClassLoader2.java create mode 100644 plugins/scalatest/ScalaTest.scala create mode 100644 plugins/scalatest/build/build.scala create mode 100644 stage1/ContextImplementation.scala create mode 100644 test/simple-fixed/Main.scala create mode 100644 test/simple-fixed/build/build.scala diff --git a/README.md b/README.md index 2485518..2240893 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ class Build(context: cbt.Context) extends PackageBuild(context){ override def artifactId = "play-json-extensions" override def dependencies = super.dependencies :+ - MavenRepository.central.resolve( + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( // encouraged way to declare dependencies ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), MavenDependency("joda-time", "joda-time", "2.9.2") diff --git a/build/build.scala b/build/build.scala index 6dfd395..1622f01 100644 --- a/build/build.scala +++ b/build/build.scala @@ -5,12 +5,14 @@ import scala.collection.immutable.Seq class Build(context: Context) extends BasicBuild(context){ // FIXME: somehow consolidate this with cbt's own boot-strapping from source. - override def dependencies = super.dependencies :+ MavenRepository.central.resolve( - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), - MavenDependency("com.typesafe.zinc","zinc","0.3.9"), - ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") - ) + override def dependencies = { + super.dependencies :+ MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + MavenDependency("com.typesafe.zinc","zinc","0.3.9"), + ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") + ) :+ BinaryDependency(new File(System.getenv("CBT_HOME")+"/compatibility"), Seq()) + } override def sources = Seq( "nailgun_launcher", "stage1", "stage2" ).map(d => projectDirectory ++ ("/" + d)) diff --git a/compatibility/ArtifactInfo.java b/compatibility/ArtifactInfo.java new file mode 100644 index 0000000..a2e6006 --- /dev/null +++ b/compatibility/ArtifactInfo.java @@ -0,0 +1,7 @@ +package cbt; + +public interface ArtifactInfo extends Dependency{ + public abstract String artifactId(); + public abstract String groupId(); + public abstract String version(); +} diff --git a/compatibility/BuildInterface.java b/compatibility/BuildInterface.java new file mode 100644 index 0000000..fea43be --- /dev/null +++ b/compatibility/BuildInterface.java @@ -0,0 +1,11 @@ +package cbt; +import java.io.*; + +public interface BuildInterface extends Dependency{ + public abstract BuildInterface copy(Context context); // needed to configure builds + public abstract String show(); // needed for debugging + public abstract String scalaVersion(); // needed to propagate scalaVersion to dependent builds + public abstract String[] crossScalaVersionsArray(); // FIXME: this probably can't use Scala classes + public abstract BuildInterface finalBuild(); // needed to propagage through build builds. Maybe we can get rid of this. + public abstract File[] triggerLoopFilesArray(); // needed for watching files across composed builds +} diff --git a/compatibility/Context.java b/compatibility/Context.java new file mode 100644 index 0000000..387c24a --- /dev/null +++ b/compatibility/Context.java @@ -0,0 +1,21 @@ +package cbt; +import java.io.*; +import java.util.concurrent.ConcurrentHashMap; + +// TODO: try to reduce the number of members +public abstract class Context{ + public abstract File projectDirectory(); + public abstract File cwd(); + public abstract String[] argsArray(); + public abstract String[] enabledLoggersArray(); + public abstract Long startCompat(); + public abstract Boolean cbtHasChangedCompat(); + public abstract String versionOrNull(); + public abstract String scalaVersionOrNull(); // needed to propagate scalaVersion to dependendee builds + public abstract ConcurrentHashMap permanentKeys(); + public abstract ConcurrentHashMap permanentClassLoaders(); + public abstract File cache(); + public abstract File cbtHome(); + public abstract File compatibilityTarget(); + public abstract BuildInterface parentBuildOrNull(); +} diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java new file mode 100644 index 0000000..d491174 --- /dev/null +++ b/compatibility/Dependency.java @@ -0,0 +1,10 @@ +package cbt; +import java.io.*; + +public interface Dependency{ + public abstract String show(); + public abstract Boolean needsUpdateCompat(); + public abstract Dependency[] dependenciesArray(); + public abstract File[] dependencyClasspathArray(); + public abstract File[] exportedClasspathArray(); +} diff --git a/compatibility/Result.java b/compatibility/Result.java new file mode 100644 index 0000000..220aa3a --- /dev/null +++ b/compatibility/Result.java @@ -0,0 +1,11 @@ +/* +package cbt; +import java.io.*; +public interface Result{ + public abstract Integer exitCode(); + public abstract OutputStream out(); + public abstract OutputStream err(); + public abstract InputStream in(); + public abstract T value(); +} +*/ diff --git a/coursier/Coursier.scala b/coursier/Coursier.scala index 48282b1..8a66aee 100644 --- a/coursier/Coursier.scala +++ b/coursier/Coursier.scala @@ -7,7 +7,7 @@ object Coursier{ import coursier._ val repositories = Seq( Cache.ivy2Local, - MavenRepository("https://repo1.maven.org/maven2") + MavenResolver("https://repo1.maven.org/maven2") ) val start = Resolution( diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java index 9c41978..b799bc0 100644 --- a/nailgun_launcher/CBTUrlClassLoader.java +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -11,7 +11,7 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ + "(\n " + Arrays.toString(getURLs()) + ",\n " - + join("\n ",getParent().toString().split("\n")) + + join("\n ",(getParent() == null?"":getParent().toString()).split("\n")) + "\n)" ); } @@ -53,7 +53,7 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ assertExist(urls); } public CbtURLClassLoader(URL[] urls){ - super(urls); + super(urls, null); assertExist(urls); } } \ No newline at end of file diff --git a/nailgun_launcher/ClassLoaderCache2.java b/nailgun_launcher/ClassLoaderCache2.java new file mode 100644 index 0000000..bf9ca3b --- /dev/null +++ b/nailgun_launcher/ClassLoaderCache2.java @@ -0,0 +1,37 @@ +package cbt; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import static java.io.File.pathSeparator; +import static cbt.Stage0Lib.*; + +final class ClassLoaderCache2{ + ConcurrentHashMap keys; + ConcurrentHashMap values; + + public ClassLoaderCache2( + ConcurrentHashMap keys, + ConcurrentHashMap values + ){ + this.keys = keys; + this.values = values; + } + + public T get( String key ){ + return values.get( + keys.get( key ) + ); + } + + public Boolean contains( String key ){ + return keys.containsKey( key ); + } + + public T put( T value, String key ){ + LockableKey2 keyObject = new LockableKey2(); + keys.put( key, keyObject ); + values.put( keyObject, value ); + return value; + } +} +class LockableKey2{} \ No newline at end of file diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java index f4d446c..7a8d033 100644 --- a/nailgun_launcher/EarlyDependencies.java +++ b/nailgun_launcher/EarlyDependencies.java @@ -10,90 +10,133 @@ import static cbt.NailgunLauncher.*; class EarlyDependencies{ /** ClassLoader for stage1 */ - ClassLoader stage1; + ClassLoader classLoader; + String[] classpathArray; /** ClassLoader for zinc */ ClassLoader zinc; - String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; - String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; - String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; - String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; - String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; - String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; - String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; - String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; - String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; - String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; - String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; - - public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ - download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); - download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); + String scalaReflect_2_11_8_File; + String scalaCompiler_2_11_8_File; + String scalaXml_1_0_5_File; + String scalaLibrary_2_11_8_File; + String zinc_0_3_9_File; + String incrementalCompiler_0_13_9_File; + String compilerInterface_0_13_9_File; + String scalaCompiler_2_10_5_File; + String sbtInterface_0_13_9_File; + String scalaReflect_2_10_5_File; + String scalaLibrary_2_10_5_File; + + public EarlyDependencies( + String mavenCache, String mavenUrl, ClassLoaderCache2 classLoaderCache, ClassLoader rootClassLoader + ) throws Exception { + scalaReflect_2_11_8_File = mavenCache + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; + scalaCompiler_2_11_8_File = mavenCache + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; + scalaXml_1_0_5_File = mavenCache + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; + scalaLibrary_2_11_8_File = mavenCache + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; + zinc_0_3_9_File = mavenCache + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; + incrementalCompiler_0_13_9_File = mavenCache + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; + compilerInterface_0_13_9_File = mavenCache + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; + scalaCompiler_2_10_5_File = mavenCache + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; + sbtInterface_0_13_9_File = mavenCache + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; + scalaReflect_2_10_5_File = mavenCache + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; + scalaLibrary_2_10_5_File = mavenCache + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; + + download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); + download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); // org.scala-lang:scala-library:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); - ClassLoader scalaLibrary_2_10_5_ = cachePut( - classLoader( scalaLibrary_2_10_5_File ), - scalaLibrary_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); + + String[] scalaLibrary_2_10_5_ClasspathArray = new String[]{scalaLibrary_2_10_5_File}; + String scalaLibrary_2_10_5_Classpath = classpath( scalaLibrary_2_10_5_ClasspathArray ); + ClassLoader scalaLibrary_2_10_5_ = + classLoaderCache.contains( scalaLibrary_2_10_5_Classpath ) + ? classLoaderCache.get( scalaLibrary_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaLibrary_2_10_5_File, rootClassLoader ), scalaLibrary_2_10_5_Classpath ); // org.scala-lang:scala-reflect:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); - ClassLoader scalaReflect_2_10_5_ = cachePut( - classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), - scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); + + String[] scalaReflect_2_10_5_ClasspathArray = new String[]{scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String scalaReflect_2_10_5_Classpath = classpath( scalaReflect_2_10_5_ClasspathArray ); + ClassLoader scalaReflect_2_10_5_ = + classLoaderCache.contains( scalaReflect_2_10_5_Classpath ) + ? classLoaderCache.get( scalaReflect_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), scalaReflect_2_10_5_Classpath ); // com.typesafe.sbt:sbt-interface:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); - ClassLoader sbtInterface_0_13_9_ = cachePut( - classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), - sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); + + String[] sbtInterface_0_13_9_ClasspathArray = new String[]{sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String sbtInterface_0_13_9_Classpath = classpath( sbtInterface_0_13_9_ClasspathArray ); + ClassLoader sbtInterface_0_13_9_ = + classLoaderCache.contains( sbtInterface_0_13_9_Classpath ) + ? classLoaderCache.get( sbtInterface_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), sbtInterface_0_13_9_Classpath ); // org.scala-lang:scala-compiler:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); - ClassLoader scalaCompiler_2_10_5_ = cachePut( - classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), - sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); + + String[] scalaCompiler_2_10_5_ClasspathArray = new String[]{sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String scalaCompiler_2_10_5_Classpath = classpath( scalaCompiler_2_10_5_ClasspathArray ); + ClassLoader scalaCompiler_2_10_5_ = + classLoaderCache.contains( scalaCompiler_2_10_5_Classpath ) + ? classLoaderCache.get( scalaCompiler_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), scalaCompiler_2_10_5_Classpath ); // com.typesafe.sbt:compiler-interface:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); - ClassLoader compilerInterface_0_13_9_ = cachePut( - classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), - compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); + + String[] compilerInterface_0_13_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String compilerInterface_0_13_9_Classpath = classpath( compilerInterface_0_13_9_ClasspathArray ); + ClassLoader compilerInterface_0_13_9_ = + classLoaderCache.contains( compilerInterface_0_13_9_Classpath ) + ? classLoaderCache.get( compilerInterface_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), compilerInterface_0_13_9_Classpath ); // com.typesafe.sbt:incremental-compiler:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); - ClassLoader incrementalCompiler_0_13_9_ = cachePut( - classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), - compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); + + String[] incrementalCompiler_0_13_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String incrementalCompiler_0_13_9_Classpath = classpath( incrementalCompiler_0_13_9_ClasspathArray ); + ClassLoader incrementalCompiler_0_13_9_ = + classLoaderCache.contains( incrementalCompiler_0_13_9_Classpath ) + ? classLoaderCache.get( incrementalCompiler_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), incrementalCompiler_0_13_9_Classpath ); // com.typesafe.zinc:zinc:0.3.9 - download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); - ClassLoader zinc_0_3_9_ = cachePut( - classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), - compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); + + String[] zinc_0_3_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String zinc_0_3_9_Classpath = classpath( zinc_0_3_9_ClasspathArray ); + ClassLoader zinc_0_3_9_ = + classLoaderCache.contains( zinc_0_3_9_Classpath ) + ? classLoaderCache.get( zinc_0_3_9_Classpath ) + : classLoaderCache.put( classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), zinc_0_3_9_Classpath ); // org.scala-lang:scala-library:2.11.8 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); - ClassLoader scalaLibrary_2_11_8_ = cachePut( - classLoader( scalaLibrary_2_11_8_File ), - scalaLibrary_2_11_8_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); + + String[] scalaLibrary_2_11_8_ClasspathArray = new String[]{scalaLibrary_2_11_8_File}; + String scalaLibrary_2_11_8_Classpath = classpath( scalaLibrary_2_11_8_ClasspathArray ); + ClassLoader scalaLibrary_2_11_8_ = + classLoaderCache.contains( scalaLibrary_2_11_8_Classpath ) + ? classLoaderCache.get( scalaLibrary_2_11_8_Classpath ) + : classLoaderCache.put( classLoader( scalaLibrary_2_11_8_File, rootClassLoader ), scalaLibrary_2_11_8_Classpath ); // org.scala-lang.modules:scala-xml_2.11:1.0.5 - download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); - ClassLoader scalaXml_1_0_5_ = cachePut( - classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), - scalaXml_1_0_5_File, scalaLibrary_2_11_8_File - ); + download(new URL(mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); + + String[] scalaXml_1_0_5_ClasspathArray = new String[]{scalaXml_1_0_5_File, scalaLibrary_2_11_8_File}; + String scalaXml_1_0_5_Classpath = classpath( scalaXml_1_0_5_ClasspathArray ); + ClassLoader scalaXml_1_0_5_ = + classLoaderCache.contains( scalaXml_1_0_5_Classpath ) + ? classLoaderCache.get( scalaXml_1_0_5_Classpath ) + : classLoaderCache.put( classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), scalaXml_1_0_5_Classpath ); - stage1 = scalaXml_1_0_5_; + classLoader = scalaXml_1_0_5_; + classpathArray = scalaXml_1_0_5_ClasspathArray; zinc = zinc_0_3_9_; } diff --git a/nailgun_launcher/MultiClassLoader2.java b/nailgun_launcher/MultiClassLoader2.java new file mode 100644 index 0000000..fadd963 --- /dev/null +++ b/nailgun_launcher/MultiClassLoader2.java @@ -0,0 +1,29 @@ +package cbt; +import java.net.*; +import java.util.*; + +public class MultiClassLoader2 extends ClassLoader{ + public ClassLoader[] parents; + public ClassLoader[] parents(){ + return this.parents; + } + public MultiClassLoader2(ClassLoader... parents){ + super(null); + this.parents = parents; + } + public Class findClass(String name) throws ClassNotFoundException{ + for(ClassLoader parent: parents){ + try{ + return parent.loadClass(name); + } catch (ClassNotFoundException e) { + if(e.getMessage() != name) throw e; + } + } + // FIXME: have a logger in Java land + // System.err.println("NOT FOUND: "+name); + return null; + } + public String toString(){ + return super.toString() + "(" + Arrays.toString(parents) +")"; + } +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 8838543..33ae4c2 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -6,6 +6,7 @@ import java.security.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static cbt.Stage0Lib.*; +import static java.io.File.pathSeparator; /** * This launcher allows to start the JVM without loading anything else permanently into its @@ -13,43 +14,40 @@ import static cbt.Stage0Lib.*; * dependencies outside the JDK. */ public class NailgunLauncher{ + /** Persistent cache for caching classloaders for the JVM life time. */ + private final static ClassLoaderCache2 classLoaderCache = new ClassLoaderCache2( + new ConcurrentHashMap(), + new ConcurrentHashMap() + ); - public static String CBT_HOME = System.getenv("CBT_HOME"); - public static String NAILGUN = System.getenv("NAILGUN"); - public static String TARGET = System.getenv("TARGET"); - public static String STAGE1 = CBT_HOME + "/stage1/"; - public static String MAVEN_CACHE = CBT_HOME + "/cache/maven"; - public static String MAVEN_URL = "https://repo1.maven.org/maven2"; - - /** - * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user - * code to improve startup time. - */ - public static ConcurrentHashMap classLoaderCacheKeys = new ConcurrentHashMap(); - public static ConcurrentHashMap classLoaderCacheValues = new ConcurrentHashMap(); - - public static SecurityManager defaultSecurityManager = System.getSecurityManager(); - - public static long lastSuccessfullCompile = 0; - static ClassLoader stage1classLoader = null; - public static ClassLoader stage2classLoader = null; + public final static SecurityManager defaultSecurityManager = System.getSecurityManager(); - public static void main(String[] args) throws ClassNotFoundException, - NoSuchMethodException, - IllegalAccessException, - InvocationTargetException, - MalformedURLException, - IOException, - NoSuchAlgorithmException { - //System.err.println("ClassLoader: "+stage1classLoader); - //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); - //System.err.println("now: "+now); - - _assert(CBT_HOME != null, CBT_HOME); - _assert(NAILGUN != null, NAILGUN); - _assert(TARGET != null, TARGET); - _assert(STAGE1 != null, STAGE1); + public static String TARGET = System.getenv("TARGET"); + private static String NAILGUN = "nailgun_launcher/"; + private static String STAGE1 = "stage1/"; + + @SuppressWarnings("unchecked") + public static Object getBuild( Object context ) throws Exception{ + BuildStage1Result res = buildStage1( + (Boolean) get(context, "cbtHasChanged"), + (Long) get(context, "start"), + ((File) get(context, "cache")).toString() + "/", + ((File) get(context, "cbtHome")).toString(), + ((File) get(context, "compatibilityTarget")).toString() + "/", + new ClassLoaderCache2( + (ConcurrentHashMap) get(context, "permanentKeys"), + (ConcurrentHashMap) get(context, "permanentClassLoaders") + ) + ); + return + res + .classLoader + .loadClass("cbt.Stage1") + .getMethod( "getBuild", Object.class, Boolean.class ) + .invoke(null, context, res.changed); + } + public static void main( String[] args ) throws Exception { Long _start = System.currentTimeMillis(); if(args[0].equals("check-alive")){ System.exit(33); @@ -58,52 +56,108 @@ public class NailgunLauncher{ String[] diff = args[0].split("\\."); long start = _start - (Long.parseLong(diff[0]) * 1000L) - Long.parseLong(diff[1]); - List stage1SourceFiles = new ArrayList(); - for( File f: new File(STAGE1).listFiles() ){ - if( f.isFile() && f.toString().endsWith(".scala") ){ - stage1SourceFiles.add(f); + + _assert(System.getenv("CBT_HOME") != null, "environment variable CBT_HOME not defined"); + String CBT_HOME = System.getenv("CBT_HOME"); + String cache = CBT_HOME + "/cache/"; + BuildStage1Result res = buildStage1( + false, start, cache, CBT_HOME, CBT_HOME + "/compatibility/" + TARGET, classLoaderCache + ); + + System.exit( + (Integer) res + .classLoader + .loadClass("cbt.Stage1") + .getMethod( + "run", + String[].class, File.class, File.class, Boolean.class, + Long.class, ConcurrentHashMap.class, ConcurrentHashMap.class + ) + .invoke( + null, + (Object) args, new File(cache), new File(CBT_HOME), res.changed, + start, classLoaderCache.keys, classLoaderCache.values + ) + ); + } + + public static BuildStage1Result buildStage1( + Boolean changed, long start, String cache, String cbtHome, String compatibilityTarget, ClassLoaderCache2 classLoaderCache + ) throws Exception { + _assert(TARGET != null, "environment variable TARGET not defined"); + String nailgunTarget = cbtHome + "/" + NAILGUN + TARGET; + String stage1Sources = cbtHome + "/" + STAGE1; + String stage1Target = stage1Sources + TARGET; + File compatibilitySources = new File(cbtHome + "/compatibility"); + String mavenCache = cache + "maven"; + String mavenUrl = "https://repo1.maven.org/maven2"; + + ClassLoader rootClassLoader = new CbtURLClassLoader( new URL[]{}, ClassLoader.getSystemClassLoader().getParent() ); // wrap for caching + EarlyDependencies earlyDeps = new EarlyDependencies(mavenCache, mavenUrl, classLoaderCache, rootClassLoader); + + List compatibilitySourceFiles = new ArrayList(); + for( File f: compatibilitySources.listFiles() ){ + if( f.isFile() && (f.toString().endsWith(".scala") || f.toString().endsWith(".java")) ){ + compatibilitySourceFiles.add(f); } } + changed = compile(changed, start, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles, defaultSecurityManager); + + ClassLoader compatibilityClassLoader; + if( classLoaderCache.contains( compatibilityTarget ) ){ + compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); + } else { + compatibilityClassLoader = classLoaderCache.put( classLoader(compatibilityTarget, rootClassLoader), compatibilityTarget ); + } - Boolean changed = lastSuccessfullCompile == 0; - for( File file: stage1SourceFiles ){ - if( file.lastModified() > lastSuccessfullCompile ){ - changed = true; - //System.err.println("File change: "+file.lastModified()); - break; - } + String[] nailgunClasspathArray = append( earlyDeps.classpathArray, nailgunTarget ); + String nailgunClasspath = classpath( nailgunClasspathArray ); + ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching + if( !classLoaderCache.contains( nailgunClasspath ) ){ + nailgunClassLoader = classLoaderCache.put( nailgunClassLoader, nailgunClasspath ); } - if(changed){ - EarlyDependencies earlyDeps = new EarlyDependencies(); - int exitCode = zinc(earlyDeps, stage1SourceFiles); - if( exitCode == 0 ){ - lastSuccessfullCompile = start; - } else { - System.exit( exitCode ); - } + String[] stage1ClasspathArray = + append( append( nailgunClasspathArray, compatibilityTarget ), stage1Target ); + String stage1Classpath = classpath( stage1ClasspathArray ); - ClassLoader nailgunClassLoader; - if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){ - nailgunClassLoader = cacheGet( NAILGUN+TARGET ); - } else { - nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP + List stage1SourceFiles = new ArrayList(); + for( File f: new File(stage1Sources).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".scala") ){ + stage1SourceFiles.add(f); } - - stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader); - stage2classLoader = null; } + changed = compile(changed, start, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles, defaultSecurityManager); - try{ - Integer exitCode = - (Integer) stage1classLoader - .loadClass("cbt.Stage1") - .getMethod("run", String[].class, ClassLoader.class, Boolean.class, Long.class) - .invoke( null, (Object) args, stage1classLoader, changed, start); - System.exit(exitCode); - }catch(Exception e){ - System.err.println(stage1classLoader); - throw e; + ClassLoader stage1classLoader; + if( !changed && classLoaderCache.contains( stage1Classpath ) ){ + stage1classLoader = classLoaderCache.get( stage1Classpath ); + } else { + stage1classLoader = + classLoaderCache.put( + classLoader( + stage1Target, + new MultiClassLoader2( + nailgunClassLoader, + compatibilityClassLoader, + earlyDeps.classLoader + ) + ), + stage1Classpath + ); } + + return new BuildStage1Result( + changed, + stage1classLoader + ); + } +} +class BuildStage1Result{ + Boolean changed; + ClassLoader classLoader; + BuildStage1Result( Boolean changed, ClassLoader classLoader ){ + this.changed = changed; + this.classLoader = classLoader; } } diff --git a/nailgun_launcher/Stage0Lib.java b/nailgun_launcher/Stage0Lib.java index d6f33e1..bd18748 100644 --- a/nailgun_launcher/Stage0Lib.java +++ b/nailgun_launcher/Stage0Lib.java @@ -8,8 +8,9 @@ import java.security.*; import java.util.*; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; import static java.io.File.pathSeparator; -import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; +import java.nio.file.*; +import java.nio.file.attribute.FileTime; public class Stage0Lib{ public static void _assert(Boolean condition, Object msg){ @@ -18,7 +19,7 @@ public class Stage0Lib{ } } - public static int runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + public static int runMain(String cls, String[] args, ClassLoader cl, SecurityManager defaultSecurityManager) throws Exception{ try{ System.setSecurityManager( new TrapSecurityManager() ); cl.loadClass(cls) @@ -32,63 +33,78 @@ public class Stage0Lib{ } throw exception; } finally { - System.setSecurityManager(NailgunLauncher.defaultSecurityManager); + System.setSecurityManager(defaultSecurityManager); } } - public static int zinc( EarlyDependencies earlyDeps, List sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ - String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File; - List zincArgs = new ArrayList( - Arrays.asList( - new String[]{ - "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, - "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, - "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, - "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, - "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, - "-cp", cp, - "-d", STAGE1+TARGET - } - ) - ); + public static Object get(Object object, String method) throws Exception{ + return object.getClass().getMethod( method ).invoke(object); + } + + public static String classpath( String... files ){ + Arrays.sort(files); + return join( pathSeparator, files ); + } - for( File f: sourceFiles ){ - zincArgs.add(f.toString()); + public static Boolean compile( + Boolean changed, Long start, String classpath, String target, + EarlyDependencies earlyDeps, List sourceFiles, SecurityManager defaultSecurityManager + ) throws Exception{ + File statusFile = new File( new File(target) + ".last-success" ); + Long lastSuccessfullCompile = statusFile.lastModified(); + for( File file: sourceFiles ){ + if( file.lastModified() > lastSuccessfullCompile ){ + changed = true; + break; + } } + if(changed){ + List zincArgs = new ArrayList( + Arrays.asList( + new String[]{ + "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, + "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, + "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, + "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, + "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, + "-cp", classpath, + "-d", target + } + ) + ); - PrintStream oldOut = System.out; - try{ - System.setOut(System.err); - return runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc ); - } finally { - System.setOut(oldOut); + for( File f: sourceFiles ){ + zincArgs.add(f.toString()); + } + + PrintStream oldOut = System.out; + try{ + System.setOut(System.err); + int exitCode = runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc, defaultSecurityManager ); + if( exitCode == 0 ){ + Files.write( statusFile.toPath(), "".getBytes()); + Files.setLastModifiedTime( statusFile.toPath(), FileTime.fromMillis(start) ); + } else { + System.exit( exitCode ); + } + } finally { + System.setOut(oldOut); + } } + return changed; } - public static ClassLoader classLoader( String file ) throws MalformedURLException{ + public static ClassLoader classLoader( String file ) throws Exception{ return new CbtURLClassLoader( new URL[]{ new URL("file:"+file) } ); } - public static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{ + public static ClassLoader classLoader( String file, ClassLoader parent ) throws Exception{ return new CbtURLClassLoader( new URL[]{ new URL("file:"+file) }, parent ); } - public static ClassLoader cacheGet( String key ){ - return classLoaderCacheValues.get( - classLoaderCacheKeys.get( key ) - ); - } - public static ClassLoader cachePut( ClassLoader classLoader, String... jars ){ - String key = join( pathSeparator, jars ); - Object keyObject = new Object(); - classLoaderCacheKeys.put( key, keyObject ); - classLoaderCacheValues.put( keyObject, classLoader ); - return classLoader; - } - - public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { + public static void download(URL urlString, Path target, String sha1) throws Exception { final Path unverified = Paths.get(target+".unverified"); if(!Files.exists(target)) { new File(target.toString()).getParentFile().mkdirs(); @@ -107,7 +123,7 @@ public class Stage0Lib{ } } - public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { + public static String sha1(byte[] bytes) throws Exception { final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); sha1.update(bytes, 0, bytes.length); return (new HexBinaryAdapter()).marshal(sha1.digest()); @@ -120,4 +136,10 @@ public class Stage0Lib{ } return result; } + + public static String[] append( String[] array, String item ){ + String[] copy = Arrays.copyOf(array, array.length + 1); + copy[array.length] = item; + return copy; + } } \ No newline at end of file diff --git a/plugins/scalatest/ScalaTest.scala b/plugins/scalatest/ScalaTest.scala new file mode 100644 index 0000000..0959087 --- /dev/null +++ b/plugins/scalatest/ScalaTest.scala @@ -0,0 +1,66 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq +import org.scalatest._ +import org.scalatest + +/* FIXME: + - Separate out SbtLayout + - Allow depending on this via a git dependency. + Probably by adding support for subfolders to "GitDependency" +*/ + +trait SbtLayout extends BasicBuild{ + outer => + override def sources = Seq( projectDirectory ++ "/src/main/scala" ) + def testSources = projectDirectory ++ "/src/test/scala" + def testDependencies: Seq[Dependency] = Nil + lazy val testBuild = + new BasicBuild(context.copy(projectDirectory = testSources)) with ScalaTest{ + override def target = outer.target + override def compileTarget = outer.scalaTarget ++ "/test-classes" + override def dependencies = (outer +: testDependencies) ++ super.dependencies + } + override def test: Option[ExitCode] = + if(testSources.exists) Some( testBuild.run ) + else None +} + +trait ScalaTest extends BasicBuild{ + override def run: ExitCode = { + import ScalaTestLib._ + val _classLoader = classLoader(context.classLoaderCache) + val suiteNames = compile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten + runSuites( suiteNames.map( loadSuite( _, _classLoader ) ) ) + ExitCode.Success + } +} + +object ScalaTestLib{ + def runSuites(suites: Seq[Suite]) = { + def color: Boolean = true + def durations: Boolean = true + def shortstacks: Boolean = true + def fullstacks: Boolean = true + def stats: Boolean = true + def testName: String = null + def configMap: ConfigMap = ConfigMap.empty + suites.foreach{ + _.execute(testName, configMap, color, durations, shortstacks, fullstacks, stats) + } + } + + def discoverSuites(discoveryPath: File, _classLoader: ClassLoader): Seq[String] = { + _classLoader + .loadClass("org.scalatest.tools.SuiteDiscoveryHelper") + .getMethod("discoverSuiteNames", classOf[List[_]], classOf[ClassLoader], classOf[Option[_]]) + .invoke(null, List(discoveryPath.string ++ "/"), _classLoader, None) + .asInstanceOf[Set[String]] + .to + } + def loadSuite(name: String, _classLoader: ClassLoader) = { + _classLoader.loadClass(name).getConstructor().newInstance().asInstanceOf[Suite] + } +} + diff --git a/plugins/scalatest/build/build.scala b/plugins/scalatest/build/build.scala new file mode 100644 index 0000000..d36d940 --- /dev/null +++ b/plugins/scalatest/build/build.scala @@ -0,0 +1,13 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context){ + override def dependencies = super.dependencies ++ Seq( + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + ScalaDependency("org.scalatest","scalatest","2.2.4") + ), + context.cbtDependency + ) +} diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 44b8d7d..e430ee1 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -4,15 +4,14 @@ import java.net._ import java.util.concurrent.ConcurrentHashMap import collection.JavaConversions._ -class ClassLoaderCache(logger: Logger){ +case class ClassLoaderCache( + logger: Logger, + private[cbt] permanentKeys: ConcurrentHashMap[String,AnyRef], + private[cbt] permanentClassLoaders: ConcurrentHashMap[AnyRef,ClassLoader] +){ val persistent = new KeyLockedLazyCache( - NailgunLauncher.classLoaderCacheKeys.asInstanceOf[ConcurrentHashMap[String,AnyRef]], - NailgunLauncher.classLoaderCacheValues.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], - Some(logger) - ) - val transient = new KeyLockedLazyCache( - new ConcurrentHashMap[String,AnyRef], - new ConcurrentHashMap[AnyRef,ClassLoader], + permanentKeys, + permanentClassLoaders, Some(logger) ) override def toString = ( @@ -20,10 +19,6 @@ class ClassLoaderCache(logger: Logger){ ++ persistent.keys.keySet.toVector.map(_.toString.split(":").mkString("\n")).sorted.mkString("\n\n","\n\n","\n\n") ++ - "---------" - ++ - transient.keys.keySet.toVector.map(_.toString.split(":").mkString("\n")).sorted.mkString("\n\n","\n\n^","\n\n") - ++ ")" ) } diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala new file mode 100644 index 0000000..d8b1382 --- /dev/null +++ b/stage1/ContextImplementation.scala @@ -0,0 +1,22 @@ +package cbt +import java.io._ +import java.util.concurrent.ConcurrentHashMap +import scala.collection.immutable.Seq +import java.lang._ + +case class ContextImplementation( + projectDirectory: File, + cwd: File, + argsArray: Array[String], + enabledLoggersArray: Array[String], + startCompat: Long, + cbtHasChangedCompat: Boolean, + versionOrNull: String, + scalaVersionOrNull: String, + permanentKeys: ConcurrentHashMap[String,AnyRef], + permanentClassLoaders: ConcurrentHashMap[AnyRef,ClassLoader], + cache: File, + cbtHome: File, + compatibilityTarget: File, + parentBuildOrNull: BuildInterface +) extends Context \ No newline at end of file diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala index aca5f74..4eff5b2 100644 --- a/stage1/KeyLockedLazyCache.scala +++ b/stage1/KeyLockedLazyCache.scala @@ -47,8 +47,16 @@ final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value <: AnyRef]( def remove( key: Key ) = keys.synchronized{ assert(keys containsKey key) val lockableKey = keys get key - keys.remove( key ) - assert(values containsKey lockableKey) - values.remove( lockableKey ) + lockableKey.synchronized{ + if(values containsKey lockableKey){ + // this is so values in the process of being replaced (which mean they have a key but no value) + // are not being removed + keys.remove( key ) + values.remove( lockableKey ) + } + } + } + def containsKey( key: Key ) = keys.synchronized{ + keys containsKey key } } diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index bfd52a7..aa31cb8 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -1,22 +1,19 @@ package cbt import scala.collection.immutable.Seq +import java.io._ import java.net._ -case class MavenRepository(url: URL){ - def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies - = new BoundMavenDependencies( Seq(url), dependencies.to ) - def resolveOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency - = BoundMavenDependency( dependency, Seq(url) ) +case class MavenResolver( cbtHasChanged: Boolean, mavenCache: File, urls: URL* ){ + def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies + = new BoundMavenDependencies( cbtHasChanged, mavenCache, urls.to, dependencies.to ) + def resolveOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency + = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) } -object MavenRepository{ - case class combine(repositories: MavenRepository*){ - def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies - = new BoundMavenDependencies( repositories.map(_.url).to, dependencies.to ) - } - def central = MavenRepository(new URL(NailgunLauncher.MAVEN_URL)) - def jcenter = MavenRepository(new URL("https://jcenter.bintray.com/releases")) - def bintray(owner: String) = MavenRepository(new URL(s"https://dl.bintray.com/$owner/maven")) +object MavenResolver{ + def central = new URL("https://repo1.maven.org/maven2") + def jcenter = new URL("https://jcenter.bintray.com/releases") + def bintray(owner: String) = new URL(s"https://dl.bintray.com/$owner/maven") private val sonatypeBase = new URL("https://oss.sonatype.org/content/repositories/") - def sonatype = MavenRepository(sonatypeBase ++ "releases") - def sonatypeSnapshots = MavenRepository(sonatypeBase ++ "snapshots") + def sonatype = sonatypeBase ++ "releases" + def sonatypeSnapshots = sonatypeBase ++ "snapshots" } diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index 74e65aa..3e3ba26 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -3,7 +3,7 @@ import java.net._ import scala.collection.immutable.Seq // do not make this a case class, required object identity equality -class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader with CachingClassLoader{ +class MultiClassLoader(final val parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader(null) with CachingClassLoader{ override def findClass(name: String) = { parents.find( parent => try{ diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 77b88a2..bf2c272 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -1,13 +1,13 @@ package cbt import java.io._ +import java.util.concurrent.ConcurrentHashMap import scala.collection.immutable.Seq import scala.collection.JavaConverters._ -import paths._ - -final case class Stage1ArgsParser(_args: Seq[String]) { +final case class Stage1ArgsParser(__args: Seq[String]) { + val _args = __args.drop(1) /** * Raw parameters including their `-D` flag. **/ @@ -41,60 +41,149 @@ case class Stage2Args( cwd: File, args: Seq[String], cbtHasChanged: Boolean, - logger: Logger, - classLoaderCache: ClassLoaderCache -) - + classLoaderCache: ClassLoaderCache, + cache: File, + cbtHome: File +){ + val ClassLoaderCache( + logger, + permanentKeys, + permanentClassLoaders + ) = classLoaderCache +} object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified } - def run(_args: Array[String], classLoader: ClassLoader, _cbtChanged: java.lang.Boolean, start: java.lang.Long): Int = { - val args = Stage1ArgsParser(_args.toVector) - val logger = new Logger(args.enabledLoggers, start) - logger.stage1(s"Stage1 start") + def getBuild( _context: java.lang.Object, _cbtChanged: java.lang.Boolean ) = { + val context = _context.asInstanceOf[Context] + val logger = new Logger( context.enabledLoggers, context.start ) + val (changed, classLoader) = buildStage2( + context.compatibilityTarget, + ClassLoaderCache( + logger, + context.permanentKeys, + context.permanentClassLoaders + ), + _cbtChanged, + context.cbtHome, + context.cache + ) + + classLoader + .loadClass("cbt.Stage2") + .getMethod( "getBuild", classOf[java.lang.Object], classOf[java.lang.Boolean] ) + .invoke(null, context, (_cbtChanged || changed): java.lang.Boolean) + } + + def buildStage2( + _compatibilityTarget: File, classLoaderCache: ClassLoaderCache, _cbtChanged: Boolean, cbtHome: File, cache: File + ): (Boolean, ClassLoader) = { + import classLoaderCache.logger val lib = new Stage1Lib(logger) import lib._ - val classLoaderCache = new ClassLoaderCache(logger) + val paths = Paths(cbtHome, cache) + import paths._ + + val stage2sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) + val cbtHasChanged = _cbtChanged || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) + + val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") + + val cbtDependency = CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, _compatibilityTarget) - val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val cbtHasChanged = _cbtChanged || lib.needsUpdate(sourceFiles, stage2StatusFile) logger.stage1("Compiling stage2 if necessary") compile( cbtHasChanged, - sourceFiles, stage2Target, stage2StatusFile, - CbtDependency().dependencyClasspath, + cbtHasChanged, + stage2sourceFiles, stage2Target, stage2StatusFile, + cbtDependency.dependencyClasspath, + mavenCache, Seq("-deprecation"), classLoaderCache, zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) logger.stage1(s"calling CbtDependency.classLoader") - if(cbtHasChanged) { - NailgunLauncher.stage2classLoader = CbtDependency().classLoader(classLoaderCache) - }else{ - classLoaderCache.transient.get( CbtDependency().classpath.string, NailgunLauncher.stage2classLoader ) + if( cbtHasChanged && classLoaderCache.persistent.containsKey( cbtDependency.classpath.string ) ) { + classLoaderCache.persistent.remove( cbtDependency.classpath.string ) } + val stage2ClassLoader = classLoaderCache.persistent.get( + cbtDependency.classpath.string, + cbtDependency.classLoader(classLoaderCache) + ) + + { + // a few classloader sanity checks + val compatibilityClassLoader = + CompatibilityDependency(cbtHasChanged, compatibilityTarget) + .classLoader(classLoaderCache) + assert( + classOf[BuildInterface].getClassLoader == compatibilityClassLoader, + classOf[BuildInterface].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ compatibilityClassLoader.toString + ) + //------------- + val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) + val stage1ClassLoader = + stage1Dependency.classLoader(classLoaderCache) + assert( + classOf[Stage1Dependency].getClassLoader == stage1ClassLoader, + classOf[Stage1Dependency].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString + ) + //------------- + assert( + Stage0Lib.get(stage2ClassLoader.getParent,"parents").asInstanceOf[Seq[ClassLoader]].contains(stage1ClassLoader), + stage1ClassLoader.toString ++ "\n\nis not contained in parents of\n\n" ++ stage2ClassLoader.toString + ) + } + + ( cbtHasChanged, stage2ClassLoader ) + } + + def run( + _args: Array[String], + cache: File, + cbtHome: File, + _cbtChanged: java.lang.Boolean, + start: java.lang.Long, + classLoaderCacheKeys: ConcurrentHashMap[String,AnyRef], + classLoaderCacheValues: ConcurrentHashMap[AnyRef,ClassLoader] + ): Int = { + val args = Stage1ArgsParser(_args.toVector) + val logger = new Logger(args.enabledLoggers, start) + logger.stage1(s"Stage1 start") + + val classLoaderCache = ClassLoaderCache( + logger, + classLoaderCacheKeys, + classLoaderCacheValues + ) + + + val (cbtHasChanged, classLoader) = buildStage2( Paths(cbtHome, cache).compatibilityTarget, classLoaderCache, _cbtChanged, cbtHome, cache ) + + val stage2Args = Stage2Args( + new File( args.args(0) ), + args.args.drop(1).toVector, + // launcher changes cause entire nailgun restart, so no need for them here + cbtHasChanged = cbtHasChanged, + classLoaderCache = classLoaderCache, + cache, + cbtHome + ) + logger.stage1(s"Run Stage2") - val cl = NailgunLauncher.stage2classLoader val exitCode = ( - cl + classLoader .loadClass( if(args.admin) "cbt.AdminStage2" else "cbt.Stage2" ) .getMethod( "run", classOf[Stage2Args] ) .invoke( null, - Stage2Args( - new File( args.args(0) ), - args.args.drop(1).toVector, - // launcher changes cause entire nailgun restart, so no need for them here - cbtHasChanged = cbtHasChanged, - logger, - classLoaderCache - ) + stage2Args ) match { case code: ExitCode => code case _ => ExitCode.Success diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index f484bbf..5fd19a8 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -1,15 +1,15 @@ package cbt -import cbt.paths._ - import java.io._ import java.lang.reflect.InvocationTargetException import java.net._ +import java.nio.charset.StandardCharsets import java.nio.file._ import java.nio.file.attribute.FileTime import javax.tools._ import java.security._ -import java.util._ +import java.util.{Set=>_,Map=>_,_} +import java.util.concurrent.ConcurrentHashMap import javax.xml.bind.annotation.adapters.HexBinaryAdapter import scala.collection.immutable.Seq @@ -31,19 +31,8 @@ object CatchTrappedExitCode{ } } -case class Context( - projectDirectory: File, - cwd: File, - args: Seq[String], - logger: Logger, - cbtHasChanged: Boolean, - classLoaderCache: ClassLoaderCache, - version: Option[String] = None, - scalaVersion: Option[String] = None -) - class BaseLib{ - def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) + def realpath(name: File) = new File(java.nio.file.Paths.get(name.getAbsolutePath).normalize.toString) } class Stage1Lib( val logger: Logger ) extends BaseLib{ @@ -140,11 +129,13 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } def compile( + cbtHasChanged: Boolean, needsRecompile: Boolean, files: Seq[File], compileTarget: File, statusFile: File, classpath: ClassPath, + mavenCache: File, scalacOptions: Seq[String] = Seq(), classLoaderCache: ClassLoaderCache, zincVersion: String, @@ -159,7 +150,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ None }else{ if( needsRecompile ){ - import MavenRepository.central + val central = MavenResolver(cbtHasChanged, mavenCache,MavenResolver.central) val zinc = central.resolveOne(MavenDependency("com.typesafe.zinc","zinc", zincVersion)) val zincDeps = zinc.transitiveDependencies @@ -167,8 +158,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ zincDeps .collect{ case d @ BoundMavenDependency( - MavenDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none), - _ + _, _, MavenDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none), _ ) => d } .headOption @@ -179,8 +169,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ zincDeps .collect{ case d @ BoundMavenDependency( - MavenDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources), - _ + _, _, MavenDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources), _ ) => d } .headOption @@ -207,6 +196,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val code = try{ + System.err.println("Compiling to " ++ compileTarget.toString) redirectOutToErr{ lib.runMain( _class, @@ -279,4 +269,93 @@ ${files.sorted.mkString(" \\\n")} MavenDependency( groupId, artifactId ++ "_" ++ scalaMajorVersion, version, classifier ) + + def cacheOnDisk[T] + ( cbtHasChanged: Boolean, cacheFile: File ) + ( deserialize: String => T ) + ( serialize: T => String ) + ( compute: => Seq[T] ) = { + if(!cbtHasChanged && cacheFile.exists){ + import collection.JavaConversions._ + Files + .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) + .toStream + .map(deserialize) + } else { + val result = compute + val string = result.map(serialize).mkString("\n") + Files.write(cacheFile.toPath, string.getBytes) + result + } + } + + def dependencyTreeRecursion(root: Dependency, indent: Int = 0): String = ( + ( " " * indent ) + ++ (if(root.needsUpdate) red(root.show) else root.show) + ++ root.dependencies.map( d => + "\n" ++ dependencyTreeRecursion(d,indent + 1) + ).mkString + ) + + def transitiveDependencies(dependency: Dependency): Seq[Dependency] = { + def linearize(deps: Seq[Dependency]): Seq[Dependency] = { + // Order is important here in order to generate the correct lineraized dependency order for EarlyDependencies + // (and maybe this as well in case we want to get rid of MultiClassLoader) + try{ + if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) + } catch{ + case e: Exception => throw new Exception(dependency.show, e) + } + } + + // FIXME: this is probably wrong too eager. + // We should consider replacing versions during traversals already + // not just replace after traversals, because that could mean we + // pulled down dependencies current versions don't even rely + // on anymore. + + val deps: Seq[Dependency] = linearize(dependency.dependencies).reverse.distinct.reverse + val hasInfo: Seq[Dependency with ArtifactInfo] = deps.collect{ case d:Dependency with ArtifactInfo => d } + val noInfo: Seq[Dependency] = deps.filter{ + case _:Dependency with ArtifactInfo => false + case _ => true + } + noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct + } + + + def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { + case d: ArtifactInfo => latest((d.groupId,d.artifactId)) + case d => d + } + + def classLoaderRecursion( dependency: Dependency, latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + val d = dependency + val dependencies = dependency.dependencies + def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( dependency.dependencies.isEmpty ){ + // wrap for caching + new cbt.URLClassLoader( ClassPath(Seq()), ClassLoader.getSystemClassLoader().getParent() ) + } else if( dependencies.size == 1 ){ + classLoaderRecursion( dependencies.head, latest, cache ) + } else{ + val cp = d.dependencyClasspath.string + if( dependencies.exists(_.needsUpdate) && cache.persistent.containsKey(cp) ){ + cache.persistent.remove(cp) + } + cache.persistent.get( + cp, + new MultiClassLoader( + dependencies.map( classLoaderRecursion(_, latest, cache) ) + ) + ) + } + } + + val a = actual( dependency, latest ) + cache.persistent.get( + a.classpath.string, + new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) + ) + } } diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 7b8b632..4594135 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -3,6 +3,7 @@ import java.io._ import java.nio.file._ import java.net._ import scala.collection.immutable.Seq +import java.util.concurrent.ConcurrentHashMap object `package`{ private val lib = new BaseLib @@ -20,4 +21,71 @@ object `package`{ def ++( s: String ): URL = new URL( url.toString ++ s ) def string = url.toString } + implicit class BuildInterfaceExtensions(build: BuildInterface){ + import build._ + def triggerLoopFiles: Seq[File] = triggerLoopFilesArray.to + def crossScalaVersions: Seq[String] = crossScalaVersionsArray.to + } + implicit class ArtifactInfoExtensions(subject: ArtifactInfo){ + import subject._ + def str = s"$groupId:$artifactId:$version" + def show = this.getClass.getSimpleName ++ s"($str)" + } + implicit class DependencyExtensions(subject: Dependency){ + import subject._ + def dependencyClasspath: ClassPath = ClassPath(dependencyClasspathArray.to) + def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.to) + def classpath = exportedClasspath ++ dependencyClasspath + def dependencies: Seq[Dependency] = dependenciesArray.to + def needsUpdate: Boolean = needsUpdateCompat + } + implicit class ContextExtensions(subject: Context){ + import subject._ + val paths = Paths(cbtHome, cache) + implicit def logger: Logger = new Logger(enabledLoggers, start) + def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( + logger, + permanentKeys, + permanentClassLoaders + ) + def cbtDependency = { + import paths._ + CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget) + } + def args: Seq[String] = argsArray.to + def enabledLoggers: Set[String] = enabledLoggersArray.to + def scalaVersion = Option(scalaVersionOrNull) + def version = Option(versionOrNull) + def parentBuild = Option(parentBuildOrNull) + def start: scala.Long = startCompat + def cbtHasChanged: scala.Boolean = cbtHasChangedCompat + + def copy( + projectDirectory: File = projectDirectory, + args: Seq[String] = args, + enabledLoggers: Set[String] = enabledLoggers, + cbtHasChanged: Boolean = cbtHasChanged, + version: Option[String] = version, + scalaVersion: Option[String] = scalaVersion, + cache: File = cache, + cbtHome: File = cbtHome, + parentBuild: Option[BuildInterface] = None + ): Context = ContextImplementation( + projectDirectory, + cwd, + args.to, + enabledLoggers.to, + startCompat, + cbtHasChangedCompat, + version.getOrElse(null), + scalaVersion.getOrElse(null), + permanentKeys, + permanentClassLoaders, + cache, + cbtHome, + compatibilityTarget, + parentBuild.getOrElse(null) + ) + } } + diff --git a/stage1/paths.scala b/stage1/paths.scala index f27e538..c16c614 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -1,17 +1,17 @@ package cbt import java.io._ -object paths{ - val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) - val mavenCache: File = cbtHome ++ "/cache/maven" +case class Paths(private val cbtHome: File, private val cache: File){ val userHome: File = new File(Option(System.getProperty("user.home")).get) - val bootstrapScala: File = cbtHome ++ "/bootstrap_scala" - val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) - val stage1: File = new File(NailgunLauncher.STAGE1) + val nailgun: File = cbtHome ++ "/nailgun_launcher" + val stage1: File = cbtHome ++ "/stage1" val stage2: File = cbtHome ++ "/stage2" - private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") + val mavenCache: File = cache ++ "/maven" + private val target = NailgunLauncher.TARGET.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) val stage2StatusFile: File = stage2Target ++ ".last-success" + val compatibility: File = cbtHome ++ "/compatibility" + val compatibilityTarget: File = compatibility ++ ("/" ++ target) + val compatibilityStatusFile: File = compatibilityTarget ++ ".last-success" val nailgunTarget: File = nailgun ++ ("/" ++ target) - val sonatypeLogin: File = cbtHome ++ "/sonatype.login" } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index ad6df23..f979247 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -5,31 +5,24 @@ import java.net._ import java.io._ import scala.collection.immutable.Seq import scala.xml._ -import paths._ import scala.concurrent._ import scala.concurrent.duration._ -trait ArtifactInfo extends Dependency{ - def artifactId: String - def groupId: String - def version: String - - protected def str = s"$groupId:$artifactId:$version" - override def show = super.show ++ s"($str)" -} -abstract class Dependency{ - implicit def logger: Logger +abstract class DependencyImplementation extends Dependency{ + implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) def needsUpdate: Boolean //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath + def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray + def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath - def exportedJars: Seq[File] - def jars: Seq[File] = exportedJars ++ dependencyJars + def dependenciesArray: Array[Dependency] = dependencies.to - def canBeCached: Boolean + def needsUpdateCompat: java.lang.Boolean = needsUpdate + /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { // FIXME: this should separate a blocking and a non-blocking EC @@ -41,7 +34,7 @@ abstract class Dependency{ .groupBy( d => (d.groupId,d.artifactId) ) .mapValues( _.head ) //, new BuildCache - ), + ), // FIXME Duration.Inf ) } @@ -52,9 +45,9 @@ abstract class Dependency{ The implementation of this method is untested and likely buggy at this stage. */ - private def exportClasspathConcurrently( - latest: Map[(String, String),ArtifactInfo]//, cache: BuildCache - )( implicit ec: ExecutionContext ): Future[ClassPath] = { + def exportClasspathConcurrently( + latest: Map[(String, String),Dependency with ArtifactInfo]//, cache: BuildCache + )( implicit ec: ExecutionContext ): Future[AnyRef] = { Future.sequence( // trigger compilation / download of all dependencies first this.dependencies.map{ d => @@ -65,7 +58,7 @@ abstract class Dependency{ } // // trigger compilation if not already triggered // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) - l.exportClasspathConcurrently( latest ) + l.exportClasspathConcurrently( latest ) // FIXME } ).map( // merge dependency classpaths into one @@ -76,168 +69,102 @@ abstract class Dependency{ exportedClasspath ) } + */ - protected def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { - case d: ArtifactInfo => latest((d.groupId,d.artifactId)) - case d => d - } - private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { - if( dependencies.isEmpty ){ - ClassLoader.getSystemClassLoader - } else if( dependencies.size == 1 ){ - dependencies.head.classLoaderRecursion( latest, cache ) - } else if( dependencies.forall(_.canBeCached) ){ - assert(transitiveDependencies.forall(_.canBeCached)) - cache.persistent.get( - dependencyClasspath.string, - new MultiClassLoader( - dependencies.map( _.classLoaderRecursion(latest, cache) ) - ) - ) - } else { - cache.transient.get( - dependencyClasspath.string, - new MultiClassLoader( - dependencies.map( _.classLoaderRecursion(latest, cache) ) - ) - ) - } - } - protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { - val a = actual( this, latest ) - ( - if( canBeCached ){ - cache.persistent - } else { - cache.transient - } - ).get( - a.classpath.string, - new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) - ) - } - def classLoader( cache: ClassLoaderCache ): ClassLoader = { + def classLoader( cache: ClassLoaderCache ): ClassLoader = { + /* if( concurrencyEnabled ){ // trigger concurrent building / downloading dependencies exportClasspathConcurrently } - classLoaderRecursion( + */ + lib.classLoaderRecursion( + this, (this +: transitiveDependencies).collect{ case d: ArtifactInfo => d }.groupBy( d => (d.groupId,d.artifactId) ).mapValues(_.head), - cache + cache // FIXME ) } // FIXME: these probably need to update outdated as well def classpath : ClassPath = exportedClasspath ++ dependencyClasspath - def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) - def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) ) + def dependencyClasspath : ClassPath = ClassPath( + transitiveDependencies + .flatMap(_.exportedClasspath.files) + .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) + ) def dependencies: Seq[Dependency] - private def linearize(deps: Seq[Dependency]): Seq[Dependency] = - // Order is important here in order to generate the correct lineraized dependency order for EarlyDependencies - // (and maybe this as well in case we want to get rid of MultiClassLoader) - if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) - private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - // FIXME: this is probably wrong too eager. - // We should consider replacing versions during traversals already - // not just replace after traversals, because that could mean we - // pulled down dependencies current versions don't even rely - // on anymore. - - val deps: Seq[Dependency] = linearize(dependencies).reverse.distinct.reverse - val hasInfo: Seq[Dependency with ArtifactInfo] = deps.collect{ case d:Dependency with ArtifactInfo => d } - val noInfo: Seq[Dependency] = deps.filter{ - case _:Dependency with ArtifactInfo => false - case _ => true - } - noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct + lib.transitiveDependencies(this) } override def show: String = this.getClass.getSimpleName // ========== debug ========== - def dependencyTree: String = dependencyTreeRecursion() - private def dependencyTreeRecursion(indent: Int = 0): String = ( - ( " " * indent ) - ++ (if(needsUpdate) lib.red(show) else show) - ++ dependencies.map( - "\n" ++ _.dependencyTreeRecursion(indent + 1) - ).mkString - ) + def dependencyTree: String = lib.dependencyTreeRecursion(this) } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenRepository.central.url)) -class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenRepository.central.url)) -class ScalaReflectDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenRepository.central.url)) +class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenResolver.central)) +class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenResolver.central)) +class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenResolver.central)) -case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => +case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd => override final val needsUpdate = false - override def canBeCached = true def targetClasspath = ClassPath(Seq()) def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq[File]() def dependencies = Seq( - new ScalaCompilerDependency(version), - new ScalaLibraryDependency(version), - new ScalaReflectDependency(version) + new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), + new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), + new ScalaReflectDependency(cbtHasChanged, mavenCache, version) ) } -case class BinaryDependency( path: File, dependencies: Seq[Dependency], canBeCached: Boolean )(implicit val logger: Logger) extends Dependency{ +case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ def exportedClasspath = ClassPath(Seq(path)) - def exportedJars = Seq[File](path) override def needsUpdate = false def targetClasspath = exportedClasspath } /** Allows to easily assemble a bunch of dependencies */ -case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ +case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = dependencies.exists(_.needsUpdate) - override def canBeCached = dependencies.forall(_.canBeCached) override def exportedClasspath = ClassPath(Seq()) - override def exportedJars = Seq() override def targetClasspath = ClassPath(Seq()) } object Dependencies{ def apply( dependencies: Dependency* )(implicit logger: Logger): Dependencies = Dependencies( dependencies.to ) } -case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations - override def canBeCached = false +case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - override def exportedJars = ???//Seq[File]() override def dependencies = Seq( - MavenRepository.central.resolve( + CompatibilityDependency(cbtHasChanged, compatibilityTarget), + MavenResolver(cbtHasChanged,mavenCache,MavenResolver.central).resolve( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), - MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) ) - // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp - override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ) = { - val a = actual( this, latest ) - cache.transient.get( - a.classpath.string, - getClass.getClassLoader - ) - } } -case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations - override def canBeCached = false +case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(compatibilityTarget) ) + override def dependencies = Seq() +} +case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq( stage2Target ) ) - override def exportedJars = ??? override def dependencies = Seq( - Stage1Dependency(), - MavenRepository.central.resolve( + Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget), + MavenResolver(cbtHasChanged, mavenCache,MavenResolver.central).resolve( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) @@ -254,16 +181,24 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( - urls: Seq[URL], mavenDependencies: Seq[MavenDependency] + cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] )(implicit logger: Logger) extends Dependencies( - mavenDependencies.map( BoundMavenDependency(_,urls) ) + mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) ) case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none -) +){ + private[cbt] def serialize = groupId ++ ":" ++ artifactId ++ ":"++ version ++ ":" ++ classifier.name.getOrElse("") +} +object MavenDependency{ + private[cbt] def deserialize = (_:String).split(":") match { + case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) + } +} +// FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( - mavenDependency: MavenDependency, repositories: Seq[URL] -)(implicit val logger: Logger) extends ArtifactInfo{ + cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] +)(implicit val logger: Logger) extends DependencyImplementation with ArtifactInfo{ val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ @@ -283,14 +218,13 @@ case class BoundMavenDependency( ) override def needsUpdate = false - override def canBeCached = dependencies.forall(_.canBeCached) private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" - override def exportedJars = Seq( jar ) + def exportedJars = Seq( jar ) override def exportedClasspath = ClassPath( exportedJars ) override def targetClasspath = exportedClasspath import scala.collection.JavaConversions._ @@ -333,6 +267,8 @@ case class BoundMavenDependency( (pomXml \ "parent").collect{ case parent => BoundMavenDependency( + cbtHasChanged: Boolean, + mavenCache, MavenDependency( (parent \ "groupId").text, (parent \ "artifactId").text, @@ -367,33 +303,36 @@ case class BoundMavenDependency( def dependencies: Seq[BoundMavenDependency] = { if(classifier == Classifier.sources) Seq() - else (pomXml \ "dependencies" \ "dependency").collect{ - case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - val artifactId = lookup(xml,_ \ "artifactId").get - val groupId = - lookup(xml,_ \ "groupId").getOrElse( - dependencyVersions - .get(artifactId).map(_._1) - .getOrElse( - throw new Exception(s"$artifactId not found in \n$dependencyVersions") + else { + lib.cacheOnDisk( + cbtHasChanged, mavenCache ++ basePath ++ ".pom.dependencies" + )( MavenDependency.deserialize )( _.serialize ){ + (pomXml \ "dependencies" \ "dependency").collect{ + case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => + val artifactId = lookup(xml,_ \ "artifactId").get + val groupId = + lookup(xml,_ \ "groupId").getOrElse( + dependencyVersions + .get(artifactId).map(_._1) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) ) - ) - val version = - lookup(xml,_ \ "version").getOrElse( - dependencyVersions - .get(artifactId).map(_._2) - .getOrElse( - throw new Exception(s"$artifactId not found in \n$dependencyVersions") + val version = + lookup(xml,_ \ "version").getOrElse( + dependencyVersions + .get(artifactId).map(_._2) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) ) - ) - BoundMavenDependency( - MavenDependency( - groupId, artifactId, version, - Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) - ), - repositories - ) - }.toVector + val classifier = Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) + MavenDependency( groupId, artifactId, version, classifier ) + }.toVector + }.map( + BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories ) + ).to + } } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { //println("lookup in "++pomUrl) diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala index 9d7dcb2..f4e61d0 100644 --- a/stage2/AdminStage2.scala +++ b/stage2/AdminStage2.scala @@ -4,7 +4,7 @@ object AdminStage2 extends Stage2Base{ def run( _args: Stage2Args ): Unit = { val args = _args.args.dropWhile(Seq("admin","direct") contains _) val lib = new Lib(_args.logger) - val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache) + val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.cbtHasChanged) new lib.ReflectObject(adminTasks){ def usage: String = "Available methods: " ++ lib.taskNames(adminTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index f189805..9086470 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -2,14 +2,25 @@ package cbt import scala.collection.immutable.Seq import java.io.{Console=>_,_} import java.nio.file._ -class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: ClassLoaderCache){ +class AdminTasks( + lib: Lib, + args: Seq[String], + cwd: File, + classLoaderCache: ClassLoaderCache, + cache: File, + cbtHome: File, + cbtHasChanged: Boolean +){ + private val paths = Paths(cbtHome, cache) + import paths._ + private val mavenCentral = MavenResolver(cbtHasChanged,mavenCache,MavenResolver.central) implicit val logger: Logger = lib.logger def resolve = { ClassPath.flatten( args(1).split(",").toVector.map{ d => val v = d.split(":") - MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath + mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath } ) } @@ -17,14 +28,14 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class args(1).split(",").toVector.map{ d => val v = d.split(":") - MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree + mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree }.mkString("\n\n") } def amm = ammonite def ammonite = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( version ) - val d = MavenRepository.central.resolveOne( + val scalac = new ScalaCompilerDependency( cbtHasChanged,mavenCache, version ) + val d = mavenCentral.resolveOne( MavenDependency( "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.7") ) @@ -36,7 +47,7 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( version ) + val scalac = new ScalaCompilerDependency( cbtHasChanged, mavenCache, version ) lib.runMain( "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader(classLoaderCache) ) @@ -49,16 +60,16 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion) val zincVersion = args.lift(3).getOrElse(constants.zincVersion) val scalaDeps = Seq( - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)), - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)) + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)), + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)) ) val scalaXml = Dependencies( - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)), - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion)) + mavenCentral.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)), + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion)) ) - val zinc = MavenRepository.central.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion)) + val zinc = mavenCentral.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion)) def valName(dep: BoundMavenDependency) = { val words = dep.artifactId.split("_").head.split("-") @@ -66,24 +77,28 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class } def jarVal(dep: BoundMavenDependency) = "_" + valName(dep) +"Jar" - def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: BoundMavenDependency => d} + def transitive(dep: Dependency) = (dep +: lib.transitiveDependencies(dep).reverse).collect{case d: BoundMavenDependency => d} def codeEach(dep: Dependency) = { transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{ deps => val d = deps.last val parents = deps.dropRight(1) - val parentString = if(parents.isEmpty) "" else ( ", " ++ valName(parents.last) ) + val parentString = if(parents.isEmpty) "rootClassLoader" else ( valName(parents.last) ) val n = valName(d) s""" // ${d.groupId}:${d.artifactId}:${d.version} - download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); - ClassLoader $n = cachePut( - classLoader( ${n}File$parentString ), - ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")} - );""" + download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); + + String[] ${n}ClasspathArray = new String[]{${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}}; + String ${n}Classpath = classpath( ${n}ClasspathArray ); + ClassLoader $n = + classLoaderCache.contains( ${n}Classpath ) + ? classLoaderCache.get( ${n}Classpath ) + : classLoaderCache.put( classLoader( ${n}File, $parentString ), ${n}Classpath );""" } } val assignments = codeEach(zinc) ++ codeEach(scalaXml) + val files = scalaDeps ++ transitive(scalaXml) ++ transitive(zinc) //{ case (name, dep) => s"$name =\n ${tree(dep, 4)};" }.mkString("\n\n ") val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies` package cbt; @@ -97,23 +112,29 @@ import static cbt.NailgunLauncher.*; class EarlyDependencies{ /** ClassLoader for stage1 */ - ClassLoader stage1; + ClassLoader classLoader; + String[] classpathArray; /** ClassLoader for zinc */ ClassLoader zinc; -${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s""" String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")} +${files.map(d => s""" String ${valName(d)}File;""").mkString("\n")} + + public EarlyDependencies( + String mavenCache, String mavenUrl, ClassLoaderCache2 classLoaderCache, ClassLoader rootClassLoader + ) throws Exception { +${files.map(d => s""" ${valName(d)}File = mavenCache + "${d.basePath}.jar";""").mkString("\n")} - public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ -${scalaDeps.map(d => s""" download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} +${scalaDeps.map(d => s""" download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} ${assignments.mkString("\n")} - stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_; + classLoader = scalaXml_${scalaXmlVersion.replace(".","_")}_; + classpathArray = scalaXml_${scalaXmlVersion.replace(".","_")}_ClasspathArray; zinc = zinc_${zincVersion.replace(".","_")}_; } } """ - val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java") + val file = nailgun ++ ("/" ++ "EarlyDependencies.java") Files.write( file.toPath, code.getBytes ) println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET ) } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 65db8a4..757e347 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -1,5 +1,4 @@ package cbt -import cbt.paths._ import java.io._ import java.net._ @@ -22,21 +21,19 @@ trait Recommended extends BasicBuild{ "-language:existentials" ) } -class BasicBuild( context: Context ) extends Build( context ) -class Build(val context: Context) extends Dependency with TriggerLoop with SbtDependencyDsl{ +class BasicBuild(val context: Context) extends DependencyImplementation with BuildInterface with TriggerLoop with SbtDependencyDsl{ // library available to builds - implicit final val logger: Logger = context.logger - implicit final val classLoaderCache: ClassLoaderCache = context.classLoaderCache - implicit final val _context = context - override final protected val lib: Lib = new Lib(logger) + implicit protected final val logger: Logger = context.logger + implicit protected final val classLoaderCache: ClassLoaderCache = context.classLoaderCache + implicit protected final val _context = context + override protected final val lib: Lib = new Lib(logger) // ========== general stuff ========== - override def canBeCached = false def enableConcurrency = false final def projectDirectory: File = lib.realpath(context.projectDirectory) assert( projectDirectory.exists, "projectDirectory does not exist: " ++ projectDirectory.string ) - final def usage: String = lib.usage(this.getClass, context) + final def usage: String = lib.usage(this.getClass, show) // ========== meta data ========== @@ -44,11 +41,15 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe final def scalaVersion = context.scalaVersion getOrElse defaultScalaVersion final def scalaMajorVersion: String = lib.scalaMajorVersion(scalaVersion) def crossScalaVersions: Seq[String] = Seq(scalaVersion, "2.10.6") - def copy(context: Context) = lib.copy(this.getClass, context).asInstanceOf[Build] + final def crossScalaVersionsArray: Array[String] = crossScalaVersions.to + + // TODO: this should probably provide a nice error message if class has constructor signature + def copy(context: Context): BuildInterface = lib.copy(this.getClass, context).asInstanceOf[BuildInterface] def zincVersion = "0.3.9" def dependencies: Seq[Dependency] = Seq( - MavenRepository.central.resolve( + // FIXME: this should probably be removed + MavenResolver(context.cbtHasChanged, context.paths.mavenCache, MavenResolver.central).resolve( "org.scala-lang" % "scala-library" % scalaVersion ) ) @@ -108,13 +109,10 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe .flatMap(_.listFiles) .filter(_.toString.endsWith(".jar")) - //def cacheJar = false override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath - override def dependencyJars : Seq[File] = localJars ++ super.dependencyJars def exportedClasspath : ClassPath = ClassPath(compile.toSeq:_*) def targetClasspath = ClassPath(Seq(compileTarget)) - def exportedJars: Seq[File] = Seq() // ========== compile, run, test ========== /** scalac options used for zinc and scaladoc */ @@ -124,15 +122,17 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe def needsUpdate: Boolean = needsUpdateCache( context.cbtHasChanged || lib.needsUpdate( sourceFiles, compileStatusFile ) - || transitiveDependencies.exists(_.needsUpdate) + || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) ) private object compileCache extends Cache[Option[File]] def compile: Option[File] = compileCache{ lib.compile( - needsUpdate, - sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, scalacOptions, - context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion + context.cbtHasChanged, + needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), + sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, + context.paths.mavenCache, scalacOptions, context.classLoaderCache, + zincVersion = zincVersion, scalaVersion = scalaVersion ) } @@ -155,6 +155,6 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe */ // ========== cbt internals ========== - private[cbt] def finalBuild = this + def finalBuild: BuildInterface = this override def show = this.getClass.getSimpleName ++ "(" ++ projectDirectory.string ++ ")" } diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index c52c3c6..bab8d88 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -1,25 +1,63 @@ package cbt import java.io.File +import java.nio.file._ import scala.collection.immutable.Seq -class BuildBuild(context: Context) extends Build(context){ - override def dependencies = Seq( CbtDependency()(context.logger) ) ++ super.dependencies +class BuildBuild(context: Context) extends BasicBuild(context){ + override def dependencies = + super.dependencies :+ context.cbtDependency def managedBuildDirectory: File = lib.realpath( projectDirectory.parent ) - val managedBuild = try{ - val managedContext = context.copy( projectDirectory = managedBuildDirectory ) - val cl = classLoader(context.classLoaderCache) - logger.composition("Loading build at "+managedContext.projectDirectory) - cl - .loadClass(lib.buildClassName) - .getConstructor(classOf[Context]) - .newInstance(managedContext) - .asInstanceOf[Build] - } catch { - case e: ClassNotFoundException if e.getMessage == lib.buildClassName => - throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory) - case e: Exception => - throw new Exception("during build: "+context.projectDirectory, e) + private object managedBuildCache extends Cache[BuildInterface] + def managedBuild = managedBuildCache{ + try{ + val managedContext = context.copy( + projectDirectory = managedBuildDirectory, + parentBuild=Some(this) + ) + val managedBuildFile = projectDirectory++"/build.scala" + logger.composition("Loading build at "++managedContext.projectDirectory.toString) + ( + if(managedBuildFile.exists){ + val contents = new String(Files.readAllBytes(managedBuildFile.toPath)) + val cbtUrl = ("cbt:"++GitDependency.GitUrl.regex++"#[a-z0-9A-Z]+").r + cbtUrl + .findFirstIn(contents) + .flatMap{ + url => + val Array(base,hash) = url.drop(4).split("#") + if(context.cbtHome.string.contains(hash)) + None + else Some{ + val checkoutDirectory = new GitDependency(base, hash).checkout + val build = new BasicBuild( context.copy( projectDirectory = checkoutDirectory ++ "/nailgun_launcher" ) ) + val cl = build + .classLoader(classLoaderCache) + // Note: cbt can't use an old version of itself for building, + // otherwise we'd have to recursively build all versions since + // the beginning. Instead CBT always needs to build the pure Java + // Launcher in the checkout with itself and then run it via reflection. + cl + .loadClass( "cbt.NailgunLauncher" ) + .getMethod( "getBuild", classOf[AnyRef] ) + .invoke( null, managedContext.copy(cbtHome=checkoutDirectory) ) + } + }.getOrElse{ + classLoader(context.classLoaderCache) + .loadClass(lib.buildClassName) + .getConstructors.head + .newInstance(managedContext) + } + } else { + new BasicBuild(managedContext) + } + ).asInstanceOf[BuildInterface] + } catch { + case e: ClassNotFoundException if e.getMessage == lib.buildClassName => + throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory) + case e: Exception => + throw new Exception("during build: "+context.projectDirectory, e) + } } override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles - override def finalBuild = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild + override def finalBuild: BuildInterface = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild } diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 8965cee..f6b6911 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -11,7 +11,8 @@ sealed abstract class ProjectProxy extends Ha{ def dependencies = Seq(delegate) } */ -trait TriggerLoop extends Dependency{ +trait TriggerLoop extends DependencyImplementation{ + final def triggerLoopFilesArray = triggerLoopFiles.toArray def triggerLoopFiles: Seq[File] } /** You likely want to use the factory method in the BasicBuild class instead of this. */ @@ -21,9 +22,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{ final override lazy val lib: Lib = new Lib(logger) private val root = lib.loadRoot( context.copy(args=Seq()) ) lazy val build = root.finalBuild - override def canBeCached = build.canBeCached def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles override final val needsUpdate = build.needsUpdate diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index 333d81c..174f9ff 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -10,11 +10,10 @@ object GitDependency{ } case class GitDependency( url: String, ref: String // example: git://github.com/cvogt/cbt.git# -)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends Dependency{ +)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{ import GitDependency._ override def lib = new Lib(logger) - override def canBeCached = dependency.canBeCached // TODO: add support for authentication via ssh and/or https // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ private val GitUrl( _, domain, path ) = url @@ -47,7 +46,6 @@ case class GitDependency( def dependencies = Seq(dependency) def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq() private[cbt] def targetClasspath = exportedClasspath def needsUpdate: Boolean = false } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index d7456e1..1b19a5e 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -1,5 +1,4 @@ package cbt -import cbt.paths._ import java.io._ import java.net._ @@ -30,7 +29,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ .newInstance(context) /** Loads Build for given Context */ - def loadDynamic(context: Context, default: Context => Build = new Build(_)): Build = { + def loadDynamic(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = { context.logger.composition( context.logger.showInvocation("Build.loadDynamic",context) ) loadRoot(context, default).finalBuild } @@ -38,7 +37,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ Loads whatever Build needs to be executed first in order to eventually build the build for the given context. This can either the Build itself, of if exists a BuildBuild or a BuildBuild for a BuildBuild and so on. */ - def loadRoot(context: Context, default: Context => Build = new Build(_)): Build = { + def loadRoot(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = { context.logger.composition( context.logger.showInvocation("Build.loadRoot",context.projectDirectory) ) def findStartDir(projectDirectory: File): File = { val buildDir = realpath( projectDirectory ++ "/build" ) @@ -73,6 +72,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } def docJar( + cbtHasChanged: Boolean, scalaVersion: String, sourceFiles: Seq[File], dependencyClasspath: ClassPath, @@ -82,7 +82,8 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ scalaMajorVersion: String, version: String, compileArgs: Seq[String], - classLoaderCache: ClassLoaderCache + classLoaderCache: ClassLoaderCache, + mavenCache: File ): Option[File] = { if(sourceFiles.isEmpty){ None @@ -98,7 +99,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - ScalaDependencies(scalaVersion)(logger).classLoader(classLoaderCache) + ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion)(logger).classLoader(classLoaderCache) ) } lib.jarFile( @@ -116,10 +117,13 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None logger.lib(s"invoke testDefault( $context )") - val exitCode: ExitCode = loadDynamic( - context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), - new BasicBuild(_) with mixins.Test - ).run.asInstanceOf[ExitCode] // FIXME + val exitCode: ExitCode = + new ReflectBuild( + loadDynamic( + context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), + new BasicBuild(_) with mixins.Test + ) + ).callNullary( Some("run") ) logger.lib(s"return testDefault( $context )") Some(exitCode) } else None @@ -147,13 +151,18 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ def taskNames(cls: Class[_]): Seq[String] = tasks(cls).keys.toVector.sorted - def usage(buildClass: Class[_], context: Context): String = { - val baseTasks = lib.taskNames(classOf[Build]) + def usage(buildClass: Class[_], show: String): String = { + val baseTasks = Seq( + classOf[BasicBuild], + classOf[PackageBuild], + classOf[PublishBuild], + classOf[Recommended] + ).flatMap(lib.taskNames).distinct.sorted val thisTasks = lib.taskNames(buildClass) diff baseTasks ( ( if( thisTasks.nonEmpty ){ - s"""Methods provided by Build ${context.projectDirectory} + s"""Methods provided by Build ${show} ${thisTasks.mkString(" ")} @@ -165,12 +174,13 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ ) ++ "\n" } - class ReflectBuild[T:scala.reflect.ClassTag](build: Build) extends ReflectObject(build){ - def usage = lib.usage(build.getClass, build.context) + class ReflectBuild[T:scala.reflect.ClassTag](build: BuildInterface) extends ReflectObject(build){ + def usage = lib.usage(build.getClass, build.show) } - abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){ + abstract class ReflectObject[T](obj: T){ def usage: String - def callNullary( taskName: Option[String] ): Unit = { + def callNullary( taskName: Option[String] ): ExitCode = { + logger.lib("Calling task " ++ taskName.toString) val ts = tasks(obj.getClass) taskName.map( NameTransformer.encode ).flatMap(ts.get).map{ method => val result: Option[Any] = Option(method.invoke(obj)) // null in case of Unit @@ -183,26 +193,30 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ scala.util.Try( value.getClass.getDeclaredMethod("toConsole") ) match { case scala.util.Success(toConsole) => println(toConsole.invoke(value)) + ExitCode.Success case scala.util.Failure(e) if Option(e.getMessage).getOrElse("") contains "toConsole" => value match { - case ExitCode(code) => System.exit(code) - case other => println( other.toString ) // no method .toConsole, using to String + case code:ExitCode => + code + case other => + println( other.toString ) // no method .toConsole, using to String + ExitCode.Success } case scala.util.Failure(e) => throw e } - }.getOrElse("") + }.getOrElse(ExitCode.Success) }.getOrElse{ taskName.foreach{ n => System.err.println(s"Method not found: $n") System.err.println("") } System.err.println(usage) - taskName.foreach{ _ => + taskName.map{ _ => ExitCode.Failure - } + }.getOrElse( ExitCode.Success ) } } } diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala index d24bf38..869af0e 100644 --- a/stage2/PackageBuild.scala +++ b/stage2/PackageBuild.scala @@ -20,9 +20,11 @@ abstract class PackageBuild(context: Context) extends BasicBuild(context) with A private object cacheDocBasicBuild extends Cache[Option[File]] def docJar: Option[File] = cacheDocBasicBuild{ - lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, scalaMajorVersion, version, scalacOptions, context.classLoaderCache ) + lib.docJar( + context.cbtHasChanged, + scalaVersion, sourceFiles, dependencyClasspath, apiTarget, + jarTarget, artifactId, scalaMajorVersion, version, + scalacOptions, context.classLoaderCache, context.paths.mavenCache + ) } - - override def jars = jar.toVector ++ dependencyJars - override def exportedJars: Seq[File] = jar.toVector } diff --git a/stage2/SbtDependencyDsl.scala b/stage2/SbtDependencyDsl.scala index 1ecf72c..d8c0786 100644 --- a/stage2/SbtDependencyDsl.scala +++ b/stage2/SbtDependencyDsl.scala @@ -1,5 +1,5 @@ package cbt -trait SbtDependencyDsl{ self: Build => +trait SbtDependencyDsl{ self: BasicBuild => /** SBT-like dependency builder DSL for syntax compatibility */ class DependencyBuilder2( groupId: String, artifactId: String, scalaVersion: Option[String] ){ def %(version: String) = scalaVersion.map( diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index fa41d79..5316e78 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -4,12 +4,23 @@ import java.io._ import scala.collection.immutable.Seq -import cbt.paths._ object Stage2 extends Stage2Base{ + def getBuild(__context: java.lang.Object, _cbtChanged: java.lang.Boolean) = { + val cl1 = __context.getClass.getClassLoader + val cl2 = classOf[Context].getClassLoader + val _context = __context.asInstanceOf[Context] + val context = _context.copy( + cbtHasChanged = _context.cbtHasChanged || _cbtChanged + ) + val first = new Lib(context.logger).loadRoot( context ) + first.finalBuild + } + def run( args: Stage2Args ): Unit = { import args.logger - + val paths = Paths(args.cbtHome,args.cache) + import paths._ val lib = new Lib(args.logger) logger.stage2(s"Stage2 start") @@ -24,11 +35,26 @@ object Stage2 extends Stage2Base{ } val task = args.args.lift( taskIndex ) - val context = Context( args.cwd, args.cwd, args.args.drop( taskIndex ), logger, args.cbtHasChanged, args.classLoaderCache ) + val context: Context = ContextImplementation( + args.cwd, + args.cwd, + args.args.drop( taskIndex ).toArray, + logger.enabledLoggers.toArray, + logger.start, + args.cbtHasChanged, + null, + null, + args.permanentKeys, + args.permanentClassLoaders, + args.cache, + args.cbtHome, + compatibilityTarget, + null + ) val first = lib.loadRoot( context ) val build = first.finalBuild - def call(build: Build) = { + def call(build: BuildInterface) = { if(cross){ build.crossScalaVersions.foreach{ v => new lib.ReflectBuild( @@ -57,7 +83,7 @@ object Stage2 extends Stage2Base{ case file if triggerFiles.exists(file.toString startsWith _.toString) => val build = lib.loadDynamic(context) - logger.loop(s"Re-running $task for " ++ build.projectDirectory.toString) + logger.loop(s"Re-running $task for " ++ build.show) call(build) } } else { diff --git a/stage2/mixins.scala b/stage2/mixins.scala index fcffd97..1383324 100644 --- a/stage2/mixins.scala +++ b/stage2/mixins.scala @@ -2,31 +2,11 @@ package cbt package mixins import scala.collection.immutable.Seq import java.io._ -trait Test extends Build{ +trait Test extends BasicBuild{ lazy val testedBuild = BuildDependency( projectDirectory.parent ) override def dependencies = Seq( testedBuild ) ++ super.dependencies override def defaultScalaVersion = testedBuild.build.scalaVersion } -trait Sbt extends Build{ - override def sources = Seq( projectDirectory ++ "/src/main/scala" ) -} trait SbtTest extends Test{ override def sources = Vector( projectDirectory.parent ++ "/src/test/scala" ) } -trait ScalaTest extends Build with Test{ - def scalaTestVersion: String - - override def dependencies = super.dependencies :+ MavenRepository.central.resolve( - "org.scalatest" %% "scalatest" % scalaTestVersion - ) - - override def run: ExitCode = { - val discoveryPath = compile.toString++"/" - context.logger.lib("discoveryPath: " ++ discoveryPath) - lib.runMain( - "org.scalatest.tools.Runner", - Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1), - classLoader(context.classLoaderCache) - ) - } -} diff --git a/test/build/build.scala b/test/build/build.scala index 29665a6..8989431 100644 --- a/test/build/build.scala +++ b/test/build/build.scala @@ -2,5 +2,5 @@ import cbt._ import java.io.File import scala.collection.immutable.Seq class Build(context: cbt.Context) extends BasicBuild(context){ - override def dependencies = Seq( CbtDependency() ) ++ super.dependencies + override def dependencies = Seq( context.cbtDependency ) ++ super.dependencies } diff --git a/test/simple-fixed/Main.scala b/test/simple-fixed/Main.scala new file mode 100644 index 0000000..1c423ca --- /dev/null +++ b/test/simple-fixed/Main.scala @@ -0,0 +1,6 @@ +import ai.x.diff +import org.eclipse.jgit.lib.Ref +import com.spotify.missinglink.ArtifactLoader +object Main extends App{ + println(diff.DiffShow.diff("a","b")) +} diff --git a/test/simple-fixed/build/build.scala b/test/simple-fixed/build/build.scala new file mode 100644 index 0000000..d088a40 --- /dev/null +++ b/test/simple-fixed/build/build.scala @@ -0,0 +1,34 @@ +import cbt._ +import cbt.extensions._ // FIXME: do not require this import +import scala.collection.immutable.Seq +import java.io.File + +// cbt:file:///Users/chris/code/cbt/#b65159f95421d9484f29327c11c0fa179eb7483f +class Build(context: cbt.Context) extends BasicBuild(context){ + override def dependencies = ( + super.dependencies + ++ + Seq( + GitDependency("https://github.com/xdotai/diff.git", "80a08bf45f7c4c3fd20c4bc6dbc9cae0072e3c0f"), + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), + MavenDependency("joda-time", "joda-time", "2.9.2"), + // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + // the below tests pom inheritance with variable substitution for pom xml tag contents + MavenDependency("com.spotify", "missinglink-core", "0.1.1") + ), + MavenResolver( + context.cbtHasChanged, + context.paths.mavenCache, + MavenResolver.central, + MavenResolver.bintray("tpolecat"), + MavenResolver.sonatypeSnapshots + ).resolve( + "org.cvogt" %% "play-json-extensions" % "0.8.0", + "org.tpolecat" %% "tut-core" % "0.4.2", + "ai.x" %% "lens" % "1.0.0-SNAPSHOT" + ) + ) + ) +} diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index 5782c2d..f54c3dc 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -7,8 +7,9 @@ class Build(context: cbt.Context) extends BasicBuild(context){ super.dependencies ++ Seq( - GitDependency("https://github.com/xdotai/diff.git", "666bbbf4dbff6fadc81c011ade7b83e91d3f9256"), - MavenRepository.central.resolve( + GitDependency("https://github.com/xdotai/diff.git", "698717469b8dd86e8570b86354892be9c0654caf"), + // FIXME: make the below less verbose + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), MavenDependency("joda-time", "joda-time", "2.9.2"), // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties @@ -21,10 +22,12 @@ class Build(context: cbt.Context) extends BasicBuild(context){ // TODO: put in a proper error message for version range not supported //MavenDependency("com.github.nikita-volkov", "sext", "0.2.4") ), - MavenRepository.combine( - MavenRepository.central, - MavenRepository.bintray("tpolecat"), - MavenRepository.sonatypeSnapshots + MavenResolver( + context.cbtHasChanged, + context.paths.mavenCache, + MavenResolver.central, + MavenResolver.bintray("tpolecat"), + MavenResolver.sonatypeSnapshots ).resolve( "org.cvogt" %% "play-json-extensions" % "0.8.0", "org.tpolecat" %% "tut-core" % "0.4.2", diff --git a/test/test.scala b/test/test.scala index 29e6afa..4cee1f1 100644 --- a/test/test.scala +++ b/test/test.scala @@ -1,14 +1,16 @@ import cbt._ -import cbt.paths._ import scala.collection.immutable.Seq +import java.util.concurrent.ConcurrentHashMap import java.io.File // micro framework object Main{ def main(_args: Array[String]): Unit = { + val start = System.currentTimeMillis val args = new Stage1ArgsParser(_args.toVector) implicit val logger: Logger = new Logger(args.enabledLoggers, System.currentTimeMillis) val lib = new Lib(logger) + val cbtHome = new File(System.getenv("CBT_HOME")) var successes = 0 var failures = 0 @@ -71,11 +73,32 @@ object Main{ logger.test( "Running tests " ++ _args.toList.toString ) + val cache = cbtHome ++ "/cache" + val mavenCache = cache ++ "/maven" + val cbtHasChanged = true + val mavenCentral = MavenResolver(cbtHasChanged, mavenCache, MavenResolver.central) + { - val noContext = Context(cbtHome ++ "/test/nothing", cbtHome, Seq(), logger, false, new ClassLoaderCache(logger)) - val b = new Build(noContext){ + val noContext = ContextImplementation( + cbtHome ++ "/test/nothing", + cbtHome, + Array(), + Array(), + start, + cbtHasChanged, + null, + null, + new ConcurrentHashMap[String,AnyRef], + new ConcurrentHashMap[AnyRef,ClassLoader], + cache, + cbtHome, + cbtHome ++ "/compatibilityTarget", + null + ) + + val b = new BasicBuild(noContext){ override def dependencies = Seq( - MavenRepository.central.resolve( + mavenCentral.resolve( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0") ) @@ -87,34 +110,34 @@ object Main{ // test that messed up artifacts crash with an assertion (which should tell the user what's up) assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", "jsch", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", null, "0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", null, "0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", "", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", "", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft%", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft%", "jsch", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("", "jsch", " 0.1.53") ).classpath } ( - MavenRepository.combine( - MavenRepository.central, MavenRepository.bintray("tpolecat") + MavenResolver( + cbtHasChanged, mavenCache, MavenResolver.central, MavenResolver.bintray("tpolecat") ).resolve( lib.ScalaDependency("org.tpolecat","tut-core","0.4.2", scalaMajorVersion="2.11") ).classpath.strings ++ - MavenRepository.sonatype.resolve( + MavenResolver(cbtHasChanged, mavenCache,MavenResolver.sonatype).resolve( MavenDependency("org.cvogt","play-json-extensions_2.11","0.8.0") ).classpath.strings ++ - MavenRepository.combine( - MavenRepository.central, MavenRepository.sonatypeSnapshots + MavenResolver( + cbtHasChanged, mavenCache, MavenResolver.central, MavenResolver.sonatypeSnapshots ).resolve( MavenDependency("ai.x","lens_2.11","1.0.0-SNAPSHOT") ).classpath.strings -- cgit v1.2.3 From 6e9faddfb0db0e7b78501cb61c46bb33887ccdcd Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Wed, 27 Apr 2016 23:45:58 -0400 Subject: Support recursively running tasks across all builds in transitive dependencies --- stage2/BasicBuild.scala | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 757e347..fb5e652 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -143,6 +143,54 @@ class BasicBuild(val context: Context) extends DependencyImplementation with Bui lib.test(context) } + def recursiveSafe(_run: BuildInterface => Any): ExitCode = { + val builds = (this +: transitiveDependencies).collect{ + case b: BuildInterface => b + } + val results = builds.map(_run) + if( + results.forall{ + case Some(_:ExitCode) => true + case None => true + case _:ExitCode => true + case other => false + } + ){ + if( + results.collect{ + case Some(c:ExitCode) => c + case c:ExitCode => c + }.filter(_ != 0) + .nonEmpty + ) ExitCode.Failure + else ExitCode.Success + } else ExitCode.Success + } + + def recursive: ExitCode = { + recursiveUnsafe(context.args.lift(1)) + } + + def recursiveUnsafe(taskName: Option[String]): ExitCode = { + recursiveSafe{ + b => + System.err.println(b.show) + lib.trapExitCode{ // FIXME: trapExitCode does not seem to work here + try{ + new lib.ReflectBuild(b).callNullary(taskName) + ExitCode.Success + } catch { + case e: Throwable => println(e.getClass); throw e + } + } + ExitCode.Success + } + } + + def c = compile + def t = test + def rt = recursiveUnsafe(Some("test")) + /* context.logger.composition(">"*80) context.logger.composition("class " ++ this.getClass.toString) -- cgit v1.2.3