diff options
Diffstat (limited to 'stage1')
-rw-r--r-- | stage1/MavenRepository.scala | 22 | ||||
-rw-r--r-- | stage1/Stage1.scala | 13 | ||||
-rw-r--r-- | stage1/Stage1Lib.scala | 75 | ||||
-rw-r--r-- | stage1/resolver.scala | 151 |
4 files changed, 158 insertions, 103 deletions
diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala new file mode 100644 index 0000000..bfd52a7 --- /dev/null +++ b/stage1/MavenRepository.scala @@ -0,0 +1,22 @@ +package cbt +import scala.collection.immutable.Seq +import java.net._ +case class MavenRepository(url: URL){ + def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies + = new BoundMavenDependencies( Seq(url), dependencies.to ) + def resolveOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency + = BoundMavenDependency( dependency, Seq(url) ) +} + +object MavenRepository{ + case class combine(repositories: MavenRepository*){ + def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies + = new BoundMavenDependencies( repositories.map(_.url).to, dependencies.to ) + } + def central = MavenRepository(new URL(NailgunLauncher.MAVEN_URL)) + def jcenter = MavenRepository(new URL("https://jcenter.bintray.com/releases")) + def bintray(owner: String) = MavenRepository(new URL(s"https://dl.bintray.com/$owner/maven")) + private val sonatypeBase = new URL("https://oss.sonatype.org/content/repositories/") + def sonatype = MavenRepository(sonatypeBase ++ "releases") + def sonatypeSnapshots = MavenRepository(sonatypeBase ++ "snapshots") +} diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 368a459..050d494 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -57,24 +57,15 @@ object Stage1{ val lib = new Stage1Lib(logger) import lib._ - - val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val changeIndicator = stage2Target ++ "/cbt/Build.class" - - val deps = Dependencies( - JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") - ) - val classLoaderCache = new ClassLoaderCache(logger) + val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) val cbtHasChanged = _cbtChanged || lib.needsUpdate(sourceFiles, stage2StatusFile) logger.stage1("Compiling stage2 if necessary") - val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) compile( cbtHasChanged, sourceFiles, stage2Target, stage2StatusFile, - nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, + CbtDependency().dependencyClasspath, Seq("-deprecation"), classLoaderCache, zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 39310b9..da9f8dd 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -56,23 +56,35 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def blue(string: String) = scala.Console.BLUE++string++scala.Console.RESET def green(string: String) = scala.Console.GREEN++string++scala.Console.RESET - def download(urlString: URL, target: File, sha1: Option[String]){ - val incomplete = Paths.get( target.string ++ ".incomplete" ); - if( !target.exists ){ - target.getParentFile.mkdirs - logger.resolver(blue("downloading ") ++ urlString.string) - logger.resolver(blue("to ") ++ target.string) - val stream = urlString.openStream - Files.copy(stream, incomplete, StandardCopyOption.REPLACE_EXISTING) - sha1.foreach{ - hash => - val expected = hash - val actual = this.sha1(Files.readAllBytes(incomplete)) - assert( expected == actual, s"$expected == $actual" ) - logger.resolver( green("verified") ++ " checksum for " ++ target.string) + def download(url: URL, target: File, sha1: Option[String]): Boolean = { + if( target.exists ){ + true + } else { + val incomplete = Paths.get( target.string ++ ".incomplete" ); + val connection = url.openConnection.asInstanceOf[HttpURLConnection] + if(connection.getResponseCode != HttpURLConnection.HTTP_OK){ + logger.resolver(blue("not found: ") ++ url.string) + false + } else { + logger.resolver(blue("downloading ") ++ url.string) + logger.resolver(blue("to ") ++ target.string) + target.getParentFile.mkdirs + val stream = connection.getInputStream + try{ + Files.copy(stream, incomplete, StandardCopyOption.REPLACE_EXISTING) + } finally { + stream.close() + } + sha1.foreach{ + hash => + val expected = hash + val actual = this.sha1(Files.readAllBytes(incomplete)) + assert( expected == actual, s"$expected == $actual" ) + logger.resolver( green("verified") ++ " checksum for " ++ target.string) + } + Files.move(incomplete, Paths.get(target.string), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + true } - stream.close - Files.move(incomplete, Paths.get(target.string), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } } @@ -137,26 +149,37 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ None }else{ if( needsRecompile ){ - val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) + import MavenRepository.central + val zinc = central.resolveOne(MavenDependency("com.typesafe.zinc","zinc", zincVersion)) val zincDeps = zinc.transitiveDependencies val sbtInterface = zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } + .collect{ case d @ + BoundMavenDependency( + MavenDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none), + _ + ) => d + } .headOption .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) .jar val compilerInterface = zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } + .collect{ case d @ + BoundMavenDependency( + MavenDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources), + _ + ) => d + } .headOption .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) .jar - val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar - val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar - val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar + val scalaLibrary = central.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion)).jar + val scalaReflect = central.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)).jar + val scalaCompiler = central.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)).jar val start = System.currentTimeMillis @@ -212,9 +235,9 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def ScalaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, - scalaVersion: String + scalaMajorVersion: String ) = - JavaDependency( - groupId, artifactId ++ "_" ++ scalaVersion, version, classifier + MavenDependency( + groupId, artifactId ++ "_" ++ scalaMajorVersion, version, classifier ) -}
\ No newline at end of file +} diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 701871c..ea20b89 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -35,7 +35,7 @@ abstract class Dependency{ def exportedJars: Seq[File] def jars: Seq[File] = exportedJars ++ dependencyJars - def canBeCached = false + def canBeCached: Boolean //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { @@ -68,7 +68,7 @@ abstract class Dependency{ d => // find out latest version of the required dependency val l = d match { - case m: JavaDependency => latest( (m.groupId,m.artifactId) ) + case m: BoundMavenDependency => latest( (m.groupId,m.artifactId) ) case _ => d } // // trigger compilation if not already triggered @@ -166,7 +166,7 @@ abstract class Dependency{ case _:ArtifactInfo => false case _ => true } - noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct + noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct } def show: String = this.getClass.getSimpleName @@ -182,9 +182,9 @@ abstract class Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-compiler",version) -class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-library",version) -class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version) +class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenRepository.central.url)) +class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenRepository.central.url)) +class ScalaReflectDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenRepository.central.url)) case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => override final val needsUpdate = false @@ -199,7 +199,7 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten ) } -case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ +case class BinaryDependency( path: File, dependencies: Seq[Dependency], canBeCached: Boolean )(implicit val logger: Logger) extends Dependency{ def exportedClasspath = ClassPath(Seq(path)) def exportedJars = Seq[File](path) override def needsUpdate = false @@ -207,16 +207,19 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici } /** Allows to easily assemble a bunch of dependencies */ -case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{ - override def dependencies = _dependencies.to - def needsUpdate = dependencies.exists(_.needsUpdate) - def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq() - def targetClasspath = ClassPath(Seq()) +case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ + override def needsUpdate = dependencies.exists(_.needsUpdate) + override def canBeCached = dependencies.forall(_.canBeCached) + override def exportedClasspath = ClassPath(Seq()) + override def exportedJars = Seq() + override def targetClasspath = ClassPath(Seq()) +} +object Dependencies{ + def apply( dependencies: Dependency* )(implicit logger: Logger): Dependencies = Dependencies( dependencies.to ) } case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations override def canBeCached = false /* private object classLoaderRecursionCache extends Cache[ClassLoader] @@ -232,23 +235,27 @@ case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) override def exportedJars = ???//Seq[File]() override def dependencies = Seq( - JavaDependency("org.scala-lang","scala-library",constants.scalaVersion), - JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + MavenRepository.central.resolve( + MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), + MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + ) ) // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ) = getClass.getClassLoader } case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations override def canBeCached = false override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq( stage2Target ) ) override def exportedJars = ??? override def dependencies = Seq( Stage1Dependency(), - JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + MavenRepository.central.resolve( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + ) ) } @@ -258,19 +265,30 @@ object Classifier{ object javadoc extends Classifier(Some("javadoc")) object sources extends Classifier(Some("sources")) } +abstract class DependenciesProxy{ -case class JavaDependency( +} +class BoundMavenDependencies( + urls: Seq[URL], mavenDependencies: Seq[MavenDependency] +)(implicit logger: Logger) extends Dependencies( + mavenDependencies.map( BoundMavenDependency(_,urls) ) +) +case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none +) +case class BoundMavenDependency( + mavenDependency: MavenDependency, repositories: Seq[URL] )(implicit val logger: Logger) extends ArtifactInfo{ + val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ - case JavaDependency.ValidIdentifier(_) => + case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid groupId: '$groupId'" ) assert( Option(artifactId).collect{ - case JavaDependency.ValidIdentifier(_) => + case BoundMavenDependency.ValidIdentifier(_) => }.nonEmpty, s"not a valid artifactId: '$artifactId'" ) @@ -280,62 +298,60 @@ case class JavaDependency( ) override def needsUpdate = false - override def canBeCached = true + override def canBeCached = dependencies.forall(_.canBeCached) private val groupPath = groupId.split("\\.").mkString("/") - def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") + protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") - private def resolverUrl:URL = new URL( - if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" - ) - private def baseUrl: URL = resolverUrl ++ basePath - private def baseFile: File = mavenCache ++ basePath - private def pomFile: File = baseFile ++ ".pom" - private def jarFile: File = baseFile ++ ".jar" //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" - private def pomUrl: URL = baseUrl ++ ".pom" - private[cbt] def jarUrl: URL = baseUrl ++ ".jar" - def exportedJars = Seq( jar ) - def exportedClasspath = ClassPath( exportedJars ) - def targetClasspath = exportedClasspath + override def exportedJars = Seq( jar ) + override def exportedClasspath = ClassPath( exportedJars ) + override def targetClasspath = exportedClasspath import scala.collection.JavaConversions._ - - def jarSha1 = { - val file = jarFile ++ ".sha1" - lib.download( jarUrl ++ ".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim + + private def resolve(suffix: String, hash: Option[String]): File = { + val file = mavenCache ++ basePath ++ "." ++ suffix + val urls = repositories.map(_ ++ basePath ++ "." ++ suffix) + urls.find( + lib.download(_, file, hash) + ).getOrElse( + throw new Exception(s"\nCannot resolve\n$this\nCan't find any of\n"++urls.mkString("\n")) + ) + file } - def pomSha1 = { - val file = pomFile++".sha1" - lib.download( pomUrl++".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim + private def resolveHash(suffix: String) = { + Files.readAllLines( + resolve( suffix ++ ".sha1", None ).toPath + ).mkString("\n").split(" ").head.trim } + + private object jarSha1Cache extends Cache[String] + def jarSha1: String = jarSha1Cache{ resolveHash("jar") } + + private object pomSha1Cache extends Cache[String] + def pomSha1: String = pomSha1Cache{ resolveHash("pom") } private object jarCache extends Cache[File] - def jar = jarCache{ - lib.download( jarUrl, jarFile, Some(jarSha1) ) - jarFile - } - def pomXml = XML.loadFile(pom.toString) + def jar: File = jarCache{ resolve("jar", Some(jarSha1)) } - def pom = { - lib.download( pomUrl, pomFile, Some(pomSha1) ) - pomFile - } + private object pomCache extends Cache[File] + def pom: File = pomCache{ resolve("pom", Some(pomSha1)) } + def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== - lazy val transitivePom: Seq[JavaDependency] = { + lazy val transitivePom: Seq[BoundMavenDependency] = { (pomXml \ "parent").collect{ case parent => - JavaDependency( - (parent \ "groupId").text, - (parent \ "artifactId").text, - (parent \ "version").text + BoundMavenDependency( + MavenDependency( + (parent \ "groupId").text, + (parent \ "artifactId").text, + (parent \ "version").text + ), + repositories )(logger) }.flatMap(_.transitivePom) :+ this } @@ -362,7 +378,7 @@ case class JavaDependency( } ).toMap - def dependencies: Seq[JavaDependency] = { + def dependencies: Seq[BoundMavenDependency] = { if(classifier == Classifier.sources) Seq() else (pomXml \ "dependencies" \ "dependency").collect{ case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => @@ -383,9 +399,12 @@ case class JavaDependency( throw new Exception(s"$artifactId not found in \n$dependencyVersions") ) ) - JavaDependency( - groupId, artifactId, version, - Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) + BoundMavenDependency( + MavenDependency( + groupId, artifactId, version, + Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) + ), + repositories ) }.toVector } @@ -411,7 +430,7 @@ case class JavaDependency( } } } -object JavaDependency{ +object BoundMavenDependency{ def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java def semanticVersionLessThan(left: String, right: String) = { // FIXME: this ignores ends when different size |