From dba77034209f7f08e5b0f894d1973686f688c2d0 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 12 Mar 2016 00:02:22 -0500 Subject: A draft implementation that runs builds concurrently (probably buggy right now). Is CBT "reactive" now ;)? --- stage1/resolver.scala | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'stage1/resolver.scala') diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 30f03e2..566b37d 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -5,6 +5,8 @@ import java.io._ import scala.collection.immutable.Seq import scala.xml._ import paths._ +import scala.concurrent._ +import scala.concurrent.duration._ private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ lazy val children = computeChildren @@ -35,8 +37,60 @@ abstract class Dependency{ def canBeCached = false def cacheDependencyClassLoader = true + //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] + def exportClasspathConcurrently: ClassPath = { + // FIXME: this should separate a blocking and a non-blocking EC + import scala.concurrent.ExecutionContext.Implicits.global + Await.result( + exportClasspathConcurrently( + transitiveDependencies + .collect{ case d: ArtifactInfo => d } + .groupBy( d => (d.groupId,d.artifactId) ) + .mapValues( _.head ) + //, new BuildCache + ), + Duration.Inf + ) + } + + def concurrencyEnabled = false + + /** + The implementation of this method is untested and likely buggy + at this stage. + */ + private object cacheExportClasspathConcurrently extends Cache[Future[ClassPath]] + private def exportClasspathConcurrently( + latest: Map[(String, String),ArtifactInfo]//, cache: BuildCache + )( implicit ec: ExecutionContext ): Future[ClassPath] = cacheExportClasspathConcurrently{ + Future.sequence( // trigger compilation / download of all dependencies first + this.dependencies.map{ + d => + // find out latest version of the required dependency + val l = d match { + case m: MavenDependency => latest( (m.groupId,m.artifactId) ) + case _ => d + } + // // trigger compilation if not already triggered + // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) + l.exportClasspathConcurrently( latest ) + } + ).map( + // merge dependency classpaths into one + ClassPath.flatten(_) + ).map( + _ => + // now that all dependencies are done, compile the code of this + exportedClasspath + ) + } + private object cacheClassLoaderBasicBuild extends Cache[URLClassLoader] def classLoader: URLClassLoader = cacheClassLoaderBasicBuild{ + if( concurrencyEnabled ){ + // trigger concurrent building / downloading dependencies + exportClasspathConcurrently + } val transitiveClassPath = transitiveDependencies.map{ case d if d.canBeCached => Left(d) case d => Right(d) -- cgit v1.2.3 From 906db6c55289d9aba5da824d5f2f3d2cae70813e Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 12 Mar 2016 16:06:14 -0500 Subject: Improved how to express dependencies Add - Constructor syntax for cross-scala-version dependencies (as rightfully requested on reddit) and encouraged over SBT's still supported operator syntax - Add support for classifiers other than "sources" --- README.md | 7 +++++- coursier/Coursier.scala | 8 +++--- stage1/Stage1Lib.scala | 30 +++++++++++++++-------- stage1/constants.scala | 3 ++- stage1/resolver.scala | 57 ++++++++++++++++++++++++------------------- stage2/AdminTasks.scala | 4 +-- stage2/BasicBuild.scala | 25 +++++++++++++------ stage2/BuildDependency.scala | 2 +- stage2/PackageBuild.scala | 1 - test/simple/build/build.scala | 9 +++++++ test/test.scala | 6 +++-- 11 files changed, 98 insertions(+), 54 deletions(-) create mode 100644 test/simple/build/build.scala (limited to 'stage1/resolver.scala') diff --git a/README.md b/README.md index 5fd89e5..36f1f50 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,12 @@ class Build(context: cbt.Context) extends PackageBuild(context){ override def groupId = "org.cvogt" override def artifactId = "play-json-extensions" override def dependencies = super.dependencies ++ Vector( - "com.typesafe.play" %% "play-json" % "2.4.4" + // encouraged way to declare dependencies + ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), + JavaDependency("joda-time", "joda-time", "2.9.2") + // also supported for SBT syntax compatibility: + // "com.typesafe.play" %% "play-json" % "2.4.4" + // "joda-time" % "joda-time % "2.9.2" ) override def compile = { println("Compiling...") diff --git a/coursier/Coursier.scala b/coursier/Coursier.scala index a2f5f39..48282b1 100644 --- a/coursier/Coursier.scala +++ b/coursier/Coursier.scala @@ -1,7 +1,7 @@ /* package cbt object Coursier{ - implicit class CoursierDependencyResolution(d: MavenDependency){ + implicit class CoursierDependencyResolution(d: JavaDependency){ import d._ def resolveCoursier = { import coursier._ @@ -12,7 +12,7 @@ object Coursier{ val start = Resolution( Set( - MavenDependency( + JavaDependency( Module(groupId, artifactId), version ) ) @@ -23,7 +23,7 @@ object Coursier{ val resolution = start.process.run(fetch).run - val errors: Seq[(MavenDependency, Seq[String])] = resolution.errors + val errors: Seq[(JavaDependency, Seq[String])] = resolution.errors if(errors.nonEmpty) throw new Exception(errors.toString) @@ -40,7 +40,7 @@ object Coursier{ case Right(file) => file }) - resolution.dependencies.map( d => cbt.MavenDependency(d.module.organization,d.module.name, d.version)).to[collection.immutable.Seq] + resolution.dependencies.map( d => cbt.JavaDependency(d.module.organization,d.module.name, d.version)).to[collection.immutable.Seq] } } } diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 67ae049..90675ae 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -38,6 +38,9 @@ class BaseLib{ class Stage1Lib( val logger: Logger ) extends BaseLib{ lib => + implicit val implicitLogger: Logger = logger + + def scalaMajorVersion(scalaMinorVersion: String) = scalaMinorVersion.split("\\.").take(2).mkString(".") // ========== reflection ========== @@ -132,26 +135,26 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ // only run zinc if files changed, for performance reasons // FIXME: this is broken, need invalidate on changes in dependencies as well if( true || needsRecompile ){ - val zinc = MavenDependency("com.typesafe.zinc","zinc", zincVersion)(logger) + val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) val zincDeps = zinc.transitiveDependencies - + val sbtInterface = zincDeps - .collect{ case d @ MavenDependency( "com.typesafe.sbt", "sbt-interface", _, false ) => d } + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } .headOption - .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies") ) + .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) .jar val compilerInterface = zincDeps - .collect{ case d @ MavenDependency( "com.typesafe.sbt", "compiler-interface", _, true ) => d } + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } .headOption - .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies") ) + .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) .jar - val scalaLibrary = MavenDependency("org.scala-lang","scala-library",scalaVersion)(logger).jar - val scalaReflect = MavenDependency("org.scala-lang","scala-reflect",scalaVersion)(logger).jar - val scalaCompiler = MavenDependency("org.scala-lang","scala-compiler",scalaVersion)(logger).jar + val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar + val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar + val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar val code = redirectOutToErr{ trapExitCode{ @@ -226,5 +229,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ code ExitCode.Success } -} + def ScalaDependency( + groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, + scalaVersion: String + ) = + JavaDependency( + groupId, artifactId ++ "_" ++ scalaVersion, version, classifier + ) +} diff --git a/stage1/constants.scala b/stage1/constants.scala index bf0943e..a14754e 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,4 +1,5 @@ package cbt object constants{ - def scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 30f03e2..a3273ed 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -80,7 +80,7 @@ abstract class Dependency{ case _:ArtifactInfo => false case _ => true } - noInfo ++ MavenDependency.removeOutdated( hasInfo ) + noInfo ++ JavaDependency.removeOutdated( hasInfo ) } def show: String = this.getClass.getSimpleName @@ -96,9 +96,9 @@ abstract class Dependency{ } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-compiler",version) -class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-library",version) -class ScalaReflectDependency (version: String)(implicit logger: Logger) extends MavenDependency("org.scala-lang","scala-reflect",version) +class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-compiler",version) +class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-library",version) +class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version) case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => final val updated = false @@ -106,9 +106,9 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten def exportedClasspath = ClassPath(Seq()) def exportedJars = Seq[File]() def dependencies = Seq( - new ScalaCompilerDependency(version)(logger), - new ScalaLibraryDependency(version)(logger), - new ScalaReflectDependency(version)(logger) + new ScalaCompilerDependency(version), + new ScalaLibraryDependency(version), + new ScalaReflectDependency(version) ) } @@ -128,27 +128,34 @@ case class CbtDependency()(implicit val logger: Logger) extends Dependency{ def exportedClasspath = ClassPath( Seq( stage2Target ) ) def exportedJars = Seq[File]() override def dependencies = Seq( - Stage1Dependency()(logger), - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger), - MavenDependency("com.lihaoyi","ammonite-repl_2.11.7","0.5.5")(logger), - MavenDependency("org.scala-lang.modules","scala-xml_2.11","1.0.5")(logger) + Stage1Dependency(), + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + lib.ScalaDependency( + "com.lihaoyi","ammonite-ops","0.5.5", scalaVersion = constants.scalaMajorVersion + ), + lib.ScalaDependency( + "org.scala-lang.modules","scala-xml","1.0.5", scalaVersion = constants.scalaMajorVersion + ) ) def updated = false // FIXME: think this through, might allow simplifications and/or optimizations } -sealed trait ClassifierBase -final case class Classifier(name: String) extends ClassifierBase -case object javadoc extends ClassifierBase -case object sources extends ClassifierBase +case class Classifier(name: Option[String]) +object Classifier{ + object none extends Classifier(None) + object javadoc extends Classifier(Some("javadoc")) + object sources extends Classifier(Some("sources")) +} -case class MavenDependency( groupId: String, artifactId: String, version: String, sources: Boolean = false )(implicit val logger: Logger) - extends ArtifactInfo{ +case class JavaDependency( + groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none +)(implicit val logger: Logger) extends ArtifactInfo{ def updated = false override def canBeCached = true private val groupPath = groupId.split("\\.").mkString("/") - def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version"++(if(sources) "-sources" else "") + def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") private def resolverUrl:URL = new URL( if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" @@ -197,25 +204,25 @@ case class MavenDependency( groupId: String, artifactId: String, version: String // ========== pom traversal ========== - lazy val pomParents: Seq[MavenDependency] = { + lazy val pomParents: Seq[JavaDependency] = { (pomXml \ "parent").collect{ case parent => - MavenDependency( + JavaDependency( (parent \ "groupId").text, (parent \ "artifactId").text, (parent \ "version").text )(logger) } } - def dependencies: Seq[MavenDependency] = { - if(sources) Seq() + def dependencies: Seq[JavaDependency] = { + if(classifier == Classifier.sources) Seq() else (pomXml \ "dependencies" \ "dependency").collect{ case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - MavenDependency( + JavaDependency( lookup(xml,_ \ "groupId").get, lookup(xml,_ \ "artifactId").get, lookup(xml,_ \ "version").get, - (xml \ "classifier").text == "sources" + Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) )(logger) }.toVector } @@ -235,7 +242,7 @@ case class MavenDependency( groupId: String, artifactId: String, version: String ) } } -object MavenDependency{ +object JavaDependency{ def semanticVersionLessThan(left: String, right: String) = { // FIXME: this ignores ends when different size val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt) diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index 70b140e..bce0ae7 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -7,7 +7,7 @@ class AdminTasks(lib: Lib, args: Array[String]){ args(1).split(",").toVector.map{ d => val v = d.split(":") - new MavenDependency(v(0),v(1),v(2))(lib.logger).classpath + new JavaDependency(v(0),v(1),v(2))(lib.logger).classpath } ) } @@ -15,7 +15,7 @@ class AdminTasks(lib: Lib, args: Array[String]){ def ammonite = { val version = args.lift(1).getOrElse(constants.scalaVersion) val scalac = new ScalaCompilerDependency( version ) - val d = MavenDependency( + val d = JavaDependency( "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.6") ) // FIXME: this does not work quite yet, throws NoSuchFileException: /ammonite/repl/frontend/ReplBridge$.class diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index bee58dd..f8431ea 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -35,7 +35,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ // ========== meta data ========== def scalaVersion: String = constants.scalaVersion - final def scalaMajorVersion: String = scalaVersion.split("\\.").take(2).mkString(".") + final def scalaMajorVersion: String = lib.scalaMajorVersion(scalaVersion) def zincVersion = "0.3.9" def dependencies: Seq[Dependency] = Seq( @@ -80,14 +80,25 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ } assertSourceDirectories() - - /** SBT-like dependency builder DSL */ - class GroupIdAndArtifactId( groupId: String, artifactId: String ){ - def %(version: String) = new MavenDependency(groupId, artifactId, version)(lib.logger) + def ScalaDependency( + groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none, + scalaVersion: String = scalaMajorVersion + ) = lib.ScalaDependency( groupId, artifactId, version, classifier, scalaVersion ) + + /** SBT-like dependency builder DSL for syntax compatibility */ + class DependencyBuilder2( groupId: String, artifactId: String, scalaVersion: Option[String] ){ + def %(version: String) = scalaVersion.map( + v => ScalaDependency(groupId, artifactId, version, scalaVersion = v) + ).getOrElse( + JavaDependency(groupId, artifactId, version) + ) } implicit class DependencyBuilder(groupId: String){ - def %%(artifactId: String) = new GroupIdAndArtifactId( groupId, artifactId++"_"++scalaMajorVersion ) - def %(artifactId: String) = new GroupIdAndArtifactId( groupId, artifactId ) + def %%(artifactId: String) = new DependencyBuilder2( groupId, artifactId, Some(scalaMajorVersion) ) + def %(artifactId: String) = new DependencyBuilder2( groupId, artifactId, None ) + } + implicit class DependencyBuilder3(d: JavaDependency){ + def %(classifier: String) = d.copy(classifier = Classifier(Some(classifier))) } final def BuildDependency(path: File) = cbt.BuildDependency( diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index f7b6b78..f400be7 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -28,7 +28,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{ final val updated = build.updated } /* -case class DependencyOr(first: BuildDependency, second: MavenDependency) extends ProjectProxy with BuildDependencyBase{ +case class DependencyOr(first: BuildDependency, second: JavaDependency) extends ProjectProxy with BuildDependencyBase{ val isFirst = new File(first.context.cwd).exists def triggerLoopFiles = if(isFirst) first.triggerLoopFiles else Seq() protected val delegate = if(isFirst) first else second diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala index 4c4e478..2866b7c 100644 --- a/stage2/PackageBuild.scala +++ b/stage2/PackageBuild.scala @@ -1,6 +1,5 @@ package cbt import java.io.File -import java.net.URL import scala.collection.immutable.Seq abstract class PackageBuild(context: Context) extends BasicBuild(context) with ArtifactInfo{ def `package`: Seq[File] = lib.concurrently( enableConcurrency )( diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala new file mode 100644 index 0000000..3f7633b --- /dev/null +++ b/test/simple/build/build.scala @@ -0,0 +1,9 @@ +import cbt._ +import scala.collection.immutable.Seq +import java.io.File +class Build(context: cbt.Context) extends BasicBuild(context){ + override def dependencies = Seq( + ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), + JavaDependency("joda-time", "joda-time", "2.9.2") + ) ++ super.dependencies +} diff --git a/test/test.scala b/test/test.scala index 7c805c9..47bd28b 100644 --- a/test/test.scala +++ b/test/test.scala @@ -67,13 +67,15 @@ object Main{ compile("nothing") usage("multi-build") compile("multi-build") + usage("simple") + compile("simple") { val noContext = Context(cbtHome ++ "/test/nothing", Seq(), logger) val b = new Build(noContext){ override def dependencies = Seq( - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger), - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0")(logger) + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0") ) } val cp = b.classpath -- cgit v1.2.3 From 17e70c8c147fbcca78eb9629bc48915e0d1ec90c Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 12 Mar 2016 19:57:48 -0500 Subject: two separate merges broke this. --- stage1/resolver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'stage1/resolver.scala') diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 39dc0eb..ccb9932 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -68,7 +68,7 @@ abstract class Dependency{ d => // find out latest version of the required dependency val l = d match { - case m: MavenDependency => latest( (m.groupId,m.artifactId) ) + case m: JavaDependency => latest( (m.groupId,m.artifactId) ) case _ => d } // // trigger compilation if not already triggered -- cgit v1.2.3