From f890b14098a588fb4e016afc6932edf89f8ef1f7 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 13:27:24 -0500 Subject: extract re-usable parts of cbt’s own build into Shared plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/build.scala | 26 +++++++++----------------- build/build/build.scala | 7 +++++++ internal/plugins/shared/Shared.scala | 13 +++++++++++++ internal/plugins/shared/build/build.scala | 5 +++++ stage2/BuildBuild.scala | 7 +++++++ 5 files changed, 41 insertions(+), 17 deletions(-) create mode 100644 build/build/build.scala create mode 100644 internal/plugins/shared/Shared.scala create mode 100644 internal/plugins/shared/build/build.scala diff --git a/build/build.scala b/build/build.scala index b4a39ea..4114bc5 100644 --- a/build/build.scala +++ b/build/build.scala @@ -1,6 +1,12 @@ import cbt._ +import cbt_internal._ + +class Build(val context: Context) extends Shared{ + override def name: String = "cbt" + override def version: String = ??? + override def description: String = "Fast, intuitive Build Tool for Scala" + override def inceptionYear: Int = 2015 -class Build(val context: Context) extends Publish{ // FIXME: somehow consolidate this with cbt's own boot-strapping from source. override def dependencies = { super.dependencies ++ Resolver(mavenCentral).bind( @@ -9,22 +15,8 @@ class Build(val context: Context) extends Publish{ ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") ) } + override def sources = Seq( "nailgun_launcher", "stage1", "stage2", "compatibility" - ).map(d => projectDirectory ++ ("/" + d)) - - def groupId: String = "org.cvogt" - - def version: String = "0.9" - override def name: String = "cbt" - - // Members declared in cbt.Publish - def description: String = "Fast, intuitive Build Tool for Scala" - def developers: Seq[cbt.Developer] = Nil - def inceptionYear: Int = 2016 - def licenses: Seq[cbt.License] = Seq( License.Apache2 ) - def organization: Option[cbt.Organization] = None - def scmConnection: String = "" - def scmUrl: String = "" - def url: java.net.URL = new java.net.URL("http://github.com/cvogt/cbt/") + ).map( projectDirectory / _ ).flatMap( _.listFiles ) } diff --git a/build/build/build.scala b/build/build/build.scala new file mode 100644 index 0000000..313b2b5 --- /dev/null +++ b/build/build/build.scala @@ -0,0 +1,7 @@ +package cbt_build.cbt.build +import cbt._ +class Build(val context: Context) extends CbtInternal{ + override def dependencies = ( + super.dependencies :+ cbtInternal.shared + ) +} diff --git a/internal/plugins/shared/Shared.scala b/internal/plugins/shared/Shared.scala new file mode 100644 index 0000000..90bc4b2 --- /dev/null +++ b/internal/plugins/shared/Shared.scala @@ -0,0 +1,13 @@ +package cbt_internal +import cbt._ +import java.net.URL +trait Shared extends SonatypeRelease with SnapshotVersion with GithubPom{ + override def user = "cvogt" + override def groupId = "org.cvogt" + override def organization = Some( Organization( "Jan Christopher Vogt", Some( new URL("http://cvogt.org") ) ) ) + override def licenses = Seq( License.Apache2 ) + override def developers = Seq(cvogt) + override def githubProject = "cbt" + + def cvogt = Developer("cvogt", "Jan Christopher Vogt", "-5", new URL("https://github.com/cvogt/")) +} diff --git a/internal/plugins/shared/build/build.scala b/internal/plugins/shared/build/build.scala new file mode 100644 index 0000000..1d6fc29 --- /dev/null +++ b/internal/plugins/shared/build/build.scala @@ -0,0 +1,5 @@ +package cbt_build.cbt_internal.library_build_plugin +import cbt._ +class Build(val context: Context) extends Plugin{ + override def dependencies = super.dependencies :+ plugins.sonatypeRelease +} diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index c312df9..d778049 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -116,3 +116,10 @@ trait BuildBuildWithoutEssentials extends BaseBuild{ if( c == p ) this else managedBuild.finalBuild( current ) } } + +trait CbtInternal extends BuildBuild{ + protected object cbtInternal{ + def shared = DirectoryDependency(context.cbtHome / "/internal/plugins/shared") + def library = DirectoryDependency(context.cbtHome / "/internal/plugins/library") + } +} -- cgit v1.2.3 From c7a5c5d7e23355ea14e8c095f750b47f308c032f Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 13:28:42 -0500 Subject: remove duplication between simple and simple-fixed test --- test/simple-fixed/Main.scala | 2 -- test/simple-fixed/build/build.scala | 18 ------------------ test/simple/Main.scala | 3 +-- test/simple/build/build.scala | 5 +---- 4 files changed, 2 insertions(+), 26 deletions(-) diff --git a/test/simple-fixed/Main.scala b/test/simple-fixed/Main.scala index 75f9349..54c764c 100644 --- a/test/simple-fixed/Main.scala +++ b/test/simple-fixed/Main.scala @@ -1,6 +1,4 @@ import lib_test.Foo -import org.eclipse.jgit.lib.Ref -import com.spotify.missinglink.ArtifactLoader object Main extends App{ println(Foo.bar) } diff --git a/test/simple-fixed/build/build.scala b/test/simple-fixed/build/build.scala index b46c337..0215c43 100644 --- a/test/simple-fixed/build/build.scala +++ b/test/simple-fixed/build/build.scala @@ -7,23 +7,5 @@ class Build(context: cbt.Context) extends BasicBuild(context){ Seq( GitDependency("https://github.com/cvogt/cbt.git", "f11b8318b85f16843d8cfa0743f64c1576614ad6", Some("test/library-test")) ) - ++ - Resolver(mavenCentral).bind( - ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), - MavenDependency("joda-time", "joda-time", "2.9.2"), - // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties - MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), - // the below tests pom inheritance with variable substitution for pom xml tag contents - MavenDependency("com.spotify", "missinglink-core", "0.1.1") - ) - ++ - Resolver( - mavenCentral, - bintray("tpolecat"), - sonatypeSnapshots - ).bind( - "org.cvogt" %% "play-json-extensions" % "0.8.0", - "ai.x" %% "lens" % "1.0.0" - ) ) } diff --git a/test/simple/Main.scala b/test/simple/Main.scala index 75f9349..c742110 100644 --- a/test/simple/Main.scala +++ b/test/simple/Main.scala @@ -1,6 +1,5 @@ -import lib_test.Foo import org.eclipse.jgit.lib.Ref import com.spotify.missinglink.ArtifactLoader object Main extends App{ - println(Foo.bar) + println("hello, world") } diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index a132dd1..517dd83 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -4,9 +4,6 @@ class Build(val context: cbt.Context) extends BaseBuild{ override def dependencies = ( super.dependencies ++ - Seq( - GitDependency("https://github.com/cvogt/cbt.git", "f11b8318b85f16843d8cfa0743f64c1576614ad6", Some("test/library-test")) - ) ++ // FIXME: make the below less verbose Resolver( mavenCentral ).bind( ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), @@ -38,6 +35,6 @@ class Build(val context: cbt.Context) extends BaseBuild{ "ai.x" %% "lens" % "1.0.0" ) ) - + def printArgs = context.args.mkString(" ") } -- cgit v1.2.3 From 0fcc91ebcd289784811a10c5bc901af644d5fc12 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 16:41:00 -0500 Subject: Make GitDependency and DirectoryDependency simply return Build objects --- .../multi-combined-example/build/build/build.scala | 4 +- stage2/BasicBuild.scala | 2 +- stage2/BuildBuild.scala | 8 +- stage2/BuildDependency.scala | 40 +++---- stage2/GitDependency.scala | 122 ++++++++++----------- 5 files changed, 82 insertions(+), 94 deletions(-) diff --git a/examples/multi-combined-example/build/build/build.scala b/examples/multi-combined-example/build/build/build.scala index b87351d..a5b0201 100644 --- a/examples/multi-combined-example/build/build/build.scala +++ b/examples/multi-combined-example/build/build/build.scala @@ -1,8 +1,8 @@ package cbt_build.cbt_examples.multi_combined_example.build import cbt._ class Build(val context: Context) extends BuildBuild{ - //println(DirectoryDependency( projectDirectory / ".." / "sub4" / "build" ).dependency.exportedClasspath) + //println(DirectoryDependency( projectDirectory / ".." / "sub4" / "build" ).exportedClasspath) override def dependencies: Seq[cbt.Dependency] = - super.dependencies :+ DirectoryDependency( projectDirectory / ".." / "sub4" / "build" ).dependency + super.dependencies :+ DirectoryDependency( projectDirectory / ".." / "sub4" / "build" ) def foo = DirectoryDependency( projectDirectory / ".." / "sub4" / "build" ) } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 0d8017f..68bda15 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -212,7 +212,7 @@ trait BaseBuild extends BuildInterface with DependencyImplementation with Trigge val testDirectory = projectDirectory / "test" if( (testDirectory / lib.buildDirectoryName / lib.buildFileName).exists ){ // FIYME: maybe we can make loadRoot(...).finalBuild an Option some - DirectoryDependency( testDirectory ).dependency + DirectoryDependency( testDirectory ) } else { new BasicBuild( context.copy(workingDirectory = testDirectory) ){ override def dependencies = Seq( diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index d778049..dead081 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -14,7 +14,7 @@ class plugins(implicit context: Context){ context.copy( workingDirectory = context.cbtHome / "plugins" / dir ) - ).dependency + ) final lazy val essentials = plugin( "essentials" ) final lazy val proguard = plugin( "proguard" ) final lazy val sbtLayout = plugin( "sbt_layout" ) @@ -66,9 +66,9 @@ trait BuildBuildWithoutEssentials extends BaseBuild{ // otherwise we'd have to recursively build all versions since // the beginning. Instead CBT always needs to build the pure Java // Launcher in the checkout with itself and then run it via reflection. - val dep = new GitDependency(base, hash, Some("nailgun_launcher")) - val ctx = managedContext.copy( cbtHome = dep.checkout ) - dep.classLoader + val build = GitDependency(base, hash, Some("nailgun_launcher")).asInstanceOf[BaseBuild] + val ctx = managedContext.copy( cbtHome = build.projectDirectory.getParentFile ) + build.classLoader .loadClass( "cbt.NailgunLauncher" ) .getMethod( "getBuild", classOf[AnyRef] ) .invoke( null, ctx ) diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 9a2918a..d7520d8 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -15,43 +15,39 @@ trait TriggerLoop extends DependencyImplementation{ def triggerLoopFiles: Seq[File] } /** You likely want to use the factory method in the BasicBuild class instead of this. */ -final case class DirectoryDependency(context: Context, pathToNestedBuild: String*) extends TriggerLoop{ - def classLoaderCache = context.classLoaderCache - override def toString = show - override def show = this.getClass.getSimpleName ++ "(" ++ context.workingDirectory.string ++ ")" - def moduleKey = this.getClass.getName ++ "("+context.workingDirectory.string+")" - lazy val logger = context.logger - override lazy val lib: Lib = new Lib(logger) - def transientCache = context.transientCache - private lazy val root = lib.loadRoot( context ) - lazy val dependency: Dependency = { +object DirectoryDependency{ + def apply(context: Context, pathToNestedBuild: String*): BuildInterface = { + val lib: Lib = new Lib(context.logger) + // TODO: move this into finalBuild probably - def selectNestedBuild( build: Dependency, names: Seq[String], previous: Seq[String] ): Dependency = { + // TODO: unify this with lib.callReflective + def selectNestedBuild( build: BuildInterface, names: Seq[String], previous: Seq[String] ): BuildInterface = { names.headOption.map{ name => if( lib.taskNames(build.getClass).contains(name) ){ val method = build.getClass.getMethod(name) val returnType = method.getReturnType - if( classOf[Dependency] isAssignableFrom returnType ){ + if( classOf[BuildInterface] isAssignableFrom returnType ){ selectNestedBuild( - method.invoke(build).asInstanceOf[Dependency], + method.invoke(build).asInstanceOf[BuildInterface], names.tail, previous :+ name ) } else { - throw new RuntimeException( s"Expected subtype of Dependency, found $returnType for " + previous.mkString(".") + " in " + show ) + throw new RuntimeException( + s"Expected subtype of BuildInterface, found $returnType for " + previous.mkString(".") + " in " + build + ) } } else { - throw new RuntimeException( (previous :+ name).mkString(".") + " not found in " + show ) + throw new RuntimeException( (previous :+ name).mkString(".") + " not found in " + build ) } }.getOrElse( build ) } - selectNestedBuild( root.finalBuild(context.workingDirectory), pathToNestedBuild, Nil ) + selectNestedBuild( + lib.loadRoot( context ).finalBuild(context.workingDirectory), + pathToNestedBuild, + Nil + ) } - def exportedClasspath = ClassPath() - def dependencies = Seq(dependency) - def triggerLoopFiles = root.triggerLoopFiles - def lastModified = dependency.lastModified - def targetClasspath = ClassPath() } /* case class DependencyOr(first: DirectoryDependency, second: JavaDependency) extends ProjectProxy with DirectoryDependencyBase{ @@ -59,4 +55,4 @@ case class DependencyOr(first: DirectoryDependency, second: JavaDependency) exte def triggerLoopFiles = if(isFirst) first.triggerLoopFiles else Seq() protected val delegate = if(isFirst) first else second } -*/ \ No newline at end of file +*/ diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index f2ac7a6..028401b 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -9,70 +9,68 @@ import org.eclipse.jgit.lib.Ref object GitDependency{ val GitUrl = "(git:|https:|file:/)//([^/]+)/(.+)".r -} -case class GitDependency( - url: String, ref: String, subDirectory: Option[String] = None, // example: git://github.com/cvogt/cbt.git# - pathToNestedBuild: Seq[String] = Seq() -)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{ - import GitDependency._ - override def lib = new Lib(logger) - def classLoaderCache = context.classLoaderCache - def moduleKey = ( - this.getClass.getName - ++ "(" ++ url ++ subDirectory.map("/" ++ _).getOrElse("") ++ "#" ++ ref - ++ ", " - ++ pathToNestedBuild.mkString(", ") - ++ ")" - ) - def transientCache = context.transientCache - // TODO: add support for authentication via ssh and/or https - // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ - private val GitUrl( _, domain, path ) = url + def apply( + url: String, ref: String, subDirectory: Option[String] = None, // example: git://github.com/cvogt/cbt.git# + pathToNestedBuild: Seq[String] = Seq() + )(implicit context: Context ): BuildInterface = { + // TODO: add support for authentication via ssh and/or https + // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ + val GitUrl( _, domain, path ) = url + val credentialsFile = context.workingDirectory ++ "/git.login" + def authenticate(_git: CloneCommand) = + if(!credentialsFile.exists){ + _git + } else { + val (user, password) = { + // TODO: implement safer method than reading credentials from plain text file + val c = new String(readAllBytes(credentialsFile.toPath)).split("\n").head.trim.split(":") + (c(0), c.drop(1).mkString(":")) + } + _git.setCredentialsProvider( new UsernamePasswordCredentialsProvider(user, password) ) + } - private val credentialsFile = context.workingDirectory ++ "/git.login" + val logger = context.logger - private def authenticate(_git: CloneCommand) = - if(!credentialsFile.exists){ - _git - } else { - val (user, password) = { - // TODO: implement safer method than reading credentials from plain text file - val c = new String(readAllBytes(credentialsFile.toPath)).split("\n").head.trim.split(":") - (c(0), c.drop(1).mkString(":")) - } - _git.setCredentialsProvider( new UsernamePasswordCredentialsProvider(user, password) ) - } + def moduleKey = ( + this.getClass.getName + ++ "(" ++ url ++ subDirectory.map("/" ++ _).getOrElse("") ++ "#" ++ ref + ++ ", " + ++ pathToNestedBuild.mkString(", ") + ++ ")" + ) - def checkout: File = taskCache[GitDependency]("checkout").memoize{ - val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" - val _git = if(checkoutDirectory.exists){ - logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") - val _git = new Git(new FileRepository(checkoutDirectory ++ "/.git")) - val actualRef = _git.getRepository.getBranch - if(actualRef != ref){ - logger.git(s"actual ref '$actualRef' does not match expected ref '$ref' - fetching and checking out") - _git.fetch().call() - _git.checkout().setName(ref).call - } - _git - } else { - logger.git(s"Cloning $url into $checkoutDirectory") - val _git = authenticate( - Git - .cloneRepository() - .setURI(url) - .setDirectory(checkoutDirectory) - ).call() + val taskCache = new PerClassCache(context.transientCache, moduleKey)(logger) - logger.git(s"Checking out ref $ref") - _git.checkout().setName(ref).call() - _git + def checkout: File = taskCache[Dependency]("checkout").memoize{ + val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" + val _git = if(checkoutDirectory.exists){ + logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") + val _git = new Git(new FileRepository(checkoutDirectory ++ "/.git")) + val actualRef = _git.getRepository.getBranch + if(actualRef != ref){ + logger.git(s"actual ref '$actualRef' does not match expected ref '$ref' - fetching and checking out") + _git.fetch().call() + _git.checkout().setName(ref).call + } + _git + } else { + logger.git(s"Cloning $url into $checkoutDirectory") + val _git = authenticate( + Git + .cloneRepository() + .setURI(url) + .setDirectory(checkoutDirectory) + ).call() + + logger.git(s"Checking out ref $ref") + _git.checkout().setName(ref).call() + _git + } + val actualRef = _git.getRepository.getBranch + assert( actualRef == ref, s"actual ref '$actualRef' does not match expected ref '$ref'") + checkoutDirectory } - val actualRef = _git.getRepository.getBranch - assert( actualRef == ref, s"actual ref '$actualRef' does not match expected ref '$ref'") - checkoutDirectory - } - def dependency = taskCache[GitDependency]("dependency").memoize{ + DirectoryDependency( context.copy( workingDirectory = checkout ++ subDirectory.map("/" ++ _).getOrElse("") @@ -80,10 +78,4 @@ case class GitDependency( pathToNestedBuild: _* ) } - - def dependencies = Seq(dependency) - - def exportedClasspath = ClassPath() - private[cbt] def targetClasspath = exportedClasspath - def lastModified: Long = dependency.lastModified } -- cgit v1.2.3 From 1346ab59395e262486d42a24406ef9d942c90962 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 16:47:42 -0500 Subject: ability to only enable slow tests on demand (e.g. on circle) --- circle.yml | 4 ++-- test/test.scala | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/circle.yml b/circle.yml index 0a6f9d5..92b8983 100644 --- a/circle.yml +++ b/circle.yml @@ -13,7 +13,7 @@ dependencies: test: override: - rm ~/.gitconfig # avoid url replacement breaking jgit - - ./cbt direct test.run: + - ./cbt direct test.run slow: timeout: 1800 - - ./cbt test.run: + - ./cbt test.run slow: timeout: 1800 diff --git a/test/test.scala b/test/test.scala index 59a85ac..45315ce 100644 --- a/test/test.scala +++ b/test/test.scala @@ -15,10 +15,20 @@ object Main{ val lib = new Lib(logger) val cbtHome = new File(System.getenv("CBT_HOME")) + + val slow = ( + System.getenv("CIRCLECI") != null // enable only on circle + || args.args.contains("slow") + ) + val compat = !args.args.contains("no-compat") + + if(!slow) System.err.println( "Skipping slow tests" ) + if(!compat) System.err.println( "Skipping cbt version compatibility tests" ) + var successes = 0 var failures = 0 def assertException[T:scala.reflect.ClassTag](msg: String = "")(code: => Unit)(implicit logger: Logger) = { - try{ + try{ code assert(false, msg) }catch{ case _:AssertionError => } @@ -187,9 +197,11 @@ object Main{ usage("simple") compile("simple") clean("simple") - usage("simple-fixed") - compile("simple-fixed") - + if( compat ){ + usage("simple-fixed") + compile("simple-fixed") + } + compile("../plugins/sbt_layout") compile("../plugins/scalafmt") compile("../plugins/scalajs") @@ -200,8 +212,12 @@ object Main{ compile("../examples/scalafmt-example") compile("../examples/scalariform-example") compile("../examples/scalatest-example") - compile("../examples/scalajs-react-example/js") - compile("../examples/scalajs-react-example/jvm") + if(slow){ + compile("../examples/scalajs-react-example/js") + compile("../examples/scalajs-react-example/jvm") + compile("../examples/scalajs-plain-example/js") + compile("../examples/scalajs-plain-example/jvm") + } compile("../examples/multi-standalone-example") compile("../examples/multi-combined-example") if(sys.props("java.version").startsWith("1.7")){ @@ -209,19 +225,23 @@ object Main{ } else { compile("../examples/dotty-example") task("run","../examples/dotty-example") - task("dottydoc","../examples/dotty-example") + if(slow){ + task("dottydoc","../examples/dotty-example") + } + } + if(slow){ + task("compile","../examples/scalajs-react-example/js") + task("fullOpt.compile","../examples/scalajs-react-example/js") } - task("compile","../examples/scalajs-react-example/js") - task("fullOpt.compile","../examples/scalajs-react-example/js") compile("../examples/uber-jar-example") - - { + + if( compat ){ val res = task("docJar","simple-fixed-cbt") assert( res.out endsWith "simple-fixed-cbt_2.11-0.1-javadoc.jar\n", res.out ) assert( res.err contains "model contains", res.err ) assert( res.err endsWith "documentable templates\n", res.err ) } - + { val res = runCbt("simple", Seq("printArgs","1","2","3")) assert(res.exit0) -- cgit v1.2.3 From 163bbc8f1d78750196b30a46a888e7e65ace5454 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 16:50:27 -0500 Subject: add developer documentation about cbt version compatibility --- doc/cbt-developer/version-compatibility.md | 50 +++++++++++ doc/design.md | 139 +++++++++++++++++++++++++++++ doc/design.txt | 139 ----------------------------- 3 files changed, 189 insertions(+), 139 deletions(-) create mode 100644 doc/cbt-developer/version-compatibility.md create mode 100644 doc/design.md delete mode 100644 doc/design.txt diff --git a/doc/cbt-developer/version-compatibility.md b/doc/cbt-developer/version-compatibility.md new file mode 100644 index 0000000..986bb37 --- /dev/null +++ b/doc/cbt-developer/version-compatibility.md @@ -0,0 +1,50 @@ +## Why does CBT version compatiblity matter? +CBT allows fixing the version used using the `// cbt: ` annotation. +This means that the CBT version you installed will download the +CBT version referenced there and use that instead. CBT also +allows composing builds, which require different CBT versions. +In order for all of this to work, different CBT versions need to +be able to talk to one another. + +This chapter is about how the current solution works and about +the pitfalls involved which can make a new version of CBT +incompatible with older versions. + +## How can compatibility be broken? +The current solution mostly relies on the Java interfaces in +the `compability/` folder. Changing the Java interface in a +non-backwards-compatible way means making the CBT incompatible with +older versions. Java 8 default methods make this a whole lot easier +and this is the main reason CBT relies on Java 8. But we also +want to keep this interfaces as small as possible in order to +minimize the risk. + +However there are more things that can break compatibility when changed: +- the format of the `// cbt: ` version string +- the name and format of Build classes (or files) that CBT looks for +- communication between versions via reflection in particular + - how the TrapSecurityManager of each CBT version talks to the + installed TrapSecurityManager via reflection + +## How to detect accidental breakages? + +CBT's tests have a few tests with `// cbt: ` annotations and some +reference libraries as Git dependencies that use these annotations. +Many incompatibilities will lead to these tests failing, either with +a compilation error against the `compatiblity/` interfaces or with +a runtime exception or unexpected behavior in case other things are +broken. + +## How an we improve the situation in the long run? + +In the long run we should think about how to reduce the risk +or sometimes even unavoidability of incompatibilities. +The unavoidability mostly stems from limitations of what Java +interfaces can express. However Java 8 interfaces mostly work +fairly well. We should consider using them for the cases that +currently use reflection instead of reflection. + +If Java 8 interfaces still turn out to be a problem in the long run, +we could consider an interface that we control completely, e.g. +an internal serialization formation, which CBT versions use to talk +to each other. diff --git a/doc/design.md b/doc/design.md new file mode 100644 index 0000000..28bbdaa --- /dev/null +++ b/doc/design.md @@ -0,0 +1,139 @@ +# Design + +This chapter explains some of CBT's less obvious design decisions and +the reasons behind them. Hopefully this answers some of the questions +often unanswered when trying to understand a new tool. + + +## Why does CBT use inheritance? + +First of all CBT uses classes, because needs some place to put tasks and +name them. Methods in classes are one way to do it. Methods calling each +other allows to effectively build a graph of dependent tasks. + +Inheritance on top of that allows patching this graph to insert additional +steps, e.g. formatting the code before compiling it. Patching of the task +graph is what you do with sbt andso with CBT via inheritance/overrides. + +This was also discussed in gitter here: https://gitter.im/cvogt/cbt?at=58a95663de50490822e869e5 + +Taking a graph an contrinuously patching it can be confusing, which is +why inheritance is confusing. You can even build non-terminating call +cycles. CBT's logic is not coupled to the inheritance layer. You can +write CBT builds without inheritance. They require a bit more code, but +may end up easier to maintain and understand. + +## Task composition and aborting failed tasks and dependents + +In CBT build tasks are methods. Task can depend on each +other by invoking each other. E.g. `package` is a method that invokes `compile`. +Build tasks can fail. By convention, if a task fails it is expected to throw +an Exception in order to also abort the execution of dependent tasks. When +`compile` finds compile errors it throws an exception and thereby also +drops out of the `package` execution pth. CBT catches and handles the +exception and returns control back to the user. + +This design was chosen for simplicity and because build code lives in a +world with exceptions anyways as a lot of it is directly or indirectly +using java libraries. We might reconsider this design at some point. Or not. +The hope is that Exceptions are without major drawbacks and more +approachable to newcomers than monadic error handling, which would require +wrapper types and value composition via .map and for-expressions. Not using +a Monad however also means that CBT cannot reason about task composition +automatically, which means parallel task execution is not automatic, but +manual and opt-in where wanted. + +## Why do CBT plugins use case classes where methods seem simpler? + +In CBT plugins you may see + +``` +trait SomePlugin{ + def doSomething = SomePluginLib(...context).doSomething( a, b, ... ) +} +case class SomePluginLib(...context...)(implicit ...more context...){ + case class doSomething(a: A, b: B, ...){ + def apply = { + // logic + } + } +} +``` + +Why this? This seems much simpler: + +``` +trait SomePlugin{ + def doSomething = SomePluginLib.doSomething( a, b, ... )(...context) +} +object SomePluginLib + def doSomething(a: A, b: B, ...)(...context...)(implicit ...more context...) = { + // logic + } +} +``` + +The reason is that the former allows easy patching while +the second does not. Let's pretend your build is this: +``` +class Build(val context: Context) extends SomePlugin{ + override def doSomething = super.doSomething.copy( b = someOtherB ) +} +``` +Such a simple replacement of `b` while keeping all other arguments would +not be easily possible if doSomething was a def not a case class. + +## What is newBuild and why do we need it? + +Methods in a class can call each other and thereby effectively form a graph. +Subclasses can patch this graph. Sometimes you want to create several +different variants of a graph. Let's say one building artifacts with a +-SNAPSHOT version and another one building a stable release. Or one with +scalaVersion 2.10 and one with scalaVersion 2.11. Or one optmizing your +code and one not optimizing it. You can subclass your build once for +each alternative. That's fine if you know which class to subclass, but +plugins for your build do not know what your concrete build class is +and cannot produce subclasses for it. Example + +``` +class Build(val context: Context){ + // this works fine + def with210 = new Build(context){ + override def scalaVersion = "2.10" + } + def with211 = new Build(context){ + override def scalaVersion = "2.11" + } +} +``` +Imagine however +``` +// defined by you: +class Build(val context: Context) extends CrossVersionPlugin +// defined by the plugin author: +trait CrossVersionPlugin{ + // this does not compile, Build does not exists when the plugin is compiled + def with210 = new Build(context){ + override def scalaVersion = "2.10" + } + def with211 = new Build(context){ + override def scalaVersion = "2.11" + } +} +``` +Luckily CBT can cheat because it has a compiler available. +``` +// defined by the plugin author: +trait CrossVersionPlugin{ + // method `newBuild` generates a new subclass of Build at runtime and creates an instance. + def with210 = newBuild{""" + override def scalaVersion = "2.10" + """} + def with211 = newBuild{""" + override def scalaVersion = "2.11" + """} +} +``` + +Problem solved. In fact this allows for a very, very flexible way of +creating differents variants of your build. diff --git a/doc/design.txt b/doc/design.txt deleted file mode 100644 index 28bbdaa..0000000 --- a/doc/design.txt +++ /dev/null @@ -1,139 +0,0 @@ -# Design - -This chapter explains some of CBT's less obvious design decisions and -the reasons behind them. Hopefully this answers some of the questions -often unanswered when trying to understand a new tool. - - -## Why does CBT use inheritance? - -First of all CBT uses classes, because needs some place to put tasks and -name them. Methods in classes are one way to do it. Methods calling each -other allows to effectively build a graph of dependent tasks. - -Inheritance on top of that allows patching this graph to insert additional -steps, e.g. formatting the code before compiling it. Patching of the task -graph is what you do with sbt andso with CBT via inheritance/overrides. - -This was also discussed in gitter here: https://gitter.im/cvogt/cbt?at=58a95663de50490822e869e5 - -Taking a graph an contrinuously patching it can be confusing, which is -why inheritance is confusing. You can even build non-terminating call -cycles. CBT's logic is not coupled to the inheritance layer. You can -write CBT builds without inheritance. They require a bit more code, but -may end up easier to maintain and understand. - -## Task composition and aborting failed tasks and dependents - -In CBT build tasks are methods. Task can depend on each -other by invoking each other. E.g. `package` is a method that invokes `compile`. -Build tasks can fail. By convention, if a task fails it is expected to throw -an Exception in order to also abort the execution of dependent tasks. When -`compile` finds compile errors it throws an exception and thereby also -drops out of the `package` execution pth. CBT catches and handles the -exception and returns control back to the user. - -This design was chosen for simplicity and because build code lives in a -world with exceptions anyways as a lot of it is directly or indirectly -using java libraries. We might reconsider this design at some point. Or not. -The hope is that Exceptions are without major drawbacks and more -approachable to newcomers than monadic error handling, which would require -wrapper types and value composition via .map and for-expressions. Not using -a Monad however also means that CBT cannot reason about task composition -automatically, which means parallel task execution is not automatic, but -manual and opt-in where wanted. - -## Why do CBT plugins use case classes where methods seem simpler? - -In CBT plugins you may see - -``` -trait SomePlugin{ - def doSomething = SomePluginLib(...context).doSomething( a, b, ... ) -} -case class SomePluginLib(...context...)(implicit ...more context...){ - case class doSomething(a: A, b: B, ...){ - def apply = { - // logic - } - } -} -``` - -Why this? This seems much simpler: - -``` -trait SomePlugin{ - def doSomething = SomePluginLib.doSomething( a, b, ... )(...context) -} -object SomePluginLib - def doSomething(a: A, b: B, ...)(...context...)(implicit ...more context...) = { - // logic - } -} -``` - -The reason is that the former allows easy patching while -the second does not. Let's pretend your build is this: -``` -class Build(val context: Context) extends SomePlugin{ - override def doSomething = super.doSomething.copy( b = someOtherB ) -} -``` -Such a simple replacement of `b` while keeping all other arguments would -not be easily possible if doSomething was a def not a case class. - -## What is newBuild and why do we need it? - -Methods in a class can call each other and thereby effectively form a graph. -Subclasses can patch this graph. Sometimes you want to create several -different variants of a graph. Let's say one building artifacts with a --SNAPSHOT version and another one building a stable release. Or one with -scalaVersion 2.10 and one with scalaVersion 2.11. Or one optmizing your -code and one not optimizing it. You can subclass your build once for -each alternative. That's fine if you know which class to subclass, but -plugins for your build do not know what your concrete build class is -and cannot produce subclasses for it. Example - -``` -class Build(val context: Context){ - // this works fine - def with210 = new Build(context){ - override def scalaVersion = "2.10" - } - def with211 = new Build(context){ - override def scalaVersion = "2.11" - } -} -``` -Imagine however -``` -// defined by you: -class Build(val context: Context) extends CrossVersionPlugin -// defined by the plugin author: -trait CrossVersionPlugin{ - // this does not compile, Build does not exists when the plugin is compiled - def with210 = new Build(context){ - override def scalaVersion = "2.10" - } - def with211 = new Build(context){ - override def scalaVersion = "2.11" - } -} -``` -Luckily CBT can cheat because it has a compiler available. -``` -// defined by the plugin author: -trait CrossVersionPlugin{ - // method `newBuild` generates a new subclass of Build at runtime and creates an instance. - def with210 = newBuild{""" - override def scalaVersion = "2.10" - """} - def with211 = newBuild{""" - override def scalaVersion = "2.11" - """} -} -``` - -Problem solved. In fact this allows for a very, very flexible way of -creating differents variants of your build. -- cgit v1.2.3 From 3f68499ec4768e2ae1bfe2e390ba66a90b190fc0 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 17:01:30 -0500 Subject: Document CBT plugin style guide and adjust Scalafmt plugin accordingly --- doc/plugin-author-guide.md | 105 +++++++++++++++++++++++++++++++++++ libraries/proguard/build/build.scala | 2 +- plugins/scalafmt/Scalafmt.scala | 21 +++---- 3 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 doc/plugin-author-guide.md diff --git a/doc/plugin-author-guide.md b/doc/plugin-author-guide.md new file mode 100644 index 0000000..db0eede --- /dev/null +++ b/doc/plugin-author-guide.md @@ -0,0 +1,105 @@ +## How to write an idiomatic CBT plugin? + +Write a small library that could be fully used outside of CBT's +build classes and handles all the use cases you need. For example + +``` +object MyLibrary{ + def doSomething( ... ) = // do something here +} +``` + +Publish it as a library and you might be done right here. + +If your library requires configuration information commonly found +in your build, like the sourceFiles, groupId, scalaVersion or else, +consider offering a mixin trait, that pre-configures your library +for user convenience. (Consider publishing them separately if that +allows people to use your library outside of CBT with fewer +dependencies.) Here is an example of a library with an +accompanying mixin trait configuring the library for CBT. + +``` +package my.library +object MyLibrary{ + case class DoSomething( scalaVersion: String, ... ){ + case class config( targetFile: File, affectBehavior: Boolean = true, ... ){ + def apply = // really do something here + } + } +} + +package my.plugin +trait MyLibrary extends BaseBuild{ + def doSomething = MyLibrary.DoSomething(scalaVersion, ...).config( scalaTarget / "my.file" ) +} +``` + +* Note: Do not override any common method like `compile` or `test` in a public plugin. * +* Instead document recommendations where users should hook in your custom methods. * +* This will help users understand their own builds. * + +See how we only define a single `def doSomething` in the MyLibrary +trait? We did not define things like `def doSomethingTargetFile`. +Instead we have a case class defined in the library which a user +can .copy as needed to customize configuration. A user build could +look like this: + +``` +class Build(val context: Context) extends MyLibrary{ + override def doSomething = super.doSomething.copy( + affectBehavior = false + ) +} +``` + +As you can see a user can use .copy to override default behavior +of your library. + +This nesting allows us to keep the global namespace small, which +helps us lower the risk of global name clashes between different +libraries. It also makes it clearer that `affectBehavior` is +something specific to `MyLibary` which should help making builds +easier to understand. Further this nesting means we don't need to +encode namespaces in the names themselves, but use Scala's language +features for that, which allows us to keep names nice and concise. + +You might wonder why there is a case class `DoSomething` rather +than `scalaVersion` just being another parameter in case class +`config`. Nesting case classes like this is a pattern that given +the way we use them allows us to make it slightly harder to +modify some parameters (the ones on the outer case class) than +others (the ones on the inner case class). Why? Some are likely +to need user customization, others are likely to break stuff if +they are touched. Example: The scalaVersion is probably something +you want to configure once consistently across your entire build. +Otherwise you might end up accidentally packaging scala-2.11 +compiled class files as a jar with a `_2.10` artifact id. +Changing which targetFile `doSomething` writes to however is +something you should be able to safely change. Since we defined +`doSomething` as +`def doSomething = MyLibrary.DoSomething(...).config( ... )` +overriding behavior in user code with .copy only affects +the inner case class because super.doSomething is an instance +of that: +``` + override def doSomething = super.doSomething.copy( + affectBehavior = false + ) +``` +Overriding things in class `DoSomething` is possible by creating +an entire new instance of the outer one, but slightly harder +preventing users from accidentally doing the wrong thing. + +Obviously this decisions what's dangerous to override and what +is not can be a judgment call and not 100% clear. + +A few more conventions for more uniform plugin designs: +If you only have one outer case class in yur plugin `object`, +call the case class `apply` instead of `DoSomething`. If you +need multiple (because you basically have several commands, +each with their own private configuration), give it a name +representing the operation, e.g. `compile` or `doc`. If there +is only one inner case class inside of anothe case class, +call it `config`, give it a name representing the operation, +e.g. `compile` or `doc`. diff --git a/libraries/proguard/build/build.scala b/libraries/proguard/build/build.scala index c781ce2..3ca38b5 100644 --- a/libraries/proguard/build/build.scala +++ b/libraries/proguard/build/build.scala @@ -15,7 +15,7 @@ class Build(val context: Context) extends Scalafmt{ } override def scalafmt = super.scalafmt.copy( - config = super.scalafmt.lib.cbtRecommendedConfig, + config = Scalafmt.cbtRecommendedConfig, whiteSpaceInParenthesis = true ) diff --git a/plugins/scalafmt/Scalafmt.scala b/plugins/scalafmt/Scalafmt.scala index 9d42cbd..5535964 100644 --- a/plugins/scalafmt/Scalafmt.scala +++ b/plugins/scalafmt/Scalafmt.scala @@ -12,16 +12,15 @@ import java.nio.file._ trait Scalafmt extends BaseBuild { /** Reformat scala source code according to `scalafmtConfig` rules */ def scalafmt = { - val scalafmtLib = new ScalafmtLib(lib) - scalafmtLib.format( sourceFiles ).config( - scalafmtLib.loadConfig( + Scalafmt.apply( lib, sourceFiles.filter(_.string endsWith ".scala") ).config( + Scalafmt.loadConfig( projectDirectory.toPath ) getOrElse ScalafmtConfig.default ) } } -class ScalafmtLib(lib: Lib){ scalafmtLib => +object Scalafmt{ def userHome = Option( System.getProperty("user.home") ).map(Paths.get(_)) /** Tries to load config from .scalafmt.conf in given directory or fallback directory */ @@ -34,22 +33,18 @@ class ScalafmtLib(lib: Lib){ scalafmtLib => .flatMap ( file => StyleCache.getStyleForFile(file.toString) ) } - case class format(files: Seq[File]){ - /** - * @param whiteSpaceInParenthesis more of a hack to make up for missing support in Scalafmt. Does not respect alignment and maxColumn. - */ + case class apply( lib: Lib, files: Seq[File] ){ + /** @param whiteSpaceInParenthesis more of a hack to make up for missing support in Scalafmt. Does not respect alignment and maxColumn. */ case class config( - config: ScalafmtConfig, - whiteSpaceInParenthesis: Boolean = false + config: ScalafmtConfig, whiteSpaceInParenthesis: Boolean = false ) extends (() => Seq[File]){ - def lib = scalafmtLib def apply = { - val (successes, errors) = scalafmtLib.lib.transformFilesOrError( + val (successes, errors) = lib.transformFilesOrError( files, org.scalafmt.Scalafmt.format(_, config) match { case Formatted.Success(formatted) => Right( if( whiteSpaceInParenthesis ){ - scalafmtLib.whiteSpaceInParenthesis(formatted) + Scalafmt.whiteSpaceInParenthesis(formatted) } else formatted ) case Formatted.Failure( e ) => Left( e ) -- cgit v1.2.3 From f0a16297ffe7719df9a7ba5dc7f75182d6371852 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 18:00:39 -0500 Subject: build package names that are easier to distinguish in stack traces --- stage2/Scaffold.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stage2/Scaffold.scala b/stage2/Scaffold.scala index 4420866..30ea73b 100644 --- a/stage2/Scaffold.scala +++ b/stage2/Scaffold.scala @@ -43,6 +43,11 @@ class Scaffold( logger: Logger ){ ) } + private[cbt] def buildPackageFromDirectory(directory: File) = { + val parts = packageFromDirectory(directory).split("\\.") + ((parts.head ++ "_build") +: parts.tail).mkString(".") + } + def createMain( projectDirectory: File ): Unit = { @@ -59,7 +64,7 @@ object Main{ def createBuild( projectDirectory: File ): Unit = { - createFile(projectDirectory, lib.buildDirectoryName++"/"++lib.buildFileName, s"""package cbt_build.${packageFromDirectory(projectDirectory)} + createFile(projectDirectory, lib.buildDirectoryName++"/"++lib.buildFileName, s"""package ${buildPackageFromDirectory(projectDirectory)} import cbt._ class Build(val context: Context) extends BaseBuild{ override def dependencies = ( -- cgit v1.2.3 From be9e36b89fd0a266010aa98fa4a3ff9d8572fce9 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 18:01:52 -0500 Subject: Google Java Format plugin --- plugins/google-java-format/GoogleJavaFormat.scala | 34 +++++++++++++++++++++++ plugins/google-java-format/Immutable.java | 5 ++++ plugins/google-java-format/build/build.scala | 9 ++++++ stage2/BuildBuild.scala | 1 + 4 files changed, 49 insertions(+) create mode 100644 plugins/google-java-format/GoogleJavaFormat.scala create mode 100644 plugins/google-java-format/Immutable.java create mode 100644 plugins/google-java-format/build/build.scala diff --git a/plugins/google-java-format/GoogleJavaFormat.scala b/plugins/google-java-format/GoogleJavaFormat.scala new file mode 100644 index 0000000..cbccb94 --- /dev/null +++ b/plugins/google-java-format/GoogleJavaFormat.scala @@ -0,0 +1,34 @@ +package cbt + +import java.io.File +import java.nio.file.Files._ +import java.nio.file._ + +import com.google.googlejavaformat.java._ + +trait GoogleJavaFormat extends BaseBuild { + def googleJavaFormat() = GoogleJavaFormat.apply( lib, sourceFiles.filter(_.string endsWith ".java") ).format +} + +object GoogleJavaFormat{ + case class apply( lib: Lib, files: Seq[File] ){ + /** @param whiteSpaceInParenthesis more of a hack to make up for missing support in Scalafmt. Does not respect alignment and maxColumn. */ + def format = { + val (successes, errors) = lib.transformFilesOrError( files, in => + try{ + Right( new Formatter().formatSource(in) ) + } catch { + case e: FormatterException => Left( e ) + } + ) + if(errors.nonEmpty) + throw new RuntimeException( + "Google Java Format failed to parse some files:\n" ++ errors.map{ + case (file, error) => file.string ++ ":" ++ error.toString + }.mkString("\n"), + errors.head._2 + ) + successes + } + } +} diff --git a/plugins/google-java-format/Immutable.java b/plugins/google-java-format/Immutable.java new file mode 100644 index 0000000..5b3ff44 --- /dev/null +++ b/plugins/google-java-format/Immutable.java @@ -0,0 +1,5 @@ +package com.google.errorprone.annotations; +// to suppress warning +// "Class com.google.errorprone.annotations.Immutable not found - continuing with a stub." +// there probably is a better solution +public class Immutable{} diff --git a/plugins/google-java-format/build/build.scala b/plugins/google-java-format/build/build.scala new file mode 100644 index 0000000..50bc423 --- /dev/null +++ b/plugins/google-java-format/build/build.scala @@ -0,0 +1,9 @@ +import cbt._ + +class Build(val context: Context) extends Plugin { + override def dependencies = + super.dependencies ++ + Resolver( mavenCentral ).bind( + MavenDependency( "com.google.googlejavaformat", "google-java-format", "1.3" ) + ) +} diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index dead081..0c2f95c 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -16,6 +16,7 @@ class plugins(implicit context: Context){ ) ) final lazy val essentials = plugin( "essentials" ) + final lazy val googleJavaFormat = plugin( "google-java-format" ) final lazy val proguard = plugin( "proguard" ) final lazy val sbtLayout = plugin( "sbt_layout" ) final lazy val scalafmt = plugin( "scalafmt" ) -- cgit v1.2.3 From cf859909fbcf577bac92d9f35f5d11e638a66177 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 11 Mar 2017 18:02:17 -0500 Subject: fix broken format string --- stage2/BuildBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index 0c2f95c..9d3458d 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -32,7 +32,7 @@ trait BuildBuildWithoutEssentials extends BaseBuild{ assert( projectDirectory.getName === lib.buildDirectoryName, - "You can't extend ${lib.buildBuildClassName} in: " + projectDirectory + "/" + lib.buildDirectoryName + s"You can't extend ${lib.buildBuildClassName} in: " + projectDirectory + "/" + lib.buildDirectoryName ) protected final val managedContext = context.copy( -- cgit v1.2.3